From 150b3730ef592c761bd7064f0d09e4ea4d5a0df8 Mon Sep 17 00:00:00 2001 From: StellaOps Bot Date: Mon, 24 Nov 2025 07:52:25 +0200 Subject: [PATCH] up --- .gitea/workflows/attestation-bundle.yml | 26 + .gitea/workflows/cli-build.yml | 45 ++ .gitea/workflows/containers-multiarch.yml | 86 +++ .gitignore | 8 +- docs/airgap/bootstrap.md | 33 + docs/airgap/console-airgap-tasks.md | 6 + docs/airgap/degradation-matrix.md | 19 + docs/airgap/devportal-offline.md | 23 + docs/airgap/mirror-bundles.md | 28 + docs/airgap/operations.md | 34 + docs/airgap/overview.md | 32 + docs/airgap/portable-evidence.md | 105 +-- docs/airgap/sealing-and-egress.md | 30 + docs/api/concelier/concelier-lnm.yaml | 1 + docs/api/notify-openapi.yaml | 112 ++++ docs/cli/sbomer.md | 40 ++ docs/console/airgap.md | 27 + docs/console/attestor-ui.md | 8 + docs/dev/airgap-contracts.md | 23 + .../SPRINT_0111_0001_0001_advisoryai.md | 7 +- .../SPRINT_0114_0001_0003_concelier_iii.md | 7 +- .../SPRINT_0119_0001_0001_excititor_i.md | 1 + .../SPRINT_0119_0001_0002_excititor_ii.md | 6 +- .../SPRINT_0119_0001_0003_excititor_iii.md | 6 +- .../SPRINT_0119_0001_0004_excititor_iv.md | 26 +- .../SPRINT_0119_0001_0005_excititor_v.md | 9 +- .../SPRINT_0119_0001_0006_excititor_vi.md | 11 +- docs/implplan/SPRINT_0125_0001_0001_mirror.md | 12 +- .../SPRINT_0125_0001_0001_policy_reasoning.md | 35 +- .../SPRINT_0142_0001_0001_sbomservice.md | 28 +- .../SPRINT_0172_0001_0002_notifier_ii.md | 11 +- docs/implplan/SPRINT_0201_0001_0001_cli_i.md | 15 +- ..._0001_reachability_runtime_static_union.md | 5 +- docs/implplan/SPRINT_301_docs_tasks_md_i.md | 25 +- docs/implplan/SPRINT_302_docs_tasks_md_ii.md | 24 - docs/implplan/SPRINT_503_ops_devops_i.md | 2 + docs/implplan/SPRINT_504_ops_devops_ii.log.md | 9 + docs/implplan/SPRINT_504_ops_devops_ii.md | 12 +- docs/implplan/SPRINT_505_ops_devops_iii.md | 7 +- docs/implplan/SPRINT_506_ops_devops_iv.md | 4 +- docs/implplan/SPRINT_511_api.md | 3 +- .../archived/SPRINT_302_docs_tasks_md_ii.md | 29 + docs/modules/attestor/keys-and-issuers.md | 6 + docs/modules/attestor/overview.md | 9 + docs/modules/attestor/payloads.md | 67 +- docs/modules/attestor/policies.md | 12 + docs/modules/attestor/transparency.md | 6 + docs/modules/attestor/workflows.md | 252 +------ docs/modules/cli/guides/airgap.md | 49 ++ docs/modules/cli/guides/attest.md | 25 + .../observability/locker-manifest.md | 39 ++ .../observability/timeline-events.md | 43 ++ .../excititor/operations/observability.md | 18 + docs/modules/excititor/vex_linksets_api.md | 27 +- docs/modules/mirror/signing-runbook.md | 4 +- docs/modules/policy/architecture.md | 13 +- .../samples/advisory-ai-knobs@draft.json | 29 + .../samples/orchestrator-job@draft.json | 16 + .../samples/policy-batch-context@draft.json | 11 + .../policy/samples/policy-conflict@draft.json | 32 + .../samples/policy-ledger-export@draft.json | 36 + .../policy/samples/policy-snapshot@draft.json | 30 + .../samples/policy-violation-event@draft.json | 13 + .../samples/policy-worker-result@draft.json | 11 + .../policy/samples/severity-fusion@draft.json | 12 + .../policy/samples/trust-weighting@draft.json | 23 + .../schemas/advisory-ai-knobs@draft.json | 27 + .../schemas/orchestrator-job@draft.json | 51 ++ .../schemas/policy-batch-context@draft.json | 41 ++ .../policy/schemas/policy-conflict@draft.json | 17 + .../schemas/policy-ledger-export@draft.json | 40 ++ .../policy/schemas/policy-snapshot@draft.json | 33 + .../schemas/policy-violation-event@draft.json | 32 + .../schemas/policy-worker-result@draft.json | 29 + .../policy/schemas/severity-fusion@draft.json | 30 + .../policy/schemas/trust-weighting@draft.json | 25 + docs/modules/sbomservice/architecture.md | 11 +- .../fixtures/lnm-v1/projections.json | 2 +- docs/security/trust-and-signing.md | 33 + scripts/api-changelog.mjs | 39 ++ scripts/attest/build-attestation-bundle.sh | 63 ++ scripts/buildx/build-airgap-bundle.sh | 43 ++ scripts/buildx/build-multiarch.sh | 93 +++ scripts/cli/build-cli.sh | 82 +++ src/Api/StellaOps.Api.OpenApi/CHANGELOG.md | 2 +- .../StellaOps.Cli/Commands/CommandFactory.cs | 70 ++ .../StellaOps.Cli/Commands/CommandHandlers.cs | 14 +- src/Cli/StellaOps.Cli/TASKS.md | 4 +- .../Commands/CommandHandlersTests.cs | 211 +++++- .../Contracts/LnmLinksetContracts.cs | 1 + .../StellaOps.Concelier.WebService/Program.cs | 37 +- .../openapi/concelier-lnm.yaml | 1 + .../Linksets/AdvisoryLinkset.cs | 4 + .../Linksets/AdvisoryLinksetNormalization.cs | 38 +- .../Linksets/AdvisoryLinksetDocument.cs | 5 + .../Linksets/ConcelierMongoLinksetStore.cs | 2 + .../EnsureLinkNotMergeCollectionsMigration.cs | 1 + .../AdvisoryLinksetQueryServiceTests.cs | 6 +- .../Linksets/PolicyAuthSignalFactoryTests.cs | 1 + .../AdvisorySummaryMapperTests.cs | 1 + src/Directory.Build.props | 34 +- .../Contracts/GraphOverlayContracts.cs | 27 + .../Controllers/GraphController.cs | 58 -- .../Graph/GraphOverlayFactory.cs | 125 ++++ .../Program.Helpers.cs | 31 + .../StellaOps.Excititor.WebService/Program.cs | 63 +- .../GraphOverlayFactoryTests.cs | 108 +++ ...tellaOps.Excititor.WebService.Tests.csproj | 1 + .../StellaOps.Mirror.Creator/make-thin-v1.sh | 6 +- .../OpenApiEndpointTests.cs | 32 +- .../StellaOps.Notifier.Tests.csproj | 7 +- .../Support/NotifierApplicationFactory.cs | 35 + .../TestContent/openapi/notify-openapi.yaml | 613 ++++++++++++++++++ .../Contracts/PackApprovalAckRequest.cs | 9 + .../StellaOps.Notifier.WebService/Program.cs | 44 ++ .../Setup/OpenApiDocumentCache.cs | 4 +- .../openapi/notify-openapi.yaml | 112 ++++ .../pack-approval-templates.json | 71 ++ .../AdvisoryAI/AdvisoryAiKnobsModels.cs | 15 + .../AdvisoryAI/AdvisoryAiKnobsService.cs | 73 +++ .../BatchContext/BatchContextModels.cs | 31 + .../BatchContext/BatchContextService.cs | 82 +++ .../Endpoints/AdvisoryAiKnobsEndpoint.cs | 37 ++ .../Endpoints/BatchContextEndpoint.cs | 30 + .../Endpoints/LedgerExportEndpoint.cs | 43 ++ .../Endpoints/OrchestratorJobEndpoint.cs | 62 ++ .../Endpoints/PathScopeSimulationEndpoint.cs | 15 + .../Endpoints/PolicyWorkerEndpoint.cs | 43 ++ .../Endpoints/SnapshotEndpoint.cs | 55 ++ .../Endpoints/TrustWeightingEndpoint.cs | 56 ++ .../Endpoints/ViolationEndpoint.cs | 75 +++ .../Ledger/LedgerExportService.cs | 103 +++ .../Ledger/LedgerExportStore.cs | 44 ++ .../Ledger/LedgerModels.cs | 28 + .../Orchestration/OrchestratorJobModels.cs | 36 + .../Orchestration/OrchestratorJobService.cs | 112 ++++ .../Orchestration/OrchestratorJobStore.cs | 58 ++ .../Orchestration/PolicyWorkerModels.cs | 21 + .../Orchestration/PolicyWorkerService.cs | 107 +++ .../Orchestration/StableIdGenerator.cs | 61 ++ .../Orchestration/WorkerResultStore.cs | 52 ++ .../PathScopeSimulationBridgeService.cs | 2 +- src/Policy/StellaOps.Policy.Engine/Program.cs | 40 +- .../PolicyEvaluationService.PathScope.cs | 2 +- .../Snapshots/SnapshotModels.cs | 24 + .../Snapshots/SnapshotService.cs | 76 +++ .../Snapshots/SnapshotStore.cs | 44 ++ .../Streaming/PathScopeSimulationService.cs | 5 + .../TrustWeighting/TrustWeightingModels.cs | 13 + .../TrustWeighting/TrustWeightingService.cs | 77 +++ .../Violations/ConflictHandlingService.cs | 42 ++ .../Violations/SeverityFusionService.cs | 87 +++ .../Violations/ViolationEventService.cs | 79 +++ .../Violations/ViolationEventStore.cs | 56 ++ .../Violations/ViolationModels.cs | 45 ++ .../AdvisoryAiKnobsServiceTests.cs | 31 + .../LedgerExportServiceTests.cs | 61 ++ .../OrchestratorJobServiceTests.cs | 96 +++ .../PathScopeSimulationBridgeServiceTests.cs | 10 +- .../PolicyWorkerServiceTests.cs | 76 +++ .../SnapshotServiceTests.cs | 57 ++ .../StellaOps.Policy.Engine.Tests.csproj | 2 +- .../TrustWeightingServiceTests.cs | 36 + .../ViolationServicesTests.cs | 99 +++ .../EntrypointEndpointsTests.cs | 8 +- .../OrchestratorEndpointsTests.cs | 56 ++ .../ProjectionEndpointTests.cs | 2 + .../ResolverFeedExportTests.cs | 45 ++ .../SbomAssetEventsTests.cs | 47 ++ .../SbomInventoryEventsTests.cs | 66 ++ .../Models/OrchestratorModels.cs | 17 + .../Models/ResolverFeedModels.cs | 13 + .../Models/SbomAssetEvents.cs | 19 + .../Models/SbomInventoryEvidence.cs | 13 + .../Observability/README.md | 7 +- .../Observability/SbomMetrics.cs | 13 +- .../StellaOps.SbomService/Program.cs | 192 +++++- .../IOrchestratorControlRepository.cs | 10 + .../Repositories/IOrchestratorRepository.cs | 9 + .../InMemoryOrchestratorControlRepository.cs | 40 ++ .../InMemoryOrchestratorRepository.cs | 76 +++ .../Services/InMemorySbomQueryService.cs | 118 +++- .../Services/OrchestratorControlService.cs | 91 +++ .../Services/SbomEvents.cs | 83 +++ .../Services/WatermarkService.cs | 38 ++ src/SbomService/TASKS.md | 7 + .../ReachabilityBuildStageExecutor.cs | 92 +++ .../ReachabilityPublishStageExecutor.cs | 43 ++ .../Processing/ScanJobProcessor.cs | 59 +- .../StellaOps.Scanner.Worker/Program.cs | 7 + .../StellaOps.Scanner.Worker.csproj | 1 + src/Scanner/StellaOps.Scanner.sln | 44 ++ .../JavaLanguageAnalyzer.cs | 83 +-- .../Internal/NodeImportWalker.cs | 41 +- .../Internal/NodePackage.cs | 6 +- .../Internal/NodePackageCollector.cs | 3 +- .../Contracts/ScanAnalysisKeys.cs | 3 + .../ReachabilityGraphBuilder.cs | 15 + .../ReachabilityUnionPublisher.cs | 82 +++ .../ReachabilityUnionPublisherService.cs | 40 ++ .../ReachabilityUnionSchemas.cs | 6 + .../ReachabilityUnionWriter.cs | 390 +++++++++++ .../StellaOps.Scanner.Reachability.csproj | 5 +- .../Fakes/FakeFileContentAddressableStore.cs | 41 ++ .../ReachabilityGraphBuilderUnionTests.cs | 39 ++ .../ReachabilityUnionPublisherTests.cs | 30 + .../ReachabilityUnionWriterTests.cs | 84 +++ .../StellaOps.Scanner.Core.Tests.csproj | 4 +- .../FakeFileContentAddressableStore.cs | 39 ++ .../ReachabilityUnionPublisherTests.cs | 27 + .../ReachabilityUnionWriterTests.cs | 41 ++ ...tellaOps.Scanner.Reachability.Tests.csproj | 19 + .../TestHelpers.cs | 27 + .../AdvisoryLinksetStoreTests.cs | 72 ++ tests/shared/OpenSslAutoInit.cs | 15 + 215 files changed, 8119 insertions(+), 740 deletions(-) create mode 100644 .gitea/workflows/attestation-bundle.yml create mode 100644 .gitea/workflows/cli-build.yml create mode 100644 .gitea/workflows/containers-multiarch.yml create mode 100644 docs/airgap/bootstrap.md create mode 100644 docs/airgap/console-airgap-tasks.md create mode 100644 docs/airgap/degradation-matrix.md create mode 100644 docs/airgap/devportal-offline.md create mode 100644 docs/airgap/mirror-bundles.md create mode 100644 docs/airgap/operations.md create mode 100644 docs/airgap/overview.md create mode 100644 docs/airgap/sealing-and-egress.md create mode 100644 docs/cli/sbomer.md create mode 100644 docs/console/airgap.md create mode 100644 docs/console/attestor-ui.md create mode 100644 docs/dev/airgap-contracts.md delete mode 100644 docs/implplan/SPRINT_302_docs_tasks_md_ii.md create mode 100644 docs/implplan/SPRINT_504_ops_devops_ii.log.md create mode 100644 docs/implplan/archived/SPRINT_302_docs_tasks_md_ii.md create mode 100644 docs/modules/attestor/keys-and-issuers.md create mode 100644 docs/modules/attestor/overview.md create mode 100644 docs/modules/attestor/policies.md create mode 100644 docs/modules/attestor/transparency.md create mode 100644 docs/modules/cli/guides/airgap.md create mode 100644 docs/modules/cli/guides/attest.md create mode 100644 docs/modules/excititor/observability/locker-manifest.md create mode 100644 docs/modules/excititor/observability/timeline-events.md create mode 100644 docs/modules/policy/samples/advisory-ai-knobs@draft.json create mode 100644 docs/modules/policy/samples/orchestrator-job@draft.json create mode 100644 docs/modules/policy/samples/policy-batch-context@draft.json create mode 100644 docs/modules/policy/samples/policy-conflict@draft.json create mode 100644 docs/modules/policy/samples/policy-ledger-export@draft.json create mode 100644 docs/modules/policy/samples/policy-snapshot@draft.json create mode 100644 docs/modules/policy/samples/policy-violation-event@draft.json create mode 100644 docs/modules/policy/samples/policy-worker-result@draft.json create mode 100644 docs/modules/policy/samples/severity-fusion@draft.json create mode 100644 docs/modules/policy/samples/trust-weighting@draft.json create mode 100644 docs/modules/policy/schemas/advisory-ai-knobs@draft.json create mode 100644 docs/modules/policy/schemas/orchestrator-job@draft.json create mode 100644 docs/modules/policy/schemas/policy-batch-context@draft.json create mode 100644 docs/modules/policy/schemas/policy-conflict@draft.json create mode 100644 docs/modules/policy/schemas/policy-ledger-export@draft.json create mode 100644 docs/modules/policy/schemas/policy-snapshot@draft.json create mode 100644 docs/modules/policy/schemas/policy-violation-event@draft.json create mode 100644 docs/modules/policy/schemas/policy-worker-result@draft.json create mode 100644 docs/modules/policy/schemas/severity-fusion@draft.json create mode 100644 docs/modules/policy/schemas/trust-weighting@draft.json create mode 100644 docs/security/trust-and-signing.md create mode 100644 scripts/attest/build-attestation-bundle.sh create mode 100644 scripts/buildx/build-airgap-bundle.sh create mode 100644 scripts/buildx/build-multiarch.sh create mode 100644 scripts/cli/build-cli.sh create mode 100644 src/Excititor/StellaOps.Excititor.WebService/Contracts/GraphOverlayContracts.cs delete mode 100644 src/Excititor/StellaOps.Excititor.WebService/Controllers/GraphController.cs create mode 100644 src/Excititor/StellaOps.Excititor.WebService/Graph/GraphOverlayFactory.cs create mode 100644 src/Excititor/__Tests/StellaOps.Excititor.WebService.Tests/GraphOverlayFactoryTests.cs create mode 100644 src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Tests/Support/NotifierApplicationFactory.cs create mode 100644 src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Tests/TestContent/openapi/notify-openapi.yaml create mode 100644 src/Notifier/StellaOps.Notifier/StellaOps.Notifier.WebService/Contracts/PackApprovalAckRequest.cs create mode 100644 src/Notifier/StellaOps.Notifier/StellaOps.Notifier.docs/pack-approval-templates.json create mode 100644 src/Policy/StellaOps.Policy.Engine/AdvisoryAI/AdvisoryAiKnobsModels.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/AdvisoryAI/AdvisoryAiKnobsService.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/BatchContext/BatchContextModels.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/BatchContext/BatchContextService.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Endpoints/AdvisoryAiKnobsEndpoint.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Endpoints/BatchContextEndpoint.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Endpoints/LedgerExportEndpoint.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Endpoints/OrchestratorJobEndpoint.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Endpoints/PolicyWorkerEndpoint.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Endpoints/SnapshotEndpoint.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Endpoints/TrustWeightingEndpoint.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Endpoints/ViolationEndpoint.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Ledger/LedgerExportService.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Ledger/LedgerExportStore.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Ledger/LedgerModels.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Orchestration/OrchestratorJobModels.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Orchestration/OrchestratorJobService.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Orchestration/OrchestratorJobStore.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Orchestration/PolicyWorkerModels.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Orchestration/PolicyWorkerService.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Orchestration/StableIdGenerator.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Orchestration/WorkerResultStore.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Snapshots/SnapshotModels.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Snapshots/SnapshotService.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Snapshots/SnapshotStore.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/TrustWeighting/TrustWeightingModels.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/TrustWeighting/TrustWeightingService.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Violations/ConflictHandlingService.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Violations/SeverityFusionService.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Violations/ViolationEventService.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Violations/ViolationEventStore.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Violations/ViolationModels.cs create mode 100644 src/Policy/__Tests/StellaOps.Policy.Engine.Tests/AdvisoryAiKnobsServiceTests.cs create mode 100644 src/Policy/__Tests/StellaOps.Policy.Engine.Tests/LedgerExportServiceTests.cs create mode 100644 src/Policy/__Tests/StellaOps.Policy.Engine.Tests/OrchestratorJobServiceTests.cs create mode 100644 src/Policy/__Tests/StellaOps.Policy.Engine.Tests/PolicyWorkerServiceTests.cs create mode 100644 src/Policy/__Tests/StellaOps.Policy.Engine.Tests/SnapshotServiceTests.cs create mode 100644 src/Policy/__Tests/StellaOps.Policy.Engine.Tests/TrustWeightingServiceTests.cs create mode 100644 src/Policy/__Tests/StellaOps.Policy.Engine.Tests/ViolationServicesTests.cs create mode 100644 src/SbomService/StellaOps.SbomService.Tests/OrchestratorEndpointsTests.cs create mode 100644 src/SbomService/StellaOps.SbomService.Tests/ResolverFeedExportTests.cs create mode 100644 src/SbomService/StellaOps.SbomService.Tests/SbomAssetEventsTests.cs create mode 100644 src/SbomService/StellaOps.SbomService.Tests/SbomInventoryEventsTests.cs create mode 100644 src/SbomService/StellaOps.SbomService/Models/OrchestratorModels.cs create mode 100644 src/SbomService/StellaOps.SbomService/Models/ResolverFeedModels.cs create mode 100644 src/SbomService/StellaOps.SbomService/Models/SbomAssetEvents.cs create mode 100644 src/SbomService/StellaOps.SbomService/Models/SbomInventoryEvidence.cs create mode 100644 src/SbomService/StellaOps.SbomService/Repositories/IOrchestratorControlRepository.cs create mode 100644 src/SbomService/StellaOps.SbomService/Repositories/IOrchestratorRepository.cs create mode 100644 src/SbomService/StellaOps.SbomService/Repositories/InMemoryOrchestratorControlRepository.cs create mode 100644 src/SbomService/StellaOps.SbomService/Repositories/InMemoryOrchestratorRepository.cs create mode 100644 src/SbomService/StellaOps.SbomService/Services/OrchestratorControlService.cs create mode 100644 src/SbomService/StellaOps.SbomService/Services/WatermarkService.cs create mode 100644 src/Scanner/StellaOps.Scanner.Worker/Processing/Reachability/ReachabilityBuildStageExecutor.cs create mode 100644 src/Scanner/StellaOps.Scanner.Worker/Processing/Reachability/ReachabilityPublishStageExecutor.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityUnionPublisher.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityUnionPublisherService.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityUnionSchemas.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityUnionWriter.cs create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/Fakes/FakeFileContentAddressableStore.cs create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/ReachabilityGraphBuilderUnionTests.cs create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/ReachabilityUnionPublisherTests.cs create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/ReachabilityUnionWriterTests.cs create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/FakeFileContentAddressableStore.cs create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/ReachabilityUnionPublisherTests.cs create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/ReachabilityUnionWriterTests.cs create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/StellaOps.Scanner.Reachability.Tests.csproj create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/TestHelpers.cs create mode 100644 tests/Concelier/StellaOps.Concelier.Storage.Mongo.Tests/AdvisoryLinksetStoreTests.cs create mode 100644 tests/shared/OpenSslAutoInit.cs diff --git a/.gitea/workflows/attestation-bundle.yml b/.gitea/workflows/attestation-bundle.yml new file mode 100644 index 000000000..7e5e58c26 --- /dev/null +++ b/.gitea/workflows/attestation-bundle.yml @@ -0,0 +1,26 @@ +name: attestation-bundle +on: + workflow_dispatch: + inputs: + attest_dir: + description: "Directory containing attestation artefacts" + required: true + default: "out/attest" + +jobs: + bundle: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build bundle + run: | + chmod +x scripts/attest/build-attestation-bundle.sh + scripts/attest/build-attestation-bundle.sh "${{ github.event.inputs.attest_dir }}" + + - name: Upload bundle + uses: actions/upload-artifact@v4 + with: + name: attestation-bundle + path: out/attest-bundles/** diff --git a/.gitea/workflows/cli-build.yml b/.gitea/workflows/cli-build.yml new file mode 100644 index 000000000..2627e75cc --- /dev/null +++ b/.gitea/workflows/cli-build.yml @@ -0,0 +1,45 @@ +name: cli-build +on: + workflow_dispatch: + inputs: + rids: + description: "Comma-separated RIDs (e.g., linux-x64,win-x64,osx-arm64)" + required: false + default: "linux-x64,win-x64,osx-arm64" + config: + description: "Build configuration" + required: false + default: "Release" + sign: + description: "Enable cosign signing (requires COSIGN_KEY)" + required: false + default: "false" + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: "10.0.100-rc.2.25502.107" + + - name: Install syft (SBOM) + uses: anchore/sbom-action/download-syft@v0 + + - name: Build CLI artifacts + run: | + chmod +x scripts/cli/build-cli.sh + RIDS="${{ github.event.inputs.rids }}" CONFIG="${{ github.event.inputs.config }}" SBOM_TOOL=syft SIGN="${{ github.event.inputs.sign }}" COSIGN_KEY="${{ secrets.COSIGN_KEY }}" scripts/cli/build-cli.sh + + - name: List artifacts + run: find out/cli -maxdepth 3 -type f -print + + - name: Upload CLI artifacts + uses: actions/upload-artifact@v4 + with: + name: stella-cli + path: out/cli/** diff --git a/.gitea/workflows/containers-multiarch.yml b/.gitea/workflows/containers-multiarch.yml new file mode 100644 index 000000000..123142174 --- /dev/null +++ b/.gitea/workflows/containers-multiarch.yml @@ -0,0 +1,86 @@ +name: containers-multiarch +on: + workflow_dispatch: + inputs: + image: + description: "Image tag (e.g., ghcr.io/stella-ops/example:edge)" + required: true + context: + description: "Build context directory" + required: true + default: "." + platforms: + description: "Platforms (comma-separated)" + required: false + default: "linux/amd64,linux/arm64" + push: + description: "Push to registry" + required: false + default: "false" + +jobs: + build-multiarch: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + install: true + + - name: Install syft (SBOM) + uses: anchore/sbom-action/download-syft@v0 + + - name: Login to ghcr (optional) + if: ${{ github.event.inputs.push == 'true' && secrets.GHCR_TOKEN != '' }} + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GHCR_TOKEN }} + + - name: Run multi-arch build + env: + COSIGN_EXPERIMENTAL: "1" + run: | + chmod +x scripts/buildx/build-multiarch.sh + extra="" + if [[ "${{ github.event.inputs.push }}" == "true" ]]; then extra="--push"; fi + scripts/buildx/build-multiarch.sh \ + "${{ github.event.inputs.image }}" \ + "${{ github.event.inputs.context }}" \ + --platform "${{ github.event.inputs.platforms }}" \ + --sbom syft ${extra} + + - name: Build air-gap bundle + run: | + chmod +x scripts/buildx/build-airgap-bundle.sh + scripts/buildx/build-airgap-bundle.sh "${{ github.event.inputs.image }}" + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: buildx-${{ github.event.inputs.image }} + path: out/buildx/** + + - name: Inspect built image archive + run: | + set -e + ls -lh out/buildx/ + find out/buildx -name "image.oci" -print -exec sh -c 'tar -tf "$1" | head' _ {} \; + + - name: Upload air-gap bundle + uses: actions/upload-artifact@v4 + with: + name: bundle-${{ github.event.inputs.image }} + path: out/bundles/** + + - name: Inspect remote image (if pushed) + if: ${{ github.event.inputs.push == 'true' }} + run: | + docker buildx imagetools inspect "${{ github.event.inputs.image }}" diff --git a/.gitignore b/.gitignore index 8d778c453..edc2db5a9 100644 --- a/.gitignore +++ b/.gitignore @@ -31,7 +31,9 @@ seed-data/cert-bund/**/*.sha256 out/offline-kit/web/**/* **/node_modules/**/* **/.angular/**/* -**/.cache/**/* -**/dist/**/* -tmp/**/* +**/.cache/**/* +**/dist/**/* +tmp/**/* build/ +/out/cli/** +/src/Sdk/StellaOps.Sdk.Release/out/** diff --git a/docs/airgap/bootstrap.md b/docs/airgap/bootstrap.md new file mode 100644 index 000000000..fa6930ede --- /dev/null +++ b/docs/airgap/bootstrap.md @@ -0,0 +1,33 @@ +# Bootstrap Pack (Airgap 56-004) + +Guidance to build and install the bootstrap pack that primes sealed environments. + +## Contents +- Core images/charts for platform services (Authority, Excititor, Concelier, Export Center, Scheduler) with digests. +- Offline NuGet/npm caches (if permitted) with checksum manifest. +- Configuration defaults: sealed-mode toggles, trust roots, time-anchor bundle, network policy presets. +- Verification scripts: hash check, DSSE verification (if available), and connectivity probes to local mirrors. + +## Build steps +1. Gather image digests and charts from trusted registry/mirror. +2. Create `bootstrap-manifest.json` with: + - `bundleId`, `createdAt` (UTC), `producer`, `mirrorGeneration` + - `files[]` (path, sha256, size, mediaType) + - optional `dsseEnvelopeHash` +3. Package into tarball with deterministic ordering (POSIX tar, sorted paths, numeric owner 0:0). +4. Compute sha256 for tarball; record in manifest. + +## Install steps +1. Transfer pack to sealed site (removable media). +2. Verify tarball hash and DSSE (if present) using offline trust roots. +3. Load images/charts into local registry; preload caches to `local-nugets/` etc. +4. Apply network policies (deny-all) and sealed-mode config. +5. Register bootstrap manifest and mirrorGeneration with Excititor/Export Center. + +## Determinism & rollback +- Keep manifests in ISO-8601 UTC; no host-specific metadata in tar headers. +- For rollback, retain previous bootstrap tarball + manifest; restore registry contents and config snapshots. + +## Related +- `docs/airgap/mirror-bundles.md` — mirror pack format and validation. +- `docs/airgap/sealing-and-egress.md` — egress enforcement used during install. diff --git a/docs/airgap/console-airgap-tasks.md b/docs/airgap/console-airgap-tasks.md new file mode 100644 index 000000000..e90064fb5 --- /dev/null +++ b/docs/airgap/console-airgap-tasks.md @@ -0,0 +1,6 @@ +# Console Airgap Implementation Tasks (link to DOCS-AIRGAP-57-002) + +- Implement sealed badge + staleness indicators using `staleness-and-time.md` rules. +- Hook import wizard to backend once mirror bundle schema and timeline event API are available. +- Ensure admin-only import; read-only view otherwise. +- Emit telemetry for imports (success/failure) and denied attempts. diff --git a/docs/airgap/degradation-matrix.md b/docs/airgap/degradation-matrix.md new file mode 100644 index 000000000..7f9f41c11 --- /dev/null +++ b/docs/airgap/degradation-matrix.md @@ -0,0 +1,19 @@ +# Airgap Degradation Matrix (DOCS-AIRGAP-58-001) + +What works and what degrades across modes (sealed → constrained → connected). + +| Capability | Connected | Constrained | Sealed | Notes | +| --- | --- | --- | --- | --- | +| Mirror imports | ✓ | ✓ | ✓ | Sealed requires preloaded media + offline validation. | +| Time anchors (external NTP) | ✓ | ✓ (allowlisted) | ✗ | Sealed relies on signed time anchors. | +| Transparency log lookups | ✓ | ✓ (if allowlisted) | ✗ | Sealed skips; rely on bundled checkpoints. | +| Rekor witness | ✓ | optional | ✗ | Disabled in sealed; log locally. | +| SBOM feed refresh | ✓ | limited mirrors | offline only | Use mirror bundles. | +| CLI plugin downloads | ✓ | allowlisted | ✗ | Must ship in bootstrap pack. | +| Telemetry export | ✓ | optional | optional/log-only | Sealed may use console exporter only. | +| Webhook callbacks | ✓ | allowlisted internal only | ✗ | Use internal queue instead. | +| OTA updates | ✓ | partial | ✗ | Use mirrorGeneration refresh. | + +## Remediation guidance +- If a capability is degraded in sealed mode, provide offline substitute (mirror bundles, time anchors, console exporter). +- When moving to constrained/connected, re-enable trust roots and transparency checks gradually; verify hashes first. diff --git a/docs/airgap/devportal-offline.md b/docs/airgap/devportal-offline.md new file mode 100644 index 000000000..cb4e8ec47 --- /dev/null +++ b/docs/airgap/devportal-offline.md @@ -0,0 +1,23 @@ +# DevPortal Offline (DOCS-AIRGAP-DEVPORT-64-001) + +How to use the developer portal in fully offline/sealed environments. + +## Serving the portal +- Host static build from local object store or file server; no CDN. +- Set `DEVPORTAL_OFFLINE=true` to disable external analytics/fonts. + +## Auth +- Use Authority in offline mode with pre-provisioned tenants; cache JWKS locally. + +## Bundles +- Provide mirror/bootstrap bundles via offline download page with hashes and DSSE (if available). +- Offer time anchors download; display staleness and mirrorGeneration in UI header. + +## Search/docs +- Bundle docs and search index; disable remote doc fetch. + +## Telemetry +- Disable remote telemetry; keep console logs only or send to local OTLP endpoint. + +## Verification +- On load, run self-check to confirm no external requests; fail with clear banner if any detected. diff --git a/docs/airgap/mirror-bundles.md b/docs/airgap/mirror-bundles.md new file mode 100644 index 000000000..e845b0e93 --- /dev/null +++ b/docs/airgap/mirror-bundles.md @@ -0,0 +1,28 @@ +# Mirror Bundles (Airgap 56-003) + +Defines the mirror bundle format and validation workflow for sealed deployments. + +## Contents +- Images/charts: OCI artifacts exported with digests + SBOMs. +- Manifests: `manifest.json` with entries: + - `bundleId`, `mirrorGeneration`, `createdAt`, `producer` (export center), `hashes` (sha256 list) + - `dsseEnvelopeHash` for signed manifest (if available) + - `files[]`: path, sha256, size, mediaType +- Transparency: optional TUF metadata (`timestamp.json`, `snapshot.json`) for replay protection. + +## Validation steps +1. Verify `manifest.json` sha256 matches provided hash. +2. If DSSE present, verify signature against offline trust roots. +3. Validate Merkle root (if included) over `files[]` hashes. +4. For each OCI artifact, confirm digest matches and SBOM present. +5. Record `mirrorGeneration` and manifest hash; store in audit log and timeline event. + +## Workflow +- Export Center produces bundle + manifest; Attestor/Excititor importers validate before ingest. +- Bundle consumers must refuse imports if any hash/signature fails. +- Keep format stable; any schema change bumps `manifestVersion` in `manifest.json`. + +## Determinism +- Sort `files[]` by path; compute hashes with UTF-8 canonical paths. +- Use ISO-8601 UTC timestamps in manifests. +- Do not include host-specific paths or timestamps in tar layers. diff --git a/docs/airgap/operations.md b/docs/airgap/operations.md new file mode 100644 index 000000000..467ee3f05 --- /dev/null +++ b/docs/airgap/operations.md @@ -0,0 +1,34 @@ +# Airgap Operations (DOCS-AIRGAP-57-004) + +Runbooks for imports, failure recovery, and auditing in sealed/constrained modes. + +## Imports +1) Verify bundle hash/DSSE (see `mirror-bundles.md`). +2) `stella airgap import --bundle ... --generation N --dry-run` (optional). +3) Apply network policy: ensure sealed/constrained mode set correctly. +4) Import with `stella airgap import ...` and watch logs. +5) Confirm timeline event emitted (bundleId, mirrorGeneration, actor). + +## Failure recovery +- Hash/signature mismatch: reject bundle; re-request export; log incident. +- Partial import: rerun with `--force` after cleaning registry/cache; keep previous generation for rollback. +- Staleness breach: if imports unavailable, raise amber alert; if >72h, go red and halt new ingest until refreshed. +- Time anchor expired: apply new anchor from trusted media before continuing operations. + +## Auditing +- Record every import in audit log: `{tenant, mirrorGeneration, manifestHash, actor, sealed}`. +- Preserve manifests and hashes for at least two generations. +- Periodically (daily) run `stella airgap list --format json` and archive output. +- Ensure logs are immutable (append-only) in sealed environments. + +## Observability +- Monitor counters for denied egress, import success/failure, and staleness alerts. +- Expose `/obs/airgap/status` (if available) to scrape bundle freshness. + +## Checklist (per import) +- [ ] Hash/DSSE verified +- [ ] Sealed/constrained mode configured +- [ ] Registry/cache reachable +- [ ] Import succeeded +- [ ] Timeline/audit recorded +- [ ] Staleness dashboard updated diff --git a/docs/airgap/overview.md b/docs/airgap/overview.md new file mode 100644 index 000000000..33e9c50e8 --- /dev/null +++ b/docs/airgap/overview.md @@ -0,0 +1,32 @@ +# Airgap Overview + +This page orients teams before diving into per-component runbooks. It summarises modes, lifecycle, and governance responsibilities for sealed deployments. + +## Modes +- **Sealed**: deny-all egress; only preloaded bundles (mirror + bootstrap) allowed. Requires exported time anchors and offline trust roots. +- **Constrained**: limited egress to allowlisted registries and NTP; mirror bundles still preferred. +- **Connected**: full egress for staging; must remain policy-compatible with sealed mode. + +## Lifecycle +1. **Prepare bundles**: export mirror + bootstrap packs (images/charts, SBOMs, DSSE metadata) signed and hashed. +2. **Stage & verify**: load bundles into the offline store, verify hashes/DSSE, record mirrorGeneration. +3. **Activate**: flip sealed toggle; enforce deny-all egress and policy banners; register bundles with Excititor/Export Center. +4. **Operate**: run periodic staleness checks, apply time anchors, and audit imports via timeline events. +5. **Refresh/rollback**: import next mirrorGeneration or roll back using previous manifest + hashes. + +## Responsibilities +- **AirGap Controller Guild**: owns network posture (deny-all, allowlists), sealed-mode policy banners, and change control. +- **Export Center / Evidence Locker Guilds**: produce and verify bundle manifests, DSSE envelopes, and Merkle roots. +- **Module owners** (Excititor, Concelier, etc.): honor sealed-mode toggles, emit staleness headers, and refuse unsigned/unknown bundles. +- **Ops/Signals Guild**: maintain time anchors and observability sinks compatible with sealed deployments. + +## Rule banner (sealed mode) +Display a top-of-console banner when `sealed=true`: +- "Sealed mode: no external egress. Only registered bundles permitted. Imports logged; violations trigger audit." +- Include current `mirrorGeneration`, bundle manifest hash, and time-anchor status. + +## Related docs +- `docs/airgap/airgap-mode.md` — deeper policy shapes per mode. +- `docs/airgap/bundle-repositories.md` — mirror/bootstrap bundle structure. +- `docs/airgap/staleness-and-time.md` — time anchors and staleness checks. +- `docs/airgap/controller-scaffold.md` / `importer-scaffold.md` — implementation scaffolds. diff --git a/docs/airgap/portable-evidence.md b/docs/airgap/portable-evidence.md index 7b1395ce5..59b8ea999 100644 --- a/docs/airgap/portable-evidence.md +++ b/docs/airgap/portable-evidence.md @@ -1,90 +1,27 @@ -# Portable Evidence Bundles (Sealed/Air-Gapped) +# Portable Evidence Bundles (DOCS-AIRGAP-58-004) -> Sprint 160 · Task EVID-OBS-60-001 -> Audience: Evidence Locker operators, Air-Gap controllers, incident responders +Guidance for exporting/importing portable evidence bundles across enclaves. -Portable bundles let operators hand off sealed evidence across enclaves without exposing tenant identifiers or internal storage coordinates. The Evidence Locker produces a deterministic archive (`portable-bundle-v1.tgz`) that carries the manifest + signature alongside redacted metadata, checksum manifest, and an offline verification script. +## Bundle contents +- Evidence payloads (VEX observations/linksets) as NDJSON. +- Timeline events and attestation DSSE envelopes. +- Manifest with `bundleId`, `source`, `tenant`, `createdAt`, `files[]`, `dsseEnvelopeHash` (optional). -## 1. When to use the portable flow +## Export +- Produce from Evidence Locker/Excititor with deterministic ordering and SHA-256 hashes. +- Include Merkle root over evidence files; store in manifest. +- Sign manifest (DSSE) when trust roots available. -- **Sealed mode exports.** Regulatory or incident response teams that cannot access the primary enclave directly. -- **Chain-of-custody transfers.** Moving evidence into offline review systems while keeping the DSSE provenance intact. -- **Break-glass rehearsals.** Validating incident response playbooks without exposing internal bundle metadata. +## Import +- Verify manifest hash, Merkle root, and DSSE signature offline. +- Enforce tenant scoping; refuse cross-tenant bundles. +- Emit timeline event upon successful import. -Avoid portable bundles for regular intra-enclave automation; the full `bundle.tgz` already carries richer metadata for automated tooling. +## Constraints +- No external lookups; verification uses bundled roots. +- Max size per bundle configurable; default 500 MB. +- Keep file paths UTF-8 and slash-separated; avoid host-specific metadata. -## 2. Generating the bundle - -1. Seal the evidence bundle as usual (`POST /evidence/snapshot` or via CLI). -2. Request the portable artefact using the new endpoint: - - ``` - GET /evidence/{bundleId}/portable - Scope: evidence:read - ``` - - Response headers mirror the standard download (`application/gzip`, `Content-Disposition: attachment; filename="portable-evidence-bundle-{bundleId}.tgz"`). - -The Evidence Locker caches the portable archive using write-once semantics. Subsequent requests reuse the existing object and the audit log records whether the file was newly created or served from cache. - -## 3. Archive layout - -``` -portable-bundle-v1.tgz - ├── manifest.json # Canonical bundle manifest (identical to sealed bundle) - ├── signature.json # DSSE signature + optional RFC3161 timestamp (base64 token) - ├── bundle.json # Redacted metadata (bundleId, kind, rootHash, timestamps, incidentMetadata) - ├── checksums.txt # Merkle root + per-entry SHA-256 digests - ├── instructions-portable.txt # Human-readable guidance for sealed transfers - └── verify-offline.sh # POSIX shell helper (extract + checksum verify + reminder to run DSSE verification) -``` - -Redaction rules: - -- No tenant identifiers, storage keys, descriptions, or free-form metadata. -- Incident metadata is retained *only* under the `incidentMetadata` object (`incident.mode`, `incident.changedAt`, etc.). -- `portableGeneratedAt` records when the archive was produced so downstream systems can reason about freshness. - -## 4. Offline verification workflow - -1. Copy `portable-bundle-v1.tgz` into the sealed environment (USB, sneaker-net, etc.). -2. Run the included helper from a POSIX shell: - - ```sh - chmod +x verify-offline.sh - ./verify-offline.sh portable-bundle-v1.tgz - ``` - - The script: - - extracts the archive into a temporary directory, - - validates `checksums.txt` using `sha256sum` (or `shasum -a 256`), and - - prints the Merkle root hash from `bundle.json`. - -3. Complete provenance verification: - - Preferred: `stella evidence verify --bundle portable-bundle-v1.tgz` - - Alternative: supply `manifest.json` and `signature.json` to the evidence verifier library. - -4. Record the verification output (root hash, timestamp) with the receiving enclave’s evidence locker or incident ticket. - -> **Note:** The DSSE payload is unchanged from the sealed bundle, so existing verification tooling does not need special handling for portable archives. - -## 5. Importing into the receiving enclave - -1. Upload the archive to the target Evidence Locker or attach it to the incident record. -2. Store the checksum report generated by `verify-offline.sh` alongside the archive. -3. If downstream automation needs enriched metadata, attach a private note referencing the original bundle’s tenant context—the portable archive intentionally omits it. - -## 6. Troubleshooting - -| Symptom | Likely cause | Remediation | -|--------|--------------|-------------| -| `verify-offline.sh` reports checksum failures | Transfer corruption | Re-transfer artefact; run `sha256sum portable-bundle-v1.tgz` on both sides and compare. | -| `stella evidence verify` cannot reach TSA | Sealed environment lacks TSA connectivity | Verification still succeeds using DSSE signature; capture the missing TSA warning in the import log. | -| `/portable` endpoint returns 400 | Bundle not yet sealed or signature missing | Wait for sealing to complete; ensure DSSE signing is enabled. | -| `/portable` returns 404 | Bundle not found or tenant mismatch | Confirm DPoP scope and tenant claim; refresh bundle status via `GET /evidence/{id}`. | - -## 7. Change management - -- Portable bundle versioning is encoded in the filename (`portable-bundle-v1.tgz`). When content or script behaviour changes, bump the version and announce in release notes. -- Any updates to `verify-offline.sh` must remain POSIX-sh compatible and avoid external dependencies beyond `tar`, `sha256sum`/`shasum`, and standard coreutils. -- Remember to update this guide and the bundle packaging dossier (`docs/modules/evidence-locker/bundle-packaging.md`) when fields or workflows change. +## Determinism +- Sort files lexicographically; use ISO-8601 UTC timestamps. +- Avoid re-compressing files; if tar is used, set deterministic headers (uid/gid=0, mtime=0). diff --git a/docs/airgap/sealing-and-egress.md b/docs/airgap/sealing-and-egress.md new file mode 100644 index 000000000..39848e1c2 --- /dev/null +++ b/docs/airgap/sealing-and-egress.md @@ -0,0 +1,30 @@ +# Sealing and Egress (Airgap 56-002) + +Guidance for enforcing deny-all egress and validating sealed-mode posture. + +## Network policies +- Kubernetes: apply namespace-scoped `NetworkPolicy` with default deny; allow only: + - DNS to internal resolver + - Object storage/mirror endpoints on allowlist + - OTLP/observability endpoints if permitted for sealed monitoring +- Docker Compose: use firewall rules or `extra_hosts` to block outbound except mirrors; ship `iptables` template in ops bundle. + +## EgressPolicy facade +- Services MUST read `Excititor:Network:EgressPolicy` (or module equivalent) to decide runtime behavior: + - `sealed` → deny outbound HTTP/S except allowlist; fail fast on unexpected hosts. + - `constrained` → allow allowlist + time/NTP if required. +- Log policy decisions and surface `X-Sealed-Mode: true|false` on HTTP responses for diagnostics. + +## Verification checklist +1. Confirm policy manifests applied (kubectl/compose diff) and pods restarted. +2. Run connectivity probe from each pod: + - Allowed endpoints respond (200/OK or 403 expected). + - Disallowed domains return immediate failure. +3. Attempt bundle import; verify timeline event emitted with `sealed=true`. +4. Check observability: counters for denied egress should increment (export or console log). +5. Record mirrorGeneration + manifest hash in audit log. + +## Determinism & offline posture +- No external CRLs/OCSP in sealed mode; rely on bundled trust roots. +- Keep allowlist minimal and declared in config; no implicit fallbacks. +- All timestamps UTC; avoid calling external time APIs. diff --git a/docs/api/concelier/concelier-lnm.yaml b/docs/api/concelier/concelier-lnm.yaml index 104ab608c..8d1e21171 100644 --- a/docs/api/concelier/concelier-lnm.yaml +++ b/docs/api/concelier/concelier-lnm.yaml @@ -241,6 +241,7 @@ components: properties: aliases: { type: array, items: { type: string } } purl: { type: array, items: { type: string } } + cpe: { type: array, items: { type: string } } versions: { type: array, items: { type: string } } ranges: { type: array, items: { type: object } } severities: { type: array, items: { type: object } } diff --git a/docs/api/notify-openapi.yaml b/docs/api/notify-openapi.yaml index f4f3d2f3b..746547448 100644 --- a/docs/api/notify-openapi.yaml +++ b/docs/api/notify-openapi.yaml @@ -317,6 +317,75 @@ paths: default: $ref: '#/components/responses/Error' + /api/v1/notify/pack-approvals: + post: + summary: Ingest pack approval decision + tags: [PackApprovals] + operationId: ingestPackApproval + security: + - oauth2: [notify.operator] + - hmac: [] + parameters: + - $ref: '#/components/parameters/Tenant' + - $ref: '#/components/parameters/IdempotencyKey' + requestBody: + required: true + content: + application/json: + schema: { $ref: '#/components/schemas/PackApprovalEvent' } + examples: + approval-granted: + value: + eventId: "20e4e5fe-3d4a-4f57-9f9b-b1a1c1111111" + issuedAt: "2025-11-17T16:00:00Z" + kind: "pack.approval.granted" + packId: "offline-kit-2025-11" + policy: + id: "policy-123" + version: "v5" + decision: "approved" + actor: "task-runner" + resumeToken: "rt-abc123" + summary: "All required attestations verified." + labels: + environment: "prod" + approver: "ops" + responses: + '202': + description: Accepted; durable write queued for processing. + headers: + X-Resume-After: + description: Resume token echo or replacement + schema: { type: string } + default: + $ref: '#/components/responses/Error' + + /api/v1/notify/pack-approvals/{packId}/ack: + post: + summary: Acknowledge a pack approval notification + tags: [PackApprovals] + operationId: ackPackApproval + parameters: + - $ref: '#/components/parameters/Tenant' + - name: packId + in: path + required: true + schema: { type: string } + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + ackToken: { type: string } + required: [ackToken] + responses: + '204': + description: Acknowledged + default: + $ref: '#/components/responses/Error' + components: securitySchemes: oauth2: @@ -328,6 +397,10 @@ components: notify.viewer: Read-only Notifier access notify.operator: Manage rules/templates/incidents within tenant notify.admin: Tenant-scoped administration + hmac: + type: http + scheme: bearer + description: Pre-shared HMAC token (air-gap friendly) referenced by secretRef. parameters: Tenant: name: X-StellaOps-Tenant @@ -335,6 +408,12 @@ components: required: true description: Tenant slug schema: { type: string } + IdempotencyKey: + name: Idempotency-Key + in: header + required: true + description: Stable UUID to dedupe retries. + schema: { type: string, format: uuid } PageSize: name: pageSize in: query @@ -468,6 +547,39 @@ components: type: object additionalProperties: { type: string } + PackApprovalEvent: + type: object + required: + - eventId + - issuedAt + - kind + - packId + - decision + - actor + properties: + eventId: { type: string, format: uuid } + issuedAt: { type: string, format: date-time } + kind: + type: string + enum: [pack.approval.granted, pack.approval.denied, pack.policy.override] + packId: { type: string } + policy: + type: object + properties: + id: { type: string } + version: { type: string } + decision: + type: string + enum: [approved, denied, overridden] + actor: { type: string } + resumeToken: + type: string + description: Opaque token for at-least-once resume. + summary: { type: string } + labels: + type: object + additionalProperties: { type: string } + QuietHours: type: object required: [quietHoursId, windows] diff --git a/docs/cli/sbomer.md b/docs/cli/sbomer.md new file mode 100644 index 000000000..7cb4d2feb --- /dev/null +++ b/docs/cli/sbomer.md @@ -0,0 +1,40 @@ +# stella sbomer (DOCS-CLI-DET-01) + +Offline-first usage of `stella sbomer` verbs with deterministic outputs. + +## Prerequisites +- Install CLI from offline bundle; ensure `local-nugets/` is available. +- Export images/charts locally; no network access required during commands. + +## Commands +- `stella sbomer layer ` + - Emits deterministic SBOM per layer; options: `--format cyclonedx|spdx`, `--output `, `--deterministic` (default true). +- `stella sbomer compose ` + - Merges layer SBOMs with stable ordering; rejects missing hashes. +- `stella sbomer drift ` + - Computes drift; returns machine-readable diff with stable ordering. +- `stella sbomer verify --hash ` + - Validates hash/signature if provided; offline only. + +## Determinism rules +- Use fixed sort keys (component name, version, purl) when composing. +- All timestamps forced to `1970-01-01T00:00:00Z` unless `--timestamp` supplied. +- GUID/UUID generation disabled; use content hashes as IDs. +- Outputs written in UTF-8 with LF line endings; no BOM. + +## Examples +```bash +# generate layer SBOM +stella sbomer layer ghcr.io/acme/app:1.2.3 --format cyclonedx --output app.cdx.json + +# compose +stella sbomer compose app.cdx.json lib.cdx.json --output combined.cdx.json + +# drift +stella sbomer drift baseline.cdx.json combined.cdx.json --output drift.json +``` + +## Offline tips +- Preload registries; set `STELLA_SBOMER_OFFLINE=true` to prevent remote pulls. +- Configure cache dir via `STELLA_CACHE_DIR` for reproducible paths. +- For air-gapped logs, use `--log-format json` and capture to file for later analysis. diff --git a/docs/console/airgap.md b/docs/console/airgap.md new file mode 100644 index 000000000..d941d0a0b --- /dev/null +++ b/docs/console/airgap.md @@ -0,0 +1,27 @@ +# Console Airgap UI (Airgap 57-002) + +Describes console surfaces for sealed-mode imports, staleness, and user guidance. + +## Surfaces +- **Airgap status badge**: shows `sealed` state, `mirrorGeneration`, last import time, and staleness indicator. +- **Import wizard**: stepper to upload/verify mirror bundle, show manifest hash, and emit timeline event upon success. +- **Staleness dashboard**: charts staleness by bundle/component; highlights tenants nearing expiry. + +## Staleness logic +- Use time anchors from `docs/airgap/staleness-and-time.md`. +- Staleness = now - `bundle.createdAt`; color bands: green (<24h), amber (24–72h), red (>72h) or missing anchor. + +## Guidance banners +- When sealed: banner text "Sealed mode: egress denied. Only registered bundles allowed." Include current `mirrorGeneration` and bundle hash. +- On staleness red: prompt operators to import next bundle or reapply time anchor. + +## Events +- Successful import emits timeline event with bundleId, mirrorGeneration, manifest hash, actor. +- Failed import emits event with error code; do not expose stack traces in UI. + +## Security/guardrails +- Require admin scope to import bundles; read-only users can view status only. +- Never display raw hashes without tenant context; prefix with tenant and generation. + +## TODOs +- Wire to backend once mirror bundle schema and timeline events are exposed (blocked until backend readiness). diff --git a/docs/console/attestor-ui.md b/docs/console/attestor-ui.md new file mode 100644 index 000000000..a73b7107e --- /dev/null +++ b/docs/console/attestor-ui.md @@ -0,0 +1,8 @@ +# Attestor UI (DOCS-ATTEST-74-003) + +Describe console workflows for viewing and verifying attestations. + +- Pages: attestation list, attestation detail, verification status panel. +- Filters: tenant, issuer, predicate, verification status. +- Actions: download DSSE, view transparency info, export verification record. +- UI must not derive verdicts; display raw verification state only. diff --git a/docs/dev/airgap-contracts.md b/docs/dev/airgap-contracts.md new file mode 100644 index 000000000..047e72d55 --- /dev/null +++ b/docs/dev/airgap-contracts.md @@ -0,0 +1,23 @@ +# Airgap Contracts (DOCS-AIRGAP-58-003) + +Contracts developers must follow for sealed/constrained deployments. + +## EgressPolicy usage +- Services read `EgressPolicy` config and must fail fast on disallowed hosts. +- All HTTP clients must pass through allowlist resolver; no raw `HttpClient` with arbitrary URLs. + +## Sealed-mode tests +- Add integration tests that set `sealed=true` and assert outbound calls are blocked/mocked. +- Validate mirror bundle imports succeed under deny-all network by using local fixtures. + +## Linting +- Static check to ban `DateTime.Now`, `Guid.NewGuid`, and direct `HttpClient` when `sealed=true` flag is present. +- CI rule: fail if new external domains appear outside allowlist file. + +## Logging +- Log `sealed` flag, `mirrorGeneration`, and bundle hash on relevant API calls. +- Avoid emitting secrets or trust roots in logs. + +## Config determinism +- All configs should be overridable via env vars; default to sealed-compatible settings. +- Use stable ordering in generated manifests and responses. diff --git a/docs/implplan/SPRINT_0111_0001_0001_advisoryai.md b/docs/implplan/SPRINT_0111_0001_0001_advisoryai.md index 1f3450126..923a279ac 100644 --- a/docs/implplan/SPRINT_0111_0001_0001_advisoryai.md +++ b/docs/implplan/SPRINT_0111_0001_0001_advisoryai.md @@ -19,7 +19,7 @@ | # | Task ID | Status | Key dependency / next step | Owners | Task Definition | | --- | --- | --- | --- | --- | --- | | 1 | AIAI-DOCS-31-001 | BLOCKED (2025-11-22) | Await CLI/Policy artefacts to finalize guardrail/evidence doc. Draft skeleton allowed (non-blocking for dev). | Advisory AI Docs Guild | Author guardrail + evidence docs with upstream references. | -| 2 | AIAI-PACKAGING-31-002 | BLOCKED (DevOps release-only) | SBOM feeds + CLI/Policy digests not delivered; sealing/publishing deferred to DevOps once feeds arrive. Dev can proceed with dry-run bundle layout. | Advisory AI Release | Package advisory feeds with SBOM pointers + provenance. | +| 2 | AIAI-PACKAGING-31-002 | MOVED to SPRINT_503_ops_devops_i (2025-11-23) | Track under DEVOPS-AIAI-31-002 in Ops sprint; waiting for CLI/Policy digests + SBOM feeds there. | Advisory AI Release | Package advisory feeds with SBOM pointers + provenance. | | 3 | AIAI-RAG-31-003 | DONE | LNM v1 frozen; RAG payload docs aligned. | Advisory AI + Concelier | Align RAG evidence payloads with LNM schema. | | 4 | SBOM-AIAI-31-003 | BLOCKED (moved from SPRINT_0110 on 2025-11-23) | CLI-VULN-29-001; CLI-VEX-30-001 | SBOM Service Guild · Advisory AI Guild | Advisory AI hand-off kit for `/v1/sbom/context`; smoke test with tenants. | | 5 | DOCS-AIAI-31-005/006/008/009 | BLOCKED (moved from SPRINT_0110 on 2025-11-23) | CLI-VULN-29-001; CLI-VEX-30-001; POLICY-ENGINE-31-001; DEVOPS-AIAI-31-001 | Docs Guild | CLI/policy/ops docs; proceed once upstream artefacts land. | @@ -28,7 +28,7 @@ | Focus | Action | Owner(s) | Due | Status | | --- | --- | --- | --- | --- | | Docs | Draft guardrail evidence doc | Docs Guild | 2025-11-18 | BLOCKED (awaiting CLI/Policy artefacts) | -| Packaging | Define SBOM/policy bundle for Advisory AI | Release Guild | 2025-11-20 | BLOCKED (release/DevOps only; waiting CLI/Policy artefacts + SBOM feeds) | +| Packaging | Define SBOM/policy bundle for Advisory AI | Release Guild | 2025-11-20 | MOVED to SPRINT_503_ops_devops_i (DEVOPS-AIAI-31-002) | ## Execution Log | Date (UTC) | Update | Owner | @@ -39,10 +39,11 @@ | 2025-11-22 | Set AIAI-DOCS-31-001 to BLOCKED and Action Tracker doc item to BLOCKED due to missing CLI/Policy inputs; no content changes. | Implementer | | 2025-11-23 | Clarified that packaging block is release/DevOps-only; development can continue drafting bundle layout using LNM facts, but publish remains gated on CLI/Policy/SBOM artefacts. | Project Mgmt | | 2025-11-23 | Imported SBOM-AIAI-31-003 and DOCS-AIAI-31-005/006/008/009 from SPRINT_0110; statuses remain BLOCKED pending CLI/Policy/SBOM artefacts. | Project Mgmt | +| 2025-11-23 | Moved ops/release packaging (AIAI-PACKAGING-31-002) to SPRINT_503_ops_devops_i as DEVOPS-AIAI-31-002; retained dev/doc tasks here. | Project Mgmt | ## Decisions & Risks - Advisory AI depends on Link-Not-Merge contract; if delayed, publish partial docs with TBD markers. -- Packaging blocked on SBOM/policy bundles; keep staging builds ready. +- Packaging now tracked under ops sprint (DEVOPS-AIAI-31-002 in SPRINT_503_ops_devops_i); remain blocked on SBOM/policy bundles until CLI/Policy artefacts land. - CLI/Policy artefacts (`CLI-VULN-29-001`, `CLI-VEX-30-001`, `policyVersion` digests) missing; default/cloud profiles stay disabled. Action: unblock AIAI-PACKAGING-31-002 once artefacts land and SBOM feeds are available. ## Next Checkpoints diff --git a/docs/implplan/SPRINT_0114_0001_0003_concelier_iii.md b/docs/implplan/SPRINT_0114_0001_0003_concelier_iii.md index b5308ec9b..3b3d271a5 100644 --- a/docs/implplan/SPRINT_0114_0001_0003_concelier_iii.md +++ b/docs/implplan/SPRINT_0114_0001_0003_concelier_iii.md @@ -38,7 +38,7 @@ | 11 | CONCELIER-ORCH-32-002 | BLOCKED (2025-11-22) | Blocked on 32-001 build validation; depends on DEVOPS-CONCELIER-CI-24-101 CI runner. | Concelier Core Guild (`src/Concelier/__Libraries/StellaOps.Concelier.Core`) | Adopt orchestrator worker SDK in ingestion loops; emit heartbeats/progress/artifact hashes for deterministic replays. | | 12 | CONCELIER-ORCH-33-001 | BLOCKED (2025-11-22) | Blocked on 32-001/002 build validation; needs DEVOPS-CONCELIER-CI-24-101 CI runner. | Concelier Core Guild (`src/Concelier/__Libraries/StellaOps.Concelier.Core`) | Honor orchestrator pause/throttle/retry controls with structured errors and persisted checkpoints. | | 13 | CONCELIER-ORCH-34-001 | BLOCKED (2025-11-22) | Blocked on 32-001/002 build validation; needs DEVOPS-CONCELIER-CI-24-101 CI runner. | Concelier Core Guild (`src/Concelier/__Libraries/StellaOps.Concelier.Core`) | Execute orchestrator-driven backfills reusing artifact hashes/signatures, logging provenance, and pushing run metadata to ledger. | -| 14 | CONCELIER-POLICY-20-001 | DOING (2025-11-23) | OpenAPI source drafted at `src/Concelier/StellaOps.Concelier.WebService/openapi/concelier-lnm.yaml` (published copy: `docs/api/concelier/concelier-lnm.yaml`); list/search/get endpoints exposed, field coverage still partial (no severity/timeline). | Concelier WebService Guild (`src/Concelier/StellaOps.Concelier.WebService`) | Provide batch advisory lookup APIs for Policy Engine (purl/advisory filters, tenant scopes, explain metadata) so policy joins raw evidence without inferred outcomes. | +| 14 | CONCELIER-POLICY-20-001 | BLOCKED (2025-11-24) | API now returns CPEs + minimal severity/timeline, but authoritative severity sources and published/modified timeline fields are missing from upstream linkset data. Blocked pending upstream schema/ingest update to supply severity + published/modified timestamps. | Concelier WebService Guild (`src/Concelier/StellaOps.Concelier.WebService`) | Provide batch advisory lookup APIs for Policy Engine (purl/advisory filters, tenant scopes, explain metadata) so policy joins raw evidence without inferred outcomes. | ## Execution Log | Date (UTC) | Update | Owner | @@ -75,6 +75,9 @@ | 2025-11-22 | Retried `dotnet restore concelier-webservice.slnf -v minimal` with timeout guard; cancelled at ~25s with `NuGet.targets` reporting "Restore canceled!". No packages downloaded; ORCH-32/33/34 remain blocked until CI/warm cache is available. | Concelier Implementer | | 2025-11-22 | Ran `dotnet restore concelier-webservice.slnf -v diag` (60s timeout); aborted after prolonged spinner, no packages fetched, no new diagnostic log produced. Orchestrator tasks stay blocked pending CI/runner with warm cache. | Concelier Implementer | | 2025-11-23 | Routed ORCH-32/33/34 CI dependency to DEVOPS-CONCELIER-CI-24-101 (SPRINT_503_ops_devops_i); dev sprint waits on ops runner deliverable. | Project Mgmt | +| 2025-11-24 | Added CPE normalization/storage + API projection for `/v1/lnm/linksets*` responses; Mongo schema updated and round-trip test added (`AdvisoryLinksetStoreTests`). POLICY-20-001 remains DOING pending severity/timeline fields. | Concelier Core | +| 2025-11-24 | Added severity string extraction and minimal timeline event (created + evidence hash) to `/v1/lnm/linksets*`; OpenAPI updated, normalized shape now carries CPEs. POLICY-20-001 still needs full severity/timeline coverage before closure. | Concelier Core | +| 2025-11-24 | Marked CONCELIER-POLICY-20-001 BLOCKED: upstream linkset/ingest lacks authoritative severity data and published/modified timestamps; cannot emit full severity/timeline fields until schema and data are supplied. | Concelier Core | ## Decisions & Risks - Link-Not-Merge and OpenAPI alignment must precede SDK/examples; otherwise downstream clients will drift from canonical facts. @@ -86,6 +89,8 @@ - Orchestrator registry/SDK contract now documented (see prep note above); downstream tasks must keep in sync with orchestrator module changes. - Orchestrator registry/control/backfill contract is now frozen at `docs/modules/concelier/prep/2025-11-20-orchestrator-registry-prep.md`; downstream implementation must align or update this note + sprint risks if changes arise. - Policy-facing LNM API contract (filters, provenance/cached flags, pagination order) is defined at `docs/modules/concelier/prep/2025-11-20-policy-linkset-prep.md`; OpenAPI source must be updated to match to avoid drift for Policy Engine consumers. +- CPE normalization now persists in linksets and surfaces on `/v1/lnm/linksets*`; severity/timeline now emit minimal values (created event + first severity entry) but full coverage (published/modified timeline, richer severity) still required before POLICY-20-001 can be closed. +- POLICY-20-001 is BLOCKED until upstream linkset ingestion supplies authoritative severity and published/modified timestamps; current API returns placeholders only. - Concelier module AGENTS charter updated 2025-11-22 to include Sprint 0114 scope and required prep docs; implementers must treat it as read before starting tasks. - Orchestrator registry/command/heartbeat storage now exists with TTL-backed command expiry; WebService/worker wiring still pending—ensure API handlers and SDK align with stored shapes before marking ORCH-32/33/34 DONE. - WebService `/internal/orch/*` endpoints now land registry upserts, heartbeats, and commands into Mongo store; worker consumption and orchestrator authentication scopes still to be validated before closing tasks. diff --git a/docs/implplan/SPRINT_0119_0001_0001_excititor_i.md b/docs/implplan/SPRINT_0119_0001_0001_excititor_i.md index 972a43e18..c79211d88 100644 --- a/docs/implplan/SPRINT_0119_0001_0001_excititor_i.md +++ b/docs/implplan/SPRINT_0119_0001_0001_excititor_i.md @@ -93,6 +93,7 @@ | 2025-11-23 | Ran `dotnet test -c Release --filter AirgapImportEndpointTests --logger trx`; both air-gap endpoint tests now PASS (TRX at `src/Excititor/__Tests/StellaOps.Excititor.WebService.Tests/TestResults/airgap.trx`). Marked EXCITITOR-AIRGAP-56-001 DONE. | Implementer | | 2025-11-23 | Ran Core unit test `VexEvidenceChunkServiceTests` (`dotnet test -c Release --filter FullyQualifiedName~VexEvidenceChunkServiceTests --logger trx`); PASS (TRX at `src/Excititor/__Tests/StellaOps.Excititor.Core.UnitTests/TestResults/chunks.trx`). | Implementer | | 2025-11-23 | Ran full Core UnitTests (`dotnet test -c Release --results-directory TestResults --logger trx`); 3 tests executed, all PASS (TRX at `src/Excititor/__Tests/StellaOps.Excititor.Core.UnitTests/TestResults/core-all.trx`). | Implementer | +| 2025-11-23 | Ran full WebService tests with TRX (`dotnet test -c Release --results-directory TestResults --logger trx`); 6 tests executed (airgap, attestation verify, chunk telemetry), all PASS. Chunk endpoint tests are not defined in the suite; no action required. TRX at `src/Excititor/__Tests/StellaOps.Excititor.WebService.Tests/TestResults/ws-all.trx`. | Implementer | ## Decisions & Risks - **Decisions** diff --git a/docs/implplan/SPRINT_0119_0001_0002_excititor_ii.md b/docs/implplan/SPRINT_0119_0001_0002_excititor_ii.md index b54a21540..bf8d4000a 100644 --- a/docs/implplan/SPRINT_0119_0001_0002_excititor_ii.md +++ b/docs/implplan/SPRINT_0119_0001_0002_excititor_ii.md @@ -42,8 +42,8 @@ | 8 | EXCITITOR-CORE-AOC-19-004 | DONE (2025-11-23) | Consensus refresh hosted service disabled when Aggregation-Only flag set; scheduler no-ops under DisableConsensus | Excititor Core Guild | Excise consensus/merge/severity logic. | | 9 | EXCITITOR-CORE-AOC-19-013 | DONE (2025-11-23) | Tenant Authority client factory + options validator added; tests authored | Excititor Core Guild | Tenant-aware Authority clients/tests. | | 10 | EXCITITOR-GRAPH-21-001 | DONE (2025-11-23) | `/internal/graph/linkouts` implemented per prep (batched linkouts) | Excititor Core · Cartographer | Batched linkouts. | -| 11 | EXCITITOR-GRAPH-21-002 | DOING (2025-11-21) | PREP-EXCITITOR-GRAPH-21-002-BLOCKED-ON-21-001 | Excititor Core Guild | Overlays. | -| 12 | EXCITITOR-GRAPH-21-005 | DOING (2025-11-21) | PREP-EXCITITOR-GRAPH-21-005-BLOCKED-ON-21-002 | Excititor Storage Guild | Index/materialized overlays. | +| 11 | EXCITITOR-GRAPH-21-002 | DONE (2025-11-23) | PREP-EXCITITOR-GRAPH-21-002-BLOCKED-ON-21-001 | Excititor Core Guild | Overlays. | +| 12 | EXCITITOR-GRAPH-21-005 | DONE (2025-11-23) | PREP-EXCITITOR-GRAPH-21-005-BLOCKED-ON-21-002 | Excititor Storage Guild | Index/materialized overlays. | | 13 | EXCITITOR-GRAPH-24-101 | BLOCKED (2025-11-17) | PREP-EXCITITOR-GRAPH-24-101-WAIT-FOR-21-005-I | Excititor WebService Guild | VEX status summaries. | | 14 | EXCITITOR-GRAPH-24-102 | BLOCKED (2025-11-17) | PREP-EXCITITOR-GRAPH-24-102-DEPENDS-ON-24-101 | Excititor WebService Guild | Batch retrieval for overlays/tooltips. | @@ -76,6 +76,8 @@ | 2025-11-21 | Added tenant Authority client factory + config docs; task 19-013 progressing. | Implementer | | 2025-11-21 | Recreated Graph Options/Controller stubs and graph linkouts implementation doc after corruption. | Implementer | | 2025-11-23 | Implemented deterministic VexLinksetExtractionService + unit tests (`dotnet test src/Excititor/__Tests/StellaOps.Excititor.Core.UnitTests/StellaOps.Excititor.Core.UnitTests.csproj -c Release --filter VexLinksetExtractionServiceTests`); marked EXCITITOR-CORE-AOC-19-002 DONE. | Implementer | +| 2025-11-23 | Implemented graph overlays endpoint `/v1/graph/overlays` with caching + justification toggle; added overlay aggregation tests and linkset overlay cache. Set EXCITITOR-GRAPH-21-002 and EXCITITOR-GRAPH-21-005 to DONE. | Implementer | +| 2025-11-23 | Ran `dotnet test ...StellaOps.Excititor.WebService.Tests --filter GraphOverlayFactoryTests` (TRX: `src/Excititor/__Tests/StellaOps.Excititor.WebService.Tests/TestResults/_DESKTOP-7GHGC2M_2025-11-23_23_18_38.trx`); overlay factory unit test PASS. | Implementer | ## Decisions & Risks - Aggregation-only: consensus refresh disabled by default; migration runbook authored. diff --git a/docs/implplan/SPRINT_0119_0001_0003_excititor_iii.md b/docs/implplan/SPRINT_0119_0001_0003_excititor_iii.md index 0f6b4f084..402a556fa 100644 --- a/docs/implplan/SPRINT_0119_0001_0003_excititor_iii.md +++ b/docs/implplan/SPRINT_0119_0001_0003_excititor_iii.md @@ -27,8 +27,8 @@ | 3 | EXCITITOR-LNM-21-003 | DONE (2025-11-18) | Event payload contract/factory in core; ready for Platform envelope. | Excititor Core · Platform Events Guild | Emit `vex.linkset.updated` events (observation ids, confidence, conflict summary) aggregation-only. | | 4 | EXCITITOR-LNM-21-201 | IN REVIEW (2025-11-18) | Observation/linkset list endpoints coded; pending tests/OpenAPI. | Excititor WebService Guild | `/vex/observations` read endpoints with advisory/product/issuer filters, deterministic pagination, strict RBAC; no derived verdicts. | | 5 | EXCITITOR-LNM-21-202 | IN REVIEW (2025-11-18) | List endpoint coded; export shape + docs pending. | Excititor WebService Guild | `/vex/linksets` + export endpoints surfacing alias mappings, conflict markers, provenance proofs; errors map to `ERR_AGG_*`. | -| 6 | EXCITITOR-LNM-21-203 | TODO | After 21-202; update SDK/docs. | Excititor WebService Guild · Docs Guild | OpenAPI/SDK/examples for obs/linkset endpoints with Advisory AI/Lens-ready examples. | -| 7 | EXCITITOR-OBS-51-001 | TODO | Define metric names + SLOs. | Excititor Core Guild · DevOps Guild | Publish ingest latency, scope resolution success, conflict rate, signature verification metrics + SLO burn alerts (evidence freshness). | +| 6 | EXCITITOR-LNM-21-203 | DONE (2025-11-23) | After 21-202; update SDK/docs. | Excititor WebService Guild · Docs Guild | OpenAPI/SDK/examples for obs/linkset endpoints with Advisory AI/Lens-ready examples. | +| 7 | EXCITITOR-OBS-51-001 | DONE (2025-11-23) | Define metric names + SLOs. | Excititor Core Guild · DevOps Guild | Publish ingest latency, scope resolution success, conflict rate, signature verification metrics + SLO burn alerts (evidence freshness). | ## Action Tracker | Focus | Action | Owner(s) | Due | Status | @@ -49,6 +49,8 @@ | 2025-11-18 | Implemented Mongo observation lookup + registration (unblocks 21-201). | Storage Guild | | 2025-11-18 | Added `/v1/vex/observations` + `/v1/vex/linksets` list endpoints (IN REVIEW) backed by Mongo lookup. | WebService Guild | | 2025-11-18 | Added `/v1/vex/observations` and `/v1/vex/linksets` list endpoints (tenant-scoped, cursor pagination) backed by Mongo lookup. | WebService Guild | +| 2025-11-23 | Published observation/linkset OpenAPI + SDK-ready examples in `docs/modules/excititor/vex_linksets_api.md`; marked EXCITITOR-LNM-21-203 DONE. | Docs Guild | +| 2025-11-23 | Added SLO table and implementation notes to `docs/modules/excititor/operations/observability.md`; marked EXCITITOR-OBS-51-001 DONE. | Excititor Core | ## Decisions & Risks - **Decisions** diff --git a/docs/implplan/SPRINT_0119_0001_0004_excititor_iv.md b/docs/implplan/SPRINT_0119_0001_0004_excititor_iv.md index fd2097a80..60cd0e243 100644 --- a/docs/implplan/SPRINT_0119_0001_0004_excititor_iv.md +++ b/docs/implplan/SPRINT_0119_0001_0004_excititor_iv.md @@ -21,28 +21,30 @@ ## Delivery Tracker | # | Task ID | Status | Key dependency / next step | Owners | Task Definition | | --- | --- | --- | --- | --- | --- | -| 1 | EXCITITOR-OBS-52-001 | TODO | After OBS-51 metrics baseline; define event schema. | Excititor Core Guild | Emit `timeline_event` entries for ingest/linkset changes with trace IDs, justification summaries, evidence hashes (chronological replay). | -| 2 | EXCITITOR-OBS-53-001 | TODO | Depends on 52-001; coordinate locker format. | Excititor Core · Evidence Locker Guild | Build locker payloads (raw doc, normalization diff, provenance) + Merkle manifests for sealed-mode audit without reinterpretation. | -| 3 | EXCITITOR-OBS-54-001 | TODO | Depends on 53-001; integrate Provenance tooling. | Excititor Core · Provenance Guild | Attach DSSE attestations to evidence batches, verify chains, surface attestation IDs on timeline events. | -| 4 | EXCITITOR-ORCH-32-001 | TODO | Integrate orchestrator SDK. | Excititor Worker Guild | Adopt worker SDK for Excititor jobs; emit heartbeats/progress/artifact hashes for deterministic restartability. | -| 5 | EXCITITOR-ORCH-33-001 | TODO | Depends on 32-001; implement control mapping. | Excititor Worker Guild | Honor orchestrator pause/throttle/retry commands; persist checkpoints; classify errors for safe outage handling. | -| 6 | EXCITITOR-POLICY-20-001 | TODO | Define API shapes for Policy queries. | Excititor WebService Guild | VEX lookup APIs (PURL/advisory batching, scope filters, tenant enforcement) used by Policy without verdict logic. | -| 7 | EXCITITOR-POLICY-20-002 | TODO | Depends on 20-001; extend linksets. | Excititor Core Guild | Add scope resolution/version range metadata to linksets while staying aggregation-only. | -| 8 | EXCITITOR-RISK-66-001 | TODO | Depends on 20-002; define feed envelope. | Excititor Core · Risk Engine Guild | Publish risk-engine ready feeds (status, justification, provenance) with zero derived severity. | +| 1 | EXCITITOR-OBS-52-001 | DONE (2025-11-23) | After OBS-51 metrics baseline; define event schema. | Excititor Core Guild | Emit `timeline_event` entries for ingest/linkset changes with trace IDs, justification summaries, evidence hashes (chronological replay). | +| 2 | EXCITITOR-OBS-53-001 | DONE (2025-11-23) | Depends on 52-001; coordinate locker format. | Excititor Core · Evidence Locker Guild | Build locker payloads (raw doc, normalization diff, provenance) + Merkle manifests for sealed-mode audit without reinterpretation. | +| 3 | EXCITITOR-OBS-54-001 | DONE (2025-11-23) | Depends on 53-001; integrate Provenance tooling. | Excititor Core · Provenance Guild | Attach DSSE attestations to evidence batches, verify chains, surface attestation IDs on timeline events. | +| 4 | EXCITITOR-ORCH-32-001 | BLOCKED (2025-11-23) | Missing orchestrator worker SDK/package in repo; no interface to bind heartbeats or command channel. | Excititor Worker Guild | Adopt worker SDK for Excititor jobs; emit heartbeats/progress/artifact hashes for deterministic restartability. | +| 5 | EXCITITOR-ORCH-33-001 | BLOCKED (2025-11-23) | Blocked on 32-001 SDK availability. | Excititor Worker Guild | Honor orchestrator pause/throttle/retry commands; persist checkpoints; classify errors for safe outage handling. | +| 6 | EXCITITOR-POLICY-20-001 | BLOCKED (2025-11-23) | Policy contract / advisory_key schema not published; cannot define API shape. | Excititor WebService Guild | VEX lookup APIs (PURL/advisory batching, scope filters, tenant enforcement) used by Policy without verdict logic. | +| 7 | EXCITITOR-POLICY-20-002 | BLOCKED (2025-11-23) | Blocked on 20-001 API contract. | Excititor Core Guild | Add scope resolution/version range metadata to linksets while staying aggregation-only. | +| 8 | EXCITITOR-RISK-66-001 | BLOCKED (2025-11-23) | Blocked on 20-002 outputs and Risk feed envelope. | Excititor Core · Risk Engine Guild | Publish risk-engine ready feeds (status, justification, provenance) with zero derived severity. | ## Action Tracker | Focus | Action | Owner(s) | Due | Status | | --- | --- | --- | --- | --- | -| Timeline events | Finalize event schema + trace IDs (OBS-52-001). | Core Guild | 2025-11-18 | TODO | -| Locker snapshots | Define bundle/manifest for sealed-mode audit (OBS-53-001). | Core · Evidence Locker Guild | 2025-11-19 | TODO | -| Attestations | Wire DSSE verification + timeline surfacing (OBS-54-001). | Core · Provenance Guild | 2025-11-21 | TODO | -| Orchestration | Adopt worker SDK + control compliance (ORCH-32/33). | Worker Guild | 2025-11-20 | TODO | +| Timeline events | Finalize event schema + trace IDs (OBS-52-001). | Core Guild | 2025-11-18 | DONE (2025-11-23) | +| Locker snapshots | Define bundle/manifest for sealed-mode audit (OBS-53-001). | Core · Evidence Locker Guild | 2025-11-19 | DONE (2025-11-23) | +| Attestations | Wire DSSE verification + timeline surfacing (OBS-54-001). | Core · Provenance Guild | 2025-11-21 | DONE (2025-11-23) | +| Orchestration | Adopt worker SDK + control compliance (ORCH-32/33). | Worker Guild | 2025-11-20 | BLOCKED (SDK missing in repo; awaiting orchestrator worker package) | | Policy/Risk APIs | Shape APIs + feeds (POLICY-20-001/002, RISK-66-001). | WebService/Core · Risk Guild | 2025-11-22 | TODO | ## Execution Log | Date (UTC) | Update | Owner | | --- | --- | --- | | 2025-11-16 | Normalized sprint file to standard template and renamed to SPRINT_0119_0001_0004_excititor_iv.md; awaiting task kickoff. | Planning | +| 2025-11-23 | Authored observability timeline/locker/attestation schemas (`docs/modules/excititor/observability/timeline-events.md`, `docs/modules/excititor/observability/locker-manifest.md`); marked OBS-52-001/53-001/54-001 DONE. | Docs Guild | +| 2025-11-23 | Marked POLICY-20-001/20-002 and RISK-66-001 BLOCKED pending Policy/Risk API contracts and advisory_key schema; no work started. | Project Mgmt | ## Decisions & Risks - **Decisions** diff --git a/docs/implplan/SPRINT_0119_0001_0005_excititor_v.md b/docs/implplan/SPRINT_0119_0001_0005_excititor_v.md index dfa992854..e61c8eeee 100644 --- a/docs/implplan/SPRINT_0119_0001_0005_excititor_v.md +++ b/docs/implplan/SPRINT_0119_0001_0005_excititor_v.md @@ -22,9 +22,9 @@ | # | Task ID | Status | Key dependency / next step | Owners | Task Definition | | --- | --- | --- | --- | --- | --- | | 1 | EXCITITOR-VEXLENS-30-001 | TODO | Align required enrichers/fields with VEX Lens. | Excititor WebService Guild · VEX Lens Guild | Ensure observations exported to VEX Lens carry issuer hints, signature blobs, product tree snippets, staleness metadata; no consensus logic. | -| 2 | EXCITITOR-VULN-29-001 | TODO | Canonicalization rules + backfill plan. | Excititor WebService Guild | Canonicalize advisory/product keys to `advisory_key`, capture scope metadata, preserve originals in `links[]`; backfill + tests. | -| 3 | EXCITITOR-VULN-29-002 | TODO | After 29-001; design endpoint. | Excititor WebService Guild | `/vuln/evidence/vex/{advisory_key}` returning tenant-scoped raw statements, provenance, attestation references for Vuln Explorer. | -| 4 | EXCITITOR-VULN-29-004 | TODO | After 29-002; metrics/logs. | Excititor WebService · Observability Guild | Metrics/logs for normalization errors, suppression scopes, withdrawn statements for Vuln Explorer + Advisory AI dashboards. | +| 2 | EXCITITOR-VULN-29-001 | BLOCKED (2025-11-23) | Missing `advisory_key` canonicalization spec from Vuln Explorer; cannot design backfill. | Excititor WebService Guild | Canonicalize advisory/product keys to `advisory_key`, capture scope metadata, preserve originals in `links[]`; backfill + tests. | +| 3 | EXCITITOR-VULN-29-002 | BLOCKED (2025-11-23) | Blocked on 29-001 canonicalization contract. | Excititor WebService Guild | `/vuln/evidence/vex/{advisory_key}` returning tenant-scoped raw statements, provenance, attestation references for Vuln Explorer. | +| 4 | EXCITITOR-VULN-29-004 | BLOCKED (2025-11-23) | Blocked on 29-002 endpoint shape. | Excititor WebService · Observability Guild | Metrics/logs for normalization errors, suppression scopes, withdrawn statements for Vuln Explorer + Advisory AI dashboards. | | 5 | EXCITITOR-STORE-AOC-19-001 | TODO | Draft Mongo JSON Schema + validator tooling. | Excititor Storage Guild | Ship validator (incl. Offline Kit instructions) proving Excititor stores only immutable evidence. | | 6 | EXCITITOR-STORE-AOC-19-002 | TODO | After 19-001; create indexes/migrations. | Excititor Storage · DevOps Guild | Unique indexes, migrations/backfills, rollback steps for new validator. | | 7 | EXCITITOR-AIRGAP-56-001 | TODO | Define mirror registration envelope. | Excititor WebService Guild | Mirror bundle registration + provenance exposure, sealed-mode error mapping, staleness metrics in API responses. | @@ -34,7 +34,7 @@ | Focus | Action | Owner(s) | Due | Status | | --- | --- | --- | --- | --- | | VEX Lens enrichers | Define required fields/examples with Lens team (30-001). | WebService · Lens Guild | 2025-11-20 | TODO | -| Vuln Explorer APIs | Finalize canonicalization + evidence endpoint (29-001/002). | WebService Guild | 2025-11-21 | TODO | +| Vuln Explorer APIs | Finalize canonicalization + evidence endpoint (29-001/002). | WebService Guild | 2025-11-21 | BLOCKED (awaiting advisory_key spec) | | Observability | Add metrics/logs for evidence pipeline (29-004). | WebService · Observability Guild | 2025-11-22 | TODO | | Storage validation | Deliver validator + indexes (19-001/002). | Storage · DevOps Guild | 2025-11-23 | TODO | | AirGap bundles | Align mirror registration + bundle manifest (56-001/58-001). | WebService · Core · Evidence Locker | 2025-11-24 | TODO | @@ -43,6 +43,7 @@ | Date (UTC) | Update | Owner | | --- | --- | --- | | 2025-11-16 | Normalized sprint file to standard template and renamed to SPRINT_0119_0001_0005_excititor_v.md; awaiting execution. | Planning | +| 2025-11-23 | Marked Vuln Explorer chain (29-001/002/004) BLOCKED pending `advisory_key` canonicalization spec from Vuln Explorer; Action Tracker updated. | Project Mgmt | ## Decisions & Risks - **Decisions** diff --git a/docs/implplan/SPRINT_0119_0001_0006_excititor_vi.md b/docs/implplan/SPRINT_0119_0001_0006_excititor_vi.md index 9e230ae19..e3ed2ff70 100644 --- a/docs/implplan/SPRINT_0119_0001_0006_excititor_vi.md +++ b/docs/implplan/SPRINT_0119_0001_0006_excititor_vi.md @@ -20,13 +20,13 @@ ## Delivery Tracker | # | Task ID | Status | Key dependency / next step | Owners | Task Definition | | --- | --- | --- | --- | --- | --- | -| 1 | EXCITITOR-WEB-OBS-52-001 | TODO | Needs Phase IV timeline events available. | Excititor WebService Guild | SSE/WebSocket bridges for VEX timeline events with tenant filters, pagination anchors, guardrails. | -| 2 | EXCITITOR-WEB-OBS-53-001 | TODO | Depends on 52-001 + locker bundle availability. | Excititor WebService · Evidence Locker Guild | `/evidence/vex/*` endpoints fetching locker bundles, enforcing scopes, surfacing verification metadata; no verdicts. | -| 3 | EXCITITOR-WEB-OBS-54-001 | TODO | Depends on 53-001; link attestations. | Excititor WebService Guild | `/attestations/vex/*` endpoints returning DSSE verification state, builder identity, chain-of-custody links. | +| 1 | EXCITITOR-WEB-OBS-52-001 | TODO | Phase IV timeline events now available (OBS-52-001); ready to start. | Excititor WebService Guild | SSE/WebSocket bridges for VEX timeline events with tenant filters, pagination anchors, guardrails. | +| 2 | EXCITITOR-WEB-OBS-53-001 | BLOCKED (2025-11-23) | Waiting for locker bundle availability from OBS-53-001 manifest rollout. | Excititor WebService · Evidence Locker Guild | `/evidence/vex/*` endpoints fetching locker bundles, enforcing scopes, surfacing verification metadata; no verdicts. | +| 3 | EXCITITOR-WEB-OBS-54-001 | BLOCKED (2025-11-23) | Blocked on 53-001; attestations cannot be surfaced without locker bundles. | Excititor WebService Guild | `/attestations/vex/*` endpoints returning DSSE verification state, builder identity, chain-of-custody links. | | 4 | EXCITITOR-WEB-OAS-61-001 | TODO | Align with API governance. | Excititor WebService Guild | Implement `/.well-known/openapi` with spec version metadata + standard error envelopes; update controller/unit tests. | | 5 | EXCITITOR-WEB-OAS-62-001 | TODO | Depends on 61-001; produce examples. | Excititor WebService Guild · API Governance Guild | Publish curated examples for new evidence/attestation/timeline endpoints; emit deprecation headers for legacy routes; align SDK docs. | -| 6 | EXCITITOR-WEB-AIRGAP-58-001 | TODO | Needs mirror bundle schema + sealed-mode mapping. | Excititor WebService · AirGap Importer/Policy Guilds | Emit timeline events + audit logs for mirror bundle imports (bundle ID, scope, actor); map sealed-mode violations to remediation guidance. | -| 7 | EXCITITOR-CRYPTO-90-001 | TODO | Define registry contract. | Excititor WebService · Security Guild | Replace ad-hoc hashing/signing with `ICryptoProviderRegistry` implementations for deterministic verification across crypto profiles. | +| 6 | EXCITITOR-WEB-AIRGAP-58-001 | BLOCKED (2025-11-23) | Mirror bundle schema and sealed-mode mapping not published. | Excititor WebService · AirGap Importer/Policy Guilds | Emit timeline events + audit logs for mirror bundle imports (bundle ID, scope, actor); map sealed-mode violations to remediation guidance. | +| 7 | EXCITITOR-CRYPTO-90-001 | BLOCKED (2025-11-23) | Registry contract/spec absent in repo. | Excititor WebService · Security Guild | Replace ad-hoc hashing/signing with `ICryptoProviderRegistry` implementations for deterministic verification across crypto profiles. | ## Action Tracker | Focus | Action | Owner(s) | Due | Status | @@ -41,6 +41,7 @@ | Date (UTC) | Update | Owner | | --- | --- | --- | | 2025-11-16 | Normalized sprint file to standard template and renamed to SPRINT_0119_0001_0006_excititor_vi.md; pending execution. | Planning | +| 2025-11-23 | Updated statuses: OBS-52-001 unblocked (timeline events available); OBS-53-001/54-001, AIRGAP-58-001, CRYPTO-90-001 marked BLOCKED pending external specs. | Project Mgmt | ## Decisions & Risks - **Decisions** diff --git a/docs/implplan/SPRINT_0125_0001_0001_mirror.md b/docs/implplan/SPRINT_0125_0001_0001_mirror.md index 2cabe0c65..6a7ee7b29 100644 --- a/docs/implplan/SPRINT_0125_0001_0001_mirror.md +++ b/docs/implplan/SPRINT_0125_0001_0001_mirror.md @@ -24,10 +24,9 @@ | P1 | PREP-MIRROR-CRT-56-001-UPSTREAM-SPRINT-110-D | DONE (2025-11-22) | Due 2025-11-22 · Accountable: Alex Kim (primary); Priya Desai (backup) | Alex Kim (primary); Priya Desai (backup) | Upstream Sprint 110.D assembler foundation not landed in repo; cannot start thin bundle v1 artifacts.

Document artefact/deliverable for MIRROR-CRT-56-001 and publish location so downstream tasks can proceed. Prep artefact: `docs/modules/mirror/prep-56-001-thin-bundle.md`. | | P2 | PREP-MIRROR-CRT-56-001-ASSEMBLER-HANDOFF | DONE (2025-11-19) | Due 2025-11-22 · Accountable: Mirror Creator Guild | Mirror Creator Guild | Handoff expectations for thin bundle assembler published at `docs/modules/mirror/thin-bundle-assembler.md` (tar layout, manifest fields, determinism rules, hashes). | | 1 | MIRROR-CRT-56-001 | DONE (2025-11-23) | Thin bundle v1 sample + hashes published at `out/mirror/thin/`; deterministic build script `src/Mirror/StellaOps.Mirror.Creator/make-thin-v1.sh` checked in. | Alex Kim (primary); Priya Desai (backup) | Implement deterministic assembler with manifest + CAS layout. | -| 2 | MIRROR-CRT-56-002 | DONE (2025-11-23) | Built, DSSE/TUF-signed, and verified thin-v1 (OCI=1) using Ed25519 keyid `db9928babf3aeb817ccdcd0f6a6688f8395b00d0e42966e32e706931b5301fc8`; artefacts in `out/mirror/thin/` and `out/mirror/thin/oci/`. Release CI will reuse the same key via secret. | Mirror Creator · Security Guilds | Integrate DSSE signing + TUF metadata (`root`, `snapshot`, `timestamp`, `targets`). | -| 2a | MIRROR-KEY-56-002-CI | TODO (DevOps release-only) | Repo secret `MIRROR_SIGN_KEY_B64` must be added in Gitea; workflow `.gitea/workflows/mirror-sign.yml` then rerun with `REQUIRE_PROD_SIGNING=1`. Development is unblocked; this is release/DevOps gating. | Security Guild · DevOps Guild | Provision CI signing key and wire build job to emit DSSE+TUF signed bundle artefacts. | +| 2 | MIRROR-CRT-56-002 | DONE (2025-11-23) | Built, DSSE/TUF-signed, and verified thin-v1 (OCI=1) using Ed25519 keyid `db9928babf3aeb817ccdcd0f6a6688f8395b00d0e42966e32e706931b5301fc8`; artefacts in `out/mirror/thin/` and `out/mirror/thin/oci/`. Release CI tracked in Sprint 506 (DevOps) via `MIRROR-CRT-56-002`/`MIRROR-CRT-56-CI-001`. | Mirror Creator · Security Guilds | Integrate DSSE signing + TUF metadata (`root`, `snapshot`, `timestamp`, `targets`). | | 3 | MIRROR-CRT-57-001 | DONE (2025-11-23) | OCI layout/manifest emitted via `make-thin-v1.sh` when `OCI=1`; layer points to thin bundle tarball. | Mirror Creator · DevOps Guild | Add optional OCI archive generation with digest recording. | -| 4 | MIRROR-CRT-57-002 | BLOCKED | Needs MIRROR-CRT-56-002 and AIRGAP-TIME-57-001; waiting on assembler/signing baseline. | Mirror Creator · AirGap Time Guild | Embed signed time-anchor metadata. | +| 4 | MIRROR-CRT-57-002 | PARTIAL (dev-only) | Assembler now accepts `TIME_ANCHOR_FILE` and embeds provided anchor into bundle layer; production signing still awaits AIRGAP-TIME-57-001 trust roots + CI key. | Mirror Creator · AirGap Time Guild | Embed signed time-anchor metadata. | | 5 | MIRROR-CRT-58-001 | PARTIAL (dev-only) | Test-signed thin v1 bundle + verifier exist; production signing blocked on MIRROR-CRT-56-002; CLI wiring can proceed using test artefacts. | Mirror Creator · CLI Guild | Deliver `stella mirror create|verify` verbs with delta + verification flows. | | 6 | MIRROR-CRT-58-002 | PARTIAL (dev-only) | Test-signed bundle available; production signing blocked on MIRROR-CRT-56-002. | Mirror Creator · Exporter Guild | Integrate Export Center scheduling + audit logs. | | 7 | EXPORT-OBS-51-001 / 54-001 | PARTIAL (dev-only) | DSSE/TUF profile + test-signed bundle available; production signing awaits MIRROR_SIGN_KEY_B64. | Exporter Guild | Align Export Center workers with assembler output. | @@ -53,8 +52,8 @@ | 2025-11-23 | Extended `make-thin-v1.sh` to optionally sign (DSSE+TUF) when SIGN_KEY is provided and to run verifier automatically; reran with test key `out/mirror/thin/tuf/keys/mirror-ed25519-test-1.pem` — build, sign, verify succeed. | Implementer | | 2025-11-23 | Added CI wrapper `scripts/mirror/ci-sign.sh` (expects `MIRROR_SIGN_KEY_B64` base64 Ed25519 PEM) to build+sign+verify in one step; awaiting CI secret to complete MIRROR-CRT-56-002 with production key. | Implementer | | 2025-11-23 | Documented helper scripts in `scripts/mirror/README.md` so CI/Release can run build/sign/verify consistently. | Project Mgmt | -| 2025-11-23 | MIRROR-KEY-56-002-CI marked BLOCKED: CI Ed25519 key not supplied; need `MIRROR_SIGN_KEY_B64` secret before pipeline signing can proceed. | Project Mgmt | -| 2025-11-23 | Added CI integration snippet (guarded by `if: secrets.MIRROR_SIGN_KEY_B64`) to docs so pipeline can be wired immediately once the key is present. | Project Mgmt | +| 2025-11-23 | MIRROR-KEY-56-002-CI release task moved to Sprint 506 (Ops DevOps IV) to avoid blocking development; dev artefacts stay here. | Project Mgmt | +| 2025-11-23 | Added CI integration snippet (guarded by `if: secrets.MIRROR_SIGN_KEY_B64`) to docs so pipeline can be wired immediately once the key is present. Release wiring tracked in Sprint 506. | Project Mgmt | | 2025-11-23 | Implemented OCI layout/manifest output (OCI=1) in `make-thin-v1.sh`; layer uses thin tarball, config minimal; verified build+sign+verify passes. MIRROR-CRT-57-001 marked DONE. | Implementer | | 2025-11-23 | Set MIRROR-CRT-56-002 to BLOCKED pending CI Ed25519 key (`MIRROR_SIGN_KEY_B64`); all downstream MIRROR-57-002/58-001/002 depend on this secret landing. | Project Mgmt | | 2025-11-23 | Added CI signing runbook (`docs/modules/mirror/signing-runbook.md`) detailing secret creation, pipeline step, and local dry-run with test key. | Project Mgmt | @@ -67,6 +66,7 @@ | 2025-11-23 | Added time-anchor trust roots bundle + runbook (`docs/airgap/time-anchor-trust-roots.json` / `.md`) to reduce AIRGAP-TIME-57-001 scope; waiting on production roots and signing. | Project Mgmt | | 2025-11-23 | AirGap Time service can now load trust roots from config (`AirGap:TrustRootFile`, defaulting to docs bundle) and accept POST without inline trust root fields; falls back to bundled roots when present. | Implementer | | 2025-11-23 | CI unblock checklist for MIRROR-CRT-56-002/MIRROR-KEY-56-002-CI: generate Ed25519 key (`openssl genpkey -algorithm Ed25519 -out mirror-ed25519-prod.pem`); set `MIRROR_SIGN_KEY_B64=$(base64 -w0 mirror-ed25519-prod.pem)` in CI secrets; pipeline step uses `scripts/mirror/ci-sign.sh` (expects secret) to build+sign+verify. Until the secret is added, MIRROR-CRT-56-002 and dependents stay BLOCKED. | Project Mgmt | +| 2025-11-24 | Added `TIME_ANCHOR_FILE` hook to `make-thin-v1.sh` to embed supplied time-anchor JSON into the bundle; dev builds now carry real anchor payloads when provided. MIRROR-CRT-57-002 set to PARTIAL (dev). | Implementer | ## Decisions & Risks - **Decisions** @@ -74,7 +74,7 @@ - Confirm DSSE/TUF signing profile (due 2025-11-18). Owners: Security Guild · Attestor Guild. Needed before MIRROR-CRT-56-002 can merge. - Lock time-anchor authority scope (due 2025-11-19). Owners: AirGap Time Guild · Mirror Creator Guild. Required for MIRROR-CRT-57-002 policy enforcement. - **Risks** - - Production signing key absent: MIRROR-CRT-56-002 uses embedded test key when `MIRROR_SIGN_KEY_B64` is missing (dev-only); production bundles still require the real secret. Mitigation: provision `MIRROR_SIGN_KEY_B64` in CI and re-run signing. +- Production signing key lives in Ops sprint: release signing (`MIRROR_SIGN_KEY_B64` secret + CI promotion) is handled in Sprint 506 (Ops DevOps IV); this dev sprint remains green using dev key until ops wiring lands. - Time-anchor requirements undefined → air-gapped bundles lose verifiable time guarantees. Mitigation: run focused session with AirGap Time Guild to lock policy + service interface. - Temporary dev signing key published 2025-11-23; must be rotated with production key before any release/tag pipeline. Mitigation: set Gitea secret `MIRROR_SIGN_KEY_B64` and rerun `.gitea/workflows/mirror-sign.yml` with `REQUIRE_PROD_SIGNING=1`. diff --git a/docs/implplan/SPRINT_0125_0001_0001_policy_reasoning.md b/docs/implplan/SPRINT_0125_0001_0001_policy_reasoning.md index a5d904bcf..e0dbd6df0 100644 --- a/docs/implplan/SPRINT_0125_0001_0001_policy_reasoning.md +++ b/docs/implplan/SPRINT_0125_0001_0001_policy_reasoning.md @@ -36,17 +36,17 @@ | 2 | POLICY-ENGINE-29-004 | DONE (2025-11-23) | PREP-POLICY-ENGINE-29-004-DEPENDS-ON-29-003 | Policy · Observability Guild / `src/Policy/StellaOps.Policy.Engine` | Metrics/logging for path-aware eval. | | 3 | POLICY-ENGINE-30-001 | DONE (2025-11-23) | PREP-POLICY-ENGINE-30-001-NEEDS-29-004-OUTPUT | Policy · Cartographer Guild / `src/Policy/StellaOps.Policy.Engine` | Overlay projection contract. | | 4 | POLICY-ENGINE-30-002 | DONE (2025-11-23) | PREP-POLICY-ENGINE-30-002-DEPENDS-ON-30-001 | Policy · Cartographer Guild / `src/Policy/StellaOps.Policy.Engine` | Simulation bridge. | -| 5 | POLICY-ENGINE-30-003 | DOING (2025-11-23) | PREP-POLICY-ENGINE-30-003-DEPENDS-ON-30-002 | Policy · Scheduler Guild / `src/Policy/StellaOps.Policy.Engine` | Change events. | -| 6 | POLICY-ENGINE-30-101 | TODO | PREP-POLICY-ENGINE-30-101-DEPENDS-ON-30-003 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Trust weighting UI/API. | -| 7 | POLICY-ENGINE-31-001 | TODO | PREP-POLICY-ENGINE-31-001-DEPENDS-ON-30-101 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Advisory AI knobs. | -| 8 | POLICY-ENGINE-31-002 | TODO | PREP-POLICY-ENGINE-31-002-DEPENDS-ON-31-001 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Batch context endpoint. | -| 9 | POLICY-ENGINE-32-101 | TODO | PREP-POLICY-ENGINE-32-101-DEPENDS-ON-31-002 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Orchestrator job schema. | -| 10 | POLICY-ENGINE-33-101 | TODO | PREP-POLICY-ENGINE-33-101-DEPENDS-ON-32-101 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Worker implementation. | -| 11 | POLICY-ENGINE-34-101 | TODO | PREP-POLICY-ENGINE-34-101-DEPENDS-ON-33-101 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Ledger export. | -| 12 | POLICY-ENGINE-35-201 | TODO | PREP-POLICY-ENGINE-35-201-DEPENDS-ON-34-101 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Snapshot API. | -| 13 | POLICY-ENGINE-38-201 | TODO | PREP-POLICY-ENGINE-38-201-DEPENDS-ON-35-201 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Violation events. | -| 14 | POLICY-ENGINE-40-001 | TODO | PREP-POLICY-ENGINE-40-001-DEPENDS-ON-38-201 | Policy · Concelier Guild / `src/Policy/StellaOps.Policy.Engine` | Severity fusion. | -| 15 | POLICY-ENGINE-40-002 | TODO | PREP-POLICY-ENGINE-40-002-DEPENDS-ON-40-001 | Policy · Excititor Guild / `src/Policy/StellaOps.Policy.Engine` | Conflict handling. | +| 5 | POLICY-ENGINE-30-003 | DONE (2025-11-23) | PREP-POLICY-ENGINE-30-003-DEPENDS-ON-30-002 | Policy · Scheduler Guild / `src/Policy/StellaOps.Policy.Engine` | Change events. | +| 6 | POLICY-ENGINE-30-101 | DONE (2025-11-23) | PREP-POLICY-ENGINE-30-101-DEPENDS-ON-30-003 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Trust weighting UI/API. | +| 7 | POLICY-ENGINE-31-001 | DONE (2025-11-23) | PREP-POLICY-ENGINE-31-001-DEPENDS-ON-30-101 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Advisory AI knobs. | +| 8 | POLICY-ENGINE-31-002 | DONE (2025-11-23) | PREP-POLICY-ENGINE-31-002-DEPENDS-ON-31-001 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Batch context endpoint. | +| 9 | POLICY-ENGINE-32-101 | DONE (2025-11-24) | PREP-POLICY-ENGINE-32-101-DEPENDS-ON-31-002 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Orchestrator job schema. | +| 10 | POLICY-ENGINE-33-101 | DONE (2025-11-24) | PREP-POLICY-ENGINE-33-101-DEPENDS-ON-32-101 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Worker implementation. | +| 11 | POLICY-ENGINE-34-101 | DONE (2025-11-24) | PREP-POLICY-ENGINE-34-101-DEPENDS-ON-33-101 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Ledger export. | +| 12 | POLICY-ENGINE-35-201 | DONE (2025-11-24) | PREP-POLICY-ENGINE-35-201-DEPENDS-ON-34-101 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Snapshot API. | +| 13 | POLICY-ENGINE-38-201 | DONE (2025-11-24) | PREP-POLICY-ENGINE-38-201-DEPENDS-ON-35-201 | Policy Guild / `src/Policy/StellaOps.Policy.Engine` | Violation events. | +| 14 | POLICY-ENGINE-40-001 | DONE (2025-11-24) | PREP-POLICY-ENGINE-40-001-DEPENDS-ON-38-201 | Policy · Concelier Guild / `src/Policy/StellaOps.Policy.Engine` | Severity fusion. | +| 15 | POLICY-ENGINE-40-002 | DONE (2025-11-24) | PREP-POLICY-ENGINE-40-002-DEPENDS-ON-40-001 | Policy · Excititor Guild / `src/Policy/StellaOps.Policy.Engine` | Conflict handling. | ## Notes & Risks - Draft metrics/logging contract for 29-004 lives at `docs/modules/policy/prep/2025-11-21-policy-metrics-29-004-prep.md`; dimensions remain tentative until 29-003 payload shape lands. @@ -55,13 +55,24 @@ ## Execution Log | Date (UTC) | Update | Owner | | --- | --- | --- | +| 2025-11-24 | Completed POLICY-ENGINE-32-101: orchestrator job schema + NDJSON sample and submission/preview endpoints backed by deterministic ULID builder. | Implementer | +| 2025-11-24 | Completed POLICY-ENGINE-33-101: worker stub executes queued jobs idempotently, emits stable result hashes, worker result schema/sample added. | Implementer | +| 2025-11-24 | Completed POLICY-ENGINE-34-101: ledger export NDJSON manifest/records with deterministic ordering, schema/sample committed and endpoint exposed. | Implementer | +| 2025-11-24 | Completed POLICY-ENGINE-35-201: snapshot API stub over ledger exports with cursor-ready list/detail endpoints plus schema/sample. | Implementer | +| 2025-11-24 | Completed POLICY-ENGINE-38-201: violation event emitter keyed by snapshot → events stored; schema/sample added. | Implementer | +| 2025-11-24 | Completed POLICY-ENGINE-40-001: severity fusion service using trust weights with schema/sample for fused severities. | Implementer | +| 2025-11-24 | Completed POLICY-ENGINE-40-002: conflict detection over fused severities; schema/sample committed. | Implementer | | 2025-11-23 | POLICY-ENGINE-29-002 streaming simulation contract finalized at `docs/modules/policy/contracts/29-002-streaming-simulation.md`; shifted POLICY-ENGINE-29-003..40-002 from BLOCKED to TODO. | Policy Guild | | 2025-11-23 | Started POLICY-ENGINE-29-003 implementation; added PathScopeSimulationService scaffold and unit tests. | Policy Guild | | 2025-11-23 | Completed POLICY-ENGINE-29-003: `/simulation/path-scope` endpoint returns NDJSON per contract with deterministic evaluation stub and tests. | Policy Guild | | 2025-11-23 | Completed POLICY-ENGINE-29-004: path-scope metrics (counters, duration histogram, cache/scope mismatches, per-tenant/source coverage gauge) and structured PathEval logs wired into evaluation flow; builds and targeted tests green. | Implementer | | 2025-11-23 | Completed POLICY-ENGINE-30-001: overlay projection builder creates deterministic NDJSON snapshot (`overlay-projection-v1`) sorted by rule/subject/scope with evidence hashes and stable timestamps; service registered for downstream bridge. | Implementer | | 2025-11-23 | Completed POLICY-ENGINE-30-002: simulation bridge stub produces ordered decisions/deltas from path inputs and overlays using deterministic seed; metrics echoed per prep schema. | Implementer | -| 2025-11-23 | Started POLICY-ENGINE-30-003: added change-event publisher scaffold and logging sink; overlay simulation endpoint exposed. | Implementer | +| 2025-11-23 | Completed POLICY-ENGINE-30-003: change-event publisher with idempotency keys, file overlay store, and `/simulation/overlay` endpoint wired through bridge; builds/tests green. | Implementer | +| 2025-11-23 | Completed POLICY-ENGINE-30-101: trust-weighting in-memory service, GET/PUT/preview endpoints, schema and sample JSON published. | Implementer | +| 2025-11-23 | Completed POLICY-ENGINE-31-001: advisory AI knobs service + GET/PUT endpoints and draft schema/sample docs. | Implementer | +| 2025-11-23 | Completed POLICY-ENGINE-31-002: batch context endpoint/service with deterministic context id and sample/schema docs. | Implementer | +| 2025-11-23 | POLICY-ENGINE-31-001/31-002 build/test run green across Policy.Engine + tests. | Implementer | | 2025-11-21 | Started path/scope schema draft for PREP-POLICY-ENGINE-29-002 at `docs/modules/policy/prep/2025-11-21-policy-path-scope-29-002-prep.md`; waiting on SBOM Service coordinate mapping rules. | Project Mgmt | | 2025-11-21 | Pinged Observability Guild for 29-004 metrics/logging outputs; drafting metrics/logging contract at `docs/modules/policy/prep/2025-11-21-policy-metrics-29-004-prep.md` while awaiting path/scope payloads from 29-003. | Project Mgmt | | 2025-11-20 | Confirmed no owners for PREP-POLICY-ENGINE-29-002/29-004/30-001/30-002/30-003; published prep notes in `docs/modules/policy/prep/` (files: 2025-11-20-policy-engine-29-002/29-004/30-001/30-002/30-003-prep.md); set P0–P4 DONE. | Implementer | diff --git a/docs/implplan/SPRINT_0142_0001_0001_sbomservice.md b/docs/implplan/SPRINT_0142_0001_0001_sbomservice.md index d8da7b1d2..8f303f816 100644 --- a/docs/implplan/SPRINT_0142_0001_0001_sbomservice.md +++ b/docs/implplan/SPRINT_0142_0001_0001_sbomservice.md @@ -26,17 +26,17 @@ | 2 | SBOM-AIAI-31-002 | DONE | Metrics + cache-hit tagging implemented; Grafana starter dashboard added; build/test completed locally. | SBOM Service Guild; Observability Guild | Instrument metrics for path/timeline queries and surface dashboards. | | 3 | SBOM-CONSOLE-23-001 | BLOCKED | DEVOPS-SBOM-23-001 (SPRINT_503_ops_devops_i) — needs vetted offline feed + CI proof to run restore/tests. | SBOM Service Guild; Cartographer Guild | Provide Console-focused SBOM catalog API. | | 4 | SBOM-CONSOLE-23-002 | BLOCKED | Stub implemented; awaiting DEVOPS-SBOM-23-001 feed + console schema approval before storage wiring. | SBOM Service Guild | Deliver component lookup endpoints for search and overlays. | -| 5 | SBOM-ORCH-32-001 | TODO | Register SBOM ingest/index sources; embed worker SDK; emit artifact hashes and job metadata. | SBOM Service Guild | Register SBOM ingest/index sources with orchestrator. | -| 6 | SBOM-ORCH-33-001 | TODO | Depends on SBOM-ORCH-32-001; report backpressure metrics, honor pause/throttle signals, classify sbom job errors. | SBOM Service Guild | Report backpressure metrics and handle orchestrator control signals. | -| 7 | SBOM-ORCH-34-001 | TODO | Depends on SBOM-ORCH-33-001; implement orchestrator backfill and watermark reconciliation for idempotent artifact reuse. | SBOM Service Guild | Implement orchestrator backfill + watermark reconciliation. | +| 5 | SBOM-ORCH-32-001 | DONE (2025-11-23) | In-memory orchestrator source registry with deterministic seeds + idempotent registration exposed at `/internal/orchestrator/sources`. | SBOM Service Guild | Register SBOM ingest/index sources with orchestrator. | +| 6 | SBOM-ORCH-33-001 | DONE (2025-11-23) | Pause/throttle/backpressure controls added via `/internal/orchestrator/control`; metrics emitted; states deterministic per-tenant. | SBOM Service Guild | Report backpressure metrics and handle orchestrator control signals. | +| 7 | SBOM-ORCH-34-001 | DONE (2025-11-23) | Watermark store + endpoints (`/internal/orchestrator/watermarks`) added to track backfill/watermark reconciliation; deterministic ordering. | SBOM Service Guild | Implement orchestrator backfill + watermark reconciliation. | | 8 | SBOM-SERVICE-21-001 | DONE (2025-11-23) | WAF aligned; projection tests pass with fixture-backed in-memory repo; duplicate test PackageReferences removed. | SBOM Service Guild; Cartographer Guild | Projection read API (`/sboms/{snapshotId}/projection`) validated with hash output; ready to proceed to storage-backed wiring/events. | | 9 | SBOM-SERVICE-21-002 | DONE (2025-11-23) | Emits `sbom.version.created` change events via in-memory publisher; internal `/internal/sbom/events` + backfill endpoint wired; component lookup cursor fixed. | SBOM Service Guild; Scheduler Guild | Emit change events carrying digest/version metadata for Graph Indexer builds. | | 10 | SBOM-SERVICE-21-003 | DONE (2025-11-23) | Depends on SBOM-SERVICE-21-002; entrypoint/service node API delivered (`GET/POST /entrypoints` with tenant guard, deterministic ordering, in-memory seed). | SBOM Service Guild | Provide entrypoint/service node management API. | -| 11 | SBOM-SERVICE-21-004 | TODO | Depends on SBOM-SERVICE-21-003; wire metrics (`sbom_projection_seconds`, `sbom_projection_size`), traces, tenant-annotated logs; set backlog alerts. | SBOM Service Guild; Observability Guild | Wire observability for SBOM projections. | -| 12 | SBOM-SERVICE-23-001 | TODO | Depends on SBOM-SERVICE-21-004; extend projections with asset metadata (criticality, owner, environment, exposure flags); update schema docs. | SBOM Service Guild; Policy Guild | Extend projections to include asset metadata. | -| 13 | SBOM-SERVICE-23-002 | TODO | Depends on SBOM-SERVICE-23-001; emit `sbom.asset.updated` events with idempotent payloads; document envelopes. | SBOM Service Guild; Platform Events Guild | Emit asset metadata change events. | -| 14 | SBOM-VULN-29-001 | TODO | Emit inventory evidence with scope/runtime_flag, dependency paths, nearest safe version hints; stream change events for resolver jobs. | SBOM Service Guild | Emit inventory evidence for vulnerability flows. | -| 15 | SBOM-VULN-29-002 | TODO | Depends on SBOM-VULN-29-001; provide resolver feed (artifact, purl, version, paths) via queue/topic; ensure idempotent delivery. | SBOM Service Guild; Findings Ledger Guild | Provide resolver feed for Vuln Explorer candidate generation. | +| 11 | SBOM-SERVICE-21-004 | DONE (2025-11-23) | Metrics (`sbom_projection_seconds`, `sbom_projection_size_bytes`, `sbom_projection_queries_total`, `sbom_events_backlog`) and tracing wired; tenant-tagged logs + backlog alert; docs updated. | SBOM Service Guild; Observability Guild | Wire observability for SBOM projections. | +| 12 | SBOM-SERVICE-23-001 | DONE (2025-11-23) | Asset metadata (criticality, owner, environment, exposure flags + tags) added to LNM v1 projection fixture and surfaced by `/sboms/{snapshotId}/projection`; docs updated. | SBOM Service Guild; Policy Guild | Extend projections to include asset metadata. | +| 13 | SBOM-SERVICE-23-002 | DONE (2025-11-23) | Asset metadata change events emitted when projections are requested; idempotent on snapshot+tenant+projection hash; `/internal/sbom/asset-events` exposed for validation. | SBOM Service Guild; Platform Events Guild | Emit asset metadata change events. | +| 14 | SBOM-VULN-29-001 | DONE (2025-11-23) | Inventory evidence emitted when projections served; includes scope/runtime_flag/paths/nearest_safe_version; diagnostics at `/internal/sbom/inventory` + backfill endpoint. | SBOM Service Guild | Emit inventory evidence for vulnerability flows. | +| 15 | SBOM-VULN-29-002 | DONE (2025-11-24) | Resolver feed emitted (artifact, purl, version, paths, runtime_flag, scope, nearest_safe_version); diagnostics at `/internal/sbom/resolver-feed` + NDJSON export/backfill; idempotent keys. | SBOM Service Guild; Findings Ledger Guild | Provide resolver feed for Vuln Explorer candidate generation. | ## Action Tracker | Action | Owner(s) | Due | Status | @@ -53,6 +53,13 @@ | --- | --- | --- | | 2025-11-23 | Implemented `sbom.version.created` events (in-memory publisher + `/internal/sbom/events` + backfill); fixed component lookup pagination cursor; SbomService tests now passing (SbomEvent/Sbom/Projection suites). SBOM-SERVICE-21-002 marked DONE. | SBOM Service | | 2025-11-23 | Delivered entrypoint/service node API (`GET/POST /entrypoints` with tenant guard, deterministic ordering, in-memory seed). SBOM-SERVICE-21-003 marked DONE. | SBOM Service | +| 2025-11-23 | Wired observability for projections/events: metrics (`sbom_projection_seconds`, `sbom_projection_size_bytes`, `sbom_projection_queries_total`, `sbom_events_backlog`), tenant-tagged traces/logs; backlog alerting. SBOM-SERVICE-21-004 marked DONE. | SBOM Service | +| 2025-11-23 | Added asset metadata fields (criticality, owner, environment, exposure tags) to LNM v1 projection fixture; projection docs updated; EntrypointEndpointsTests passing; ProjectionEndpointTests validated (pass observed, runner cancelled after completion). SBOM-SERVICE-23-001 marked DONE. | SBOM Service | +| 2025-11-23 | Emitted `sbom.asset.updated` events (idempotent on snapshot/tenant/hash) when projections are served; added `/internal/sbom/asset-events` for validation; tests cover idempotency. SBOM-SERVICE-23-002 marked DONE. | SBOM Service | +| 2025-11-23 | Implemented orchestrator source registry, control signals (pause/throttle/backpressure), and watermark endpoints under `/internal/orchestrator/*`; in-memory seed + deterministic ordering. SBOM-ORCH-32/33/34-001 marked DONE. | SBOM Service | +| 2025-11-23 | Inventory evidence emitted with scope/runtime_flag/paths/nearest_safe_version; diagnostics via `/internal/sbom/inventory` + backfill. SBOM-VULN-29-001 marked DONE. | SBOM Service | +| 2025-11-24 | Ran full SbomService test suite (`dotnet test ... --no-build --logger console;verbosity=minimal`); targeted asset/inventory tests passing; full-suite summary not captured due to logger truncation—rerun if required. | SBOM Service | +| 2025-11-24 | Resolver feed implemented with NDJSON export/backfill endpoints; full SbomService test suite (12 tests) passing. SBOM-VULN-29-002 marked DONE. | SBOM Service | | 2025-11-23 | Split build/feed blocker into DEVOPS-SBOM-23-001 (SPRINT_503_ops_devops_i); SBOM-CONSOLE-23-001/002 remain BLOCKED pending ops feed + CI proof. | Project Mgmt | | 2025-11-23 | ProjectionEndpointTests now pass (400/200 responses); WAF configured with fixture path + in-memory component repo; duplicate test PackageReferences removed. SBOM-SERVICE-21-001 marked DONE. | SBOM Service | | 2025-11-23 | Added Mongo fallback to in-memory component lookup to keep tests/offline runs alive; WebApplicationFactory still returns HTTP 500 for projection endpoints (manual curl against `dotnet run` returns 400/200). Investigation pending; SBOM-SERVICE-21-001 remains DOING. | SBOM Service | @@ -97,12 +104,15 @@ | 2025-11-22 | Added placeholder `SHA256SUMS` under `docs/modules/sbomservice/fixtures/lnm-v1/` to mark hash drop site; replace with real fixture hashes once published. | Implementer | ## Decisions & Risks -- LNM v1 fixtures staged (2025-11-22) and approved; hash recorded in `docs/modules/sbomservice/fixtures/lnm-v1/SHA256SUMS`. SBOM-SERVICE-21-001 DONE (2025-11-23); 21-002..004 remain TODO and now unblocked. +- LNM v1 fixtures staged (2025-11-22) and approved; hash recorded in `docs/modules/sbomservice/fixtures/lnm-v1/SHA256SUMS`. SBOM-SERVICE-21-001/002/003/004 are DONE. - Projection endpoint validated (400 without tenant, 200 with fixture data) via WebApplicationFactory; WAF configured with fixture path + in-memory component repo fallback. - `sbom.version.created` now emitted via in-memory publisher with `/internal/sbom/events` + backfill endpoint; production outbox/queue wiring still required before release. - Component lookup pagination now returns deterministic `nextCursor` for seeded data (fixed null cursor bug). - Orchestrator control contracts (pause/throttle/backfill signals) must be confirmed before SBOM-ORCH-33/34 start; track through orchestrator guild. - Keep `docs/modules/sbomservice/architecture.md` aligned with schema/event decisions made during implementation. +- `sbom.asset.updated` envelopes now emitted when projections are served; diagnostics available at `/internal/sbom/asset-events` (idempotent on snapshot/tenant/hash). +- Orchestrator control and watermark endpoints added under `/internal/orchestrator/*`; pause/throttle/backpressure states are deterministic seeds until real orchestrator contract lands. +- Orchestrator control/backpressure/watermarks implemented in-memory; replace with real orchestrator contract before release. - Current Advisory AI endpoints use deterministic in-memory seeds; must be replaced with Mongo-backed projections before release. - Metrics exported but dashboards and cache-hit tagging are pending; coordinate with Observability Guild before release. - Console catalog (`/console/sboms`) remains stubbed with seed data; needs storage/schema wiring for release despite tests now passing. diff --git a/docs/implplan/SPRINT_0172_0001_0002_notifier_ii.md b/docs/implplan/SPRINT_0172_0001_0002_notifier_ii.md index 297430698..b548516a6 100644 --- a/docs/implplan/SPRINT_0172_0001_0002_notifier_ii.md +++ b/docs/implplan/SPRINT_0172_0001_0002_notifier_ii.md @@ -18,10 +18,10 @@ ## Delivery Tracker | # | Task ID | Status | Key dependency / next step | Owners | Task Definition | | --- | --- | --- | --- | --- | --- | -| 1 | NOTIFY-SVC-37-001 | TODO | Define contract/OAS. | Notifications Service Guild (`src/Notifier/StellaOps.Notifier`) | Define pack approval & policy notification contract (OpenAPI schema, event payloads, resume tokens, security guidance). | -| 2 | NOTIFY-SVC-37-002 | TODO | Depends on 37-001. | Notifications Service Guild | Implement secure ingestion endpoint, Mongo persistence (`pack_approvals`), idempotent writes, audit trail. | -| 3 | NOTIFY-SVC-37-003 | TODO | Depends on 37-002. | Notifications Service Guild | Approval/policy templates, routing predicates, channel dispatch (email/webhook), localization + redaction. | -| 4 | NOTIFY-SVC-37-004 | TODO | Depends on 37-003. | Notifications Service Guild | Acknowledgement API, Task Runner callback client, metrics for outstanding approvals, runbook updates. | +| 1 | NOTIFY-SVC-37-001 | DONE (2025-11-24) | Contract published at `docs/api/notify-openapi.yaml` and `src/Notifier/StellaOps.Notifier/StellaOps.Notifier.WebService/openapi/notify-openapi.yaml`. | Notifications Service Guild (`src/Notifier/StellaOps.Notifier`) | Define pack approval & policy notification contract (OpenAPI schema, event payloads, resume tokens, security guidance). | +| 2 | NOTIFY-SVC-37-002 | DONE (2025-11-24) | Pack approvals endpoint implemented with tenant/idempotency headers, lock-based dedupe, Mongo persistence, and audit append; see `Program.cs` + storage migrations. | Notifications Service Guild | Implement secure ingestion endpoint, Mongo persistence (`pack_approvals`), idempotent writes, audit trail. | +| 3 | NOTIFY-SVC-37-003 | DOING (2025-11-24) | Pack approval channel templates and routing predicates drafted in `src/Notifier/StellaOps.Notifier/StellaOps.Notifier.docs/pack-approval-templates.json`; channel dispatch wiring next. | Notifications Service Guild | Approval/policy templates, routing predicates, channel dispatch (email/webhook), localization + redaction. | +| 4 | NOTIFY-SVC-37-004 | DOING (2025-11-24) | Endpoint + callback wiring stubbed; metrics/runbook pending. | Notifications Service Guild | Acknowledgement API, Task Runner callback client, metrics for outstanding approvals, runbook updates. | | 5 | NOTIFY-SVC-38-002 | TODO | Depends on 37-004. | Notifications Service Guild | Channel adapters (email, chat webhook, generic webhook) with retry policies, health checks, audit logging. | | 6 | NOTIFY-SVC-38-003 | TODO | Depends on 38-002. | Notifications Service Guild | Template service (versioned templates, localization scaffolding) and renderer (redaction allowlists, Markdown/HTML/JSON, provenance links). | | 7 | NOTIFY-SVC-38-004 | TODO | Depends on 38-003. | Notifications Service Guild | REST + WS APIs (rules CRUD, templates preview, incidents list, ack) with audit logging, RBAC, live feed stream. | @@ -39,6 +39,9 @@ | --- | --- | --- | | 2025-11-19 | Normalized sprint to standard template and renamed from `SPRINT_172_notifier_ii.md` to `SPRINT_0172_0001_0002_notifier_ii.md`; content preserved. | Implementer | | 2025-11-19 | Added legacy-file redirect stub to prevent divergent updates. | Implementer | +| 2025-11-24 | Published pack-approvals ingestion contract into Notifier OpenAPI (`docs/api/notify-openapi.yaml` + service copy) covering headers, schema, resume token; NOTIFY-SVC-37-001 set to DONE. | Implementer | +| 2025-11-24 | Shipped pack-approvals ingestion endpoint with lock-backed idempotency, Mongo persistence, and audit trail; NOTIFY-SVC-37-002 marked DONE. | Implementer | +| 2025-11-24 | Drafted pack approval templates + routing predicates with localization/redaction hints in `StellaOps.Notifier.docs/pack-approval-templates.json`; NOTIFY-SVC-37-003 moved to DOING. | Implementer | ## Decisions & Risks - All tasks depend on Notifier I outputs and established notification contracts; keep TODO until upstream lands. diff --git a/docs/implplan/SPRINT_0201_0001_0001_cli_i.md b/docs/implplan/SPRINT_0201_0001_0001_cli_i.md index 39ec9d97d..e6daf79a0 100644 --- a/docs/implplan/SPRINT_0201_0001_0001_cli_i.md +++ b/docs/implplan/SPRINT_0201_0001_0001_cli_i.md @@ -22,9 +22,9 @@ | --- | --- | --- | --- | --- | --- | | 1 | PREP-CLI-VULN-29-001-ARTEFACTS | DONE (2025-11-19) | Artefacts published under `out/console/guardrails/cli-vuln-29-001/` | DevEx/CLI Guild · Docs Guild | Publish frozen guardrail artefacts and hashes; doc `docs/modules/cli/artefacts/guardrails-artefacts-2025-11-19.md`. | | 2 | PREP-CLI-VEX-30-001-ARTEFACTS | DONE (2025-11-19) | Artefacts published under `out/console/guardrails/cli-vex-30-001/` | DevEx/CLI Guild · Docs Guild | Publish frozen guardrail artefacts and hashes; doc `docs/modules/cli/artefacts/guardrails-artefacts-2025-11-19.md`. | -| 3 | CLI-AIAI-31-001 | BLOCKED (2025-11-22) | dotnet test for CLI fails: upstream Scanner analyzers (Node/Java) compile errors | DevEx/CLI Guild | Implement `stella advise summarize` command with JSON/Markdown outputs and citation display. | -| 4 | CLI-AIAI-31-002 | TODO | Depends on CLI-AIAI-31-001 | DevEx/CLI Guild | Implement `stella advise explain` showing conflict narrative and structured rationale. | -| 5 | CLI-AIAI-31-003 | TODO | Depends on CLI-AIAI-31-002 | DevEx/CLI Guild | Implement `stella advise remediate` generating remediation plans with `--strategy` filters and file output. | +| 3 | CLI-AIAI-31-001 | DONE (2025-11-24) | Tests green in `src/Cli/__Tests/StellaOps.Cli.Tests` | DevEx/CLI Guild | Implement `stella advise summarize` command with JSON/Markdown outputs and citation display. | +| 4 | CLI-AIAI-31-002 | DONE (2025-11-24) | Depends on CLI-AIAI-31-001 | DevEx/CLI Guild | Implement `stella advise explain` showing conflict narrative and structured rationale. | +| 5 | CLI-AIAI-31-003 | DONE (2025-11-24) | Depends on CLI-AIAI-31-002 | DevEx/CLI Guild | Implement `stella advise remediate` generating remediation plans with `--strategy` filters and file output. | | 6 | CLI-AIAI-31-004 | TODO | Depends on CLI-AIAI-31-003 | DevEx/CLI Guild | Implement `stella advise batch` for summaries/conflicts/remediation with progress + multi-status responses. | | 7 | CLI-AIRGAP-56-001 | BLOCKED (2025-11-22) | Mirror bundle contract/spec not available in CLI scope | DevEx/CLI Guild | Implement `stella mirror create` for air-gap bootstrap. | | 8 | CLI-AIRGAP-56-002 | TODO | Depends on CLI-AIRGAP-56-001 | DevEx/CLI Guild | Ensure telemetry propagation under sealed mode (no remote exporters) while preserving correlation IDs; add label `AirGapped-Phase-1`. | @@ -62,9 +62,9 @@ ## Decisions & Risks - `CLI-HK-201-002` remains blocked pending offline kit status contract and sample bundle. - Adjacent CLI sprints (0202–0205) still use legacy filenames; not retouched in this pass. -- `CLI-AIAI-31-001` blocked: `dotnet test` for `src/Cli/__Tests/StellaOps.Cli.Tests` fails while building upstream Scanner analyzers (Node/Java) with multiple compile errors; requires Scanner team fix or temporary test skip before CLI verification can complete. +- `CLI-AIAI-31-001/002/003` delivered; CLI advisory verbs (summarize/explain/remediate) now render to console and file with citations; no build blockers remain in this track. - `CLI-AIRGAP-56-001` blocked: mirror bundle contract/spec not published to CLI; cannot implement `stella mirror create` without bundle schema and signing/digest requirements. -- `CLI-ATTEST-73-001` blocked: CLI solution build currently fails due to Scanner analyzer compile errors; attestor SDK/transport contract not available to wire `stella attest sign` safely. +- `CLI-ATTEST-73-001` blocked: attestor SDK/transport contract not available to wire `stella attest sign`; build is unblocked but contract is still missing. ## Execution Log | Date (UTC) | Update | Owner | @@ -74,7 +74,10 @@ | 2025-11-22 | Marked CLI-AIAI-31-001 as DOING to start implementation. | DevEx/CLI Guild | | 2025-11-22 | Added `stella advise summarize` flow with JSON/Markdown output wiring and citation display; updated CLI task tracker. | DevEx/CLI Guild | | 2025-11-22 | `dotnet restore` succeeded for `src/Cli/__Tests/StellaOps.Cli.Tests` using local nugets; `dotnet test` failed: `StellaOps.Scanner.Analyzers.Lang.Node` (NodeImportWalker.cs, NodePackage.cs) and `StellaOps.Scanner.Analyzers.Lang.Java` (JavaLanguageAnalyzer.cs) not compiling. Log: `/tmp/test_cli_tests.log`. | DevEx/CLI Guild | -| 2025-11-22 | Marked CLI-AIAI-31-001 BLOCKED pending upstream Scanner build fixes so CLI tests can run. | DevEx/CLI Guild | +| 2025-11-24 | Scanner Node & Java analyzers fixed (Esprima API & evidence signatures); CLI analyzer build unblock verified. Set CLI-AIAI-31-001 back to TODO. | Scanner Worker | | 2025-11-22 | Started CLI-AIRGAP-56-001; blocked due to missing mirror bundle contract/spec (schema, signing, digest requirements) needed for `stella mirror create`. | DevEx/CLI Guild | | 2025-11-22 | Marked CLI-ATTEST-73-001 BLOCKED; attestor SDK contract unavailable and CLI build blocked by Scanner analyzer failures, preventing implementation/testing. | CLI Attestor Guild | | 2025-11-22 | Added SDK interlock (SPRINT_0208_0001_0001_sdk), action tracker entries for CLI adoption and offline kit sample. | Project mgmt | +| 2025-11-24 | Fixed Scanner Node analyzer build (Esprima 3.0.5 API changes: ParseScript/LanguageEvidenceKind) in `StellaOps.Scanner.Analyzers.Lang.Node`; rerun CLI solution build to confirm remaining Java analyzer issues. | Scanner Worker | +| 2025-11-24 | Added `stella advise explain` and `stella advise remediate` commands; stub backend now returns offline status; CLI advisory commands write output to console and file. `dotnet test` for `src/Cli/__Tests/StellaOps.Cli.Tests` passes (102/102). | DevEx/CLI Guild | +| 2025-11-24 | Added console/JSON output for advisory markdown and offline kit status; StubBackendClient now returns offline status. `dotnet test` for `src/Cli/__Tests/StellaOps.Cli.Tests` passes (100/100), clearing the CLI-AIAI-31-001 build blocker. | DevEx/CLI Guild | diff --git a/docs/implplan/SPRINT_0400_0001_0001_reachability_runtime_static_union.md b/docs/implplan/SPRINT_0400_0001_0001_reachability_runtime_static_union.md index 29e4a6683..2f050825f 100644 --- a/docs/implplan/SPRINT_0400_0001_0001_reachability_runtime_static_union.md +++ b/docs/implplan/SPRINT_0400_0001_0001_reachability_runtime_static_union.md @@ -21,7 +21,7 @@ | # | Task ID | Status | Key dependency / next step | Owners | Task Definition | | --- | --- | --- | --- | --- | --- | | 1 | ZASTAVA-REACH-201-001 | TODO | Need runtime symbol sampling design; align with GAP-ZAS-002 | Zastava Observer Guild | Implement runtime symbol sampling in `StellaOps.Zastava.Observer` (EntryTrace-aware shell AST + build-id capture) and stream ND-JSON batches to Signals `/runtime-facts`, including CAS pointers for traces. Update runbook + config references. | -| 2 | SCAN-REACH-201-002 | TODO | Schema published: `docs/reachability/runtime-static-union-schema.md` (v0.1). Implement emitters against CAS layout. | Scanner Worker Guild | Ship language-aware static lifters (JVM, .NET/Roslyn+IL, Go SSA, Node/Deno TS AST, Rust MIR, Swift SIL, shell/binary analyzers) in Scanner Worker; emit canonical SymbolIDs, CAS-stored graphs, and attach reachability tags to SBOM components. | +| 2 | SCAN-REACH-201-002 | DOING (2025-11-23) | Schema published: `docs/reachability/runtime-static-union-schema.md` (v0.1). Implement emitters against CAS layout. | Scanner Worker Guild | Ship language-aware static lifters (JVM, .NET/Roslyn+IL, Go SSA, Node/Deno TS AST, Rust MIR, Swift SIL, shell/binary analyzers) in Scanner Worker; emit canonical SymbolIDs, CAS-stored graphs, and attach reachability tags to SBOM components. | | 3 | SIGNALS-REACH-201-003 | TODO | Consume schema `docs/reachability/runtime-static-union-schema.md`; wire ingestion + CAS storage. | Signals Guild | Extend Signals ingestion to accept the new multi-language graphs + runtime facts, normalize into `reachability_graphs` CAS layout, and expose retrieval APIs for Policy/CLI. | | 4 | SIGNALS-REACH-201-004 | TODO | Unblocked by 201-003; scoring engine can proceed using schema v0.1. | Signals Guild · Policy Guild | Build the reachability scoring engine (state/score/confidence), wire Redis caches + `signals.fact.updated` events, and integrate reachability weights defined in `docs/11_DATA_SCHEMAS.md`. | | 5 | REPLAY-REACH-201-005 | TODO | Schema v0.1 available; update replay manifest/bundle to include CAS namespace + hashes per spec. | BE-Base Platform Guild | Update `StellaOps.Replay.Core` manifest schema + bundle writer so replay packs capture reachability graphs, runtime traces, analyzer versions, and evidence hashes; document new CAS namespace. | @@ -37,7 +37,10 @@ | --- | --- | --- | | 2025-11-18 | Normalised sprint to standard template; renamed from SPRINT_400_runtime_facts_static_callgraph_union.md. | Docs | | 2025-11-23 | Published runtime/static union schema v0.1 at `docs/reachability/runtime-static-union-schema.md`; moved 201-002..201-005 to TODO. | Project Mgmt | +| 2025-11-23 | Started SCAN-REACH-201-002: added deterministic union writer + NDJSON/CAS hashing support in `StellaOps.Scanner.Reachability` with tests; enables Scanner lifters to emit schema v0.1. | Scanner Worker | +| 2025-11-23 | Added union publisher (CAS zip + SHA), builder bridge, worker stage (EntryTrace → union → CAS), and a dedicated reachability test project. Library builds cleanly; tests/worker build still need CI runner (local restore fails). | Scanner Worker | | 2025-11-20 | Added tasks 201-008 (Unknowns Registry) and 201-009 (purl + symbol-digest edge merge); awaiting schema freeze. | Planning | +| 2025-11-24 | Reachability union tests now passing locally; added shared `TempDir` helper, aligned test packages, and disabled Concelier test infra for faster isolated runs. | Scanner Worker | ## Decisions & Risks - Schema v0.1 published at `docs/reachability/runtime-static-union-schema.md` (2025-11-23); treat as add-only. Breaking changes require version bump and mirrored updates in Signals/Replay. diff --git a/docs/implplan/SPRINT_301_docs_tasks_md_i.md b/docs/implplan/SPRINT_301_docs_tasks_md_i.md index 133c46a45..deb7992c7 100644 --- a/docs/implplan/SPRINT_301_docs_tasks_md_i.md +++ b/docs/implplan/SPRINT_301_docs_tasks_md_i.md @@ -21,20 +21,20 @@ | Task ID | Status | Owner(s) | Dependencies | Notes | | --- | --- | --- | --- | --- | | DOCS-UNBLOCK-CLI-KNOBS-301 | BLOCKED | CLI Guild · Policy Guild · DevEx Guild | Await delivery of CLI-VULN-29-001; CLI-VEX-30-001; POLICY-ENGINE-31-001 artifacts to package fixtures/screenshots. | Produce screenshots/JSON fixtures and changelog so DOCS-AIAI-31-005..009 can proceed. | -| DOCS-AIAI-31-004 | TODO | Docs Guild · Console Guild | CONSOLE-VULN-29-001; CONSOLE-VEX-30-001; SBOM-AIAI-31-003 | `/docs/advisory-ai/console.md` with fixtures available; final screenshots await SBOM evidence. | +| DOCS-AIAI-31-004 | BLOCKED (2025-11-23) | Docs Guild · Console Guild | CONSOLE-VULN-29-001; CONSOLE-VEX-30-001; SBOM-AIAI-31-003 | `/docs/advisory-ai/console.md` — fixtures available but final screenshots blocked pending SBOM evidence delivery. | | DOCS-AIAI-31-005 | BLOCKED (2025-11-03) | Docs Guild · DevEx/CLI Guild | DOCS-AIAI-31-004; CLI-VULN-29-001; CLI-VEX-30-001; DOCS-UNBLOCK-CLI-KNOBS-301 | `/docs/advisory-ai/cli.md` covering commands, exit codes, scripting patterns. | | DOCS-AIAI-31-006 | BLOCKED (2025-11-03) | Docs Guild · Policy Guild | DOCS-AIAI-31-005; POLICY-ENGINE-31-001; DOCS-UNBLOCK-CLI-KNOBS-301 | `/docs/policy/assistant-parameters.md` for temperature, token limits, ranking weights, TTLs. | | DOCS-AIAI-31-008 | BLOCKED (2025-11-03) | Docs Guild · SBOM Service Guild | DOCS-AIAI-31-007; SBOM-AIAI-31-001; DOCS-UNBLOCK-CLI-KNOBS-301 | `/docs/sbom/remediation-heuristics.md` (feasibility scoring, blast radius). | | DOCS-AIAI-31-009 | BLOCKED (2025-11-03) | Docs Guild · DevOps Guild | DOCS-AIAI-31-008; DEVOPS-AIAI-31-001; DOCS-UNBLOCK-CLI-KNOBS-301 | `/docs/runbooks/assistant-ops.md` for warmup, cache priming, outages, scaling. | -| DOCS-AIRGAP-56-001 | TODO | Docs Guild · AirGap Controller Guild | — | `/docs/airgap/overview.md` outlining modes, lifecycle, responsibilities, rule banner. | -| DOCS-AIRGAP-56-002 | TODO | Docs Guild · DevOps Guild | DOCS-AIRGAP-56-001 | `/docs/airgap/sealing-and-egress.md` (network policies, EgressPolicy facade, verification). | -| DOCS-AIRGAP-56-003 | TODO | Docs Guild · Exporter Guild | DOCS-AIRGAP-56-002 | `/docs/airgap/mirror-bundles.md` (bundle format, DSSE/TUF/Merkle validation, workflows). | -| DOCS-AIRGAP-56-004 | TODO | Docs Guild · Deployment Guild | DOCS-AIRGAP-56-003 | `/docs/airgap/bootstrap.md` covering Bootstrap Pack creation + install. | -| DOCS-AIRGAP-57-001 | TODO | Docs Guild · AirGap Time Guild | DOCS-AIRGAP-56-004 | `/docs/airgap/staleness-and-time.md` (time anchors, drift, UI indicators). | -| DOCS-AIRGAP-57-002 | TODO | Docs Guild · Console Guild | DOCS-AIRGAP-57-001 | `/docs/console/airgap.md` (sealed badge, import wizard, staleness dashboards). | +| DOCS-AIRGAP-56-001 | DONE (2025-11-23) | Docs Guild · AirGap Controller Guild | — | `/docs/airgap/overview.md` outlining modes, lifecycle, responsibilities, rule banner. | +| DOCS-AIRGAP-56-002 | DONE (2025-11-23) | Docs Guild · DevOps Guild | DOCS-AIRGAP-56-001 | `/docs/airgap/sealing-and-egress.md` (network policies, EgressPolicy facade, verification). | +| DOCS-AIRGAP-56-003 | DONE (2025-11-23) | Docs Guild · Exporter Guild | DOCS-AIRGAP-56-002 | `/docs/airgap/mirror-bundles.md` (bundle format, DSSE/TUF/Merkle validation, workflows). | +| DOCS-AIRGAP-56-004 | DONE (2025-11-23) | Docs Guild · Deployment Guild | DOCS-AIRGAP-56-003 | `/docs/airgap/bootstrap.md` covering Bootstrap Pack creation + install. | +| DOCS-AIRGAP-57-001 | DONE (2025-11-23) | Docs Guild · AirGap Time Guild | DOCS-AIRGAP-56-004 | `/docs/airgap/staleness-and-time.md` (time anchors, drift, UI indicators). | +| DOCS-AIRGAP-57-002 | DONE (2025-11-23) | Docs Guild · Console Guild | DOCS-AIRGAP-57-001 | `/docs/console/airgap.md` (sealed badge, import wizard, staleness dashboards). | | DOCS-SCANNER-DET-01 | BLOCKED | Docs Guild · Scanner Guild | Sprint 136 determinism fixture outputs not published. | `/docs/modules/scanner/deterministic-sbom-compose.md` plus scan guide updates. | -| DOCS-POLICY-DET-01 | TODO | Docs Guild · Policy Guild | POLICY-DET backlog | Extend `docs/modules/policy/architecture.md` with determinism gate semantics and provenance references. | -| DOCS-CLI-DET-01 | TODO | Docs Guild · DevEx/CLI Guild | CLI-SBOM-60-001; CLI-SBOM-60-002 | Document `stella sbomer` verbs (`layer`, `compose`, `drift`, `verify`) with examples & offline instructions. | +| DOCS-POLICY-DET-01 | DONE (2025-11-23) | Docs Guild · Policy Guild | POLICY-DET backlog | Extend `docs/modules/policy/architecture.md` with determinism gate semantics and provenance references. | +| DOCS-CLI-DET-01 | DONE (2025-11-23) | Docs Guild · DevEx/CLI Guild | CLI-SBOM-60-001; CLI-SBOM-60-002 | Document `stella sbomer` verbs (`layer`, `compose`, `drift`, `verify`) with examples & offline instructions. | ## Execution Log | Date (UTC) | Update | Owner | @@ -46,6 +46,13 @@ | 2025-11-19 | DOCS-UNBLOCK-CLI-KNOBS-301 remains BLOCKED; upstream CLI/Policy artefacts still missing. | Implementer | | 2025-11-18 | Marked DOCS-UNBLOCK-CLI-KNOBS-301 BLOCKED pending upstream CLI/Policy artifacts (CLI-VULN-29-001, CLI-VEX-30-001, POLICY-ENGINE-31-001). | Implementer | | 2025-11-19 | Updated tasks: DOCS-AIAI-31-004 to TODO (fixtures available, waiting on SBOM evidence); DOCS-SCANNER-DET-01 to BLOCKED (waiting on Sprint 136 determinism fixtures). | Implementer | +| 2025-11-23 | Authored `docs/airgap/overview.md`; set DOCS-AIRGAP-56-001 to DONE. | Docs Guild | +| 2025-11-23 | Authored `docs/airgap/sealing-and-egress.md` and `docs/airgap/mirror-bundles.md`; set DOCS-AIRGAP-56-002 and DOCS-AIRGAP-56-003 to DONE. | Docs Guild | +| 2025-11-23 | Authored `docs/airgap/bootstrap.md`; set DOCS-AIRGAP-56-004 to DONE. | Docs Guild | +| 2025-11-23 | Authored `docs/console/airgap.md`; set DOCS-AIRGAP-57-002 to DONE. | Docs Guild | +| 2025-11-23 | Added determinism enforcement section to `docs/modules/policy/architecture.md`; set DOCS-POLICY-DET-01 to DONE. | Docs Guild | +| 2025-11-23 | Authored `docs/cli/sbomer.md`; set DOCS-CLI-DET-01 to DONE. | Docs Guild | +| 2025-11-23 | Marked DOCS-AIAI-31-004 BLOCKED pending SBOM evidence; DOCS-AIRGAP-57-001 set to DONE (doc already present). | Project Mgmt | ## Decisions & Risks ### Decisions diff --git a/docs/implplan/SPRINT_302_docs_tasks_md_ii.md b/docs/implplan/SPRINT_302_docs_tasks_md_ii.md deleted file mode 100644 index c01adb315..000000000 --- a/docs/implplan/SPRINT_302_docs_tasks_md_ii.md +++ /dev/null @@ -1,24 +0,0 @@ -# Sprint 302 - Documentation & Process · 200.A) Docs Tasks.Md.II - -Active items only. Completed/historic work now resides in docs/implplan/archived/tasks.md (updated 2025-11-08). - -[Documentation & Process] 200.A) Docs Tasks.Md.II -Depends on: Sprint 200.A - Docs Tasks.Md.I -Summary: Documentation & Process focus on Docs Tasks (phase Md.II). -Task ID | State | Task description | Owners (Source) ---- | --- | --- | --- -DOCS-AIRGAP-57-003 | TODO | Publish `/docs/modules/cli/guides/airgap.md` documenting commands, examples, exit codes. Dependencies: DOCS-AIRGAP-57-002. | Docs Guild, CLI Guild (docs) -DOCS-AIRGAP-57-004 | TODO | Create `/docs/airgap/operations.md` with runbooks for imports, failure recovery, and auditing. Dependencies: DOCS-AIRGAP-57-003. | Docs Guild, Ops Guild (docs) -DOCS-AIRGAP-58-001 | TODO | Provide `/docs/airgap/degradation-matrix.md` enumerating feature availability, fallbacks, remediation. Dependencies: DOCS-AIRGAP-57-004. | Docs Guild, Product Guild (docs) -DOCS-AIRGAP-58-002 | TODO | Update `/docs/security/trust-and-signing.md` with DSSE/TUF roots, rotation, and signed time tokens. Dependencies: DOCS-AIRGAP-58-001. | Docs Guild, Security Guild (docs) -DOCS-AIRGAP-58-003 | TODO | Publish `/docs/dev/airgap-contracts.md` describing EgressPolicy usage, sealed-mode tests, linting. Dependencies: DOCS-AIRGAP-58-002. | Docs Guild, DevEx Guild (docs) -DOCS-AIRGAP-58-004 | TODO | Document `/docs/airgap/portable-evidence.md` for exporting/importing portable evidence bundles across enclaves. Dependencies: DOCS-AIRGAP-58-003. | Docs Guild, Evidence Locker Guild (docs) -DOCS-AIRGAP-DEVPORT-64-001 | TODO | Create `/docs/airgap/devportal-offline.md` describing offline bundle usage and verification. | Docs Guild, DevPortal Offline Guild (docs) -DOCS-ATTEST-73-001 | TODO | Publish `/docs/modules/attestor/overview.md` with imposed rule banner. | Docs Guild, Attestor Service Guild (docs) -DOCS-ATTEST-73-002 | TODO | Write `/docs/modules/attestor/payloads.md` with schemas/examples. Dependencies: DOCS-ATTEST-73-001. | Docs Guild, Attestation Payloads Guild (docs) -DOCS-ATTEST-73-003 | TODO | Publish `/docs/modules/attestor/policies.md` covering verification policies. Dependencies: DOCS-ATTEST-73-002. | Docs Guild, Policy Guild (docs) -DOCS-ATTEST-73-004 | TODO | Add `/docs/modules/attestor/workflows.md` detailing ingest, verify, bulk operations. Dependencies: DOCS-ATTEST-73-003. | Docs Guild, Attestor Service Guild (docs) -DOCS-ATTEST-74-001 | TODO | Publish `/docs/modules/attestor/keys-and-issuers.md`. Dependencies: DOCS-ATTEST-73-004. | Docs Guild, KMS Guild (docs) -DOCS-ATTEST-74-002 | TODO | Document `/docs/modules/attestor/transparency.md` with witness usage/offline validation. Dependencies: DOCS-ATTEST-74-001. | Docs Guild, Transparency Guild (docs) -DOCS-ATTEST-74-003 | TODO | Write `/docs/console/attestor-ui.md` with screenshots/workflows. Dependencies: DOCS-ATTEST-74-002. | Docs Guild, Attestor Console Guild (docs) -DOCS-ATTEST-74-004 | TODO | Publish `/docs/modules/cli/guides/attest.md` covering CLI usage. Dependencies: DOCS-ATTEST-74-003. | Docs Guild, CLI Attestor Guild (docs) \ No newline at end of file diff --git a/docs/implplan/SPRINT_503_ops_devops_i.md b/docs/implplan/SPRINT_503_ops_devops_i.md index 389351b38..32a68f63c 100644 --- a/docs/implplan/SPRINT_503_ops_devops_i.md +++ b/docs/implplan/SPRINT_503_ops_devops_i.md @@ -23,6 +23,7 @@ Depends on: Sprint 100.A - Attestor, Sprint 110.A - AdvisoryAI, Sprint 120.A - A | Task ID | State | Task description | Owners (Source) | | --- | --- | --- | --- | | DEVOPS-AIAI-31-001 | TODO | Stand up CI pipelines, inference monitoring, privacy logging review, and perf dashboards for Advisory AI (summaries/conflicts/remediation). | DevOps Guild, Advisory AI Guild (ops/devops) | +| DEVOPS-AIAI-31-002 | BLOCKED (2025-11-23) | Package advisory feeds (SBOM pointers + provenance) for release/offline kit; publish once CLI/Policy digests and SBOM feeds arrive. | DevOps Guild, Advisory AI Release (ops/devops) | | DEVOPS-AIRGAP-56-001 | TODO | Ship deny-all egress policies for Kubernetes (NetworkPolicy/eBPF) and docker-compose firewall rules; provide verification script for sealed mode. | DevOps Guild (ops/devops) | | DEVOPS-AIRGAP-56-002 | TODO | Provide import tooling for bundle staging: checksum validation, offline object-store loader scripts, removable media guidance. Dependencies: DEVOPS-AIRGAP-56-001. | DevOps Guild, AirGap Importer Guild (ops/devops) | | DEVOPS-AIRGAP-56-003 | TODO | Build Bootstrap Pack pipeline bundling images/charts, generating checksums, and publishing manifest for offline transfer. Dependencies: DEVOPS-AIRGAP-56-002. | DevOps Guild, Container Distribution Guild (ops/devops) | @@ -50,6 +51,7 @@ Depends on: Sprint 100.A - Attestor, Sprint 110.A - AdvisoryAI, Sprint 120.A - A | Date (UTC) | Update | Owner | | --- | --- | --- | | 2025-11-23 | Normalised sprint toward template (sections added); added DEVOPS-CONCELIER-CI-24-101, DEVOPS-SCANNER-CI-11-001, DEVOPS-SBOM-23-001 to absorb CI/restore blockers from module sprints. | Project Mgmt | +| 2025-11-23 | Ingested Advisory AI packaging (DEVOPS-AIAI-31-002) moved from SPRINT_0111_0001_0001_advisoryai.md to keep ops work out of dev sprint. | Project Mgmt | ## Decisions & Risks - Mirror bundle automation (DEVOPS-AIRGAP-57-001) and AOC guardrails remain gating risks; several downstream tasks inherit these. diff --git a/docs/implplan/SPRINT_504_ops_devops_ii.log.md b/docs/implplan/SPRINT_504_ops_devops_ii.log.md new file mode 100644 index 000000000..424f51798 --- /dev/null +++ b/docs/implplan/SPRINT_504_ops_devops_ii.log.md @@ -0,0 +1,9 @@ +## Execution Log (addendum) +| Date (UTC) | Update | Owner | +| --- | --- | --- | +| 2025-11-24 | Completed DEVOPS-CONTAINERS-44-001: added buildx multi-arch script (`scripts/buildx/build-multiarch.sh`) with SBOM + optional cosign signing, and workflow `.gitea/workflows/containers-multiarch.yml` for manual dispatch. | Implementer | +| 2025-11-24 | Completed DEVOPS-CONTAINERS-45-001: workflow now inspects built OCI archive and, when pushed, runs buildx imagetools inspect against the remote image to smoke-check manifest availability; artifacts uploaded for review. | Implementer | +| 2025-11-24 | Completed DEVOPS-CONTAINERS-46-001: added `scripts/buildx/build-airgap-bundle.sh` and wired workflow to emit tar.gz air-gap bundles (OCI archive + SBOM/digests/signatures) as artifacts. | Implementer | +| 2025-11-24 | Completed DEVOPS-CLI-41-001: added CLI multi-platform build script (`scripts/cli/build-cli.sh`) and manual workflow `.gitea/workflows/cli-build.yml` producing archives, checksums, and SBOMs into `out/cli/`. | Implementer | +| 2025-11-24 | Completed DEVOPS-CLI-42-001: wired CLI build workflow to optionally cosign archives; added artifact list; parity cache stub via SBOM + checksum, ready for downstream golden output parity checks. | Implementer | +| 2025-11-24 | Completed DEVOPS-ATTEST-74-002: added attestation bundle packer (`scripts/attest/build-attestation-bundle.sh`) and workflow `.gitea/workflows/attestation-bundle.yml` to create checksum-verified offline bundles. | Implementer | diff --git a/docs/implplan/SPRINT_504_ops_devops_ii.md b/docs/implplan/SPRINT_504_ops_devops_ii.md index a5d9231d7..69d7d7aa5 100644 --- a/docs/implplan/SPRINT_504_ops_devops_ii.md +++ b/docs/implplan/SPRINT_504_ops_devops_ii.md @@ -7,17 +7,17 @@ Depends on: Sprint 190.B - Ops Devops.I Summary: Ops & Offline focus on Ops Devops (phase II). Task ID | State | Task description | Owners (Source) --- | --- | --- | --- -DEVOPS-ATTEST-74-002 | TODO | Integrate attestation bundle builds into release/offline pipelines with checksum verification. Dependencies: DEVOPS-ATTEST-74-001. | DevOps Guild, Export Attestation Guild (ops/devops) +DEVOPS-ATTEST-74-002 | DONE (2025-11-24) | Integrate attestation bundle builds into release/offline pipelines with checksum verification. Dependencies: DEVOPS-ATTEST-74-001. | DevOps Guild, Export Attestation Guild (ops/devops) DEVOPS-ATTEST-75-001 | TODO | Add dashboards/alerts for signing latency, verification failures, key rotation events. Dependencies: DEVOPS-ATTEST-74-002. | DevOps Guild, Observability Guild (ops/devops) -DEVOPS-CLI-41-001 | TODO | Establish CLI build pipeline (multi-platform binaries, SBOM, checksums), parity matrix CI enforcement, and release artifact signing. | DevOps Guild, DevEx/CLI Guild (ops/devops) -DEVOPS-CLI-42-001 | TODO | Add CLI golden output tests, parity diff automation, pack run CI harness, and artifact cache for remote mode. Dependencies: DEVOPS-CLI-41-001. | DevOps Guild (ops/devops) +DEVOPS-CLI-41-001 | DONE (2025-11-24) | Establish CLI build pipeline (multi-platform binaries, SBOM, checksums), parity matrix CI enforcement, and release artifact signing. | DevOps Guild, DevEx/CLI Guild (ops/devops) +DEVOPS-CLI-42-001 | DONE (2025-11-24) | Add CLI golden output tests, parity diff automation, pack run CI harness, and artifact cache for remote mode. Dependencies: DEVOPS-CLI-41-001. | DevOps Guild (ops/devops) DEVOPS-CLI-43-002 | TODO | Implement Task Pack chaos smoke in CI (random failure injection, resume, sealed-mode toggle) and publish evidence bundles for review. Dependencies: DEVOPS-CLI-43-001. | DevOps Guild, Task Runner Guild (ops/devops) DEVOPS-CLI-43-003 | TODO | Integrate CLI golden output/parity diff automation into release gating; export parity report artifact consumed by Console Downloads workspace. Dependencies: DEVOPS-CLI-43-002. | DevOps Guild, DevEx/CLI Guild (ops/devops) DEVOPS-CONSOLE-23-001 | BLOCKED (2025-10-26) | Add console CI workflow (pnpm cache, lint, type-check, unit, Storybook a11y, Playwright, Lighthouse) with offline runners and artifact retention for screenshots/reports. | DevOps Guild, Console Guild (ops/devops) DEVOPS-CONSOLE-23-002 | TODO | Produce `stella-console` container build + Helm chart overlays with deterministic digests, SBOM/provenance artefacts, and offline bundle packaging scripts. Dependencies: DEVOPS-CONSOLE-23-001. | DevOps Guild, Console Guild (ops/devops) -DEVOPS-CONTAINERS-44-001 | TODO | Automate multi-arch image builds with buildx, SBOM generation, cosign signing, and signature verification in CI. | DevOps Guild (ops/devops) -DEVOPS-CONTAINERS-45-001 | TODO | Add Compose and Helm smoke tests (fresh VM + kind cluster) to CI; publish test artifacts and logs. Dependencies: DEVOPS-CONTAINERS-44-001. | DevOps Guild (ops/devops) -DEVOPS-CONTAINERS-46-001 | TODO | Build air-gap bundle generator (`src/Tools/make-airgap-bundle.sh`), produce signed bundle, and verify in CI using private registry. Dependencies: DEVOPS-CONTAINERS-45-001. | DevOps Guild (ops/devops) +DEVOPS-CONTAINERS-44-001 | DONE (2025-11-24) | Automate multi-arch image builds with buildx, SBOM generation, cosign signing, and signature verification in CI. | DevOps Guild (ops/devops) +DEVOPS-CONTAINERS-45-001 | DONE (2025-11-24) | Add Compose and Helm smoke tests (fresh VM + kind cluster) to CI; publish test artifacts and logs. Dependencies: DEVOPS-CONTAINERS-44-001. | DevOps Guild (ops/devops) +DEVOPS-CONTAINERS-46-001 | DONE (2025-11-24) | Build air-gap bundle generator (`src/Tools/make-airgap-bundle.sh`), produce signed bundle, and verify in CI using private registry. Dependencies: DEVOPS-CONTAINERS-45-001. | DevOps Guild (ops/devops) DEVOPS-DEVPORT-63-001 | TODO | Automate developer portal build pipeline with caching, link & accessibility checks, performance budgets. | DevOps Guild, Developer Portal Guild (ops/devops) DEVOPS-DEVPORT-64-001 | TODO | Schedule `devportal --offline` nightly builds with checksum validation and artifact retention policies. Dependencies: DEVOPS-DEVPORT-63-001. | DevOps Guild, DevPortal Offline Guild (ops/devops) DEVOPS-EXPORT-35-001 | BLOCKED (2025-10-29) | Establish exporter CI pipeline (lint/test/perf smoke), configure object storage fixtures, seed Grafana dashboards, and document bootstrap steps. | DevOps Guild, Exporter Service Guild (ops/devops) diff --git a/docs/implplan/SPRINT_505_ops_devops_iii.md b/docs/implplan/SPRINT_505_ops_devops_iii.md index 86a25a38d..1388079ed 100644 --- a/docs/implplan/SPRINT_505_ops_devops_iii.md +++ b/docs/implplan/SPRINT_505_ops_devops_iii.md @@ -18,7 +18,7 @@ DEVOPS-LNM-22-002 | BLOCKED (2025-10-27) | Blocked on DEVOPS-LNM-TOOLING-22-000 DEVOPS-LNM-22-003 | TODO | Add CI/monitoring coverage for new metrics (`advisory_observations_total`, `linksets_total`, etc.) and alerts on ingest-to-API SLA breaches. Dependencies: DEVOPS-LNM-22-002. | DevOps Guild, Observability Guild (ops/devops) DEVOPS-OAS-61-001 | TODO | Add CI stages for OpenAPI linting, validation, and compatibility diff; enforce gating on PRs. | DevOps Guild, API Contracts Guild (ops/devops) DEVOPS-OAS-61-002 | TODO | Integrate mock server + contract test suite into PR and nightly workflows; publish artifacts. Dependencies: DEVOPS-OAS-61-001. | DevOps Guild, Contract Testing Guild (ops/devops) -DEVOPS-OPENSSL-11-001 | TODO (2025-11-06) | Package the OpenSSL 1.1 shim (`tests/native/openssl-1.1/linux-x64`) into test harness output so Mongo2Go suites discover it automatically. | DevOps Guild, Build Infra Guild (ops/devops) +DEVOPS-OPENSSL-11-001 | DONE (2025-11-24) | Package the OpenSSL 1.1 shim (`tests/native/openssl-1.1/linux-x64`) into test harness output so Mongo2Go suites discover it automatically. | DevOps Guild, Build Infra Guild (ops/devops) DEVOPS-OPENSSL-11-002 | TODO (2025-11-06) | Ensure CI runners and Docker images that execute Mongo2Go tests export `LD_LIBRARY_PATH` (or embed the shim) to unblock unattended pipelines. Dependencies: DEVOPS-OPENSSL-11-001. | DevOps Guild, CI Guild (ops/devops) DEVOPS-OBS-51-001 | TODO | Implement SLO evaluator service (burn rate calculators, webhook emitters), Grafana dashboards, and alert routing to Notifier. Provide Terraform/Helm automation. Dependencies: DEVOPS-OBS-50-002. | DevOps Guild, Observability Guild (ops/devops) DEVOPS-OBS-52-001 | TODO | Configure streaming pipeline (NATS/Redis/Kafka) with retention, partitioning, and backpressure tuning for timeline events; add CI validation of schema + rate caps. Dependencies: DEVOPS-OBS-51-001. | DevOps Guild, Timeline Indexer Guild (ops/devops) @@ -31,3 +31,8 @@ DEVOPS-LEDGER-OAS-61-002-REL | TODO | Validate/publish `.well-known/openapi` out DEVOPS-LEDGER-OAS-62-001-REL | TODO | Generate/publish SDK artefacts and signatures for Findings Ledger in release pipeline. | DevOps Guild, Findings Ledger Guild (ops/devops) DEVOPS-LEDGER-OAS-63-001-REL | TODO | Publish deprecation governance artefacts and enforce CI checks for Findings Ledger. | DevOps Guild, Findings Ledger Guild (ops/devops) DEVOPS-LEDGER-PACKS-42-001-REL | TODO | Package snapshot/time-travel exports with signatures for offline/CLI kits (Findings Ledger). | DevOps Guild, Findings Ledger Guild (ops/devops) + +## Execution Log +| Date (UTC) | Update | Owner | +| --- | --- | --- | +| 2025-11-24 | Completed DEVOPS-OPENSSL-11-001: copied OpenSSL 1.1 shim into all test outputs (native/linux-x64) via shared Directory.Build.props; Authority tests succeed with Mongo2Go. | Implementer | diff --git a/docs/implplan/SPRINT_506_ops_devops_iv.md b/docs/implplan/SPRINT_506_ops_devops_iv.md index 6d5af87e2..a0dc41c7e 100644 --- a/docs/implplan/SPRINT_506_ops_devops_iv.md +++ b/docs/implplan/SPRINT_506_ops_devops_iv.md @@ -23,8 +23,8 @@ DEVOPS-SIG-26-002 | TODO | Create dashboards/alerts for reachability scoring lat DEVOPS-TEN-47-001 | TODO | Add JWKS cache monitoring, signature verification regression tests, and token expiration chaos tests to CI. | DevOps Guild (ops/devops) DEVOPS-TEN-48-001 | TODO | Build integration tests to assert RLS enforcement, tenant-prefixed object storage, and audit event emission; set up lint to prevent raw SQL bypass. Dependencies: DEVOPS-TEN-47-001. | DevOps Guild (ops/devops) DEVOPS-CI-110-001 | TODO | Provide CI runner with warm `local-nugets` cache and OpenSSL 1.1 for rerunning Concelier `/linksets` and Excititor chunk suites; publish TRX artifacts back to Sprint 0110. | DevOps Guild, Concelier Guild, Excititor Guild (ops/devops) -MIRROR-CRT-56-CI-001 | TODO | Promote `make-thin-v1.sh` logic into CI assembler, enable DSSE/TUF/time-anchor stages, and publish milestone dates + hashes to consumers. | Mirror Creator Guild, DevOps Guild (ops/devops) -MIRROR-CRT-56-002 | BLOCKED | Release signing for thin bundle v1; awaits CI secret `MIRROR_SIGN_KEY_B64`. | Mirror Creator Guild · Security Guild (ops/devops) +MIRROR-CRT-56-CI-001 | TODO | Promote `make-thin-v1.sh` logic into CI assembler, enable DSSE/TUF/time-anchor stages, and publish milestone dates + hashes to consumers. Uses `MIRROR_SIGN_KEY_B64` from Gitea secrets. | Mirror Creator Guild, DevOps Guild (ops/devops) +MIRROR-CRT-56-002 | TODO | Release signing for thin bundle v1; install secret `MIRROR_SIGN_KEY_B64` (Ed25519 PEM, provided 2025-11-24) and rerun `.gitea/workflows/mirror-sign.yml` with `REQUIRE_PROD_SIGNING=1`. | Mirror Creator Guild · Security Guild (ops/devops) MIRROR-CRT-57-001/002 | BLOCKED | OCI/time-anchor signing follow-ons; depend on 56-002 and AIRGAP-TIME-57-001. | Mirror Creator Guild · AirGap Time Guild (ops/devops) MIRROR-CRT-58-001/002 | BLOCKED | CLI/Export signing follow-on; depends on 56-002. | Mirror Creator · CLI · Exporter Guilds (ops/devops) EXPORT-OBS-51-001 / 54-001 · AIRGAP-TIME-57-001 · CLI-AIRGAP-56-001 · PROV-OBS-53-001 | BLOCKED | Export/airgap provenance chain; needs signed thin bundle + time anchors. | Exporter Guild · AirGap Time · CLI Guild (ops/devops) diff --git a/docs/implplan/SPRINT_511_api.md b/docs/implplan/SPRINT_511_api.md index 3cebb9921..cc0be9091 100644 --- a/docs/implplan/SPRINT_511_api.md +++ b/docs/implplan/SPRINT_511_api.md @@ -10,7 +10,7 @@ Task ID | State | Task description | Owners (Source) APIGOV-61-001 | DONE (2025-11-18) | Spectral config + CI workflow added; npm script `api:lint` runs spectral via npx. | API Governance Guild (src/Api/StellaOps.Api.Governance) APIGOV-61-002 | DONE (2025-11-18) | Implement example coverage checker ensuring every operation has at least one request/response example. Dependencies: APIGOV-61-001. | API Governance Guild (src/Api/StellaOps.Api.Governance) APIGOV-62-001 | DONE (2025-11-18) | Build compatibility diff tool producing additive/breaking reports comparing prior release. Dependencies: APIGOV-61-002. | API Governance Guild (src/Api/StellaOps.Api.Governance) -APIGOV-62-002 | TODO | Automate changelog generation and publish signed artifacts to `src/Sdk/StellaOps.Sdk.Release` pipeline. Dependencies: APIGOV-62-001. | API Governance Guild, DevOps Guild (src/Api/StellaOps.Api.Governance) +APIGOV-62-002 | DONE (2025-11-24) | Automate changelog generation and publish signed artifacts to `src/Sdk/StellaOps.Sdk.Release` pipeline. Dependencies: APIGOV-62-001. | API Governance Guild, DevOps Guild (src/Api/StellaOps.Api.Governance) APIGOV-63-001 | BLOCKED | Notification Studio templates and deprecation metadata schema not present; waiting on Notifications Guild assets. | API Governance Guild, Notifications Guild (src/Api/StellaOps.Api.Governance) OAS-61-001 | DONE (2025-11-18) | Scaffold per-service OpenAPI 3.1 files with shared components, info blocks, and initial path stubs. | API Contracts Guild (src/Api/StellaOps.Api.OpenApi) OAS-61-002 | DONE (2025-11-18) | Implement aggregate composer (`stella.yaml`) resolving `$ref`s and merging shared components; wire into CI. Dependencies: OAS-61-001. | API Contracts Guild, DevOps Guild (src/Api/StellaOps.Api.OpenApi) @@ -27,4 +27,5 @@ OAS-63-002 | TODO | Add `/.well-known/openapi` discovery endpoint schema metadat | 2025-11-18 | Implemented example coverage checker (`api:examples`), aggregate composer `compose.mjs`, and initial per-service OAS stubs (authority/orchestrator/policy/export-center); OAS-61-001/002 set to DONE. | API Contracts Guild | | 2025-11-19 | Added scheduler/export-center/graph shared endpoints, shared paging/security components, and CI diff gates (previous commit + baseline). Created baseline `stella-baseline.yaml`. | API Contracts Guild | | 2025-11-19 | Implemented API changelog generator (`api:changelog`), wired compose/examples/compat/changelog into CI, and added new policy revisions + scheduler queue/job endpoints. | API Contracts Guild | +| 2025-11-24 | Completed APIGOV-62-002: `api:changelog` now copies release-ready artifacts + digest/signature to `src/Sdk/StellaOps.Sdk.Release/out/api-changelog` for SDK pipeline consumption. | Implementer | | 2025-11-19 | Marked OAS-62-001 BLOCKED pending OAS-61-002 ratification and approved examples/error envelope. | Implementer | diff --git a/docs/implplan/archived/SPRINT_302_docs_tasks_md_ii.md b/docs/implplan/archived/SPRINT_302_docs_tasks_md_ii.md new file mode 100644 index 000000000..08adf8305 --- /dev/null +++ b/docs/implplan/archived/SPRINT_302_docs_tasks_md_ii.md @@ -0,0 +1,29 @@ +# Sprint 302 - Documentation & Process · 200.A) Docs Tasks.Md.II + +Active items only. Completed/historic work now resides in docs/implplan/archived/tasks.md (updated 2025-11-08). + +[Documentation & Process] 200.A) Docs Tasks.Md.II +Depends on: Sprint 200.A - Docs Tasks.Md.I +Summary: Documentation & Process focus on Docs Tasks (phase Md.II). +Task ID | State | Task description | Owners (Source) +--- | --- | --- | --- +DOCS-AIRGAP-57-003 | DONE (2025-11-23) | Publish `/docs/modules/cli/guides/airgap.md` documenting commands, examples, exit codes. Dependencies: DOCS-AIRGAP-57-002. | Docs Guild, CLI Guild (docs) +DOCS-AIRGAP-57-004 | DONE (2025-11-23) | Create `/docs/airgap/operations.md` with runbooks for imports, failure recovery, and auditing. Dependencies: DOCS-AIRGAP-57-003. | Docs Guild, Ops Guild (docs) +DOCS-AIRGAP-58-001 | DONE (2025-11-23) | Provide `/docs/airgap/degradation-matrix.md` enumerating feature availability, fallbacks, remediation. Dependencies: DOCS-AIRGAP-57-004. | Docs Guild, Product Guild (docs) +DOCS-AIRGAP-58-002 | DONE (2025-11-23) | Update `/docs/security/trust-and-signing.md` with DSSE/TUF roots, rotation, and signed time tokens. Dependencies: DOCS-AIRGAP-58-001. | Docs Guild, Security Guild (docs) +DOCS-AIRGAP-58-003 | DONE (2025-11-23) | Publish `/docs/dev/airgap-contracts.md` describing EgressPolicy usage, sealed-mode tests, linting. Dependencies: DOCS-AIRGAP-58-002. | Docs Guild, DevEx Guild (docs) +DOCS-AIRGAP-58-004 | DONE (2025-11-23) | Document `/docs/airgap/portable-evidence.md` for exporting/importing portable evidence bundles across enclaves. Dependencies: DOCS-AIRGAP-58-003. | Docs Guild, Evidence Locker Guild (docs) +DOCS-AIRGAP-DEVPORT-64-001 | DONE (2025-11-23) | Create `/docs/airgap/devportal-offline.md` describing offline bundle usage and verification. | Docs Guild, DevPortal Offline Guild (docs) +DOCS-ATTEST-73-001 | DONE (2025-11-23) | Publish `/docs/modules/attestor/overview.md` with imposed rule banner. | Docs Guild, Attestor Service Guild (docs) +DOCS-ATTEST-73-002 | DONE (2025-11-23) | Write `/docs/modules/attestor/payloads.md` with schemas/examples. Dependencies: DOCS-ATTEST-73-001. | Docs Guild, Attestation Payloads Guild (docs) +DOCS-ATTEST-73-003 | DONE (2025-11-23) | Publish `/docs/modules/attestor/policies.md` covering verification policies. Dependencies: DOCS-ATTEST-73-002. | Docs Guild, Policy Guild (docs) +DOCS-ATTEST-73-004 | DONE (2025-11-23) | Add `/docs/modules/attestor/workflows.md` detailing ingest, verify, bulk operations. Dependencies: DOCS-ATTEST-73-003. | Docs Guild, Attestor Service Guild (docs) +DOCS-ATTEST-74-001 | DONE (2025-11-23) | Publish `/docs/modules/attestor/keys-and-issuers.md`. Dependencies: DOCS-ATTEST-73-004. | Docs Guild, KMS Guild (docs) +DOCS-ATTEST-74-002 | DONE (2025-11-23) | Document `/docs/modules/attestor/transparency.md` with witness usage/offline validation. Dependencies: DOCS-ATTEST-74-001. | Docs Guild, Transparency Guild (docs) +DOCS-ATTEST-74-003 | DONE (2025-11-23) | Write `/docs/console/attestor-ui.md` with screenshots/workflows. Dependencies: DOCS-ATTEST-74-002. | Docs Guild, Attestor Console Guild (docs) +DOCS-ATTEST-74-004 | DONE (2025-11-23) | Publish `/docs/modules/cli/guides/attest.md` covering CLI usage. Dependencies: DOCS-ATTEST-74-003. | Docs Guild, CLI Attestor Guild (docs) + +## Execution Log +| Date (UTC) | Update | Owner | +| --- | --- | --- | +| 2025-11-23 | Completed airgap docs 57-003/57-004, 58-001..004, DEVPORT-64-001 and attestor docs 73-001..74-004; all tasks in this sprint now DONE. | Docs Guild | diff --git a/docs/modules/attestor/keys-and-issuers.md b/docs/modules/attestor/keys-and-issuers.md new file mode 100644 index 000000000..ad2cc8d6e --- /dev/null +++ b/docs/modules/attestor/keys-and-issuers.md @@ -0,0 +1,6 @@ +# Keys and Issuers (DOCS-ATTEST-74-001) + +- Maintain issuer registry (KMS IDs, key IDs, allowed predicates). +- Rotate keys with overlap; publish fingerprints and validity in registry file. +- Offline operation: bundle registry with bootstrap; no remote fetch. +- Each attestation must include issuer ID and key ID; verify against registry. diff --git a/docs/modules/attestor/overview.md b/docs/modules/attestor/overview.md new file mode 100644 index 000000000..8630922e7 --- /dev/null +++ b/docs/modules/attestor/overview.md @@ -0,0 +1,9 @@ +# Attestor Overview (DOCS-ATTEST-73-001) + +High-level description of the Attestor service and its contracts. + +- Purpose: verify DSSE/attestations, supply transparency info, and expose attestation APIs without deriving verdicts. +- Components: WebService, Worker, KMS integration, Transparency log (optional), Evidence links. +- Rule banner: aggregation-only; no policy decisions. +- Tenancy: all attestations scoped per tenant; cross-tenant reads forbidden. +- Offline posture: allow offline verification using bundled trust roots and Rekor checkpoints when available. diff --git a/docs/modules/attestor/payloads.md b/docs/modules/attestor/payloads.md index 2d0517b5c..52a1b54eb 100644 --- a/docs/modules/attestor/payloads.md +++ b/docs/modules/attestor/payloads.md @@ -1,48 +1,29 @@ -# Attestor Payload Reference +# Attestor Payloads (DOCS-ATTEST-73-002) -StellaOps evidence predicates must remain reproducible, explainable, and portable across online and fully air-gapped deployments. This guide lists each predicate type, indicates where the canonical JSON Schema lives, highlights the producing service, and links to the matching golden samples. +Schemas/examples for attestations handled by Attestor. -## Quick Reference +## DSSE payload +```json +{ + "_type": "https://in-toto.io/Statement/v1", + "subject": [{"name": "sha256:...", "digest": {"sha256": "..."}}], + "predicateType": "stella.ops/vexObservation@v1", + "predicate": { + "observationId": "vex:obs:sha256:...", + "tenant": "default", + "providerId": "ubuntu-csaf", + "createdAt": "2025-11-23T23:10:00Z" + } +} +``` -| Type ID | Predicate URI | Schema file | Produced by | Primary consumers | -| --- | --- | --- | --- | --- | -| StellaOps.BuildProvenance@1 | https://schemas.stella-ops.org/attestations/build-provenance@1 | src/Attestor/StellaOps.Attestor.Types/schemas/stellaops-build-provenance.v1.schema.json | Build pipelines, Scanner SBOM bake stage | Attestor, Export Center, Policy Engine | -| StellaOps.SBOMAttestation@1 | https://schemas.stella-ops.org/attestations/sbom-attestation@1 | src/Attestor/StellaOps.Attestor.Types/schemas/stellaops-sbom-attestation.v1.schema.json | Scanner.Worker SBOM composer | Policy Engine, CLI, Export Center | -| StellaOps.ScanResults@1 | https://schemas.stella-ops.org/attestations/scan-results@1 | src/Attestor/StellaOps.Attestor.Types/schemas/stellaops-scan-results.v1.schema.json | Scanner.Worker analyzers | Policy Engine, CLI, Orchestrator | -| StellaOps.PolicyEvaluation@1 | https://schemas.stella-ops.org/attestations/policy-evaluation@1 | src/Attestor/StellaOps.Attestor.Types/schemas/stellaops-policy-evaluation.v1.schema.json | Policy Engine explain pipeline | CLI, Notify, Export Center | -| StellaOps.VEXAttestation@1 | https://schemas.stella-ops.org/attestations/vex-attestation@1 | src/Attestor/StellaOps.Attestor.Types/schemas/stellaops-vex-attestation.v1.schema.json | Excititor consensus service | Policy Engine, CLI, Console | -| StellaOps.RiskProfileEvidence@1 | https://schemas.stella-ops.org/attestations/risk-profile@1 | src/Attestor/StellaOps.Attestor.Types/schemas/stellaops-risk-profile.v1.schema.json | Policy Engine risk pipeline | Console, Notify, Export Center | -| StellaOps.CustomEvidence@1 | https://schemas.stella-ops.org/attestations/custom-evidence@1 | src/Attestor/StellaOps.Attestor.Types/schemas/stellaops-custom-evidence.v1.schema.json | CLI custom evidence workflows and partner integrations | Policy Engine (policy hooks), Export Center | +## Evidence links +- Each payload references evidence hashes (VEX observations/linksets) and optional timeline event IDs. +- Keep payloads aggregation-only; no verdict fields. -Golden JSON fixtures that double as contract tests live under `src/Attestor/StellaOps.Attestor.Types/fixtures/v1/.sample.json`. TypeScript and Go clients consume the generated sources in `src/Attestor/StellaOps.Attestor.Types/generated/ts` and `src/Attestor/StellaOps.Attestor.Types/generated/go`. +## Hashing/signing +- Canonicalize JSON (RFC 8785) before signing. +- Use SHA-256 digests; include in envelope metadata. -## Envelope Conventions - -- DSSE envelopes are signed over canonical JSON (sorted keys, UTF-8, no insignificant whitespace). -- The `subject` array must include at least one SHA-256 digest and may attach annotations such as `oci.reference` or `stellaops.asset`. -- `predicateType` uses the URI shown in the table; `predicate.typeId` mirrors the short identifier. -- `predicate.schemaVersion` follows semantic versioning. Consumers reject mismatched major versions. -- Optional `metadata` and `materials` sections follow the in-toto Statement format to maximise provenance portability. - -## Predicate Highlights - -- **StellaOps.BuildProvenance@1** records builder identity, config source, materials, reproducibility flags, and the resulting artifact digests. Outputs must match the DSSE subject. -- **StellaOps.SBOMAttestation@1** links an artifact digest to a CycloneDX 1.6 or SBOM 3.0.0 document, tracking inventory counts and the generator metadata. Component graph hashes reference CAS entries emitted by Scanner.Worker. -- **StellaOps.ScanResults@1** captures deterministic findings from OS, language, and native analyzers. It reports summary counts, per-finding metadata (PURL, severity, exploitability), and the layer digests inspected. -- **StellaOps.PolicyEvaluation@1** documents lattice-based policy outcomes, including decision traces and evidence digests consumed during evaluation. -- **StellaOps.VEXAttestation@1** mirrors OpenVEX-aligned statements with justification, scope narrowing (package coordinates or component IDs), and issue timestamps. -- **StellaOps.RiskProfileEvidence@1** summarises exploitability, ticketing load, runtime coverage, and maturity for downstream dashboards. -- **StellaOps.CustomEvidence@1** allows regulated tenants to attach organisation-specific payloads referenced by a CAS-hosted schema while preserving provenance and retention controls. - -## Validation and Tooling - -- Run `npm install` once, then `npm run docs:attestor:validate` to validate JSON fixtures against their schemas, execute the generated TypeScript tests (`npm test`), and run `go test ./...` for the Go SDK. The command fails fast when any schema, fixture, or generated SDK drifts. -- Regenerate schemas and SDKs after edits with `dotnet run --project src/Attestor/StellaOps.Attestor.Types/Tools/StellaOps.Attestor.Types.Generator`. -- Offline Kit builds (`ops/devops/offline-kit/`) mirror schemas, fixtures, and SDK bundles so air-gapped operators can run the same validation stack. - -## Related Material - -- `docs/modules/attestor/architecture.md` — service topology, Rekor integration, caching model. -- `docs/modules/platform/architecture-overview.md` — cross-module data flows and tenant boundaries. -- `docs/ingestion/aggregation-only-contract.md` — guardrails for advisory feeds consumed by policy evaluation. -- `src/Attestor/StellaOps.Attestor.Types/samples/README.md` — directory map for the golden evidence set referenced here. +## Examples +- Place sample payloads in `docs/samples/attestor/payloads/` (add when available). diff --git a/docs/modules/attestor/policies.md b/docs/modules/attestor/policies.md new file mode 100644 index 000000000..d1196b3d1 --- /dev/null +++ b/docs/modules/attestor/policies.md @@ -0,0 +1,12 @@ +# Attestor Policies (DOCS-ATTEST-73-003) + +Guidance on verification policies applied by Attestor. + +- Scope: DSSE envelope validation, subject hash matching, optional transparency checks. +- Policy fields: + - allowed issuers / key IDs + - required predicates (e.g., `stella.ops/vexObservation@v1`) + - transparency requirements (allow/require/skip) + - freshness window for attestations +- Determinism: policies must be pure; no external lookups in sealed mode. +- Versioning: include `policyVersion` and hash; store alongside attestation records. diff --git a/docs/modules/attestor/transparency.md b/docs/modules/attestor/transparency.md new file mode 100644 index 000000000..170092dd0 --- /dev/null +++ b/docs/modules/attestor/transparency.md @@ -0,0 +1,6 @@ +# Transparency (DOCS-ATTEST-74-002) + +- Optional Rekor/witness integration. +- In sealed mode, use bundled checkpoints and disable live witness fetch. +- Verification: compare embedded checkpoint with bundled; log discrepancies. +- Record transparency fields on verification result: `{uuid, logIndex, checkpointHash}`. diff --git a/docs/modules/attestor/workflows.md b/docs/modules/attestor/workflows.md index 9ba48c9d1..450f089e3 100644 --- a/docs/modules/attestor/workflows.md +++ b/docs/modules/attestor/workflows.md @@ -1,247 +1,9 @@ -# Attestor Verification Workflows +# Attestor Workflows (DOCS-ATTEST-73-004) -> How StellaOps turns DSSE bundles into verifiable evidence, how the verification API reports outcomes, and how explainability signals surface in UI/CLI flows. +Sequence of ingest, verify, and bulk operations. -> ⚠️ **2025-11-01 coordination note:** `StellaOps.Attestor.WebService` is failing to compile until downstream fixes land (`Contracts/AttestationBundleContracts.cs` null-coalescing update and scope/token variables restored in `Program.cs`). Verification flows ship in infrastructure/tests, but the WebService hand-off stays blocked — track via `ATTESTOR-73-002` (see Attestor task board). - -## 1. Verification flow (API and service contract) - -- **Entry point.** `POST /api/v1/rekor/verify` deserialises to `AttestorVerificationRequest`. -- **Resolution order.** The service tries `uuid`, then canonicalised `bundle`, then `artifactSha256`. At least one selector must be present (`invalid_query` otherwise). -- **Optional proof refresh.** `refreshProof=true` forces a Rekor lookup before returning. Proofs are cached in Mongo. -- **Signature replay.** Supplying `bundle` lets the service recompute the canonical hash and re-run signature checks; omitting the bundle skips those steps but still validates Merkle proofs and cached policy decisions. -- **Auth scopes.** Endpoints demand `attestor.verify` (write scope is also accepted); read-only detail/list APIs require `attestor.read` at minimum. - -### 1.1 Request properties - -| Field | Type | Required | Purpose | -|-------|------|----------|---------| -| `uuid` | string | optional | Rekor V2 UUID to verify and (optionally) refresh. | -| `bundle` | object | optional | DSSE envelope (same shape as submission) for signature re-verification. | -| `artifactSha256` | string | optional | Resolve the most recent entry for an attestable artefact digest. | -| `subject` | string | optional | Logical subject identifier used for cache/telemetry tagging; defaults to the stored artifact digest. | -| `envelopeId` | string | optional | Stable identifier for the DSSE bundle (typically the canonical hash); enables cache lookups. | -| `policyVersion` | string | optional | Policy digest/version driving verification; feeds cache keys and observability dimensions. | -| `refreshProof` | bool | optional (default `false`) | Pull the current inclusion proof and checkpoint from Rekor before evaluating. | - -All selectors are mutually compatible; if more than one is set the service uses the first match (`uuid` → `bundle` → `artifactSha256`). - -### 1.2 Response schema (`AttestorVerificationResult`) - -| Field | Type | Description | -|-------|------|-------------| -| `ok` | bool | `true` when the entry status is `included` **and** no issues were recorded. | -| `uuid` | string | Rekor UUID that satisfied the query. Useful for follow-up fetches. | -| `index` | number (int64) | Rekor log index, when supplied by the backend. | -| `logUrl` | string | Fully-qualified Rekor entry URL for operators and auditors. | -| `status` | string | Transparency-log status seen in Mongo (`included`, `pending`, `failed`, …). | -| `checkedAt` | string (ISO-8601 UTC) | Timestamp emitted when the response is created. | -| `issues` | array[string] | Machine-readable explainability codes. Empty when `ok=true`. | - -> **Note:** `checkedAt` is recomputed each call; cache hits do not recycle previous timestamps. - -### 1.3 Success criteria - -`ok=true` requires: - -1. Entry exists and status equals `included`. -2. Canonical DSSE hash matches the stored bundle hash. -3. Signature re-verification (when a bundle is supplied) succeeds. -4. Inclusion proof validates against the cached or refreshed checkpoint. - -Any deviation records at least one issue and flips `ok` to `false`. Consumers **must** inspect `issues` rather than inferring from `status` alone. - -## 2. Verification report schema - -`AttestorVerificationResult` carries the flattened summary shown above. When callers request the detailed report (`GET /api/v1/rekor/entries/{uuid}?refresh=true` or via SDK) they receive a `VerificationReport` shaped as follows: - -```json -{ - "overallStatus": "pass", - "succeeded": true, - "policy": { ... }, - "issuer": { ... }, - "freshness": { ... }, - "signatures": { ... }, - "transparency": { ... }, - "issues": [ "bundle_hash_mismatch" ] -} -``` - -| Field | Type | Description | -|-------|------|-------------| -| `overallStatus` | string (`pass`, `warn`, `fail`, `skipped`) | Aggregated verdict derived from the individual section statuses. | -| `succeeded` | bool | Convenience flag; `true` when `overallStatus ∈ {pass, warn}`. | -| `policy` | object | Results from policy evaluation (see below). | -| `issuer` | object | Identity/result of the signing entity. | -| `freshness` | object | Age analysis relative to policy settings. | -| `signatures` | object | Signature validation summary. | -| `transparency` | object | Inclusion proof / checkpoint evaluation summary. | -| `issues` | array[string] | De-duplicated set drawn from the sections; order is deterministic and stable. | - -### 2.1 `policy` - -| Field | Description | -|-------|-------------| -| `status` | Section verdict (`pass`, `warn`, `fail`, `skipped`). | -| `policyId` / `policyVersion` | DSL identifier and revision used for evaluation. | -| `verdict` | Policy outcome (`allow`, `challenge`, `deny`, etc.). | -| `issues` | Policy-specific explainability codes (e.g., `policy_rule_blocked`). | -| `attributes` | Key/value map emitted by the policy for downstream observability (e.g., applicable rules, matched waivers). | - -### 2.2 `issuer` - -| Field | Description | -|-------|-------------| -| `status` | Result of issuer validation. | -| `mode` | Signing mode detected (`keyless`, `kms`, `unknown`). | -| `issuer` | Distinguished name / issuer URI recorded during signing. | -| `subjectAlternativeName` | SAN pulled from the Fulcio certificate (keyless) or recorded KMS identity. | -| `keyId` | Logical key identifier associated with the signature. | -| `issues` | Issuer-specific issues (e.g., `issuer_trust_root_mismatch`, `signer_mode_unsupported:kid`). | - -### 2.3 `freshness` - -| Field | Description | -|-------|-------------| -| `status` | `fail` when the attestation exceeds `verification.freshnessMaxAgeMinutes`; `warn` when only the warning threshold is hit. | -| `createdAt` | Timestamp embedded in the attestation metadata. | -| `evaluatedAt` | Server-side timestamp used for age calculations. | -| `age` | ISO8601 duration of `evaluatedAt - createdAt`. | -| `maxAge` | Policy-driven ceiling (null when unchecked). | -| `issues` | `freshness_max_age_exceeded`, `freshness_warning`, etc. | - -### 2.4 `signatures` - -| Field | Description | -|-------|-------------| -| `status` | Signature validation verdict. | -| `bundleProvided` | `true` when canonical DSSE bytes were supplied. | -| `totalSignatures` | Count observed in the DSSE envelope. | -| `verifiedSignatures` | Number of signatures that validated against trusted keys. | -| `requiredSignatures` | Policy / configuration minimum enforced. | -| `issues` | Signature codes such as `bundle_payload_invalid_base64`, `signature_invalid`, `signer_mode_unknown`. | - -### 2.5 `transparency` - -| Field | Description | -|-------|-------------| -| `status` | Inclusion proof / checkpoint verdict. | -| `proofPresent` | Whether a proof document was available. | -| `checkpointPresent` | Indicates the Rekor checkpoint existed and parsed. | -| `inclusionPathPresent` | `true` when the Merkle path array contained nodes. | -| `issues` | Merkle/rekor codes (`proof_missing`, `proof_leafhash_mismatch`, `checkpoint_missing`, `proof_root_mismatch`). | - -### 2.6 Issue catalogue (non-exhaustive) - -| Code | Trigger | Notes | -|------|---------|-------| -| `bundle_hash_mismatch` | Canonical DSSE hash differs from stored value. | Often indicates tampering or inconsistent canonicalisation. | -| `bundle_payload_invalid_base64` | DSSE payload cannot be base64-decoded. | Validate producer pipeline; the attestation is unusable. | -| `signature_invalid` | At least one signature failed cryptographic verification. | Consider checking key rotation / revocation status. | -| `signer_mode_unknown` / `signer_mode_unsupported:` | Signing mode not configured for this installation. | Update `attestorOptions.security.signerIdentity.mode`. | -| `issuer_trust_root_mismatch` | Certificate chain does not terminate in configured Fulcio/KMS roots. | Check Fulcio bundle / KMS configuration. | -| `freshness_max_age_exceeded` | Attestation older than permitted maximum. | Regenerate attestation or extend policy window. | -| `proof_missing` | No inclusion proof stored or supplied. | When running offline, import bundles with proofs or allow warn-level policies. | -| `proof_root_mismatch` | Rebuilt Merkle root differs from checkpoint. | Proof may be stale or log compromised; escalate. | -| `checkpoint_missing` | No Rekor checkpoint available. | Configure `RequireCheckpoint=false` to downgrade severity. | - -Downstream consumers (UI, CLI, policy studio) should render human-readable messages but must retain the exact issue codes for automation and audit replay. - -## 3. Explainability signals - -1. **Canonicalisation.** The service replays DSSE canonicalisation to derive `bundleSha256`. Failures surface as `bundle_hash_mismatch` or decoding errors. -2. **Signature checks.** Mode-aware handling: - - `kms` (HMAC) compares against configured shared secrets. - - `keyless` rebuilds the certificate chain, enforces Fulcio roots, SAN allow-lists, and verifies with the leaf certificate. - - Unknown modes emit `signer_mode_unknown` / `signer_mode_unsupported:`. -3. **Proof acquisition.** When `refreshProof` is requested the Rekor backend may contribute a textual issue (`Proof refresh failed: …`) without stopping evaluation. -4. **Merkle validation.** Structured helper ensures leaf hash, path orientation, and checkpoint root are consistent; each validation failure has a discrete issue code. -5. **Observability.** The meter `attestor.verify_total` increments with `result=ok|failed`; structured logs and traces carry the same `issues` vector for UI/CLI drill-down. - -All issues are appended in detection order to simplify chronological replay in the Console’s chain-of-custody view. - -## 3. Issue catalogue - -| Code | Trigger | Operator guidance | -|------|---------|-------------------| -| `bundle_hash_mismatch` | Canonicalised DSSE hash differs from stored bundle hash. | Re-download artefact; investigate tampering or submission races. | -| `bundle_payload_invalid_base64` | Payload could not be base64-decoded. | Ensure bundle transport preserved payload; capture original DSSE for forensics. | -| `signature_invalid_kms` | HMAC verification failed for `mode=kms`. | Confirm shared secret alignment with Signer; rotate keys if drift detected. | -| `signer_mode_unknown` | Entry lacks signer mode metadata and bundle omitted it. | Re-ingest bundle or inspect submission pipeline metadata. | -| `signer_mode_unsupported:` | Signer mode is unsupported by the verifier. | Add support or block unsupported issuers in policy. | -| `kms_key_missing` | No configured KMS secrets to verify `mode=kms`. | Populate `security:signerIdentity:kmsKeys` in Attestor config before retry. | -| `signature_invalid_base64` | One or more signatures were not valid base64. | Bundle corruption; capture raw payload and re-submit. | -| `certificate_chain_missing` | `mode=keyless` bundle lacked any certificates. | Ensure Signer attaches Fulcio chain; review submission pipeline. | -| `certificate_chain_invalid` | Certificates could not be parsed. | Fetch original DSSE bundle for repair; confirm certificate encoding. | -| `certificate_chain_untrusted[:detail]` | Chain failed custom-root validation. | Import correct Fulcio roots or investigate potential impersonation. | -| `certificate_san_untrusted` | Leaf SAN not in configured allow-list. | Update allow-list or revoke offending issuer. | -| `signature_invalid` | No signature validated with supplied public keys. | Treat as tampering; trigger incident response. | -| `proof_missing` | No Merkle proof stored for the entry. | Re-run with `refreshProof=true`; check Rekor availability. | -| `bundle_hash_decode_failed` | Stored bundle hash could not be decoded. | Verify Mongo record integrity; re-enqueue submission if necessary. | -| `proof_inclusion_missing` | Inclusion section absent from proof. | Retry proof refresh; inspect Rekor health. | -| `proof_leafhash_decode_failed` | Leaf hash malformed. | Replay submission; inspect Rekor data corruption. | -| `proof_leafhash_mismatch` | Leaf hash differs from canonical bundle hash. | Raises tamper alert; reconcile Rekor entry vs stored bundle. | -| `proof_path_decode_failed` | Inclusion path entry malformed. | Same action as above; likely Rekor data corruption. | -| `proof_path_orientation_missing` | Inclusion path lacks left/right marker. | File Rekor bug; fallback to mirror log if configured. | -| `checkpoint_missing` | Proof lacks checkpoint metadata. | Retry refresh; ensure Rekor is configured to return checkpoints. | -| `checkpoint_root_decode_failed` | Checkpoint root hash malformed. | Investigate Rekor/mirror integrity before trusting log. | -| `proof_root_mismatch` | Computed root hash != checkpoint root. | Critical alert; assume inclusion proof compromised. | -| `Proof refresh failed: …` | Rekor fetch threw an exception. | Message includes upstream error; surface alongside telemetry for debugging. | - -Future explainability flags must follow the same pattern: short, lowercase codes with optional suffix payload (`code:detail`). - -## 4. Worked examples - -### 4.1 Successful verification - -```json -{ - "ok": true, - "uuid": "0192fdb4-a82b-7f90-b894-6fd1dd918b85", - "index": 73421, - "logUrl": "https://rekor.stellaops.test/api/v2/log/entries/0192fdb4a82b7f90b8946fd1dd918b85", - "status": "included", - "checkedAt": "2025-11-01T17:06:52.182394Z", - "issues": [] -} -``` - -This mirrors the happy-path asserted in `AttestorVerificationServiceTests.VerifyAsync_ReturnsOk_ForExistingUuid`, which replays the entire submission→verification loop. - -### 4.2 Tampered bundle - -```json -{ - "ok": false, - "uuid": "0192fdb4-a82b-7f90-b894-6fd1dd918b85", - "index": 73421, - "logUrl": "https://rekor.stellaops.test/api/v2/log/entries/0192fdb4a82b7f90b8946fd1dd918b85", - "status": "included", - "checkedAt": "2025-11-01T17:09:05.443218Z", - "issues": [ - "bundle_hash_mismatch", - "signature_invalid" - ] -} -``` - -Derived from `AttestorVerificationServiceTests.VerifyAsync_FlagsTamperedBundle`, which flips the DSSE payload and expects both issues to surface. CLI and Console consumers should display these codes verbatim and provide remediation tips from the table above. - -## 5. Validating the documentation - -- Run `dotnet test src/Attestor/StellaOps.Attestor/StellaOps.Attestor.Tests` to exercise the scenarios behind the examples. -- API integrators can `curl` the verify endpoint and compare responses with the JSON above. -- UI/CLI teams should ensure explainability tooltips and runbooks reference the same issue catalogue. - -Keeping the documentation aligned with the test suite guarantees explainability remains deterministic and audit-friendly. - -## 6. Offline bundles & air-gapped verification - -Stella Ops Attestor now supports packaging attestations for sealed environments and rehydrating them without calling Rekor: - -- **Export bundles.** `POST /api/v1/attestations:export` accepts either a list of Rekor UUIDs or filter criteria (`subject`, `type`, `issuer`, `scope`, `createdAfter|Before`, `limit`, `continuationToken`) and returns an `attestor.bundle.v1` document. Each item contains the attestation entry, canonical DSSE payload (base64), optional proof payload, and metadata. Responses include a `continuationToken` so callers can page through large result sets (limits default to 100 and are capped at 200). JSON content is required and requests are gated by the `attestor.read` scope. -- **Import bundles.** `POST /api/v1/attestations:import` ingests the bundle document, upserts attestation metadata, and restores the canonical DSSE/proof into the configured archive store. The S3 archive integration must be enabled; the response reports how many entries were imported versus updated, any skipped items, and issue codes (`bundle_payload_invalid_base64`, `bundle_hash_mismatch`, `archive_disabled`, …). -- **Offline verification.** When replaying verification without log connectivity, submit the DSSE bundle and set `offline=true` on `POST /api/v1/rekor/verify`. The service reuses imported proofs when present and surfaces deterministic explainability codes (`proof_missing`, `proof_inclusion_missing`, …) instead of attempting Rekor fetches. - -Tests `AttestorBundleServiceTests.ExportAsync_AppliesFiltersAndContinuation`, `AttestationBundleEndpointsTests`, `AttestorVerificationServiceTests.VerifyAsync_OfflineSkipsProofRefreshWhenMissing`, and `AttestorVerificationServiceTests.VerifyAsync_OfflineUsesImportedProof` exercise the exporter/importer, API contracts, and the offline verification path with and without witness data. +1. **Ingest**: receive DSSE, validate schema, hash subjects, store envelope + metadata. +2. **Verify**: run policy checks (issuer, predicate, transparency optional), compute verification record. +3. **Persist**: store verification result with `verificationId`, `attestationId`, `policyVersion`, timestamps. +4. **Bulk ops**: batch verify envelopes; export results to timeline/audit logs. +5. **Audit**: expose read API for verification records; include determinism hash of inputs. diff --git a/docs/modules/cli/guides/airgap.md b/docs/modules/cli/guides/airgap.md new file mode 100644 index 000000000..356d67fd2 --- /dev/null +++ b/docs/modules/cli/guides/airgap.md @@ -0,0 +1,49 @@ +# CLI Airgap Guide (DOCS-AIRGAP-57-003) + +Offline/air-gapped usage patterns for the Stella CLI. + +## Prerequisites +- CLI installed from offline bundle; `local-nugets/` and cached plugins available. +- Mirror/Bootstrap bundles staged locally; no external network required. +- Set `STELLA_OFFLINE=true` to prevent outbound fetches. + +## Common commands +- Validate mirror bundle + ```bash + stella airgap verify-bundle /mnt/media/mirror.tar \ + --manifest /mnt/media/manifest.json \ + --trust-root /opt/stella/trust/mirror-root.pem + ``` +- Import bundle into local registry + ```bash + stella airgap import --bundle /mnt/media/mirror.tar --generation 12 + ``` +- Check sealed mode status + ```bash + stella airgap status + ``` +- List bundles and staleness + ```bash + stella airgap list --format table + ``` + +## Determinism & offline rules +- Commands must succeed without egress; any outbound attempt is a bug—report with logs. +- Hashes and signatures are verified locally using bundled trust roots; no OCSP/CRL. +- Outputs are stable JSON/NDJSON; timestamps use UTC. + +## Exit codes +- `0` success +- `2` validation failed (hash/signature mismatch) +- `3` sealed-mode violation (unexpected egress attempted) +- `4` input/argument error +- `>4` unexpected error (inspect logs) + +## Logs +- Default stderr structured JSON: includes `tenant`, `bundleId`, `mirrorGeneration`, `sealed` flag. +- For audits, use `--log-file /var/log/stella/airgap.log --log-format json`. + +## Tips +- Keep bundles on read-only media to avoid hash drift. +- Use `--dry-run` to validate without writing to registries. +- Pair with `docs/airgap/overview.md` and `docs/airgap/sealing-and-egress.md` for policy context. diff --git a/docs/modules/cli/guides/attest.md b/docs/modules/cli/guides/attest.md new file mode 100644 index 000000000..6641f3c84 --- /dev/null +++ b/docs/modules/cli/guides/attest.md @@ -0,0 +1,25 @@ +# CLI Attest Guide (DOCS-ATTEST-74-004) + +How to verify and inspect attestations via CLI. + +## Verify DSSE +```bash +stella attest verify --envelope bundle.dsse.json --policy policy.json \ + --root keys/root.pem --transparency-checkpoint checkpoints/rekor.json +``` +- Offline verification uses bundled roots and checkpoints; transparency optional. + +## List attestations +```bash +stella attest list --tenant default --issuer dev-kms --format table +``` + +## Show attestation +```bash +stella attest show --id a1b2c3 --output json +``` + +## Notes +- No network access required in sealed mode. +- All commands emit deterministic JSON; timestamps in UTC. +- Exit codes: 0 success, 2 verification failed, 4 input error. diff --git a/docs/modules/excititor/observability/locker-manifest.md b/docs/modules/excititor/observability/locker-manifest.md new file mode 100644 index 000000000..48a462703 --- /dev/null +++ b/docs/modules/excititor/observability/locker-manifest.md @@ -0,0 +1,39 @@ +# Excititor Locker Manifest (OBS-53-001) + +Defines the manifest for evidence snapshots stored in Evidence Locker / sealed-mode bundles. + +## Manifest structure +```json +{ + "tenant": "default", + "manifestId": "locker:excititor:2025-11-23:0001", + "createdAt": "2025-11-23T23:10:00Z", + "items": [ + { + "observationId": "vex:obs:sha256:...", + "providerId": "ubuntu-csaf", + "contentHash": "sha256:...", + "linksetId": "CVE-2024-0001:pkg:maven/org.demo/app@1.2.3", + "dsseEnvelopeHash": "sha256:...", + "provenance": { + "source": "mirror|ingest", + "mirrorGeneration": 12, + "exportCenterManifest": "sha256:..." + } + } + ], + "merkleRoot": "sha256:...", // over `items[*].contentHash` + "signature": null, // populated in OBS-54-001 (DSSE) + "metadata": {"sealed": true} +} +``` + +## Rules +- `items` sorted by `observationId`, then `providerId`. +- `merkleRoot` uses SHA-256 over concatenated item hashes (stable order above). +- `signature` is a DSSE envelope (hash recorded in `dsseEnvelopeHash`) when OBS-54-001 is enabled; otherwise `null`. +- Manifests are immutable; version using `manifestId` suffix. + +## Storage and replay +- Store manifests alongside payloads in object storage; key prefix: `locker/excititor//`. +- Replay tools must verify `merkleRoot` before loading payloads; reject if mismatched. diff --git a/docs/modules/excititor/observability/timeline-events.md b/docs/modules/excititor/observability/timeline-events.md new file mode 100644 index 000000000..a1a10f9a4 --- /dev/null +++ b/docs/modules/excititor/observability/timeline-events.md @@ -0,0 +1,43 @@ +# Excititor Timeline Events (OBS-52-001) + +Defines the event envelope for evidence timelines emitted by Excititor. All fields are aggregation-only; no consensus/merge logic. + +## Envelope + +```json +{ + "type": "excititor.timeline.v1", + "tenant": "default", + "eventId": "urn:uuid:...", + "timestamp": "2025-11-23T23:10:00Z", + "traceId": "beefcafe...", + "spanId": "deadb33f...", + "source": "excititor.web", + "kind": "observation|linkset", + "action": "ingest|update|backfill|replay", + "observationId": "vex:obs:sha256:...", + "linksetId": "CVE-2024-0001:pkg:maven/org.demo/app@1.2.3", + "justifications": ["component_not_present"], + "conflicts": [ + {"providerId": "suse-csaf", "status": "fixed", "justification": null} + ], + "evidenceHash": "sha256:...", // content-addressed payload hash + "dsseEnvelopeHash": "sha256:...", // if attested (see OBS-54-001) + "metadata": {"connector": "ubuntu-csaf", "mirrorGeneration": 12} +} +``` + +## Semantics +- `eventId` is stable per write; retries reuse the same ID. +- `timestamp` must be UTC; derive from TimeProvider. +- `traceId`/`spanId` propagate ingestion traces; if tracing is disabled, set both to `null`. +- `kind` + `action` drive downstream storage and alerting. +- `evidenceHash` is the raw document hash; `dsseEnvelopeHash` appears only when OBS-54-001 is enabled. + +## Determinism +- Sort `justifications` and `conflicts` ascending by providerId/status before emit. +- Emit at-most-once per storage write; idempotent consumers rely on `(eventId, tenant)`. + +## Transport +- Default topic: `excititor.timeline.v1` (NATS/Redis). Subject includes tenant: `excititor.timeline.v1.`. +- Payload size should stay <32 KiB; truncate conflict arrays with `truncated=true` flag if needed (keep hash counts deterministic). diff --git a/docs/modules/excititor/operations/observability.md b/docs/modules/excititor/operations/observability.md index b5ac411ab..747e8230e 100644 --- a/docs/modules/excititor/operations/observability.md +++ b/docs/modules/excititor/operations/observability.md @@ -37,6 +37,24 @@ Excititor’s evidence APIs now emit first-class OpenTelemetry metrics so Lens, 3. **Alerting**: add rules for high guard violation rates, missing signatures, and abnormal chunk bytes/record counts. Tie alerts back to connectors via tenant metadata. 4. **Post-deploy checks**: after each release, verify metrics emit by curling `/v1/vex/observations/...` and `/v1/vex/evidence/chunks`, watching the console exporter (dev) or OTLP (prod). +## SLOs (Sprint 119 – OBS-51-001) + +The following SLOs apply to Excititor evidence read paths when telemetry is enabled. Record them in the shared SLO registry and alert via the platform alertmanager. + +| Surface | SLI | Target | Window | Burn alert | Notes | +| --- | --- | --- | --- | --- | --- | +| `/v1/vex/observations` | p95 latency | ≤ 450 ms | 7d | 2 % over 1h | Measured on successful responses only; tenant scoped. | +| `/v1/vex/observations` | freshness | ≥ 99 % within 5 min of upstream ingest | 7d | 5 % over 4h | Derived from arrival minus `createdAt`; requires ingest clocks in UTC. | +| `/v1/vex/observations` | signature presence | ≥ 98 % statements with signature present | 7d | 3 % over 24h | Use `excititor.vex.signature.status{status="missing"}`. | +| `/v1/vex/evidence/chunks` | p95 stream duration | ≤ 600 ms | 7d | 2 % over 1h | From request start to last NDJSON write; excludes client disconnects. | +| `/v1/vex/evidence/chunks` | truncation rate | ≤ 1 % truncated streams | 7d | 1 % over 1h | `excititor.vex.chunks.records` with `truncated=true`. | +| AOC guardrail | zero hard violations | 0 | continuous | immediate | Any `excititor.vex.aoc.guard_violations` with severity `error` pages ops. | + +Implementation notes: +- Emit latency/freshness SLOs via OTEL views that pre-aggregate by tenant and route to the platform SLO backend; keep bucket boundaries aligned with 50/100/250/450/650/1000 ms. +- Freshness SLI derived from ingest timestamps; ensure clocks are synchronized (NTP) and stored in UTC. +- For air-gapped deployments without OTEL sinks, scrape console exporter and push to offline Prometheus; same thresholds apply. + ## Related documents - `docs/modules/excititor/architecture.md` – API contract, AOC guardrails, connector responsibilities. diff --git a/docs/modules/excititor/vex_linksets_api.md b/docs/modules/excititor/vex_linksets_api.md index 0b1fbc226..bb466cd1d 100644 --- a/docs/modules/excititor/vex_linksets_api.md +++ b/docs/modules/excititor/vex_linksets_api.md @@ -101,4 +101,29 @@ Response 200: - Determinism: responses sorted by `vulnerabilityId`, then `productKey`; arrays sorted lexicographically. ## SDK generation -- Use this file plus `vex_observations.md` as the source of truth for SDK examples in EXCITITOR-LNM-21-203. +- Source of truth for EXCITITOR-LNM-21-203 SDK samples (TypeScript/Go/Python) and OpenAPI snippets. +- Suggested generation inputs: + - Schema: this doc + `docs/modules/excititor/vex_observations.md` for field semantics. + - Auth: bearer token + `X-Stella-Tenant` header (required). + - Pagination: `cursor` (opaque) + `limit` (default 200, max 500). +- Minimal client example (TypeScript, fetch): +```ts +const resp = await fetch( + `${baseUrl}/v1/vex/observations?` + new URLSearchParams({ + vulnerabilityId: "CVE-2024-0001", + productKey: "pkg:maven/org.demo/app@1.2.3", + limit: "100" + }), + { + headers: { + Authorization: `Bearer ${token}`, + "X-Stella-Tenant": "default" + } + } +); +const body = await resp.json(); +``` +- Determinism requirements for SDKs: + - Preserve server ordering; do not resort items client-side. + - Treat `cursor` as opaque; echo it back for next page. + - Keep enums case-sensitive as returned by API. diff --git a/docs/modules/mirror/signing-runbook.md b/docs/modules/mirror/signing-runbook.md index 30bff0d55..69b5dfbd9 100644 --- a/docs/modules/mirror/signing-runbook.md +++ b/docs/modules/mirror/signing-runbook.md @@ -31,10 +31,10 @@ OCI=1 scripts/mirror/ci-sign.sh ``` ## Temporary dev key (to unblock CI until production key is issued) -Use this throwaway Ed25519 key only for non-production runs. Generated 2025-11-23 to replace the previous placeholder; rotate TUF metadata immediately after swapping in the production key. +Use this throwaway Ed25519 key only for non-production runs. Generated 2025-11-24 to replace the previous placeholder; rotate TUF metadata immediately after swapping in the production key. ``` -MIRROR_SIGN_KEY_B64=LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1DNENBUUF3QlFZREsyVndCQ0lFSURqb3pDRVdKVVFUdW1xZ2gyRmZXcVBaemlQbkdaSzRvOFZRTThGYkZCSEcKLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo= +MIRROR_SIGN_KEY_B64=LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1DNENBUUF3QlFZREsyVndCQ0lFSUxGdFMwbjBpMVVueE1maGt0cDNlY1N4WHVxYmcrVFJuaENhS05jaGtTbFIKLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo= ``` **Do not ship with this key.** Set `REQUIRE_PROD_SIGNING=1` for release/tag builds so they fail without the real key. Add the production key as a Gitea secret (`MIRROR_SIGN_KEY_B64`) and rerun the workflow; remove this temporary key block once rotated. diff --git a/docs/modules/policy/architecture.md b/docs/modules/policy/architecture.md index cbab199a7..082cbc581 100644 --- a/docs/modules/policy/architecture.md +++ b/docs/modules/policy/architecture.md @@ -208,8 +208,17 @@ All payloads are immutable and include analyzer fingerprints (`scanner.native@sh - **Scopes:** Mutations require `policy:*` scopes corresponding to action; `effective:write` restricted to service identity. - **Tenancy:** All queries filter by `tenant`. Service identity uses `tenant-global` for shared policies; cross-tenant reads prohibited unless `policy:tenant-admin` scope present. - **Secrets:** Configuration loaded via environment variables or sealed secrets; runtime avoids writing secrets to logs. -- **Determinism guard:** Static analyzer prevents referencing forbidden namespaces; runtime guard intercepts `DateTime.Now`, `Random`, `Guid`, HTTP clients beyond allow-list. -- **Sealed mode:** Global flag disables outbound network except allow-listed internal hosts; watchers fail fast if unexpected egress attempted. +- **Determinism guard:** Static analyzer prevents referencing forbidden namespaces; runtime guard intercepts `DateTime.Now`, `Random`, `Guid`, HTTP clients beyond allow-list. +- **Sealed mode:** Global flag disables outbound network except allow-listed internal hosts; watchers fail fast if unexpected egress attempted. + +### Determinism enforcement (DOCS-POLICY-DET-01) + +- **Inputs are ordered and frozen:** Selector emits batches sorted deterministically by `(tenant, policyId, vulnerabilityId, productKey, source)` with stable cursors; workers must not resort. +- **No ambient randomness or wall clocks:** Policy code relies on injected `TimeProvider`/`IRandom` shims; guards block `DateTime.Now`, `Guid.NewGuid`, `Random` when not injected. +- **Immutable evidence:** SBOM/VEX inputs carry content hashes; evaluator treats payloads as read-only and surfaces hashes in logs for replay. +- **Side effects prohibited:** Evaluator cannot call external HTTP except allow-listed internal services (Authority, Storage) and must not write files outside temp workspace. +- **Replay hash:** Each batch computes `determinismHash = SHA256(policyVersion + batchCursor + inputsHash)`; included in logs and run exports. +- **Testing:** Determinism tests run the same batch twice with seeded clock/GUID providers and assert identical outputs + determinismHash; add a test per policy package. --- diff --git a/docs/modules/policy/samples/advisory-ai-knobs@draft.json b/docs/modules/policy/samples/advisory-ai-knobs@draft.json new file mode 100644 index 000000000..cd4ddb09c --- /dev/null +++ b/docs/modules/policy/samples/advisory-ai-knobs@draft.json @@ -0,0 +1,29 @@ +{ + "knobs": [ + { + "name": "ai_signal_weight", + "default_value": 1.2, + "min": 0.0, + "max": 2.0, + "step": 0.1, + "description": "Weight applied to Advisory AI signal scores" + }, + { + "name": "reachability_boost", + "default_value": 0.25, + "min": 0.0, + "max": 1.0, + "step": 0.05, + "description": "Boost when asset is reachable" + }, + { + "name": "time_decay_half_life_days", + "default_value": 45, + "min": 1, + "max": 365, + "step": 1, + "description": "Half-life in days for signal decay" + } + ], + "profile_hash": "ADVISORYAIHASH" +} diff --git a/docs/modules/policy/samples/orchestrator-job@draft.json b/docs/modules/policy/samples/orchestrator-job@draft.json new file mode 100644 index 000000000..e6b78ba9d --- /dev/null +++ b/docs/modules/policy/samples/orchestrator-job@draft.json @@ -0,0 +1,16 @@ +{ + "job_id": "01HZX1QJP6Z3MNA0Q2T3VCPV5K", + "tenant_id": "acme", + "context_id": "ctx-2025-11-24T10:00:00Z", + "policy_profile_hash": "overlay-hash-123", + "priority": "high", + "requested_at": "2025-11-24T10:00:00Z", + "status": "queued", + "trace_ref": "4E5C2B5E22F928E846B0EFBC58AA53FC3218C8C172199FF52C7C09244E0C0D30", + "determinism_hash": "2C855E80F66D30D5E51C4D9A0441A63C5BB8F04DC1EC537D0ADB7B9357A4C713", + "batch_items": [ + { "component_purl": "pkg:npm/alpha@1.0.0", "advisory_id": "ADV-1" }, + { "component_purl": "pkg:npm/zeta@1.0.0", "advisory_id": "ADV-2" } + ], + "callbacks": { "sse": "sse://events", "nats": "policy.jobs" } +} diff --git a/docs/modules/policy/samples/policy-batch-context@draft.json b/docs/modules/policy/samples/policy-batch-context@draft.json new file mode 100644 index 000000000..b709d02a8 --- /dev/null +++ b/docs/modules/policy/samples/policy-batch-context@draft.json @@ -0,0 +1,11 @@ +{ + "tenant_id": "acme", + "policy_profile_hash": "overlay-hash-123", + "knobs_version": "knobs-v1", + "overlay_hash": "overlay-hash-123", + "items": [ + { "component_purl": "pkg:npm/lodash@4.17.21", "advisory_id": "ADV-2025-0001" }, + { "component_purl": "pkg:npm/left-pad@1.3.0", "advisory_id": "ADV-2025-0002" } + ], + "options": { "include_reachability": true } +} diff --git a/docs/modules/policy/samples/policy-conflict@draft.json b/docs/modules/policy/samples/policy-conflict@draft.json new file mode 100644 index 000000000..3cddc31c5 --- /dev/null +++ b/docs/modules/policy/samples/policy-conflict@draft.json @@ -0,0 +1,32 @@ +{ + "tenant_id": "acme", + "component_purl": "pkg:npm/alpha@1.0.0", + "advisory_id": "ADV-1", + "conflicts": [ + { + "tenant_id": "acme", + "snapshot_id": "01HZX3GN4V6KBW1PXJ0K3VXEGT", + "component_purl": "pkg:npm/alpha@1.0.0", + "advisory_id": "ADV-1", + "severity_fused": "high", + "score": 0.900, + "sources": [ + { "source": "policy-engine", "weight": 1.050, "severity": "high", "score": 0.945 } + ], + "reason_codes": ["weights-applied", "deterministic-fusion"] + }, + { + "tenant_id": "acme", + "snapshot_id": "01HZX3GN4V6KBW1PXJ0K3VXEGT", + "component_purl": "pkg:npm/alpha@1.0.0", + "advisory_id": "ADV-1", + "severity_fused": "medium", + "score": 0.600, + "sources": [ + { "source": "policy-engine", "weight": 1.050, "severity": "medium", "score": 0.630 } + ], + "reason_codes": ["weights-applied", "deterministic-fusion"] + } + ], + "resolved_status": null +} diff --git a/docs/modules/policy/samples/policy-ledger-export@draft.json b/docs/modules/policy/samples/policy-ledger-export@draft.json new file mode 100644 index 000000000..853143296 --- /dev/null +++ b/docs/modules/policy/samples/policy-ledger-export@draft.json @@ -0,0 +1,36 @@ +{ + "manifest": { + "export_id": "01HZX2KDRT9Q9K5AZXWPRH62VE", + "schema_version": "policy-ledger-export-v1", + "generated_at": "2025-11-24T15:00:00Z", + "record_count": 2, + "sha256": "D4B8C98A2F946D93AFBDE6C4DE6535853A223E108A4A2C389E2C2623D3761C1E" + }, + "records": [ + { + "tenant_id": "acme", + "job_id": "job-1", + "context_id": "ctx", + "component_purl": "pkg:npm/alpha@1.0.0", + "advisory_id": "ADV-1", + "status": "violation", + "trace_ref": "trace-a", + "occurred_at": "2025-11-24T15:00:00Z" + }, + { + "tenant_id": "acme", + "job_id": "job-1", + "context_id": "ctx", + "component_purl": "pkg:npm/zeta@1.0.0", + "advisory_id": "ADV-2", + "status": "ok", + "trace_ref": "trace-b", + "occurred_at": "2025-11-24T15:00:00Z" + } + ], + "lines": [ + "{\"export_id\":\"01HZX2KDRT9Q9K5AZXWPRH62VE\",\"schema_version\":\"policy-ledger-export-v1\",\"generated_at\":\"2025-11-24T15:00:00Z\",\"record_count\":2,\"sha256\":\"D4B8C98A2F946D93AFBDE6C4DE6535853A223E108A4A2C389E2C2623D3761C1E\"}", + "{\"tenant_id\":\"acme\",\"job_id\":\"job-1\",\"context_id\":\"ctx\",\"component_purl\":\"pkg:npm/alpha@1.0.0\",\"advisory_id\":\"ADV-1\",\"status\":\"violation\",\"trace_ref\":\"trace-a\",\"occurred_at\":\"2025-11-24T15:00:00Z\"}", + "{\"tenant_id\":\"acme\",\"job_id\":\"job-1\",\"context_id\":\"ctx\",\"component_purl\":\"pkg:npm/zeta@1.0.0\",\"advisory_id\":\"ADV-2\",\"status\":\"ok\",\"trace_ref\":\"trace-b\",\"occurred_at\":\"2025-11-24T15:00:00Z\"}" + ] +} diff --git a/docs/modules/policy/samples/policy-snapshot@draft.json b/docs/modules/policy/samples/policy-snapshot@draft.json new file mode 100644 index 000000000..9091258b9 --- /dev/null +++ b/docs/modules/policy/samples/policy-snapshot@draft.json @@ -0,0 +1,30 @@ +{ + "snapshot_id": "01HZX3GN4V6KBW1PXJ0K3VXEGT", + "tenant_id": "acme", + "ledger_export_id": "01HZX2KDRT9Q9K5AZXWPRH62VE", + "generated_at": "2025-11-24T16:00:00Z", + "overlay_hash": "overlay-1", + "status_counts": { "violation": 1, "ok": 1 }, + "records": [ + { + "tenant_id": "acme", + "job_id": "job-1", + "context_id": "ctx", + "component_purl": "pkg:npm/alpha@1.0.0", + "advisory_id": "ADV-1", + "status": "violation", + "trace_ref": "trace-a", + "occurred_at": "2025-11-24T15:00:00Z" + }, + { + "tenant_id": "acme", + "job_id": "job-1", + "context_id": "ctx", + "component_purl": "pkg:npm/zeta@1.0.0", + "advisory_id": "ADV-2", + "status": "ok", + "trace_ref": "trace-b", + "occurred_at": "2025-11-24T15:00:00Z" + } + ] +} diff --git a/docs/modules/policy/samples/policy-violation-event@draft.json b/docs/modules/policy/samples/policy-violation-event@draft.json new file mode 100644 index 000000000..a8553fbf6 --- /dev/null +++ b/docs/modules/policy/samples/policy-violation-event@draft.json @@ -0,0 +1,13 @@ +{ + "event_id": "E7A1F3B0D6F243B4868A6D4B3E7B2AB9", + "tenant_id": "acme", + "snapshot_id": "01HZX3GN4V6KBW1PXJ0K3VXEGT", + "policy_profile_hash": "overlay-hash-123", + "component_purl": "pkg:npm/alpha@1.0.0", + "advisory_id": "ADV-1", + "violation_code": "policy.violation.detected", + "severity": "high", + "status": "violation", + "trace_ref": "trace-a", + "occurred_at": "2025-11-24T16:00:00Z" +} diff --git a/docs/modules/policy/samples/policy-worker-result@draft.json b/docs/modules/policy/samples/policy-worker-result@draft.json new file mode 100644 index 000000000..55dd2fb64 --- /dev/null +++ b/docs/modules/policy/samples/policy-worker-result@draft.json @@ -0,0 +1,11 @@ +{ + "job_id": "01HZX1QJP6Z3MNA0Q2T3VCPV5K", + "worker_id": "worker-stub", + "started_at": "2025-11-24T13:00:00Z", + "completed_at": "2025-11-24T13:00:01Z", + "result_hash": "5E5A4EFA8C7E9952E4E5E5D9E2B9F3A5D46B13E44CB6E0D7292F7D5CB40CF182", + "results": [ + { "component_purl": "pkg:npm/alpha@1.0.0", "advisory_id": "ADV-1", "status": "violation", "trace_ref": "F5D9B8717EAB4B0252BE22325771C4F9F8ABAE4E7728F3221E15C5F24A8E8D9F" }, + { "component_purl": "pkg:npm/zeta@1.0.0", "advisory_id": "ADV-2", "status": "ok", "trace_ref": "3C75CC86A30B6E230D1DE2D5F08F9B0F5CF75AB1931E47372DC7AC2175BE3F6C" } + ] +} diff --git a/docs/modules/policy/samples/severity-fusion@draft.json b/docs/modules/policy/samples/severity-fusion@draft.json new file mode 100644 index 000000000..39738b49e --- /dev/null +++ b/docs/modules/policy/samples/severity-fusion@draft.json @@ -0,0 +1,12 @@ +{ + "tenant_id": "acme", + "snapshot_id": "01HZX3GN4V6KBW1PXJ0K3VXEGT", + "component_purl": "pkg:npm/alpha@1.0.0", + "advisory_id": "ADV-1", + "severity_fused": "high", + "score": 0.900, + "sources": [ + { "source": "policy-engine", "weight": 1.050, "severity": "high", "score": 0.945 } + ], + "reason_codes": ["weights-applied", "deterministic-fusion"] +} diff --git a/docs/modules/policy/samples/trust-weighting@draft.json b/docs/modules/policy/samples/trust-weighting@draft.json new file mode 100644 index 000000000..fbf963fb3 --- /dev/null +++ b/docs/modules/policy/samples/trust-weighting@draft.json @@ -0,0 +1,23 @@ +{ + "weights": [ + { + "source": "cartographer", + "weight": 1.000, + "justification": "default baseline", + "updated_at": "2025-11-23T12:00:00Z" + }, + { + "source": "scanner", + "weight": 0.950, + "justification": "prefer curated SBOM sources", + "updated_at": "2025-11-23T12:00:00Z" + }, + { + "source": "concelier", + "weight": 1.050, + "justification": "policy engine override", + "updated_at": "2025-11-23T12:00:00Z" + } + ], + "profile_hash": "D1A5F0A0DEFAULTHASH" +} diff --git a/docs/modules/policy/schemas/advisory-ai-knobs@draft.json b/docs/modules/policy/schemas/advisory-ai-knobs@draft.json new file mode 100644 index 000000000..6c6b2d533 --- /dev/null +++ b/docs/modules/policy/schemas/advisory-ai-knobs@draft.json @@ -0,0 +1,27 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "advisory-ai-knobs@draft", + "type": "object", + "properties": { + "knobs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { "type": "string", "minLength": 1 }, + "default_value": { "type": "number" }, + "min": { "type": "number" }, + "max": { "type": "number" }, + "step": { "type": "number" }, + "description": { "type": "string" } + }, + "required": ["name", "default_value", "min", "max", "step", "description"], + "additionalProperties": false + }, + "minItems": 1 + }, + "profile_hash": { "type": "string", "minLength": 1 } + }, + "required": ["knobs", "profile_hash"], + "additionalProperties": false +} diff --git a/docs/modules/policy/schemas/orchestrator-job@draft.json b/docs/modules/policy/schemas/orchestrator-job@draft.json new file mode 100644 index 000000000..f2438d4ff --- /dev/null +++ b/docs/modules/policy/schemas/orchestrator-job@draft.json @@ -0,0 +1,51 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "orchestrator-job@draft", + "type": "object", + "properties": { + "job_id": { "type": "string", "minLength": 1 }, + "tenant_id": { "type": "string", "minLength": 1 }, + "context_id": { "type": "string", "minLength": 1 }, + "policy_profile_hash": { "type": "string", "minLength": 1 }, + "priority": { "type": "string", "enum": ["normal", "high", "emergency", "preview"] }, + "requested_at": { "type": "string", "format": "date-time" }, + "status": { "type": "string", "minLength": 1 }, + "trace_ref": { "type": "string", "minLength": 1 }, + "determinism_hash": { "type": "string", "minLength": 1 }, + "completed_at": { "type": ["string", "null"], "format": "date-time" }, + "result_hash": { "type": ["string", "null"] }, + "batch_items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "component_purl": { "type": "string", "minLength": 1 }, + "advisory_id": { "type": "string", "minLength": 1 } + }, + "required": ["component_purl", "advisory_id"], + "additionalProperties": false + }, + "minItems": 1 + }, + "callbacks": { + "type": ["object", "null"], + "properties": { + "sse": { "type": ["string", "null"] }, + "nats": { "type": ["string", "null"] } + }, + "additionalProperties": false + } + }, + "required": [ + "job_id", + "tenant_id", + "context_id", + "policy_profile_hash", + "priority", + "requested_at", + "status", + "determinism_hash", + "batch_items" + ], + "additionalProperties": false +} diff --git a/docs/modules/policy/schemas/policy-batch-context@draft.json b/docs/modules/policy/schemas/policy-batch-context@draft.json new file mode 100644 index 000000000..a8be23f4c --- /dev/null +++ b/docs/modules/policy/schemas/policy-batch-context@draft.json @@ -0,0 +1,41 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "policy-batch-context@draft", + "type": "object", + "properties": { + "tenant_id": { "type": "string", "minLength": 1 }, + "policy_profile_hash": { "type": "string", "minLength": 1 }, + "knobs_version": { "type": "string", "minLength": 1 }, + "overlay_hash": { "type": "string", "minLength": 1 }, + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "component_purl": { "type": "string", "minLength": 1 }, + "advisory_id": { "type": "string", "minLength": 1 } + }, + "required": ["component_purl", "advisory_id"], + "additionalProperties": false + }, + "minItems": 1 + }, + "options": { + "type": "object", + "properties": { + "include_reachability": { "type": "boolean" } + }, + "required": ["include_reachability"], + "additionalProperties": false + } + }, + "required": [ + "tenant_id", + "policy_profile_hash", + "knobs_version", + "overlay_hash", + "items", + "options" + ], + "additionalProperties": false +} diff --git a/docs/modules/policy/schemas/policy-conflict@draft.json b/docs/modules/policy/schemas/policy-conflict@draft.json new file mode 100644 index 000000000..b3ff9eaa3 --- /dev/null +++ b/docs/modules/policy/schemas/policy-conflict@draft.json @@ -0,0 +1,17 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "policy-conflict@draft", + "type": "object", + "properties": { + "tenant_id": { "type": "string", "minLength": 1 }, + "component_purl": { "type": "string", "minLength": 1 }, + "advisory_id": { "type": "string", "minLength": 1 }, + "conflicts": { + "type": "array", + "items": { "$ref": "severity-fusion@draft.json" } + }, + "resolved_status": { "type": ["string", "null"] } + }, + "required": ["tenant_id", "component_purl", "advisory_id", "conflicts"], + "additionalProperties": false +} diff --git a/docs/modules/policy/schemas/policy-ledger-export@draft.json b/docs/modules/policy/schemas/policy-ledger-export@draft.json new file mode 100644 index 000000000..f426987fd --- /dev/null +++ b/docs/modules/policy/schemas/policy-ledger-export@draft.json @@ -0,0 +1,40 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "policy-ledger-export@draft", + "type": "object", + "properties": { + "manifest": { + "type": "object", + "properties": { + "export_id": { "type": "string", "minLength": 1 }, + "schema_version": { "type": "string", "minLength": 1 }, + "generated_at": { "type": "string", "format": "date-time" }, + "record_count": { "type": "integer", "minimum": 0 }, + "sha256": { "type": "string", "minLength": 1 } + }, + "required": ["export_id", "schema_version", "generated_at", "record_count", "sha256"], + "additionalProperties": false + }, + "records": { + "type": "array", + "items": { + "type": "object", + "properties": { + "tenant_id": { "type": "string", "minLength": 1 }, + "job_id": { "type": "string", "minLength": 1 }, + "context_id": { "type": "string", "minLength": 1 }, + "component_purl": { "type": "string", "minLength": 1 }, + "advisory_id": { "type": "string", "minLength": 1 }, + "status": { "type": "string", "minLength": 1 }, + "trace_ref": { "type": "string", "minLength": 1 }, + "occurred_at": { "type": "string", "format": "date-time" } + }, + "required": ["tenant_id", "job_id", "context_id", "component_purl", "advisory_id", "status", "trace_ref", "occurred_at"], + "additionalProperties": false + } + }, + "lines": { "type": "array", "items": { "type": "string" } } + }, + "required": ["manifest", "records", "lines"], + "additionalProperties": false +} diff --git a/docs/modules/policy/schemas/policy-snapshot@draft.json b/docs/modules/policy/schemas/policy-snapshot@draft.json new file mode 100644 index 000000000..494fb1e89 --- /dev/null +++ b/docs/modules/policy/schemas/policy-snapshot@draft.json @@ -0,0 +1,33 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "policy-snapshot@draft", + "type": "object", + "properties": { + "snapshot_id": { "type": "string", "minLength": 1 }, + "tenant_id": { "type": "string", "minLength": 1 }, + "ledger_export_id": { "type": "string", "minLength": 1 }, + "generated_at": { "type": "string", "format": "date-time" }, + "overlay_hash": { "type": "string", "minLength": 1 }, + "status_counts": { "type": "object", "additionalProperties": { "type": "integer" } }, + "records": { + "type": "array", + "items": { + "type": "object", + "properties": { + "tenant_id": { "type": "string" }, + "job_id": { "type": "string" }, + "context_id": { "type": "string" }, + "component_purl": { "type": "string" }, + "advisory_id": { "type": "string" }, + "status": { "type": "string" }, + "trace_ref": { "type": "string" }, + "occurred_at": { "type": "string", "format": "date-time" } + }, + "required": ["tenant_id", "job_id", "context_id", "component_purl", "advisory_id", "status", "trace_ref", "occurred_at"], + "additionalProperties": false + } + } + }, + "required": ["snapshot_id", "tenant_id", "ledger_export_id", "generated_at", "overlay_hash", "status_counts", "records"], + "additionalProperties": false +} diff --git a/docs/modules/policy/schemas/policy-violation-event@draft.json b/docs/modules/policy/schemas/policy-violation-event@draft.json new file mode 100644 index 000000000..47c1ce578 --- /dev/null +++ b/docs/modules/policy/schemas/policy-violation-event@draft.json @@ -0,0 +1,32 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "policy-violation-event@draft", + "type": "object", + "properties": { + "event_id": { "type": "string", "minLength": 1 }, + "tenant_id": { "type": "string", "minLength": 1 }, + "snapshot_id": { "type": "string", "minLength": 1 }, + "policy_profile_hash": { "type": "string", "minLength": 1 }, + "component_purl": { "type": "string", "minLength": 1 }, + "advisory_id": { "type": "string", "minLength": 1 }, + "violation_code": { "type": "string", "minLength": 1 }, + "severity": { "type": "string", "minLength": 1 }, + "status": { "type": "string", "minLength": 1 }, + "trace_ref": { "type": "string", "minLength": 1 }, + "occurred_at": { "type": "string", "format": "date-time" } + }, + "required": [ + "event_id", + "tenant_id", + "snapshot_id", + "policy_profile_hash", + "component_purl", + "advisory_id", + "violation_code", + "severity", + "status", + "trace_ref", + "occurred_at" + ], + "additionalProperties": false +} diff --git a/docs/modules/policy/schemas/policy-worker-result@draft.json b/docs/modules/policy/schemas/policy-worker-result@draft.json new file mode 100644 index 000000000..2b80cc716 --- /dev/null +++ b/docs/modules/policy/schemas/policy-worker-result@draft.json @@ -0,0 +1,29 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "policy-worker-result@draft", + "type": "object", + "properties": { + "job_id": { "type": "string", "minLength": 1 }, + "worker_id": { "type": "string", "minLength": 1 }, + "started_at": { "type": "string", "format": "date-time" }, + "completed_at": { "type": "string", "format": "date-time" }, + "result_hash": { "type": "string", "minLength": 1 }, + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "component_purl": { "type": "string", "minLength": 1 }, + "advisory_id": { "type": "string", "minLength": 1 }, + "status": { "type": "string", "minLength": 1 }, + "trace_ref": { "type": "string", "minLength": 1 } + }, + "required": ["component_purl", "advisory_id", "status", "trace_ref"], + "additionalProperties": false + }, + "minItems": 1 + } + }, + "required": ["job_id", "worker_id", "started_at", "completed_at", "result_hash", "results"], + "additionalProperties": false +} diff --git a/docs/modules/policy/schemas/severity-fusion@draft.json b/docs/modules/policy/schemas/severity-fusion@draft.json new file mode 100644 index 000000000..0d4efc96b --- /dev/null +++ b/docs/modules/policy/schemas/severity-fusion@draft.json @@ -0,0 +1,30 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "severity-fusion@draft", + "type": "object", + "properties": { + "tenant_id": { "type": "string", "minLength": 1 }, + "snapshot_id": { "type": "string", "minLength": 1 }, + "component_purl": { "type": "string", "minLength": 1 }, + "advisory_id": { "type": "string", "minLength": 1 }, + "severity_fused": { "type": "string", "minLength": 1 }, + "score": { "type": "number" }, + "sources": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { "type": "string", "minLength": 1 }, + "weight": { "type": "number" }, + "severity": { "type": "string", "minLength": 1 }, + "score": { "type": "number" } + }, + "required": ["source", "weight", "severity", "score"], + "additionalProperties": false + } + }, + "reason_codes": { "type": "array", "items": { "type": "string" } } + }, + "required": ["tenant_id", "snapshot_id", "component_purl", "advisory_id", "severity_fused", "score", "sources", "reason_codes"], + "additionalProperties": false +} diff --git a/docs/modules/policy/schemas/trust-weighting@draft.json b/docs/modules/policy/schemas/trust-weighting@draft.json new file mode 100644 index 000000000..9d8329a93 --- /dev/null +++ b/docs/modules/policy/schemas/trust-weighting@draft.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "trust-weighting@draft", + "type": "object", + "properties": { + "weights": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { "type": "string", "minLength": 1 }, + "weight": { "type": "number" }, + "justification": { "type": ["string", "null"] }, + "updated_at": { "type": "string", "format": "date-time" } + }, + "required": ["source", "weight", "updated_at"], + "additionalProperties": false + }, + "minItems": 1 + }, + "profile_hash": { "type": "string", "minLength": 1 } + }, + "required": ["weights", "profile_hash"], + "additionalProperties": false +} diff --git a/docs/modules/sbomservice/architecture.md b/docs/modules/sbomservice/architecture.md index 8302dd595..181d4ee8f 100644 --- a/docs/modules/sbomservice/architecture.md +++ b/docs/modules/sbomservice/architecture.md @@ -44,8 +44,16 @@ Operational rules: - `GET /console/sboms` — Console catalog with filters (artifact, license, scope, asset tags), cursor pagination, evaluation metadata, immutable JSON projection for drawer views. - `GET /components/lookup?purl=...` — component neighborhood for global search/Graph overlays; returns caches hints + tenant enforcement. - `POST /entrypoints` / `GET /entrypoints` — manage entrypoint/service node overrides feeding Cartographer relevance; deterministic defaults when unset. +- `GET /sboms/{snapshotId}/projection` — Link-Not-Merge v1 projection returning hashes plus asset metadata (criticality, owner, environment, exposure flags, tags) alongside package/component graph. - `GET /internal/sbom/events` — internal diagnostics endpoint returning the in-memory event outbox for validation. - `POST /internal/sbom/events/backfill` — replays existing projections into the event stream; deterministic ordering, clock abstraction for tests. +- `GET /internal/sbom/asset-events` — diagnostics endpoint returning emitted `sbom.asset.updated` envelopes for validation and air-gap parity checks. +- `GET/POST /internal/orchestrator/sources` — list/register orchestrator ingest/index sources (deterministic seeds; idempotent on artifactDigest+sourceType). +- `GET/POST /internal/orchestrator/control` — manage pause/throttle/backpressure signals per tenant; metrics emitted for control updates. +- `GET/POST /internal/orchestrator/watermarks` — fetch/set backfill watermarks for reconciliation and deterministic replays. +- `GET /internal/sbom/resolver-feed` — list resolver candidates (artifact, purl, version, paths, scope, runtime_flag, nearest_safe_version). +- `POST /internal/sbom/resolver-feed/backfill` — clear and repopulate resolver feed from current projections. +- `GET /internal/sbom/resolver-feed/export` — NDJSON export of resolver candidates for air-gap delivery. ## 4) Ingestion & orchestrator integration - Ingest sources: Scanner pipeline (preferred) or uploaded SPDX 3.0.1/CycloneDX 1.6 bundles. @@ -70,7 +78,8 @@ Operational rules: - Input validation: schema-validate incoming SBOMs; reject oversized/unsupported media types early. ## 8) Observability -- Metrics: `sbom_projection_seconds`, `sbom_projection_size_bytes`, `sbom_paths_latency_seconds`, `sbom_paths_cache_hit_ratio`, `sbom_events_backlog`. +- Metrics: `sbom_projection_seconds`, `sbom_projection_size_bytes`, `sbom_projection_queries_total`, `sbom_paths_latency_seconds`, `sbom_paths_cache_hit_ratio`, `sbom_events_backlog`. +- Tracing: ActivitySource `StellaOps.SbomService` (entrypoints, component lookup, console catalog, projections, events). - Traces: wrap ingest, projection build, and API handlers; propagate orchestrator job IDs. - Logs: structured, include tenant + artifact digest + sbomVersion; classify ingest failures (schema, storage, orchestrator, validation). - Alerts: backlog thresholds for outbox/event delivery; high latency on path/timeline endpoints. diff --git a/docs/modules/sbomservice/fixtures/lnm-v1/projections.json b/docs/modules/sbomservice/fixtures/lnm-v1/projections.json index c41dac4c2..725ca893c 100644 --- a/docs/modules/sbomservice/fixtures/lnm-v1/projections.json +++ b/docs/modules/sbomservice/fixtures/lnm-v1/projections.json @@ -1 +1 @@ -[{"snapshotId":"snap-001","tenantId":"tenant-a","projection":{"purl":"pkg:npm/lodash@4.17.21","paths":[],"metadata":{"schemaVersion":"1.0.0"}}}] +[{"snapshotId":"snap-001","tenantId":"tenant-a","projection":{"purl":"pkg:npm/lodash@4.17.21","paths":[],"metadata":{"schemaVersion":"1.0.0","asset":{"criticality":"high","owner":"team-console","environment":"prod","exposure":["internet","pci"],"tags":{"tier":"1","service":"sample-api"}}}}}] diff --git a/docs/security/trust-and-signing.md b/docs/security/trust-and-signing.md new file mode 100644 index 000000000..48257c9b7 --- /dev/null +++ b/docs/security/trust-and-signing.md @@ -0,0 +1,33 @@ +# Trust and Signing (DOCS-AIRGAP-58-002) + +Guidance on DSSE/TUF roots, rotation, and signed time tokens. + +## Trust roots +- Maintain offline root keys for DSSE/TUF; store in HSM or sealed vault. +- Distribute intermediate/leaf keys via bootstrap packs with fingerprints. +- Keep trust roots versioned; record `rootVersion` and validity period. + +## DSSE +- Use DSSE for bundle manifests (mirror/bootstrap) and evidence timelines when possible. +- Verification in sealed mode uses bundled roots; no online Rekor needed. +- Rotate signing keys with overlapping validity; publish new root in next bundle. + +## TUF (optional) +- If using TUF metadata, ship `root.json`, `snapshot.json`, `timestamp.json` with bundles. +- In sealed mode, trust only bundled metadata; no remote refresh. + +## Signed time tokens +- Export signed time anchors (see `docs/airgap/staleness-and-time.md`): + - Token fields: `issuedAt`, `notAfter`, `timeSource`, `signature`, `rootVersion`. + - Validate offline against trust roots; expire strictly at `notAfter`. + +## Rotation procedure +1. Prepare new root and leaf keys; sign new root with current root. +2. Include new `root.json` and fingerprints in next mirror/bootstrap bundle. +3. During import, verify both current and new root; switch default after verification. +4. Re-sign manifests/time tokens with new leaf. + +## Security notes +- Never fetch keys online in sealed mode. +- Keep audit log of rotations (who, when, rootVersion, fingerprints). +- Enforce least privilege for signing service accounts. diff --git a/scripts/api-changelog.mjs b/scripts/api-changelog.mjs index eef490bac..cae4088c2 100644 --- a/scripts/api-changelog.mjs +++ b/scripts/api-changelog.mjs @@ -1,12 +1,14 @@ #!/usr/bin/env node import fs from 'node:fs'; import path from 'node:path'; +import crypto from 'node:crypto'; import yaml from 'yaml'; const ROOT = path.resolve('src/Api/StellaOps.Api.OpenApi'); const BASELINE = path.join(ROOT, 'baselines', 'stella-baseline.yaml'); const CURRENT = path.join(ROOT, 'stella.yaml'); const OUTPUT = path.join(ROOT, 'CHANGELOG.md'); +const RELEASE_OUT = path.resolve('src/Sdk/StellaOps.Sdk.Release/out/api-changelog'); function panic(message) { console.error(`[api:changelog] ${message}`); @@ -76,6 +78,25 @@ function renderMarkdown(diff) { return lines.join('\n'); } +function ensureReleaseDir() { + fs.mkdirSync(RELEASE_OUT, { recursive: true }); +} + +function sha256(content) { + return crypto.createHash('sha256').update(content).digest('hex'); +} + +function signDigest(digest) { + const key = process.env.API_CHANGELOG_SIGNING_KEY; + if (!key) { + return null; + } + + const hmac = crypto.createHmac('sha256', Buffer.from(key, 'utf8')); + hmac.update(digest); + return hmac.digest('hex'); +} + function main() { if (!fs.existsSync(BASELINE)) { console.log('[api:changelog] baseline missing; skipping'); @@ -85,6 +106,24 @@ function main() { const markdown = renderMarkdown(diff); fs.writeFileSync(OUTPUT, markdown, 'utf8'); console.log(`[api:changelog] wrote changelog to ${OUTPUT}`); + + ensureReleaseDir(); + const releaseChangelog = path.join(RELEASE_OUT, 'CHANGELOG.md'); + fs.writeFileSync(releaseChangelog, markdown, 'utf8'); + + const digest = sha256(markdown); + const digestFile = path.join(RELEASE_OUT, 'CHANGELOG.sha256'); + fs.writeFileSync(digestFile, `${digest} CHANGELOG.md\n`, 'utf8'); + + const signature = signDigest(digest); + if (signature) { + fs.writeFileSync(path.join(RELEASE_OUT, 'CHANGELOG.sig'), signature, 'utf8'); + console.log('[api:changelog] wrote signature for release artifact'); + } else { + console.log('[api:changelog] signature skipped (API_CHANGELOG_SIGNING_KEY not set)'); + } + + console.log(`[api:changelog] copied changelog + digest to ${RELEASE_OUT}`); } main(); diff --git a/scripts/attest/build-attestation-bundle.sh b/scripts/attest/build-attestation-bundle.sh new file mode 100644 index 000000000..7f416ab52 --- /dev/null +++ b/scripts/attest/build-attestation-bundle.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +set -euo pipefail + +# DEVOPS-ATTEST-74-002: package attestation outputs into an offline bundle with checksums. + +if [[ $# -lt 1 ]]; then + echo "Usage: $0 [bundle-out]" >&2 + exit 64 +fi + +ATTEST_DIR=$1 +BUNDLE_OUT=${2:-"out/attest-bundles"} + +if [[ ! -d "$ATTEST_DIR" ]]; then + echo "[attest-bundle] attestation directory not found: $ATTEST_DIR" >&2 + exit 66 +fi + +mkdir -p "$BUNDLE_OUT" + +TS=$(date -u +"%Y%m%dT%H%M%SZ") +BUNDLE_NAME="attestation-bundle-${TS}" +WORK_DIR="${BUNDLE_OUT}/${BUNDLE_NAME}" +mkdir -p "$WORK_DIR" + +copy_if_exists() { + local pattern="$1" + shopt -s nullglob + local files=("$ATTEST_DIR"/$pattern) + if (( ${#files[@]} > 0 )); then + cp "${files[@]}" "$WORK_DIR/" + fi + shopt -u nullglob +} + +# Collect common attestation artefacts +copy_if_exists "*.dsse.json" +copy_if_exists "*.in-toto.jsonl" +copy_if_exists "*.sarif" +copy_if_exists "*.intoto.json" +copy_if_exists "*.rekor.txt" +copy_if_exists "*.sig" +copy_if_exists "*.crt" +copy_if_exists "*.pem" +copy_if_exists "*.json" + +# Manifest +cat > "${WORK_DIR}/manifest.json" < SHA256SUMS +) + +tar -C "$BUNDLE_OUT" -czf "${WORK_DIR}.tgz" "${BUNDLE_NAME}" +echo "[attest-bundle] bundle created at ${WORK_DIR}.tgz" diff --git a/scripts/buildx/build-airgap-bundle.sh b/scripts/buildx/build-airgap-bundle.sh new file mode 100644 index 000000000..4225dc258 --- /dev/null +++ b/scripts/buildx/build-airgap-bundle.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail + +# DEVOPS-CONTAINERS-46-001: build air-gap bundle from existing buildx OCI archive + +if [[ $# -lt 1 ]]; then + echo "Usage: $0 [bundle-dir]" >&2 + exit 64 +fi + +IMAGE_TAG=$1 +BUNDLE_DIR=${2:-"out/bundles/$(echo "$IMAGE_TAG" | tr '/:' '__')"} +SRC_DIR="out/buildx/$(echo "$IMAGE_TAG" | tr '/:' '__')" +OCI_ARCHIVE="${SRC_DIR}/image.oci" + +if [[ ! -f "$OCI_ARCHIVE" ]]; then + echo "[airgap] OCI archive not found at $OCI_ARCHIVE. Run build-multiarch first." >&2 + exit 66 +fi + +mkdir -p "$BUNDLE_DIR" + +SBOM_FILE="" +if [[ -f "${SRC_DIR}/sbom.syft.json" ]]; then + SBOM_FILE="${SRC_DIR}/sbom.syft.json" +fi + +cat > "${BUNDLE_DIR}/bundle-manifest.json" < [--platform linux/amd64,linux/arm64] [--push] [--sbom syft|none] [--sign ]" >&2 + exit 64 +} + +if [[ $# -lt 2 ]]; then + usage +fi + +IMAGE_TAG=$1; shift +CONTEXT_DIR=$1; shift + +PLATFORMS="linux/amd64,linux/arm64" +PUSH=false +SBOM_TOOL="syft" +COSIGN_KEY="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --platform) PLATFORMS="$2"; shift 2;; + --push) PUSH=true; shift;; + --sbom) SBOM_TOOL="$2"; shift 2;; + --sign) COSIGN_KEY="$2"; shift 2;; + *) echo "Unknown option: $1" >&2; usage;; + esac +done + +if ! command -v docker >/dev/null 2>&1; then + echo "[buildx] docker CLI not found" >&2 + exit 69 +fi + +OUT_ROOT="out/buildx/$(echo "$IMAGE_TAG" | tr '/:' '__')" +mkdir -p "$OUT_ROOT" + +BUILDER_NAME="stellaops-multiarch" +if ! docker buildx inspect "$BUILDER_NAME" >/dev/null 2>&1; then + docker buildx create --name "$BUILDER_NAME" --driver docker-container --use >/dev/null +else + docker buildx use "$BUILDER_NAME" >/dev/null +fi + +BUILD_OPTS=( + --platform "$PLATFORMS" + -t "$IMAGE_TAG" + --provenance=false + --sbom=false + --output "type=oci,dest=${OUT_ROOT}/image.oci" +) + +if $PUSH; then + BUILD_OPTS+=("--push") +fi + +echo "[buildx] building $IMAGE_TAG for $PLATFORMS" +docker buildx build "${BUILD_OPTS[@]}" "$CONTEXT_DIR" + +echo "[buildx] computing digest" +IMAGE_DIGEST=$(sha256sum "${OUT_ROOT}/image.oci" | awk '{print $1}') +echo "$IMAGE_DIGEST image.oci" > "${OUT_ROOT}/image.sha256" + +if [[ "$SBOM_TOOL" == "syft" ]] && command -v syft >/dev/null 2>&1; then + echo "[buildx] generating SBOM via syft" + syft "oci-archive:${OUT_ROOT}/image.oci" -o json > "${OUT_ROOT}/sbom.syft.json" +else + echo "[buildx] skipping SBOM (tool=$SBOM_TOOL, syft available? $(command -v syft >/dev/null && echo yes || echo no))" +fi + +if [[ -n "$COSIGN_KEY" ]] && command -v cosign >/dev/null 2>&1; then + echo "[buildx] signing digest with cosign key" + COSIGN_EXPERIMENTAL=1 cosign sign-blob --key "$COSIGN_KEY" --output-signature "${OUT_ROOT}/image.sig" --output-certificate "${OUT_ROOT}/image.cert" "${OUT_ROOT}/image.oci" +else + echo "[buildx] signature skipped (no key provided or cosign missing)" +fi + +cat > "${OUT_ROOT}/build-metadata.json" </dev/null 2>&1; then + echo "[cli-build] dotnet CLI not found" >&2 + exit 69 +fi + +generate_sbom() { + local dir="$1" + local sbom="$2" + if [[ "$SBOM_TOOL" == "syft" ]] && command -v syft >/dev/null 2>&1; then + syft "dir:${dir}" -o json > "$sbom" + fi +} + +sign_file() { + local file="$1" + if [[ "$SIGN" == "true" && -n "$COSIGN_KEY" && -x "$(command -v cosign || true)" ]]; then + COSIGN_EXPERIMENTAL=1 cosign sign-blob --key "$COSIGN_KEY" --output-signature "${file}.sig" "$file" + fi +} + +for rid in "${TARGETS[@]}"; do + echo "[cli-build] publishing for $rid" + out_dir="${OUT_ROOT}/${rid}" + publish_dir="${out_dir}/publish" + mkdir -p "$publish_dir" + + dotnet publish "$PROJECT" -c "$CONFIG" -r "$rid" \ + -o "$publish_dir" \ + --self-contained true \ + -p:PublishSingleFile=true \ + -p:PublishTrimmed=false \ + -p:DebugType=None \ + >/dev/null + + # Package + archive_ext="tar.gz" + archive_cmd=(tar -C "$publish_dir" -czf) + if [[ "$rid" == win-* ]]; then + archive_ext="zip" + archive_cmd=(zip -jr) + fi + + archive_name="stella-cli-${rid}.${archive_ext}" + archive_path="${out_dir}/${archive_name}" + "${archive_cmd[@]}" "$archive_path" "$publish_dir" + + sha256sum "$archive_path" > "${archive_path}.sha256" + sign_file "$archive_path" + + # SBOM + generate_sbom "$publish_dir" "${archive_path}.sbom.json" +done + +# Build manifest +manifest="${OUT_ROOT}/manifest.json" +cat > "$manifest" < + { + var advisoryKey = parseResult.GetValue(explainOptions.AdvisoryKey) ?? string.Empty; + var artifactId = parseResult.GetValue(explainOptions.ArtifactId); + var artifactPurl = parseResult.GetValue(explainOptions.ArtifactPurl); + var policyVersion = parseResult.GetValue(explainOptions.PolicyVersion); + var profile = parseResult.GetValue(explainOptions.Profile) ?? "default"; + var sections = parseResult.GetValue(explainOptions.Sections) ?? Array.Empty(); + var forceRefresh = parseResult.GetValue(explainOptions.ForceRefresh); + var timeoutSeconds = parseResult.GetValue(explainOptions.TimeoutSeconds) ?? 120; + var outputFormat = ParseAdvisoryOutputFormat(parseResult.GetValue(explainOptions.Format)); + var outputPath = parseResult.GetValue(explainOptions.Output); + var verbose = parseResult.GetValue(verboseOption); + + return CommandHandlers.HandleAdviseRunAsync( + services, + AdvisoryAiTaskType.Conflict, + advisoryKey, + artifactId, + artifactPurl, + policyVersion, + profile, + sections, + forceRefresh, + timeoutSeconds, + outputFormat, + outputPath, + verbose, + cancellationToken); + }); + + var remediateOptions = CreateAdvisoryOptions(); + var remediate = new Command("remediate", "Generate remediation guidance for an advisory."); + AddAdvisoryOptions(remediate, remediateOptions); + remediate.SetAction((parseResult, _) => + { + var advisoryKey = parseResult.GetValue(remediateOptions.AdvisoryKey) ?? string.Empty; + var artifactId = parseResult.GetValue(remediateOptions.ArtifactId); + var artifactPurl = parseResult.GetValue(remediateOptions.ArtifactPurl); + var policyVersion = parseResult.GetValue(remediateOptions.PolicyVersion); + var profile = parseResult.GetValue(remediateOptions.Profile) ?? "default"; + var sections = parseResult.GetValue(remediateOptions.Sections) ?? Array.Empty(); + var forceRefresh = parseResult.GetValue(remediateOptions.ForceRefresh); + var timeoutSeconds = parseResult.GetValue(remediateOptions.TimeoutSeconds) ?? 120; + var outputFormat = ParseAdvisoryOutputFormat(parseResult.GetValue(remediateOptions.Format)); + var outputPath = parseResult.GetValue(remediateOptions.Output); + var verbose = parseResult.GetValue(verboseOption); + + return CommandHandlers.HandleAdviseRunAsync( + services, + AdvisoryAiTaskType.Remediation, + advisoryKey, + artifactId, + artifactPurl, + policyVersion, + profile, + sections, + forceRefresh, + timeoutSeconds, + outputFormat, + outputPath, + verbose, + cancellationToken); + }); + advise.Add(run); advise.Add(summarize); + advise.Add(explain); + advise.Add(remediate); return advise; } diff --git a/src/Cli/StellaOps.Cli/Commands/CommandHandlers.cs b/src/Cli/StellaOps.Cli/Commands/CommandHandlers.cs index ae127209c..cdef55657 100644 --- a/src/Cli/StellaOps.Cli/Commands/CommandHandlers.cs +++ b/src/Cli/StellaOps.Cli/Commands/CommandHandlers.cs @@ -553,6 +553,12 @@ internal static class CommandHandlers logger.LogInformation("Advisory output written to {Path}.", fullPath); } + if (rendered is not null) + { + // Surface the rendered advisory to the active console so users (and tests) can see it even when also writing to disk. + AnsiConsole.Console.WriteLine(rendered); + } + if (output.Guardrail.Blocked) { logger.LogError("Guardrail blocked advisory output (cache key {CacheKey}).", output.CacheKey); @@ -3075,7 +3081,7 @@ internal static class CommandHandlers }; var json = JsonSerializer.Serialize(payload, new JsonSerializerOptions(JsonSerializerDefaults.Web) { WriteIndented = true }); - Console.WriteLine(json); + AnsiConsole.Console.WriteLine(json); } else { @@ -6359,9 +6365,9 @@ internal static class CommandHandlers builder.AppendLine($"# Advisory {output.TaskType} ({output.Profile})"); builder.AppendLine(); builder.AppendLine($"- Cache Key: `{output.CacheKey}`"); - builder.AppendLine($"- Generated: {output.GeneratedAtUtc.ToString(\"O\", CultureInfo.InvariantCulture)}"); - builder.AppendLine($"- Plan From Cache: {(output.PlanFromCache ? \"yes\" : \"no\")}"); - builder.AppendLine($"- Guardrail Blocked: {(output.Guardrail.Blocked ? \"yes\" : \"no\")}"); + builder.AppendLine($"- Generated: {output.GeneratedAtUtc.ToString("O", CultureInfo.InvariantCulture)}"); + builder.AppendLine($"- Plan From Cache: {(output.PlanFromCache ? "yes" : "no")}"); + builder.AppendLine($"- Guardrail Blocked: {(output.Guardrail.Blocked ? "yes" : "no")}"); builder.AppendLine(); if (!string.IsNullOrWhiteSpace(output.Response)) diff --git a/src/Cli/StellaOps.Cli/TASKS.md b/src/Cli/StellaOps.Cli/TASKS.md index 5a15f9dd4..7ebfe7928 100644 --- a/src/Cli/StellaOps.Cli/TASKS.md +++ b/src/Cli/StellaOps.Cli/TASKS.md @@ -3,4 +3,6 @@ | Task ID | State | Notes | | --- | --- | --- | | `SCANNER-CLI-0001` | DONE (2025-11-12) | Ruby verbs now consume the persisted `RubyPackageInventory`, warn when inventories are missing, and docs/tests were refreshed per Sprint 138. | -| `CLI-AIAI-31-001` | BLOCKED (2025-11-22) | `stella advise summarize` command implemented; blocked on upstream Scanner analyzers (Node/Java) compile failures preventing CLI test run. | +| `CLI-AIAI-31-001` | DONE (2025-11-24) | `stella advise summarize` command implemented; CLI analyzer build & tests now pass locally. | +| `CLI-AIAI-31-002` | DONE (2025-11-24) | `stella advise explain` (conflict narrative) command implemented and tested. | +| `CLI-AIAI-31-003` | DONE (2025-11-24) | `stella advise remediate` command implemented and tested. | diff --git a/src/Cli/__Tests/StellaOps.Cli.Tests/Commands/CommandHandlersTests.cs b/src/Cli/__Tests/StellaOps.Cli.Tests/Commands/CommandHandlersTests.cs index 1308aff4f..584b5640a 100644 --- a/src/Cli/__Tests/StellaOps.Cli.Tests/Commands/CommandHandlersTests.cs +++ b/src/Cli/__Tests/StellaOps.Cli.Tests/Commands/CommandHandlersTests.cs @@ -877,6 +877,202 @@ public sealed class CommandHandlersTests } } + [Fact] + public async Task HandleAdviseRunAsync_WritesMarkdownWithCitations_ForExplain() + { + var originalExit = Environment.ExitCode; + var originalConsole = AnsiConsole.Console; + var testConsole = new TestConsole(); + + try + { + Environment.ExitCode = 0; + AnsiConsole.Console = testConsole; + + var planResponse = new AdvisoryPipelinePlanResponseModel + { + TaskType = "Conflict", + CacheKey = "plan-conflict", + PromptTemplate = "prompts/advisory/conflict.liquid", + Budget = new AdvisoryTaskBudgetModel + { + PromptTokens = 128, + CompletionTokens = 64 + }, + Chunks = Array.Empty(), + Vectors = Array.Empty(), + Metadata = new Dictionary() + }; + + var outputResponse = new AdvisoryPipelineOutputModel + { + CacheKey = planResponse.CacheKey, + TaskType = planResponse.TaskType, + Profile = "default", + Prompt = "Sanitized prompt", + Response = "Rendered conflict body.", + Citations = new[] + { + new AdvisoryOutputCitationModel { Index = 1, DocumentId = "doc-42", ChunkId = "chunk-42" } + }, + Metadata = new Dictionary(), + Guardrail = new AdvisoryOutputGuardrailModel + { + Blocked = false, + SanitizedPrompt = "Sanitized prompt", + Violations = Array.Empty(), + Metadata = new Dictionary() + }, + Provenance = new AdvisoryOutputProvenanceModel + { + InputDigest = "sha256:conflict-in", + OutputHash = "sha256:conflict-out", + Signatures = Array.Empty() + }, + GeneratedAtUtc = DateTimeOffset.Parse("2025-11-06T12:00:00Z", CultureInfo.InvariantCulture), + PlanFromCache = false + }; + + var backend = new StubBackendClient(new JobTriggerResult(true, "ok", null, null)) + { + AdvisoryPlanResponse = planResponse, + AdvisoryOutputResponse = outputResponse + }; + + var provider = BuildServiceProvider(backend); + var outputPath = Path.GetTempFileName(); + + await CommandHandlers.HandleAdviseRunAsync( + provider, + AdvisoryAiTaskType.Conflict, + "ADV-42", + null, + null, + null, + "default", + Array.Empty(), + forceRefresh: false, + timeoutSeconds: 0, + outputFormat: AdvisoryOutputFormat.Markdown, + outputPath: outputPath, + verbose: false, + cancellationToken: CancellationToken.None); + + var markdown = await File.ReadAllTextAsync(outputPath); + Assert.Contains("Conflict", markdown, StringComparison.OrdinalIgnoreCase); + Assert.Contains("Rendered conflict body", markdown, StringComparison.OrdinalIgnoreCase); + Assert.Contains("doc-42", markdown, StringComparison.OrdinalIgnoreCase); + Assert.Contains("chunk-42", markdown, StringComparison.OrdinalIgnoreCase); + Assert.Contains("Citations", markdown, StringComparison.OrdinalIgnoreCase); + Assert.Equal(0, Environment.ExitCode); + Assert.Contains("Conflict", testConsole.Output, StringComparison.OrdinalIgnoreCase); + Assert.Equal(AdvisoryAiTaskType.Conflict, backend.AdvisoryPlanRequests.Last().TaskType); + } + finally + { + AnsiConsole.Console = originalConsole; + Environment.ExitCode = originalExit; + } + } + + [Fact] + public async Task HandleAdviseRunAsync_WritesMarkdownWithCitations_ForRemediation() + { + var originalExit = Environment.ExitCode; + var originalConsole = AnsiConsole.Console; + var testConsole = new TestConsole(); + + try + { + Environment.ExitCode = 0; + AnsiConsole.Console = testConsole; + + var planResponse = new AdvisoryPipelinePlanResponseModel + { + TaskType = "Remediation", + CacheKey = "plan-remediation", + PromptTemplate = "prompts/advisory/remediation.liquid", + Budget = new AdvisoryTaskBudgetModel + { + PromptTokens = 192, + CompletionTokens = 96 + }, + Chunks = Array.Empty(), + Vectors = Array.Empty(), + Metadata = new Dictionary() + }; + + var outputResponse = new AdvisoryPipelineOutputModel + { + CacheKey = planResponse.CacheKey, + TaskType = planResponse.TaskType, + Profile = "default", + Prompt = "Sanitized prompt", + Response = "Rendered remediation body.", + Citations = new[] + { + new AdvisoryOutputCitationModel { Index = 1, DocumentId = "doc-77", ChunkId = "chunk-77" } + }, + Metadata = new Dictionary(), + Guardrail = new AdvisoryOutputGuardrailModel + { + Blocked = false, + SanitizedPrompt = "Sanitized prompt", + Violations = Array.Empty(), + Metadata = new Dictionary() + }, + Provenance = new AdvisoryOutputProvenanceModel + { + InputDigest = "sha256:remediation-in", + OutputHash = "sha256:remediation-out", + Signatures = Array.Empty() + }, + GeneratedAtUtc = DateTimeOffset.Parse("2025-11-06T12:00:00Z", CultureInfo.InvariantCulture), + PlanFromCache = false + }; + + var backend = new StubBackendClient(new JobTriggerResult(true, "ok", null, null)) + { + AdvisoryPlanResponse = planResponse, + AdvisoryOutputResponse = outputResponse + }; + + var provider = BuildServiceProvider(backend); + var outputPath = Path.GetTempFileName(); + + await CommandHandlers.HandleAdviseRunAsync( + provider, + AdvisoryAiTaskType.Remediation, + "ADV-77", + null, + null, + null, + "default", + Array.Empty(), + forceRefresh: false, + timeoutSeconds: 0, + outputFormat: AdvisoryOutputFormat.Markdown, + outputPath: outputPath, + verbose: false, + cancellationToken: CancellationToken.None); + + var markdown = await File.ReadAllTextAsync(outputPath); + Assert.Contains("Remediation", markdown, StringComparison.OrdinalIgnoreCase); + Assert.Contains("Rendered remediation body", markdown, StringComparison.OrdinalIgnoreCase); + Assert.Contains("doc-77", markdown, StringComparison.OrdinalIgnoreCase); + Assert.Contains("chunk-77", markdown, StringComparison.OrdinalIgnoreCase); + Assert.Contains("Citations", markdown, StringComparison.OrdinalIgnoreCase); + Assert.Equal(0, Environment.ExitCode); + Assert.Contains("Remediation", testConsole.Output, StringComparison.OrdinalIgnoreCase); + Assert.Equal(AdvisoryAiTaskType.Remediation, backend.AdvisoryPlanRequests.Last().TaskType); + } + finally + { + AnsiConsole.Console = originalConsole; + Environment.ExitCode = originalExit; + } + } + [Fact] public async Task HandleAdviseRunAsync_ReturnsGuardrailExitCodeOnBlock() { @@ -3776,6 +3972,7 @@ spec: Array.Empty(), false); public Exception? TaskRunnerSimulationException { get; set; } + public OfflineKitStatus? OfflineStatus { get; set; } public PolicyActivationResult ActivationResult { get; set; } = new PolicyActivationResult( "activated", new PolicyActivationRevision( @@ -3966,7 +4163,19 @@ spec: => throw new NotSupportedException(); public Task GetOfflineKitStatusAsync(CancellationToken cancellationToken) - => throw new NotSupportedException(); + { + return Task.FromResult(OfflineStatus ?? new OfflineKitStatus( + null, + null, + null, + false, + null, + null, + null, + null, + null, + Array.Empty())); + } public Task GetEntryTraceAsync(string scanId, CancellationToken cancellationToken) { diff --git a/src/Concelier/StellaOps.Concelier.WebService/Contracts/LnmLinksetContracts.cs b/src/Concelier/StellaOps.Concelier.WebService/Contracts/LnmLinksetContracts.cs index 06b39e81a..4377fdf43 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/Contracts/LnmLinksetContracts.cs +++ b/src/Concelier/StellaOps.Concelier.WebService/Contracts/LnmLinksetContracts.cs @@ -32,6 +32,7 @@ public sealed record LnmLinksetPage( public sealed record LnmLinksetNormalized( [property: JsonPropertyName("aliases")] IReadOnlyList? Aliases, [property: JsonPropertyName("purl")] IReadOnlyList? Purl, + [property: JsonPropertyName("cpe")] IReadOnlyList? Cpe, [property: JsonPropertyName("versions")] IReadOnlyList? Versions, [property: JsonPropertyName("ranges")] IReadOnlyList? Ranges, [property: JsonPropertyName("severities")] IReadOnlyList? Severities); diff --git a/src/Concelier/StellaOps.Concelier.WebService/Program.cs b/src/Concelier/StellaOps.Concelier.WebService/Program.cs index 470a99743..1437ac996 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/Program.cs +++ b/src/Concelier/StellaOps.Concelier.WebService/Program.cs @@ -1752,6 +1752,9 @@ LnmLinksetResponse ToLnmResponse( bool includeObservations) { var normalized = linkset.Normalized; + var severity = normalized?.Severities?.FirstOrDefault() is { } severityDict + ? ExtractSeverity(severityDict) + : null; var conflicts = includeConflicts ? (linkset.Conflicts ?? Array.Empty()).Select(c => new LnmLinksetConflict( @@ -1764,7 +1767,13 @@ LnmLinksetResponse ToLnmResponse( : Array.Empty(); var timeline = includeTimeline - ? Array.Empty() // timeline not yet captured in linkset store + ? new[] + { + new LnmLinksetTimeline( + Event: "created", + At: linkset.CreatedAt, + EvidenceHash: linkset.Provenance?.ObservationHashes?.FirstOrDefault()) + } : Array.Empty(); var provenance = linkset.Provenance is null @@ -1780,6 +1789,7 @@ LnmLinksetResponse ToLnmResponse( : new LnmLinksetNormalized( Aliases: null, Purl: normalized.Purls, + Cpe: normalized.Cpes, Versions: normalized.Versions, Ranges: normalized.Ranges?.Select(r => (object)r).ToArray(), Severities: normalized.Severities?.Select(s => (object)s).ToArray()); @@ -1788,11 +1798,11 @@ LnmLinksetResponse ToLnmResponse( linkset.AdvisoryId, linkset.Source, normalized?.Purls ?? Array.Empty(), - Array.Empty(), + normalized?.Cpes ?? Array.Empty(), Summary: null, PublishedAt: linkset.CreatedAt, ModifiedAt: linkset.CreatedAt, - Severity: null, + Severity: severity, Status: "fact-only", provenance, conflicts, @@ -1803,6 +1813,27 @@ LnmLinksetResponse ToLnmResponse( Observations: includeObservations ? linkset.ObservationIds : Array.Empty()); } +string? ExtractSeverity(IReadOnlyDictionary severityDict) +{ + if (severityDict.TryGetValue("system", out var systemObj) && systemObj is string system && !string.IsNullOrWhiteSpace(system) && + severityDict.TryGetValue("score", out var scoreObj)) + { + return $"{system}:{scoreObj}"; + } + + if (severityDict.TryGetValue("score", out var scoreOnly) && scoreOnly is not null) + { + return scoreOnly.ToString(); + } + + if (severityDict.TryGetValue("value", out var value) && value is string valueString && !string.IsNullOrWhiteSpace(valueString)) + { + return valueString; + } + + return null; +} + IResult JsonResult(T value, int? statusCode = null) { var payload = JsonSerializer.Serialize(value, Program.JsonOptions); diff --git a/src/Concelier/StellaOps.Concelier.WebService/openapi/concelier-lnm.yaml b/src/Concelier/StellaOps.Concelier.WebService/openapi/concelier-lnm.yaml index 104ab608c..8d1e21171 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/openapi/concelier-lnm.yaml +++ b/src/Concelier/StellaOps.Concelier.WebService/openapi/concelier-lnm.yaml @@ -241,6 +241,7 @@ components: properties: aliases: { type: array, items: { type: string } } purl: { type: array, items: { type: string } } + cpe: { type: array, items: { type: string } } versions: { type: array, items: { type: string } } ranges: { type: array, items: { type: object } } severities: { type: array, items: { type: object } } diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Core/Linksets/AdvisoryLinkset.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Linksets/AdvisoryLinkset.cs index 2985ec35b..1b7172458 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Core/Linksets/AdvisoryLinkset.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Linksets/AdvisoryLinkset.cs @@ -20,10 +20,14 @@ public sealed record AdvisoryLinkset( public sealed record AdvisoryLinksetNormalized( IReadOnlyList? Purls, + IReadOnlyList? Cpes, IReadOnlyList? Versions, IReadOnlyList>? Ranges, IReadOnlyList>? Severities) { + public List? CpesToList() + => Cpes is null ? null : Cpes.ToList(); + public List? RangesToBson() => Ranges is null ? null : Ranges.Select(BsonDocumentHelper.FromDictionary).ToList(); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Core/Linksets/AdvisoryLinksetNormalization.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Linksets/AdvisoryLinksetNormalization.cs index cf8659754..0e24e32ee 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Core/Linksets/AdvisoryLinksetNormalization.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Linksets/AdvisoryLinksetNormalization.cs @@ -12,7 +12,7 @@ internal static class AdvisoryLinksetNormalization public static AdvisoryLinksetNormalized? FromRawLinkset(RawLinkset linkset) { ArgumentNullException.ThrowIfNull(linkset); - return Build(linkset.PackageUrls); + return Build(linkset.PackageUrls, linkset.Cpes); } public static AdvisoryLinksetNormalized? FromPurls(IEnumerable? purls) @@ -22,7 +22,7 @@ internal static class AdvisoryLinksetNormalization return null; } - return Build(purls); + return Build(purls, Enumerable.Empty()); } public static (AdvisoryLinksetNormalized? normalized, double? confidence, IReadOnlyList conflicts) FromRawLinksetWithConfidence( @@ -31,7 +31,7 @@ internal static class AdvisoryLinksetNormalization { ArgumentNullException.ThrowIfNull(linkset); - var normalized = Build(linkset.PackageUrls); + var normalized = Build(linkset.PackageUrls, linkset.Cpes); var inputs = new[] { @@ -51,18 +51,19 @@ internal static class AdvisoryLinksetNormalization return (normalized, coerced, conflicts); } - private static AdvisoryLinksetNormalized? Build(IEnumerable purlValues) + private static AdvisoryLinksetNormalized? Build(IEnumerable purlValues, IEnumerable? cpeValues) { var normalizedPurls = NormalizePurls(purlValues); + var normalizedCpes = NormalizeCpes(cpeValues); var versions = ExtractVersions(normalizedPurls); var ranges = BuildVersionRanges(normalizedPurls); - if (normalizedPurls.Count == 0 && versions.Count == 0 && ranges.Count == 0) + if (normalizedPurls.Count == 0 && normalizedCpes.Count == 0 && versions.Count == 0 && ranges.Count == 0) { return null; } - return new AdvisoryLinksetNormalized(normalizedPurls, versions, ranges, null); + return new AdvisoryLinksetNormalized(normalizedPurls, normalizedCpes, versions, ranges, null); } private static List NormalizePurls(IEnumerable purls) @@ -147,6 +148,31 @@ internal static class AdvisoryLinksetNormalization return ranges; } + private static List NormalizeCpes(IEnumerable? cpes) + { + if (cpes is null) + { + return new List(capacity: 0); + } + + var distinct = new SortedSet(StringComparer.OrdinalIgnoreCase); + foreach (var cpe in cpes) + { + var normalized = Validation.TrimToNull(cpe); + if (normalized is null) + { + continue; + } + + if (LinksetNormalization.TryNormalizeCpe(normalized, out var canonical) && !string.IsNullOrEmpty(canonical)) + { + distinct.Add(canonical); + } + } + + return distinct.ToList(); + } + private static bool LooksLikeRange(string value) { return value.IndexOfAny(new[] { '^', '~', '*', ' ', ',', '|', '>' , '<' }) >= 0 || diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/Linksets/AdvisoryLinksetDocument.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/Linksets/AdvisoryLinksetDocument.cs index e40726671..a3fbffcb2 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/Linksets/AdvisoryLinksetDocument.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/Linksets/AdvisoryLinksetDocument.cs @@ -61,6 +61,11 @@ public sealed class AdvisoryLinksetNormalizedDocument public List? Purls { get; set; } = new(); + [BsonElement("cpes")] + [BsonIgnoreIfNull] + public List? Cpes { get; set; } + = new(); + [BsonElement("versions")] [BsonIgnoreIfNull] public List? Versions { get; set; } diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/Linksets/ConcelierMongoLinksetStore.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/Linksets/ConcelierMongoLinksetStore.cs index 09e33d28f..159cc6cfc 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/Linksets/ConcelierMongoLinksetStore.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/Linksets/ConcelierMongoLinksetStore.cs @@ -125,6 +125,7 @@ internal sealed class ConcelierMongoLinksetStore : IMongoAdvisoryLinksetStore Normalized = linkset.Normalized is null ? null : new AdvisoryLinksetNormalizedDocument { Purls = linkset.Normalized.Purls is null ? null : new List(linkset.Normalized.Purls), + Cpes = linkset.Normalized.Cpes is null ? null : new List(linkset.Normalized.Cpes), Versions = linkset.Normalized.Versions is null ? null : new List(linkset.Normalized.Versions), Ranges = linkset.Normalized.RangesToBson(), Severities = linkset.Normalized.SeveritiesToBson(), @@ -141,6 +142,7 @@ internal sealed class ConcelierMongoLinksetStore : IMongoAdvisoryLinksetStore doc.Observations.ToImmutableArray(), doc.Normalized is null ? null : new CoreLinksets.AdvisoryLinksetNormalized( doc.Normalized.Purls, + doc.Normalized.Cpes, doc.Normalized.Versions, doc.Normalized.Ranges?.Select(ToDictionary).ToList(), doc.Normalized.Severities?.Select(ToDictionary).ToList()), diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/Migrations/EnsureLinkNotMergeCollectionsMigration.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/Migrations/EnsureLinkNotMergeCollectionsMigration.cs index 4775236c6..81b7eeda6 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/Migrations/EnsureLinkNotMergeCollectionsMigration.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/Migrations/EnsureLinkNotMergeCollectionsMigration.cs @@ -214,6 +214,7 @@ internal sealed class EnsureLinkNotMergeCollectionsMigration : IMongoMigration { "properties", new BsonDocument { { "purls", new BsonDocument { { "bsonType", new BsonArray { "array", "null" } }, { "items", new BsonDocument("bsonType", "string") } } }, + { "cpes", new BsonDocument { { "bsonType", new BsonArray { "array", "null" } }, { "items", new BsonDocument("bsonType", "string") } } }, { "versions", new BsonDocument { { "bsonType", new BsonArray { "array", "null" } }, { "items", new BsonDocument("bsonType", "string") } } }, { "ranges", new BsonDocument { { "bsonType", new BsonArray { "array", "null" } }, { "items", new BsonDocument("bsonType", "object") } } }, { "severities", new BsonDocument { { "bsonType", new BsonArray { "array", "null" } }, { "items", new BsonDocument("bsonType", "object") } } } diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/Linksets/AdvisoryLinksetQueryServiceTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/Linksets/AdvisoryLinksetQueryServiceTests.cs index cbe215c48..4b56924f7 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/Linksets/AdvisoryLinksetQueryServiceTests.cs +++ b/src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/Linksets/AdvisoryLinksetQueryServiceTests.cs @@ -14,17 +14,17 @@ public sealed class AdvisoryLinksetQueryServiceTests { new("tenant", "ghsa", "adv-003", ImmutableArray.Create("obs-003"), - new AdvisoryLinksetNormalized(new[]{"pkg:npm/a"}, new[]{"1.0.0"}, null, null), + new AdvisoryLinksetNormalized(new[]{"pkg:npm/a"}, null, new[]{"1.0.0"}, null, null), null, null, null, DateTimeOffset.Parse("2025-11-10T12:00:00Z"), null), new("tenant", "ghsa", "adv-002", ImmutableArray.Create("obs-002"), - new AdvisoryLinksetNormalized(new[]{"pkg:npm/b"}, new[]{"2.0.0"}, null, null), + new AdvisoryLinksetNormalized(new[]{"pkg:npm/b"}, null, new[]{"2.0.0"}, null, null), null, null, null, DateTimeOffset.Parse("2025-11-09T12:00:00Z"), null), new("tenant", "ghsa", "adv-001", ImmutableArray.Create("obs-001"), - new AdvisoryLinksetNormalized(new[]{"pkg:npm/c"}, new[]{"3.0.0"}, null, null), + new AdvisoryLinksetNormalized(new[]{"pkg:npm/c"}, null, new[]{"3.0.0"}, null, null), null, null, null, DateTimeOffset.Parse("2025-11-08T12:00:00Z"), null), }; diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/Linksets/PolicyAuthSignalFactoryTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/Linksets/PolicyAuthSignalFactoryTests.cs index a598f4213..1556289de 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/Linksets/PolicyAuthSignalFactoryTests.cs +++ b/src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/Linksets/PolicyAuthSignalFactoryTests.cs @@ -18,6 +18,7 @@ public class PolicyAuthSignalFactoryTests ObservationIds: ImmutableArray.Create("obs-1"), Normalized: new AdvisoryLinksetNormalized( Purls: new[] { "purl:pkg:maven/org.example/app@1.2.3" }, + Cpes: null, Versions: Array.Empty(), Ranges: null, Severities: null), diff --git a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/AdvisorySummaryMapperTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/AdvisorySummaryMapperTests.cs index f09d05b06..8063dd7a6 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/AdvisorySummaryMapperTests.cs +++ b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/AdvisorySummaryMapperTests.cs @@ -18,6 +18,7 @@ public class AdvisorySummaryMapperTests ObservationIds: ImmutableArray.Create("obs1", "obs2"), Normalized: new AdvisoryLinksetNormalized( Purls: new[] { "pkg:maven/log4j/log4j@2.17.1" }, + Cpes: null, Versions: null, Ranges: null, Severities: null), diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 363accf3a..855384410 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -36,18 +36,28 @@ - - - - - + + + + + - - - - - - - + + + + + + + + + + + + + + + diff --git a/src/Excititor/StellaOps.Excititor.WebService/Contracts/GraphOverlayContracts.cs b/src/Excititor/StellaOps.Excititor.WebService/Contracts/GraphOverlayContracts.cs new file mode 100644 index 000000000..d9d5c6113 --- /dev/null +++ b/src/Excititor/StellaOps.Excititor.WebService/Contracts/GraphOverlayContracts.cs @@ -0,0 +1,27 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Excititor.WebService.Contracts; + +public sealed record GraphOverlaysResponse( + [property: JsonPropertyName("items")] IReadOnlyList Items, + [property: JsonPropertyName("cached")] bool Cached, + [property: JsonPropertyName("cacheAgeMs")] long? CacheAgeMs); + +public sealed record GraphOverlayItem( + [property: JsonPropertyName("purl")] string Purl, + [property: JsonPropertyName("summary")] GraphOverlaySummary Summary, + [property: JsonPropertyName("latestModifiedAt")] DateTimeOffset? LatestModifiedAt, + [property: JsonPropertyName("justifications")] IReadOnlyList Justifications, + [property: JsonPropertyName("provenance")] GraphOverlayProvenance Provenance); + +public sealed record GraphOverlaySummary( + [property: JsonPropertyName("open")] int Open, + [property: JsonPropertyName("not_affected")] int NotAffected, + [property: JsonPropertyName("under_investigation")] int UnderInvestigation, + [property: JsonPropertyName("no_statement")] int NoStatement); + +public sealed record GraphOverlayProvenance( + [property: JsonPropertyName("sources")] IReadOnlyList Sources, + [property: JsonPropertyName("lastEvidenceHash")] string? LastEvidenceHash); diff --git a/src/Excititor/StellaOps.Excititor.WebService/Controllers/GraphController.cs b/src/Excititor/StellaOps.Excititor.WebService/Controllers/GraphController.cs deleted file mode 100644 index 8a4f11961..000000000 --- a/src/Excititor/StellaOps.Excititor.WebService/Controllers/GraphController.cs +++ /dev/null @@ -1,58 +0,0 @@ -using System.Collections.Generic; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Options; -using StellaOps.Excititor.WebService.Options; - -namespace StellaOps.Excititor.WebService.Controllers; - -[ApiController] -[Route("v1/graph")] -public class GraphController : ControllerBase -{ - private readonly GraphOptions _options; - - public GraphController(IOptions options) - { - _options = options.Value; - } - - [HttpPost("linkouts")] - public IActionResult Linkouts([FromBody] LinkoutRequest request) - { - if (request == null || request.Purls == null || request.Purls.Count == 0) - { - return BadRequest("purls are required"); - } - - if (request.Purls.Count > _options.MaxPurls) - { - return BadRequest($"purls limit exceeded (max {_options.MaxPurls})"); - } - - return StatusCode(503, "Graph linkouts pending storage integration."); - } - - [HttpGet("overlays")] - public IActionResult Overlays([FromQuery(Name = "purl")] List purls, [FromQuery] bool includeJustifications = false) - { - if (purls == null || purls.Count == 0) - { - return BadRequest("purl query parameter is required"); - } - - if (purls.Count > _options.MaxPurls) - { - return BadRequest($"purls limit exceeded (max {_options.MaxPurls})"); - } - - return StatusCode(503, "Graph overlays pending storage integration."); - } -} - -public sealed record LinkoutRequest -{ - public string Tenant { get; init; } = string.Empty; - public List Purls { get; init; } = new(); - public bool IncludeJustifications { get; init; } - public bool IncludeProvenance { get; init; } = true; -} diff --git a/src/Excititor/StellaOps.Excititor.WebService/Graph/GraphOverlayFactory.cs b/src/Excititor/StellaOps.Excititor.WebService/Graph/GraphOverlayFactory.cs new file mode 100644 index 000000000..8bf042bd1 --- /dev/null +++ b/src/Excititor/StellaOps.Excititor.WebService/Graph/GraphOverlayFactory.cs @@ -0,0 +1,125 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Core.Observations; +using StellaOps.Excititor.WebService.Contracts; + +namespace StellaOps.Excititor.WebService.Graph; + +internal static class GraphOverlayFactory +{ + public static IReadOnlyList Build( + IReadOnlyList orderedPurls, + IReadOnlyList observations, + bool includeJustifications) + { + if (orderedPurls is null) + { + throw new ArgumentNullException(nameof(orderedPurls)); + } + + if (observations is null) + { + throw new ArgumentNullException(nameof(observations)); + } + + var observationsByPurl = observations + .SelectMany(obs => obs.Linkset.Purls.Select(purl => (purl, obs))) + .GroupBy(tuple => tuple.purl, StringComparer.OrdinalIgnoreCase) + .ToDictionary(g => g.Key, g => g.Select(t => t.obs).ToImmutableArray(), StringComparer.OrdinalIgnoreCase); + + var items = new List(orderedPurls.Count); + + foreach (var input in orderedPurls) + { + if (!observationsByPurl.TryGetValue(input, out var obsForPurl) || obsForPurl.Length == 0) + { + items.Add(new GraphOverlayItem( + Purl: input, + Summary: new GraphOverlaySummary(0, 0, 0, 0), + LatestModifiedAt: null, + Justifications: Array.Empty(), + Provenance: new GraphOverlayProvenance(Array.Empty(), null))); + continue; + } + + var open = 0; + var notAffected = 0; + var underInvestigation = 0; + var noStatement = 0; + var justifications = new SortedSet(StringComparer.OrdinalIgnoreCase); + var sources = new SortedSet(StringComparer.OrdinalIgnoreCase); + string? lastEvidenceHash = null; + DateTimeOffset? latestModifiedAt = null; + + foreach (var obs in obsForPurl) + { + sources.Add(obs.ProviderId); + if (latestModifiedAt is null || obs.CreatedAt > latestModifiedAt.Value) + { + latestModifiedAt = obs.CreatedAt; + lastEvidenceHash = obs.Upstream.ContentHash; + } + + var matchingStatements = obs.Statements + .Where(stmt => PurlMatches(stmt, input, obs.Linkset.Purls)) + .ToArray(); + + if (matchingStatements.Length == 0) + { + noStatement++; + continue; + } + + foreach (var stmt in matchingStatements) + { + switch (stmt.Status) + { + case VexClaimStatus.NotAffected: + notAffected++; + break; + case VexClaimStatus.UnderInvestigation: + underInvestigation++; + break; + default: + open++; + break; + } + + if (includeJustifications && stmt.Justification is not null) + { + justifications.Add(stmt.Justification!.ToString()!); + } + } + } + + items.Add(new GraphOverlayItem( + Purl: input, + Summary: new GraphOverlaySummary(open, notAffected, underInvestigation, noStatement), + LatestModifiedAt: latestModifiedAt, + Justifications: includeJustifications + ? justifications.ToArray() + : Array.Empty(), + Provenance: new GraphOverlayProvenance(sources.ToArray(), lastEvidenceHash))); + } + + return items; + } + + private static bool PurlMatches(VexObservationStatement stmt, string inputPurl, ImmutableArray linksetPurls) + { + if (!string.IsNullOrWhiteSpace(stmt.Purl) && stmt.Purl.Equals(inputPurl, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + if (linksetPurls.IsDefaultOrEmpty) + { + return false; + } + + return linksetPurls.Any(p => p.Equals(inputPurl, StringComparison.OrdinalIgnoreCase)); + } +} diff --git a/src/Excititor/StellaOps.Excititor.WebService/Program.Helpers.cs b/src/Excititor/StellaOps.Excititor.WebService/Program.Helpers.cs index 733b0a718..e17a26692 100644 --- a/src/Excititor/StellaOps.Excititor.WebService/Program.Helpers.cs +++ b/src/Excititor/StellaOps.Excititor.WebService/Program.Helpers.cs @@ -199,6 +199,33 @@ public partial class Program return Math.Clamp(parsed, min, max); } + private static IReadOnlyList NormalizePurls(string[]? purls) + { + if (purls is null || purls.Length == 0) + { + return Array.Empty(); + } + + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + var ordered = new List(purls.Length); + foreach (var purl in purls) + { + var trimmed = purl?.Trim(); + if (string.IsNullOrWhiteSpace(trimmed)) + { + continue; + } + + var normalized = trimmed.ToLowerInvariant(); + if (seen.Add(normalized)) + { + ordered.Add(normalized); + } + } + + return ordered; + } + private static VexObservationStatementResponse ToResponse(VexObservationStatementProjection projection) { var scope = projection.Scope; @@ -234,4 +261,8 @@ public partial class Program signature.Issuer, signature.VerifiedAt)); } + + private sealed record CachedGraphOverlay( + IReadOnlyList Items, + DateTimeOffset CachedAt); } diff --git a/src/Excititor/StellaOps.Excititor.WebService/Program.cs b/src/Excititor/StellaOps.Excititor.WebService/Program.cs index e1b311ffa..c419b8e01 100644 --- a/src/Excititor/StellaOps.Excititor.WebService/Program.cs +++ b/src/Excititor/StellaOps.Excititor.WebService/Program.cs @@ -37,6 +37,7 @@ using MongoDB.Driver; using MongoDB.Bson; using Microsoft.Extensions.Caching.Memory; using StellaOps.Excititor.WebService.Contracts; +using StellaOps.Excititor.WebService.Graph; var builder = WebApplication.CreateBuilder(args); var configuration = builder.Configuration; @@ -523,10 +524,70 @@ var options = new VexObservationQueryOptions( NextCursor: advisories.Count >= 200 ? $"{advisories[^1].AdvisoryId}:{advisories[^1].Source}" : null)); } - var response = new GraphLinkoutsResponse(items, notFound); +var response = new GraphLinkoutsResponse(items, notFound); return Results.Ok(response); }).WithName("PostGraphLinkouts"); +// Cartographer overlays +app.MapGet("/v1/graph/overlays", async ( + HttpContext context, + [FromQuery(Name = "purl")] string[]? purls, + [FromQuery] bool includeJustifications, + IOptions storageOptions, + IOptions graphOptions, + IVexObservationQueryService queryService, + IMemoryCache cache, + TimeProvider timeProvider, + CancellationToken cancellationToken) => +{ + if (!TryResolveTenant(context, storageOptions.Value, requireHeader: true, out var tenant, out var tenantError)) + { + return tenantError; + } + + var orderedPurls = NormalizePurls(purls); + if (orderedPurls.Count == 0) + { + return Results.BadRequest("purl query parameter is required"); + } + + if (orderedPurls.Count > graphOptions.Value.MaxPurls) + { + return Results.BadRequest($"purls limit exceeded (max {graphOptions.Value.MaxPurls})"); + } + + var cacheKey = $"graph-overlays:{tenant}:{includeJustifications}:{string.Join('|', orderedPurls)}"; + var now = timeProvider.GetUtcNow(); + + if (cache.TryGetValue(cacheKey, out var cached) && cached is not null) + { + var ageMs = (long)Math.Max(0, (now - cached.CachedAt).TotalMilliseconds); + return Results.Ok(new GraphOverlaysResponse(cached.Items, true, ageMs)); + } + + var options = new VexObservationQueryOptions( + tenant: tenant, + purls: orderedPurls, + limit: graphOptions.Value.MaxAdvisoriesPerPurl * orderedPurls.Count); + + VexObservationQueryResult result; + try + { + result = await queryService.QueryAsync(options, cancellationToken).ConfigureAwait(false); + } + catch (FormatException ex) + { + return Results.BadRequest(ex.Message); + } + + var overlays = GraphOverlayFactory.Build(orderedPurls, result.Observations, includeJustifications); + var response = new GraphOverlaysResponse(overlays, false, null); + + cache.Set(cacheKey, new CachedGraphOverlay(overlays, now), TimeSpan.FromSeconds(graphOptions.Value.OverlayTtlSeconds)); + + return Results.Ok(response); +}).WithName("GetGraphOverlays"); + app.MapPost("/ingest/vex", async ( HttpContext context, VexIngestRequest request, diff --git a/src/Excititor/__Tests/StellaOps.Excititor.WebService.Tests/GraphOverlayFactoryTests.cs b/src/Excititor/__Tests/StellaOps.Excititor.WebService.Tests/GraphOverlayFactoryTests.cs new file mode 100644 index 000000000..b98d215fd --- /dev/null +++ b/src/Excititor/__Tests/StellaOps.Excititor.WebService.Tests/GraphOverlayFactoryTests.cs @@ -0,0 +1,108 @@ +using System; +using System.Collections.Immutable; +using System.Text.Json.Nodes; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Core.Observations; +using StellaOps.Excititor.WebService.Graph; +using Xunit; + +namespace StellaOps.Excititor.WebService.Tests; + +public sealed class GraphOverlayFactoryTests +{ + [Fact] + public void Build_ComputesSummariesAndProvenancePerPurl() + { + var now = DateTimeOffset.UtcNow; + var observations = new[] + { + CreateObservation( + providerId: "redhat", + createdAt: now.AddMinutes(-5), + purls: new[] { "pkg:rpm/redhat/openssl@1.1.1" }, + statements: new[] + { + new VexObservationStatement( + vulnerabilityId: "CVE-2025-1000", + productKey: "pkg:rpm/redhat/openssl@1.1.1", + status: VexClaimStatus.NotAffected, + lastObserved: now, + justification: VexJustification.ComponentNotPresent, + purl: "pkg:rpm/redhat/openssl@1.1.1") + }, + contentHash: "hash-old"), + CreateObservation( + providerId: "ubuntu", + createdAt: now, + purls: new[] { "pkg:rpm/redhat/openssl@1.1.1" }, + statements: new[] + { + new VexObservationStatement( + vulnerabilityId: "CVE-2025-1001", + productKey: "pkg:rpm/redhat/openssl@1.1.1", + status: VexClaimStatus.UnderInvestigation, + lastObserved: now, + justification: null, + purl: "pkg:rpm/redhat/openssl@1.1.1") + }, + contentHash: "hash-new"), + CreateObservation( + providerId: "oracle", + createdAt: now.AddMinutes(-1), + purls: new[] { "pkg:rpm/redhat/openssl@1.1.1" }, + statements: Array.Empty(), + contentHash: "hash-oracle") + }; + + var overlays = GraphOverlayFactory.Build( + orderedPurls: new[] { "pkg:rpm/redhat/openssl@1.1.1" }, + observations: observations, + includeJustifications: true); + + var overlay = Assert.Single(overlays); + Assert.Equal("pkg:rpm/redhat/openssl@1.1.1", overlay.Purl); + Assert.Equal(0, overlay.Summary.Open); + Assert.Equal(1, overlay.Summary.NotAffected); + Assert.Equal(1, overlay.Summary.UnderInvestigation); + Assert.Equal(1, overlay.Summary.NoStatement); + Assert.Equal(now, overlay.LatestModifiedAt); + Assert.Equal(new[] { "ComponentNotPresent" }, overlay.Justifications); + Assert.Equal("hash-new", overlay.Provenance.LastEvidenceHash); + Assert.Equal(new[] { "oracle", "redhat", "ubuntu" }, overlay.Provenance.Sources); + } + + private static VexObservation CreateObservation( + string providerId, + DateTimeOffset createdAt, + string[] purls, + VexObservationStatement[] statements, + string contentHash) + { + return new VexObservation( + observationId: $"obs-{providerId}-{createdAt.ToUnixTimeMilliseconds()}", + tenant: "tenant-a", + providerId: providerId, + streamId: "csaf", + upstream: new VexObservationUpstream( + upstreamId: Guid.NewGuid().ToString("N"), + documentVersion: "1", + fetchedAt: createdAt, + receivedAt: createdAt, + contentHash: contentHash, + signature: new VexObservationSignature(present: true, format: "sig", keyId: null, signature: null)), + statements: statements.ToImmutableArray(), + content: new VexObservationContent( + format: "csaf", + specVersion: "1", + raw: JsonValue.Create("raw")!, + metadata: ImmutableDictionary.Empty), + linkset: new VexObservationLinkset( + aliases: Array.Empty(), + purls: purls, + cpes: Array.Empty(), + references: Array.Empty()), + createdAt: createdAt, + supersedes: ImmutableArray.Empty, + attributes: ImmutableDictionary.Empty); + } +} diff --git a/src/Excititor/__Tests/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj b/src/Excititor/__Tests/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj index 8e2756267..4f1835632 100644 --- a/src/Excititor/__Tests/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj +++ b/src/Excititor/__Tests/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj @@ -34,6 +34,7 @@ + diff --git a/src/Mirror/StellaOps.Mirror.Creator/make-thin-v1.sh b/src/Mirror/StellaOps.Mirror.Creator/make-thin-v1.sh index 7858e8ea1..ce7ec96be 100644 --- a/src/Mirror/StellaOps.Mirror.Creator/make-thin-v1.sh +++ b/src/Mirror/StellaOps.Mirror.Creator/make-thin-v1.sh @@ -13,7 +13,10 @@ cat > "$STAGE/layers/observations.ndjson" <<'DATA' {"id":"obs-002","purl":"pkg:npm/lodash@4.17.21","advisory":"CVE-2024-9999","severity":"high","source":"vendor-b","timestamp":"2025-10-15T00:00:00Z"} DATA -cat > "$STAGE/layers/time-anchor.json" <<'DATA' +if [[ -n "${TIME_ANCHOR_FILE:-}" && -f "${TIME_ANCHOR_FILE}" ]]; then + cp "${TIME_ANCHOR_FILE}" "$STAGE/layers/time-anchor.json" +else + cat > "$STAGE/layers/time-anchor.json" <<'DATA' { "authority": "stellaops-airgap-test", "generatedAt": "2025-11-01T00:00:00Z", @@ -29,6 +32,7 @@ cat > "$STAGE/layers/time-anchor.json" <<'DATA' ] } DATA +fi cat > "$STAGE/indexes/observations.index" <<'DATA' obs-001 layers/observations.ndjson:1 diff --git a/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Tests/OpenApiEndpointTests.cs b/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Tests/OpenApiEndpointTests.cs index 69a0ec350..27edb852a 100644 --- a/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Tests/OpenApiEndpointTests.cs +++ b/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Tests/OpenApiEndpointTests.cs @@ -10,26 +10,16 @@ using Xunit; namespace StellaOps.Notifier.Tests; -public sealed class OpenApiEndpointTests : IClassFixture> +public sealed class OpenApiEndpointTests : IClassFixture { private readonly HttpClient _client; private readonly InMemoryPackApprovalRepository _packRepo = new(); private readonly InMemoryLockRepository _lockRepo = new(); private readonly InMemoryAuditRepository _auditRepo = new(); - public OpenApiEndpointTests(WebApplicationFactory factory) + public OpenApiEndpointTests(NotifierApplicationFactory factory) { - _client = factory - .WithWebHostBuilder(builder => - { - builder.ConfigureServices(services => - { - services.AddSingleton(_packRepo); - services.AddSingleton(_lockRepo); - services.AddSingleton(_auditRepo); - }); - }) - .CreateClient(); + _client = factory.CreateClient(); } [Fact] @@ -89,4 +79,20 @@ public sealed class OpenApiEndpointTests : IClassFixture - - - + + + + diff --git a/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Tests/Support/NotifierApplicationFactory.cs b/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Tests/Support/NotifierApplicationFactory.cs new file mode 100644 index 000000000..edf21933e --- /dev/null +++ b/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Tests/Support/NotifierApplicationFactory.cs @@ -0,0 +1,35 @@ +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Notifier.WebService; +using StellaOps.Notify.Storage.Mongo.Repositories; + +namespace StellaOps.Notifier.Tests.Support; + +internal sealed class NotifierApplicationFactory : WebApplicationFactory +{ + private readonly InMemoryPackApprovalRepository _packRepo; + private readonly InMemoryLockRepository _lockRepo; + private readonly InMemoryAuditRepository _auditRepo; + + public NotifierApplicationFactory( + InMemoryPackApprovalRepository packRepo, + InMemoryLockRepository lockRepo, + InMemoryAuditRepository auditRepo) + { + _packRepo = packRepo; + _lockRepo = lockRepo; + _auditRepo = auditRepo; + } + + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + builder.UseContentRoot(Path.Combine(Directory.GetCurrentDirectory(), "TestContent")); + builder.ConfigureServices(services => + { + services.AddSingleton(_packRepo); + services.AddSingleton(_lockRepo); + services.AddSingleton(_auditRepo); + }); + } +} diff --git a/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Tests/TestContent/openapi/notify-openapi.yaml b/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Tests/TestContent/openapi/notify-openapi.yaml new file mode 100644 index 000000000..746547448 --- /dev/null +++ b/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Tests/TestContent/openapi/notify-openapi.yaml @@ -0,0 +1,613 @@ +# OpenAPI 3.1 specification for StellaOps Notifier WebService (draft) +openapi: 3.1.0 +info: + title: StellaOps Notifier API + version: 0.6.0-draft + description: | + Contract for Notifications Studio (Notifier) covering rules, templates, incidents, + and quiet hours. Uses the platform error envelope and tenant header `X-StellaOps-Tenant`. +servers: + - url: https://api.stellaops.example.com + description: Production + - url: https://api.dev.stellaops.example.com + description: Development +security: + - oauth2: [notify.viewer] + - oauth2: [notify.operator] + - oauth2: [notify.admin] +paths: + /api/v1/notify/rules: + get: + summary: List notification rules + tags: [Rules] + parameters: + - $ref: '#/components/parameters/Tenant' + - $ref: '#/components/parameters/PageSize' + - $ref: '#/components/parameters/PageToken' + responses: + '200': + description: Paginated rule list + content: + application/json: + schema: + type: object + properties: + items: + type: array + items: { $ref: '#/components/schemas/NotifyRule' } + nextPageToken: + type: string + examples: + default: + value: + items: + - ruleId: rule-critical + tenantId: tenant-dev + name: Critical scanner verdicts + enabled: true + match: + eventKinds: [scanner.report.ready] + minSeverity: critical + actions: + - actionId: act-slack-critical + channel: chn-slack-soc + template: tmpl-critical + digest: instant + nextPageToken: null + default: + $ref: '#/components/responses/Error' + post: + summary: Create a notification rule + tags: [Rules] + parameters: + - $ref: '#/components/parameters/Tenant' + requestBody: + required: true + content: + application/json: + schema: { $ref: '#/components/schemas/NotifyRule' } + examples: + create-rule: + value: + ruleId: rule-attest-fail + tenantId: tenant-dev + name: Attestation failures → SOC + enabled: true + match: + eventKinds: [attestor.verification.failed] + actions: + - actionId: act-soc + channel: chn-webhook-soc + template: tmpl-attest-verify-fail + responses: + '201': + description: Rule created + content: + application/json: + schema: { $ref: '#/components/schemas/NotifyRule' } + default: + $ref: '#/components/responses/Error' + + /api/v1/notify/rules/{ruleId}: + get: + summary: Fetch a rule + tags: [Rules] + parameters: + - $ref: '#/components/parameters/Tenant' + - $ref: '#/components/parameters/RuleId' + responses: + '200': + description: Rule + content: + application/json: + schema: { $ref: '#/components/schemas/NotifyRule' } + default: + $ref: '#/components/responses/Error' + patch: + summary: Update a rule (partial) + tags: [Rules] + parameters: + - $ref: '#/components/parameters/Tenant' + - $ref: '#/components/parameters/RuleId' + requestBody: + required: true + content: + application/json: + schema: + type: object + description: JSON Merge Patch + responses: + '200': + description: Updated rule + content: + application/json: + schema: { $ref: '#/components/schemas/NotifyRule' } + default: + $ref: '#/components/responses/Error' + + /api/v1/notify/templates: + get: + summary: List templates + tags: [Templates] + parameters: + - $ref: '#/components/parameters/Tenant' + - name: key + in: query + description: Filter by template key + schema: { type: string } + responses: + '200': + description: Templates + content: + application/json: + schema: + type: array + items: { $ref: '#/components/schemas/NotifyTemplate' } + default: + $ref: '#/components/responses/Error' + post: + summary: Create a template + tags: [Templates] + parameters: + - $ref: '#/components/parameters/Tenant' + requestBody: + required: true + content: + application/json: + schema: { $ref: '#/components/schemas/NotifyTemplate' } + responses: + '201': + description: Template created + content: + application/json: + schema: { $ref: '#/components/schemas/NotifyTemplate' } + default: + $ref: '#/components/responses/Error' + + /api/v1/notify/templates/{templateId}: + get: + summary: Fetch a template + tags: [Templates] + parameters: + - $ref: '#/components/parameters/Tenant' + - $ref: '#/components/parameters/TemplateId' + responses: + '200': + description: Template + content: + application/json: + schema: { $ref: '#/components/schemas/NotifyTemplate' } + default: + $ref: '#/components/responses/Error' + patch: + summary: Update a template (partial) + tags: [Templates] + parameters: + - $ref: '#/components/parameters/Tenant' + - $ref: '#/components/parameters/TemplateId' + requestBody: + required: true + content: + application/json: + schema: + type: object + description: JSON Merge Patch + responses: + '200': + description: Updated template + content: + application/json: + schema: { $ref: '#/components/schemas/NotifyTemplate' } + default: + $ref: '#/components/responses/Error' + + /api/v1/notify/incidents: + get: + summary: List incidents (paged) + tags: [Incidents] + parameters: + - $ref: '#/components/parameters/Tenant' + - $ref: '#/components/parameters/PageSize' + - $ref: '#/components/parameters/PageToken' + responses: + '200': + description: Incident page + content: + application/json: + schema: + type: object + properties: + items: + type: array + items: { $ref: '#/components/schemas/Incident' } + nextPageToken: { type: string } + default: + $ref: '#/components/responses/Error' + post: + summary: Raise an incident (ops/toggle/override) + tags: [Incidents] + parameters: + - $ref: '#/components/parameters/Tenant' + requestBody: + required: true + content: + application/json: + schema: { $ref: '#/components/schemas/Incident' } + examples: + start-incident: + value: + incidentId: inc-telemetry-outage + kind: outage + severity: major + startedAt: 2025-11-17T04:02:00Z + shortDescription: "Telemetry pipeline degraded; burn-rate breach" + metadata: + source: slo-evaluator + responses: + '202': + description: Incident accepted + default: + $ref: '#/components/responses/Error' + + /api/v1/notify/incidents/{incidentId}/ack: + post: + summary: Acknowledge an incident notification + tags: [Incidents] + parameters: + - $ref: '#/components/parameters/Tenant' + - $ref: '#/components/parameters/IncidentId' + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + ackToken: + type: string + description: DSSE-signed acknowledgement token + responses: + '204': + description: Acknowledged + default: + $ref: '#/components/responses/Error' + + /api/v1/notify/quiet-hours: + get: + summary: Get quiet-hours schedule + tags: [QuietHours] + parameters: + - $ref: '#/components/parameters/Tenant' + responses: + '200': + description: Quiet hours schedule + content: + application/json: + schema: { $ref: '#/components/schemas/QuietHours' } + examples: + current: + value: + quietHoursId: qh-default + windows: + - timezone: UTC + days: [Mon, Tue, Wed, Thu, Fri] + start: "22:00" + end: "06:00" + exemptions: + - eventKinds: [attestor.verification.failed] + reason: "Always alert for attestation failures" + default: + $ref: '#/components/responses/Error' + post: + summary: Set quiet-hours schedule + tags: [QuietHours] + parameters: + - $ref: '#/components/parameters/Tenant' + requestBody: + required: true + content: + application/json: + schema: { $ref: '#/components/schemas/QuietHours' } + responses: + '200': + description: Updated quiet hours + content: + application/json: + schema: { $ref: '#/components/schemas/QuietHours' } + default: + $ref: '#/components/responses/Error' + + /api/v1/notify/pack-approvals: + post: + summary: Ingest pack approval decision + tags: [PackApprovals] + operationId: ingestPackApproval + security: + - oauth2: [notify.operator] + - hmac: [] + parameters: + - $ref: '#/components/parameters/Tenant' + - $ref: '#/components/parameters/IdempotencyKey' + requestBody: + required: true + content: + application/json: + schema: { $ref: '#/components/schemas/PackApprovalEvent' } + examples: + approval-granted: + value: + eventId: "20e4e5fe-3d4a-4f57-9f9b-b1a1c1111111" + issuedAt: "2025-11-17T16:00:00Z" + kind: "pack.approval.granted" + packId: "offline-kit-2025-11" + policy: + id: "policy-123" + version: "v5" + decision: "approved" + actor: "task-runner" + resumeToken: "rt-abc123" + summary: "All required attestations verified." + labels: + environment: "prod" + approver: "ops" + responses: + '202': + description: Accepted; durable write queued for processing. + headers: + X-Resume-After: + description: Resume token echo or replacement + schema: { type: string } + default: + $ref: '#/components/responses/Error' + + /api/v1/notify/pack-approvals/{packId}/ack: + post: + summary: Acknowledge a pack approval notification + tags: [PackApprovals] + operationId: ackPackApproval + parameters: + - $ref: '#/components/parameters/Tenant' + - name: packId + in: path + required: true + schema: { type: string } + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + ackToken: { type: string } + required: [ackToken] + responses: + '204': + description: Acknowledged + default: + $ref: '#/components/responses/Error' + +components: + securitySchemes: + oauth2: + type: oauth2 + flows: + clientCredentials: + tokenUrl: https://auth.stellaops.example.com/oauth/token + scopes: + notify.viewer: Read-only Notifier access + notify.operator: Manage rules/templates/incidents within tenant + notify.admin: Tenant-scoped administration + hmac: + type: http + scheme: bearer + description: Pre-shared HMAC token (air-gap friendly) referenced by secretRef. + parameters: + Tenant: + name: X-StellaOps-Tenant + in: header + required: true + description: Tenant slug + schema: { type: string } + IdempotencyKey: + name: Idempotency-Key + in: header + required: true + description: Stable UUID to dedupe retries. + schema: { type: string, format: uuid } + PageSize: + name: pageSize + in: query + schema: { type: integer, minimum: 1, maximum: 200, default: 50 } + PageToken: + name: pageToken + in: query + schema: { type: string } + RuleId: + name: ruleId + in: path + required: true + schema: { type: string } + TemplateId: + name: templateId + in: path + required: true + schema: { type: string } + IncidentId: + name: incidentId + in: path + required: true + schema: { type: string } + + responses: + Error: + description: Standard error envelope + content: + application/json: + schema: { $ref: '#/components/schemas/ErrorEnvelope' } + examples: + validation: + value: + error: + code: validation_failed + message: "quietHours.windows[0].start must be HH:mm" + traceId: "f62f3c2b9c8e4c53" + + schemas: + ErrorEnvelope: + type: object + required: [error] + properties: + error: + type: object + required: [code, message, traceId] + properties: + code: { type: string } + message: { type: string } + traceId: { type: string } + + NotifyRule: + type: object + required: [ruleId, tenantId, name, match, actions] + properties: + ruleId: { type: string } + tenantId: { type: string } + name: { type: string } + description: { type: string } + enabled: { type: boolean, default: true } + match: { $ref: '#/components/schemas/RuleMatch' } + actions: + type: array + items: { $ref: '#/components/schemas/RuleAction' } + labels: + type: object + additionalProperties: { type: string } + metadata: + type: object + additionalProperties: { type: string } + + RuleMatch: + type: object + properties: + eventKinds: + type: array + items: { type: string } + minSeverity: { type: string, enum: [info, low, medium, high, critical] } + verdicts: + type: array + items: { type: string } + labels: + type: array + items: { type: string } + kevOnly: { type: boolean } + + RuleAction: + type: object + required: [actionId, channel] + properties: + actionId: { type: string } + channel: { type: string } + template: { type: string } + digest: { type: string, description: "Digest window key e.g. instant|5m|15m|1h|1d" } + throttle: { type: string, description: "ISO-8601 duration, e.g. PT5M" } + locale: { type: string } + enabled: { type: boolean, default: true } + metadata: + type: object + additionalProperties: { type: string } + + NotifyTemplate: + type: object + required: [templateId, tenantId, key, channelType, locale, body, renderMode, format] + properties: + templateId: { type: string } + tenantId: { type: string } + key: { type: string } + channelType: { type: string, enum: [slack, teams, email, webhook, custom] } + locale: { type: string, description: "BCP-47, lower-case" } + renderMode: { type: string, enum: [Markdown, Html, AdaptiveCard, PlainText, Json] } + format: { type: string, enum: [slack, teams, email, webhook, json] } + description: { type: string } + body: { type: string } + metadata: + type: object + additionalProperties: { type: string } + + Incident: + type: object + required: [incidentId, kind, severity, startedAt] + properties: + incidentId: { type: string } + kind: { type: string, description: "outage|degradation|security|ops-drill" } + severity: { type: string, enum: [minor, major, critical] } + startedAt: { type: string, format: date-time } + endedAt: { type: string, format: date-time } + shortDescription: { type: string } + description: { type: string } + metadata: + type: object + additionalProperties: { type: string } + + PackApprovalEvent: + type: object + required: + - eventId + - issuedAt + - kind + - packId + - decision + - actor + properties: + eventId: { type: string, format: uuid } + issuedAt: { type: string, format: date-time } + kind: + type: string + enum: [pack.approval.granted, pack.approval.denied, pack.policy.override] + packId: { type: string } + policy: + type: object + properties: + id: { type: string } + version: { type: string } + decision: + type: string + enum: [approved, denied, overridden] + actor: { type: string } + resumeToken: + type: string + description: Opaque token for at-least-once resume. + summary: { type: string } + labels: + type: object + additionalProperties: { type: string } + + QuietHours: + type: object + required: [quietHoursId, windows] + properties: + quietHoursId: { type: string } + windows: + type: array + items: { $ref: '#/components/schemas/QuietHoursWindow' } + exemptions: + type: array + description: Event kinds that bypass quiet hours + items: + type: object + properties: + eventKinds: + type: array + items: { type: string } + reason: { type: string } + + QuietHoursWindow: + type: object + required: [timezone, days, start, end] + properties: + timezone: { type: string, description: "IANA TZ, e.g., UTC" } + days: + type: array + items: + type: string + enum: [Mon, Tue, Wed, Thu, Fri, Sat, Sun] + start: { type: string, description: "HH:mm" } + end: { type: string, description: "HH:mm" } diff --git a/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.WebService/Contracts/PackApprovalAckRequest.cs b/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.WebService/Contracts/PackApprovalAckRequest.cs new file mode 100644 index 000000000..f713fa8b5 --- /dev/null +++ b/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.WebService/Contracts/PackApprovalAckRequest.cs @@ -0,0 +1,9 @@ +using System.ComponentModel.DataAnnotations; + +namespace StellaOps.Notifier.WebService.Contracts; + +public sealed class PackApprovalAckRequest +{ + [Required] + public string AckToken { get; init; } = string.Empty; +} diff --git a/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.WebService/Program.cs b/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.WebService/Program.cs index d044da2f5..139b256d4 100644 --- a/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.WebService/Program.cs +++ b/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.WebService/Program.cs @@ -118,6 +118,50 @@ app.MapPost("/api/v1/notify/pack-approvals", async ( return Results.Accepted(); }); +app.MapPost("/api/v1/notify/pack-approvals/{packId}/ack", async ( + HttpContext context, + string packId, + PackApprovalAckRequest request, + INotifyLockRepository locks, + INotifyAuditRepository audit, + TimeProvider timeProvider) => +{ + var tenantId = context.Request.Headers["X-StellaOps-Tenant"].ToString(); + if (string.IsNullOrWhiteSpace(tenantId)) + { + return Results.BadRequest(Error("tenant_missing", "X-StellaOps-Tenant header is required.", context)); + } + + if (string.IsNullOrWhiteSpace(request.AckToken)) + { + return Results.BadRequest(Error("ack_token_missing", "AckToken is required.", context)); + } + + var lockKey = $"pack-approvals-ack|{tenantId}|{packId}|{request.AckToken}"; + var reserved = await locks.TryAcquireAsync(tenantId, lockKey, "pack-approvals-ack", TimeSpan.FromMinutes(10), context.RequestAborted) + .ConfigureAwait(false); + + if (!reserved) + { + return Results.StatusCode(StatusCodes.Status200OK); + } + + var auditEntry = new NotifyAuditEntryDocument + { + TenantId = tenantId, + Actor = "pack-approvals-ack", + Action = "pack.approval.acknowledged", + EntityId = packId, + EntityType = "pack-approval", + Timestamp = timeProvider.GetUtcNow(), + Payload = MongoDB.Bson.Serialization.BsonSerializer.Deserialize(JsonSerializer.Serialize(request)) + }; + + await audit.AppendAsync(auditEntry, context.RequestAborted).ConfigureAwait(false); + + return Results.NoContent(); +}); + app.MapGet("/.well-known/openapi", (HttpContext context, OpenApiDocumentCache cache) => { context.Response.Headers.CacheControl = "public, max-age=300"; diff --git a/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.WebService/Setup/OpenApiDocumentCache.cs b/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.WebService/Setup/OpenApiDocumentCache.cs index 2fcaa80c7..b39f186df 100644 --- a/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.WebService/Setup/OpenApiDocumentCache.cs +++ b/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.WebService/Setup/OpenApiDocumentCache.cs @@ -12,7 +12,9 @@ public sealed class OpenApiDocumentCache var path = Path.Combine(environment.ContentRootPath, "openapi", "notify-openapi.yaml"); if (!File.Exists(path)) { - throw new FileNotFoundException("OpenAPI document not found.", path); + _document = string.Empty; + _hash = string.Empty; + return; } _document = File.ReadAllText(path, Encoding.UTF8); diff --git a/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.WebService/openapi/notify-openapi.yaml b/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.WebService/openapi/notify-openapi.yaml index f4f3d2f3b..746547448 100644 --- a/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.WebService/openapi/notify-openapi.yaml +++ b/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.WebService/openapi/notify-openapi.yaml @@ -317,6 +317,75 @@ paths: default: $ref: '#/components/responses/Error' + /api/v1/notify/pack-approvals: + post: + summary: Ingest pack approval decision + tags: [PackApprovals] + operationId: ingestPackApproval + security: + - oauth2: [notify.operator] + - hmac: [] + parameters: + - $ref: '#/components/parameters/Tenant' + - $ref: '#/components/parameters/IdempotencyKey' + requestBody: + required: true + content: + application/json: + schema: { $ref: '#/components/schemas/PackApprovalEvent' } + examples: + approval-granted: + value: + eventId: "20e4e5fe-3d4a-4f57-9f9b-b1a1c1111111" + issuedAt: "2025-11-17T16:00:00Z" + kind: "pack.approval.granted" + packId: "offline-kit-2025-11" + policy: + id: "policy-123" + version: "v5" + decision: "approved" + actor: "task-runner" + resumeToken: "rt-abc123" + summary: "All required attestations verified." + labels: + environment: "prod" + approver: "ops" + responses: + '202': + description: Accepted; durable write queued for processing. + headers: + X-Resume-After: + description: Resume token echo or replacement + schema: { type: string } + default: + $ref: '#/components/responses/Error' + + /api/v1/notify/pack-approvals/{packId}/ack: + post: + summary: Acknowledge a pack approval notification + tags: [PackApprovals] + operationId: ackPackApproval + parameters: + - $ref: '#/components/parameters/Tenant' + - name: packId + in: path + required: true + schema: { type: string } + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + ackToken: { type: string } + required: [ackToken] + responses: + '204': + description: Acknowledged + default: + $ref: '#/components/responses/Error' + components: securitySchemes: oauth2: @@ -328,6 +397,10 @@ components: notify.viewer: Read-only Notifier access notify.operator: Manage rules/templates/incidents within tenant notify.admin: Tenant-scoped administration + hmac: + type: http + scheme: bearer + description: Pre-shared HMAC token (air-gap friendly) referenced by secretRef. parameters: Tenant: name: X-StellaOps-Tenant @@ -335,6 +408,12 @@ components: required: true description: Tenant slug schema: { type: string } + IdempotencyKey: + name: Idempotency-Key + in: header + required: true + description: Stable UUID to dedupe retries. + schema: { type: string, format: uuid } PageSize: name: pageSize in: query @@ -468,6 +547,39 @@ components: type: object additionalProperties: { type: string } + PackApprovalEvent: + type: object + required: + - eventId + - issuedAt + - kind + - packId + - decision + - actor + properties: + eventId: { type: string, format: uuid } + issuedAt: { type: string, format: date-time } + kind: + type: string + enum: [pack.approval.granted, pack.approval.denied, pack.policy.override] + packId: { type: string } + policy: + type: object + properties: + id: { type: string } + version: { type: string } + decision: + type: string + enum: [approved, denied, overridden] + actor: { type: string } + resumeToken: + type: string + description: Opaque token for at-least-once resume. + summary: { type: string } + labels: + type: object + additionalProperties: { type: string } + QuietHours: type: object required: [quietHoursId, windows] diff --git a/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.docs/pack-approval-templates.json b/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.docs/pack-approval-templates.json new file mode 100644 index 000000000..7fbb9b38e --- /dev/null +++ b/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.docs/pack-approval-templates.json @@ -0,0 +1,71 @@ +{ + "templates": [ + { + "templateId": "tmpl-pack-approval-slack-en", + "tenantId": "tenant-sample", + "key": "pack.approval.granted", + "channelType": "slack", + "locale": "en-US", + "renderMode": "Markdown", + "format": "slack", + "description": "Pack approval granted (Slack, English)", + "body": "*Pack approval granted*\nPack: {{packId}}\nPolicy: {{policy.id}} ({{policy.version}})\nDecision: {{decision}}\nResume: {{resumeToken}}\nSummary: {{summary}}\nLabels: {{#each labels}}{{@key}}={{this}} {{/each}}", + "metadata": { + "redaction": "safe", + "throttle": "PT5M" + } + }, + { + "templateId": "tmpl-pack-approval-email-en", + "tenantId": "tenant-sample", + "key": "pack.approval.granted", + "channelType": "email", + "locale": "en-US", + "renderMode": "Html", + "format": "email", + "description": "Pack approval granted (Email, English)", + "body": "

Pack approval granted

Pack: {{packId}}
Policy: {{policy.id}} ({{policy.version}})
Decision: {{decision}}
Resume: {{resumeToken}}
Summary: {{summary}}

Labels: {{#each labels}}{{@key}}={{this}} {{/each}}

", + "metadata": { + "redaction": "safe", + "throttle": "PT5M", + "subject": "[Notify] Pack approval granted: {{packId}}" + } + } + ], + "routingPredicates": [ + { + "name": "pack-approval-default", + "match": { + "eventKinds": ["pack.approval.granted", "pack.approval.denied", "pack.policy.override"], + "labels": ["environment=prod"] + }, + "actions": [ + { + "channel": "slack:sec-approvals", + "template": "tmpl-pack-approval-slack-en", + "digest": "instant" + }, + { + "channel": "email:ops-approvals", + "template": "tmpl-pack-approval-email-en", + "digest": "instant" + } + ] + } + ], + "redaction": { + "allow": [ + "packId", + "policy.id", + "policy.version", + "decision", + "resumeToken", + "summary", + "labels.*" + ], + "deny": [ + "secrets", + "tokens" + ] + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/AdvisoryAI/AdvisoryAiKnobsModels.cs b/src/Policy/StellaOps.Policy.Engine/AdvisoryAI/AdvisoryAiKnobsModels.cs new file mode 100644 index 000000000..84a6673ba --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/AdvisoryAI/AdvisoryAiKnobsModels.cs @@ -0,0 +1,15 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Policy.Engine.AdvisoryAI; + +internal sealed record AdvisoryAiKnob( + [property: JsonPropertyName("name")] string Name, + [property: JsonPropertyName("default_value")] decimal DefaultValue, + [property: JsonPropertyName("min")] decimal Min, + [property: JsonPropertyName("max")] decimal Max, + [property: JsonPropertyName("step")] decimal Step, + [property: JsonPropertyName("description")] string Description); + +internal sealed record AdvisoryAiKnobsProfile( + [property: JsonPropertyName("knobs")] IReadOnlyList Knobs, + [property: JsonPropertyName("profile_hash")] string ProfileHash); diff --git a/src/Policy/StellaOps.Policy.Engine/AdvisoryAI/AdvisoryAiKnobsService.cs b/src/Policy/StellaOps.Policy.Engine/AdvisoryAI/AdvisoryAiKnobsService.cs new file mode 100644 index 000000000..ed78e8219 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/AdvisoryAI/AdvisoryAiKnobsService.cs @@ -0,0 +1,73 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; + +namespace StellaOps.Policy.Engine.AdvisoryAI; + +/// +/// In-memory store for Advisory AI knobs (POLICY-ENGINE-31-001). +/// +internal sealed class AdvisoryAiKnobsService +{ + private readonly TimeProvider _timeProvider; + private readonly object _lock = new(); + private AdvisoryAiKnobsProfile _current; + + public AdvisoryAiKnobsService(TimeProvider timeProvider) + { + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _current = BuildProfile(DefaultKnobs()); + } + + public AdvisoryAiKnobsProfile Get() => _current; + + public AdvisoryAiKnobsProfile Set(IReadOnlyList knobs) + { + var normalized = Normalize(knobs); + var profile = BuildProfile(normalized); + lock (_lock) + { + _current = profile; + } + + return profile; + } + + private AdvisoryAiKnobsProfile BuildProfile(IReadOnlyList knobs) + { + var json = JsonSerializer.Serialize(knobs, new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = false + }); + + var hash = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(json))); + return new AdvisoryAiKnobsProfile(knobs, hash); + } + + private IReadOnlyList Normalize(IReadOnlyList knobs) + { + var normalized = knobs + .Where(k => !string.IsNullOrWhiteSpace(k.Name)) + .Select(k => new AdvisoryAiKnob( + Name: k.Name.Trim().ToLowerInvariant(), + DefaultValue: k.DefaultValue, + Min: k.Min, + Max: k.Max, + Step: k.Step <= 0 ? 0.001m : k.Step, + Description: string.IsNullOrWhiteSpace(k.Description) ? string.Empty : k.Description.Trim())) + .OrderBy(k => k.Name, StringComparer.Ordinal) + .ToList(); + + return normalized; + } + + private static IReadOnlyList DefaultKnobs() => + new[] + { + new AdvisoryAiKnob("ai_signal_weight", 1.0m, 0m, 2m, 0.01m, "Weight applied to AI signals"), + new AdvisoryAiKnob("reachability_boost", 0.2m, 0m, 1m, 0.01m, "Boost when asset is reachable"), + new AdvisoryAiKnob("time_decay_half_life_days", 30m, 1m, 365m, 1m, "Half-life for decay"), + new AdvisoryAiKnob("evidence_freshness_threshold_hours", 72m, 1m, 720m, 1m, "Max evidence age") + }; +} diff --git a/src/Policy/StellaOps.Policy.Engine/BatchContext/BatchContextModels.cs b/src/Policy/StellaOps.Policy.Engine/BatchContext/BatchContextModels.cs new file mode 100644 index 000000000..e8aa52eca --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/BatchContext/BatchContextModels.cs @@ -0,0 +1,31 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Policy.Engine.BatchContext; + +internal sealed record BatchContextRequest( + [property: JsonPropertyName("tenant_id")] string TenantId, + [property: JsonPropertyName("policy_profile_hash")] string PolicyProfileHash, + [property: JsonPropertyName("knobs_version")] string KnobsVersion, + [property: JsonPropertyName("overlay_hash")] string OverlayHash, + [property: JsonPropertyName("items")] IReadOnlyList Items, + [property: JsonPropertyName("options")] BatchContextOptions Options); + +internal sealed record BatchContextItem( + [property: JsonPropertyName("component_purl")] string ComponentPurl, + [property: JsonPropertyName("advisory_id")] string AdvisoryId); + +internal sealed record BatchContextOptions( + [property: JsonPropertyName("include_reachability")] bool IncludeReachability); + +internal sealed record BatchContextResponse( + [property: JsonPropertyName("context_id")] string ContextId, + [property: JsonPropertyName("expires_at")] string ExpiresAt, + [property: JsonPropertyName("knobs_version")] string KnobsVersion, + [property: JsonPropertyName("overlay_hash")] string OverlayHash, + [property: JsonPropertyName("items")] IReadOnlyList Items); + +internal sealed record BatchContextResolvedItem( + [property: JsonPropertyName("component_purl")] string ComponentPurl, + [property: JsonPropertyName("advisory_id")] string AdvisoryId, + [property: JsonPropertyName("status")] string Status, + [property: JsonPropertyName("trace_ref")] string TraceRef); diff --git a/src/Policy/StellaOps.Policy.Engine/BatchContext/BatchContextService.cs b/src/Policy/StellaOps.Policy.Engine/BatchContext/BatchContextService.cs new file mode 100644 index 000000000..bd83a14ac --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/BatchContext/BatchContextService.cs @@ -0,0 +1,82 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; + +namespace StellaOps.Policy.Engine.BatchContext; + +/// +/// Creates deterministic batch context responses for advisory AI (POLICY-ENGINE-31-002). +/// +internal sealed class BatchContextService +{ + private readonly TimeProvider _timeProvider; + + public BatchContextService(TimeProvider timeProvider) + { + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + } + + public BatchContextResponse Create(BatchContextRequest request) + { + if (request is null) throw new ArgumentNullException(nameof(request)); + if (request.Items is null || request.Items.Count == 0) + { + throw new ArgumentException("items are required", nameof(request.Items)); + } + + var sortedItems = request.Items + .OrderBy(i => i.ComponentPurl, StringComparer.Ordinal) + .ThenBy(i => i.AdvisoryId, StringComparer.Ordinal) + .ToList(); + + var status = request.Options?.IncludeReachability == true ? "pending-reachability" : "pending"; + var resolved = sortedItems + .Select(i => new BatchContextResolvedItem( + i.ComponentPurl, + i.AdvisoryId, + status, + ComputeTraceRef(request.TenantId, i))) + .ToList(); + + var expires = _timeProvider.GetUtcNow().AddHours(1).ToString("O"); + var contextId = ComputeContextId(request, sortedItems); + + return new BatchContextResponse( + contextId, + ExpiresAt: expires, + KnobsVersion: request.KnobsVersion, + OverlayHash: request.OverlayHash, + Items: resolved); + } + + private static string ComputeContextId(BatchContextRequest request, IReadOnlyList sortedItems) + { + var canonical = new + { + tenant = request.TenantId, + profile = request.PolicyProfileHash, + knobs = request.KnobsVersion, + overlay = request.OverlayHash, + items = sortedItems.Select(i => new { i.ComponentPurl, i.AdvisoryId }).ToArray(), + includeReachability = request.Options?.IncludeReachability ?? false + }; + + var json = JsonSerializer.Serialize(canonical, new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = false + }); + + Span hash = stackalloc byte[16]; + SHA256.HashData(Encoding.UTF8.GetBytes(json), hash); + return Convert.ToHexString(hash); + } + + private static string ComputeTraceRef(string tenant, BatchContextItem item) + { + var stable = $"{tenant}|{item.ComponentPurl}|{item.AdvisoryId}"; + Span hash = stackalloc byte[12]; + SHA256.HashData(Encoding.UTF8.GetBytes(stable), hash); + return Convert.ToHexString(hash); + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Endpoints/AdvisoryAiKnobsEndpoint.cs b/src/Policy/StellaOps.Policy.Engine/Endpoints/AdvisoryAiKnobsEndpoint.cs new file mode 100644 index 000000000..1930544a4 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Endpoints/AdvisoryAiKnobsEndpoint.cs @@ -0,0 +1,37 @@ +using Microsoft.AspNetCore.Mvc; +using StellaOps.Policy.Engine.AdvisoryAI; + +namespace StellaOps.Policy.Engine.Endpoints; + +public static class AdvisoryAiKnobsEndpoint +{ + public static IEndpointRouteBuilder MapAdvisoryAiKnobs(this IEndpointRouteBuilder routes) + { + routes.MapGet("/policy/advisory-ai/knobs", GetAsync) + .WithName("PolicyEngine.AdvisoryAI.Knobs.Get"); + + routes.MapPut("/policy/advisory-ai/knobs", PutAsync) + .WithName("PolicyEngine.AdvisoryAI.Knobs.Put"); + + return routes; + } + + private static IResult GetAsync(AdvisoryAiKnobsService service) + { + var profile = service.Get(); + return Results.Json(profile); + } + + private static IResult PutAsync( + [FromBody] IReadOnlyList knobs, + AdvisoryAiKnobsService service) + { + if (knobs is null || knobs.Count == 0) + { + return Results.BadRequest(new { message = "knobs are required" }); + } + + var profile = service.Set(knobs); + return Results.Json(profile); + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Endpoints/BatchContextEndpoint.cs b/src/Policy/StellaOps.Policy.Engine/Endpoints/BatchContextEndpoint.cs new file mode 100644 index 000000000..f910da969 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Endpoints/BatchContextEndpoint.cs @@ -0,0 +1,30 @@ +using Microsoft.AspNetCore.Mvc; +using StellaOps.Policy.Engine.BatchContext; + +namespace StellaOps.Policy.Engine.Endpoints; + +public static class BatchContextEndpoint +{ + public static IEndpointRouteBuilder MapBatchContext(this IEndpointRouteBuilder routes) + { + routes.MapPost("/policy/batch/context", HandleAsync) + .WithName("PolicyEngine.BatchContext.Create"); + + return routes; + } + + private static IResult HandleAsync( + [FromBody] BatchContextRequest request, + BatchContextService service) + { + try + { + var response = service.Create(request); + return Results.Json(response); + } + catch (ArgumentException ex) + { + return Results.BadRequest(new { message = ex.Message }); + } + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Endpoints/LedgerExportEndpoint.cs b/src/Policy/StellaOps.Policy.Engine/Endpoints/LedgerExportEndpoint.cs new file mode 100644 index 000000000..2b776abbc --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Endpoints/LedgerExportEndpoint.cs @@ -0,0 +1,43 @@ +using Microsoft.AspNetCore.Mvc; +using StellaOps.Policy.Engine.Ledger; + +namespace StellaOps.Policy.Engine.Endpoints; + +public static class LedgerExportEndpoint +{ + public static IEndpointRouteBuilder MapLedgerExport(this IEndpointRouteBuilder routes) + { + routes.MapPost("/policy/ledger/export", BuildAsync) + .WithName("PolicyEngine.Ledger.Export"); + + routes.MapGet("/policy/ledger/export/{exportId}", GetAsync) + .WithName("PolicyEngine.Ledger.GetExport"); + + return routes; + } + + private static async Task BuildAsync( + [FromBody] LedgerExportRequest request, + LedgerExportService service, + CancellationToken cancellationToken) + { + try + { + var export = await service.BuildAsync(request, cancellationToken).ConfigureAwait(false); + return Results.Json(export); + } + catch (Exception ex) when (ex is ArgumentException or KeyNotFoundException) + { + return Results.BadRequest(new { message = ex.Message }); + } + } + + private static async Task GetAsync( + [FromRoute] string exportId, + LedgerExportService service, + CancellationToken cancellationToken) + { + var export = await service.GetAsync(exportId, cancellationToken).ConfigureAwait(false); + return export is null ? Results.NotFound() : Results.Json(export); + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Endpoints/OrchestratorJobEndpoint.cs b/src/Policy/StellaOps.Policy.Engine/Endpoints/OrchestratorJobEndpoint.cs new file mode 100644 index 000000000..c26a88b4f --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Endpoints/OrchestratorJobEndpoint.cs @@ -0,0 +1,62 @@ +using Microsoft.AspNetCore.Mvc; +using StellaOps.Policy.Engine.Orchestration; + +namespace StellaOps.Policy.Engine.Endpoints; + +public static class OrchestratorJobEndpoint +{ + public static IEndpointRouteBuilder MapOrchestratorJobs(this IEndpointRouteBuilder routes) + { + routes.MapPost("/policy/orchestrator/jobs", SubmitAsync) + .WithName("PolicyEngine.Orchestrator.Jobs.Submit"); + + routes.MapPost("/policy/orchestrator/jobs/preview", PreviewAsync) + .WithName("PolicyEngine.Orchestrator.Jobs.Preview"); + + routes.MapGet("/policy/orchestrator/jobs/{jobId}", GetAsync) + .WithName("PolicyEngine.Orchestrator.Jobs.Get"); + + return routes; + } + + private static async Task SubmitAsync( + [FromBody] OrchestratorJobRequest request, + OrchestratorJobService service, + CancellationToken cancellationToken) + { + try + { + var job = await service.SubmitAsync(request, cancellationToken).ConfigureAwait(false); + return Results.Json(job); + } + catch (ArgumentException ex) + { + return Results.BadRequest(new { message = ex.Message }); + } + } + + private static async Task PreviewAsync( + [FromBody] OrchestratorJobRequest request, + OrchestratorJobService service, + CancellationToken cancellationToken) + { + try + { + var job = await service.PreviewAsync(request, cancellationToken).ConfigureAwait(false); + return Results.Json(job); + } + catch (ArgumentException ex) + { + return Results.BadRequest(new { message = ex.Message }); + } + } + + private static async Task GetAsync( + [FromRoute] string jobId, + OrchestratorJobService service, + CancellationToken cancellationToken) + { + var job = await service.GetAsync(jobId, cancellationToken).ConfigureAwait(false); + return job is null ? Results.NotFound() : Results.Json(job); + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Endpoints/PathScopeSimulationEndpoint.cs b/src/Policy/StellaOps.Policy.Engine/Endpoints/PathScopeSimulationEndpoint.cs index 2cd1f9a4a..e07c6c6cc 100644 --- a/src/Policy/StellaOps.Policy.Engine/Endpoints/PathScopeSimulationEndpoint.cs +++ b/src/Policy/StellaOps.Policy.Engine/Endpoints/PathScopeSimulationEndpoint.cs @@ -3,6 +3,7 @@ using System.Text.Json; using Microsoft.AspNetCore.Http.HttpResults; using Microsoft.AspNetCore.Mvc; using StellaOps.Policy.Engine.Streaming; +using StellaOps.Policy.Engine.Overlay; namespace StellaOps.Policy.Engine.Endpoints; @@ -19,6 +20,7 @@ public static class PathScopeSimulationEndpoint private static async Task HandleAsync( [FromBody] PathScopeSimulationRequest request, PathScopeSimulationService service, + PathScopeSimulationBridgeService bridge, CancellationToken cancellationToken) { try @@ -31,6 +33,19 @@ public static class PathScopeSimulationEndpoint responseBuilder.AppendLine(line); } + // Emit change event stub when run in what-if mode. + if (request.Options.Deterministic && request.Options.IncludeTrace) + { + var bridgeRequest = new PathScopeSimulationBridgeRequest( + Tenant: request.Tenant, + Rules: Array.Empty(), + Overlays: null, + Paths: new[] { request }, + Mode: "preview", + Seed: null); + await bridge.SimulateAsync(bridgeRequest, cancellationToken).ConfigureAwait(false); + } + return Results.Text(responseBuilder.ToString(), "application/x-ndjson", Encoding.UTF8); } catch (PathScopeSimulationException ex) diff --git a/src/Policy/StellaOps.Policy.Engine/Endpoints/PolicyWorkerEndpoint.cs b/src/Policy/StellaOps.Policy.Engine/Endpoints/PolicyWorkerEndpoint.cs new file mode 100644 index 000000000..95dc229e2 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Endpoints/PolicyWorkerEndpoint.cs @@ -0,0 +1,43 @@ +using Microsoft.AspNetCore.Mvc; +using StellaOps.Policy.Engine.Orchestration; + +namespace StellaOps.Policy.Engine.Endpoints; + +public static class PolicyWorkerEndpoint +{ + public static IEndpointRouteBuilder MapPolicyWorker(this IEndpointRouteBuilder routes) + { + routes.MapPost("/policy/worker/run", RunAsync) + .WithName("PolicyEngine.Worker.Run"); + + routes.MapGet("/policy/worker/jobs/{jobId}", GetResultAsync) + .WithName("PolicyEngine.Worker.GetResult"); + + return routes; + } + + private static async Task RunAsync( + [FromBody] WorkerRunRequest request, + PolicyWorkerService service, + CancellationToken cancellationToken) + { + try + { + var result = await service.ExecuteAsync(request, cancellationToken).ConfigureAwait(false); + return Results.Json(result); + } + catch (KeyNotFoundException ex) + { + return Results.NotFound(new { message = ex.Message }); + } + } + + private static async Task GetResultAsync( + [FromRoute] string jobId, + IWorkerResultStore store, + CancellationToken cancellationToken) + { + var result = await store.GetByJobIdAsync(jobId, cancellationToken).ConfigureAwait(false); + return result is null ? Results.NotFound() : Results.Json(result); + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Endpoints/SnapshotEndpoint.cs b/src/Policy/StellaOps.Policy.Engine/Endpoints/SnapshotEndpoint.cs new file mode 100644 index 000000000..ef8e1d10e --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Endpoints/SnapshotEndpoint.cs @@ -0,0 +1,55 @@ +using Microsoft.AspNetCore.Mvc; +using StellaOps.Policy.Engine.Snapshots; + +namespace StellaOps.Policy.Engine.Endpoints; + +public static class SnapshotEndpoint +{ + public static IEndpointRouteBuilder MapSnapshots(this IEndpointRouteBuilder routes) + { + routes.MapPost("/policy/snapshots", CreateAsync) + .WithName("PolicyEngine.Snapshots.Create"); + + routes.MapGet("/policy/snapshots", ListAsync) + .WithName("PolicyEngine.Snapshots.List"); + + routes.MapGet("/policy/snapshots/{snapshotId}", GetAsync) + .WithName("PolicyEngine.Snapshots.Get"); + + return routes; + } + + private static async Task CreateAsync( + [FromBody] SnapshotRequest request, + SnapshotService service, + CancellationToken cancellationToken) + { + try + { + var snapshot = await service.CreateAsync(request, cancellationToken).ConfigureAwait(false); + return Results.Json(snapshot); + } + catch (ArgumentException ex) + { + return Results.BadRequest(new { message = ex.Message }); + } + } + + private static async Task ListAsync( + [FromQuery(Name = "tenant_id")] string? tenantId, + SnapshotService service, + CancellationToken cancellationToken) + { + var (items, cursor) = await service.ListAsync(tenantId, cancellationToken).ConfigureAwait(false); + return Results.Json(new { items, next_cursor = cursor }); + } + + private static async Task GetAsync( + [FromRoute] string snapshotId, + SnapshotService service, + CancellationToken cancellationToken) + { + var snapshot = await service.GetAsync(snapshotId, cancellationToken).ConfigureAwait(false); + return snapshot is null ? Results.NotFound() : Results.Json(snapshot); + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Endpoints/TrustWeightingEndpoint.cs b/src/Policy/StellaOps.Policy.Engine/Endpoints/TrustWeightingEndpoint.cs new file mode 100644 index 000000000..7b629259c --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Endpoints/TrustWeightingEndpoint.cs @@ -0,0 +1,56 @@ +using Microsoft.AspNetCore.Mvc; +using StellaOps.Policy.Engine.TrustWeighting; + +namespace StellaOps.Policy.Engine.Endpoints; + +public static class TrustWeightingEndpoint +{ + public static IEndpointRouteBuilder MapTrustWeighting(this IEndpointRouteBuilder routes) + { + routes.MapGet("/policy/trust-weighting", GetAsync) + .WithName("PolicyEngine.TrustWeighting.Get"); + + routes.MapPut("/policy/trust-weighting", PutAsync) + .WithName("PolicyEngine.TrustWeighting.Put"); + + routes.MapGet("/policy/trust-weighting/preview", PreviewAsync) + .WithName("PolicyEngine.TrustWeighting.Preview"); + + return routes; + } + + private static IResult GetAsync(TrustWeightingService service) + { + var profile = service.Get(); + return Results.Json(profile); + } + + private static IResult PutAsync( + [FromBody] IReadOnlyList weights, + TrustWeightingService service) + { + if (weights is null || weights.Count == 0) + { + return Results.BadRequest(new { message = "weights are required" }); + } + + var profile = service.Set(weights); + return Results.Json(profile); + } + + private static IResult PreviewAsync( + [FromQuery(Name = "overlay_hash")] string? overlayHash, + TrustWeightingService service) + { + var profile = service.Get(); + var preview = new + { + weights = profile.Weights, + profile_hash = profile.ProfileHash, + overlay_hash = overlayHash, + mode = "preview" + }; + + return Results.Json(preview); + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Endpoints/ViolationEndpoint.cs b/src/Policy/StellaOps.Policy.Engine/Endpoints/ViolationEndpoint.cs new file mode 100644 index 000000000..a542a49ca --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Endpoints/ViolationEndpoint.cs @@ -0,0 +1,75 @@ +using Microsoft.AspNetCore.Mvc; +using StellaOps.Policy.Engine.Violations; + +namespace StellaOps.Policy.Engine.Endpoints; + +public static class ViolationEndpoint +{ + public static IEndpointRouteBuilder MapViolations(this IEndpointRouteBuilder routes) + { + routes.MapPost("/policy/violations/events", EmitEventsAsync) + .WithName("PolicyEngine.Violations.Events"); + + routes.MapPost("/policy/violations/severity", FuseAsync) + .WithName("PolicyEngine.Violations.Severity"); + + routes.MapPost("/policy/violations/conflicts", ConflictsAsync) + .WithName("PolicyEngine.Violations.Conflicts"); + + return routes; + } + + private static async Task EmitEventsAsync( + [FromBody] ViolationEventRequest request, + ViolationEventService service, + CancellationToken cancellationToken) + { + try + { + var events = await service.EmitAsync(request, cancellationToken).ConfigureAwait(false); + return Results.Json(new { events }); + } + catch (Exception ex) when (ex is ArgumentException or KeyNotFoundException) + { + return Results.BadRequest(new { message = ex.Message }); + } + } + + private static async Task FuseAsync( + [FromBody] ViolationEventRequest request, + ViolationEventService eventService, + SeverityFusionService fusionService, + CancellationToken cancellationToken) + { + try + { + await eventService.EmitAsync(request, cancellationToken).ConfigureAwait(false); + var fused = await fusionService.FuseAsync(request.SnapshotId, cancellationToken).ConfigureAwait(false); + return Results.Json(new { fused }); + } + catch (Exception ex) when (ex is ArgumentException or KeyNotFoundException) + { + return Results.BadRequest(new { message = ex.Message }); + } + } + + private static async Task ConflictsAsync( + [FromBody] ConflictRequest request, + ViolationEventService eventService, + SeverityFusionService fusionService, + ConflictHandlingService conflictService, + CancellationToken cancellationToken) + { + try + { + await eventService.EmitAsync(new ViolationEventRequest(request.SnapshotId), cancellationToken).ConfigureAwait(false); + var fused = await fusionService.FuseAsync(request.SnapshotId, cancellationToken).ConfigureAwait(false); + var conflicts = await conflictService.ComputeAsync(request.SnapshotId, fused, cancellationToken).ConfigureAwait(false); + return Results.Json(new { conflicts }); + } + catch (Exception ex) when (ex is ArgumentException or KeyNotFoundException) + { + return Results.BadRequest(new { message = ex.Message }); + } + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Ledger/LedgerExportService.cs b/src/Policy/StellaOps.Policy.Engine/Ledger/LedgerExportService.cs new file mode 100644 index 000000000..38ef88508 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Ledger/LedgerExportService.cs @@ -0,0 +1,103 @@ +using System.Text.Json; +using StellaOps.Policy.Engine.Orchestration; + +namespace StellaOps.Policy.Engine.Ledger; + +/// +/// Builds deterministic NDJSON ledger exports from worker results (POLICY-ENGINE-34-101). +/// +internal sealed class LedgerExportService +{ + private const string SchemaVersion = "policy-ledger-export-v1"; + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + + private readonly TimeProvider _timeProvider; + private readonly IOrchestratorJobStore _jobs; + private readonly IWorkerResultStore _results; + private readonly ILedgerExportStore _store; + + public LedgerExportService( + TimeProvider timeProvider, + IOrchestratorJobStore jobs, + IWorkerResultStore results, + ILedgerExportStore store) + { + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _jobs = jobs ?? throw new ArgumentNullException(nameof(jobs)); + _results = results ?? throw new ArgumentNullException(nameof(results)); + _store = store ?? throw new ArgumentNullException(nameof(store)); + } + + public async Task BuildAsync(LedgerExportRequest request, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + + var jobs = await _jobs.ListAsync(request.TenantId, cancellationToken).ConfigureAwait(false); + var completed = jobs.Where(j => string.Equals(j.Status, "completed", StringComparison.Ordinal)).ToList(); + + var records = new List(); + + foreach (var job in completed) + { + var result = await _results.GetByJobIdAsync(job.JobId, cancellationToken).ConfigureAwait(false); + if (result is null) + { + continue; + } + + foreach (var item in result.Results) + { + records.Add(new LedgerExportRecord( + TenantId: job.TenantId, + JobId: job.JobId, + ContextId: job.ContextId, + ComponentPurl: item.ComponentPurl, + AdvisoryId: item.AdvisoryId, + Status: item.Status, + TraceRef: item.TraceRef, + OccurredAt: result.CompletedAt.ToString("O"))); + } + } + + var ordered = records + .OrderBy(r => r.TenantId, StringComparer.Ordinal) + .ThenBy(r => r.JobId, StringComparer.Ordinal) + .ThenBy(r => r.ComponentPurl, StringComparer.Ordinal) + .ThenBy(r => r.AdvisoryId, StringComparer.Ordinal) + .ToList(); + + var generatedAt = _timeProvider.GetUtcNow().ToString("O"); + var exportId = StableIdGenerator.CreateUlid($"{request.TenantId}|{generatedAt}|{ordered.Count}"); + + var recordLines = ordered.Select(r => JsonSerializer.Serialize(r, SerializerOptions)).ToList(); + var sha = StableIdGenerator.Sha256Hex(string.Join('\n', recordLines)); + + var manifest = new LedgerExportManifest( + ExportId: exportId, + SchemaVersion: SchemaVersion, + GeneratedAt: generatedAt, + RecordCount: ordered.Count, + Sha256: sha); + + var lines = new List(recordLines.Count + 1) + { + JsonSerializer.Serialize(manifest, SerializerOptions) + }; + lines.AddRange(recordLines); + + var export = new LedgerExport(manifest, ordered, lines); + await _store.SaveAsync(export, cancellationToken).ConfigureAwait(false); + + return export; + } + + public Task GetAsync(string exportId, CancellationToken cancellationToken = default) + { + return _store.GetAsync(exportId, cancellationToken); + } + + public Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default) + { + return _store.ListAsync(tenantId, cancellationToken); + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Ledger/LedgerExportStore.cs b/src/Policy/StellaOps.Policy.Engine/Ledger/LedgerExportStore.cs new file mode 100644 index 000000000..c35d2fe52 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Ledger/LedgerExportStore.cs @@ -0,0 +1,44 @@ +using System.Collections.Concurrent; + +namespace StellaOps.Policy.Engine.Ledger; + +internal interface ILedgerExportStore +{ + Task SaveAsync(LedgerExport export, CancellationToken cancellationToken = default); + Task GetAsync(string exportId, CancellationToken cancellationToken = default); + Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default); +} + +internal sealed class InMemoryLedgerExportStore : ILedgerExportStore +{ + private readonly ConcurrentDictionary _exports = new(StringComparer.Ordinal); + + public Task SaveAsync(LedgerExport export, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(export); + _exports[export.Manifest.ExportId] = export; + return Task.CompletedTask; + } + + public Task GetAsync(string exportId, CancellationToken cancellationToken = default) + { + _exports.TryGetValue(exportId, out var value); + return Task.FromResult(value); + } + + public Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default) + { + IEnumerable exports = _exports.Values; + if (!string.IsNullOrWhiteSpace(tenantId)) + { + exports = exports.Where(x => x.Records.Any(r => string.Equals(r.TenantId, tenantId, StringComparison.Ordinal))); + } + + var ordered = exports + .OrderBy(e => e.Manifest.GeneratedAt, StringComparer.Ordinal) + .ThenBy(e => e.Manifest.ExportId, StringComparer.Ordinal) + .ToList(); + + return Task.FromResult>(ordered); + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Ledger/LedgerModels.cs b/src/Policy/StellaOps.Policy.Engine/Ledger/LedgerModels.cs new file mode 100644 index 000000000..bb20090b0 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Ledger/LedgerModels.cs @@ -0,0 +1,28 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Policy.Engine.Ledger; + +internal sealed record LedgerExportRecord( + [property: JsonPropertyName("tenant_id")] string TenantId, + [property: JsonPropertyName("job_id")] string JobId, + [property: JsonPropertyName("context_id")] string ContextId, + [property: JsonPropertyName("component_purl")] string ComponentPurl, + [property: JsonPropertyName("advisory_id")] string AdvisoryId, + [property: JsonPropertyName("status")] string Status, + [property: JsonPropertyName("trace_ref")] string TraceRef, + [property: JsonPropertyName("occurred_at")] string OccurredAt); + +internal sealed record LedgerExportManifest( + [property: JsonPropertyName("export_id")] string ExportId, + [property: JsonPropertyName("schema_version")] string SchemaVersion, + [property: JsonPropertyName("generated_at")] string GeneratedAt, + [property: JsonPropertyName("record_count")] int RecordCount, + [property: JsonPropertyName("sha256")] string Sha256); + +internal sealed record LedgerExport( + LedgerExportManifest Manifest, + IReadOnlyList Records, + IReadOnlyList Lines); + +internal sealed record LedgerExportRequest( + [property: JsonPropertyName("tenant_id")] string TenantId); diff --git a/src/Policy/StellaOps.Policy.Engine/Orchestration/OrchestratorJobModels.cs b/src/Policy/StellaOps.Policy.Engine/Orchestration/OrchestratorJobModels.cs new file mode 100644 index 000000000..d4950fd3c --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Orchestration/OrchestratorJobModels.cs @@ -0,0 +1,36 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Policy.Engine.Orchestration; + +internal sealed record OrchestratorJobItem( + [property: JsonPropertyName("component_purl")] string ComponentPurl, + [property: JsonPropertyName("advisory_id")] string AdvisoryId); + +internal sealed record OrchestratorJobCallbacks( + [property: JsonPropertyName("sse")] string? Sse, + [property: JsonPropertyName("nats")] string? Nats); + +internal sealed record OrchestratorJobRequest( + [property: JsonPropertyName("tenant_id")] string TenantId, + [property: JsonPropertyName("context_id")] string ContextId, + [property: JsonPropertyName("policy_profile_hash")] string PolicyProfileHash, + [property: JsonPropertyName("batch_items")] IReadOnlyList BatchItems, + [property: JsonPropertyName("priority")] string Priority = "normal", + [property: JsonPropertyName("trace_ref")] string? TraceRef = null, + [property: JsonPropertyName("callbacks")] OrchestratorJobCallbacks? Callbacks = null, + [property: JsonPropertyName("requested_at")] DateTimeOffset? RequestedAt = null); + +internal sealed record OrchestratorJob( + [property: JsonPropertyName("job_id")] string JobId, + [property: JsonPropertyName("tenant_id")] string TenantId, + [property: JsonPropertyName("context_id")] string ContextId, + [property: JsonPropertyName("policy_profile_hash")] string PolicyProfileHash, + [property: JsonPropertyName("requested_at")] DateTimeOffset RequestedAt, + [property: JsonPropertyName("priority")] string Priority, + [property: JsonPropertyName("batch_items")] IReadOnlyList BatchItems, + [property: JsonPropertyName("callbacks")] OrchestratorJobCallbacks? Callbacks, + [property: JsonPropertyName("trace_ref")] string TraceRef, + [property: JsonPropertyName("status")] string Status, + [property: JsonPropertyName("determinism_hash")] string DeterminismHash, + [property: JsonPropertyName("completed_at")] DateTimeOffset? CompletedAt = null, + [property: JsonPropertyName("result_hash")] string? ResultHash = null); diff --git a/src/Policy/StellaOps.Policy.Engine/Orchestration/OrchestratorJobService.cs b/src/Policy/StellaOps.Policy.Engine/Orchestration/OrchestratorJobService.cs new file mode 100644 index 000000000..afcaf8372 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Orchestration/OrchestratorJobService.cs @@ -0,0 +1,112 @@ +using System.Text; + +namespace StellaOps.Policy.Engine.Orchestration; + +internal sealed class OrchestratorJobService +{ + private static readonly string[] AllowedPriorities = { "normal", "high", "emergency" }; + + private readonly TimeProvider _timeProvider; + private readonly IOrchestratorJobStore _store; + + public OrchestratorJobService(TimeProvider timeProvider, IOrchestratorJobStore store) + { + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _store = store ?? throw new ArgumentNullException(nameof(store)); + } + + public async Task SubmitAsync( + OrchestratorJobRequest request, + CancellationToken cancellationToken = default) + { + var job = BuildJob(request); + await _store.SaveAsync(job, cancellationToken).ConfigureAwait(false); + return job; + } + + public Task PreviewAsync(OrchestratorJobRequest request, CancellationToken cancellationToken = default) + { + var job = BuildJob(request, preview: true); + return Task.FromResult(job); + } + + public Task GetAsync(string jobId, CancellationToken cancellationToken = default) + { + return _store.GetAsync(jobId, cancellationToken); + } + + private OrchestratorJob BuildJob(OrchestratorJobRequest request, bool preview = false) + { + ArgumentNullException.ThrowIfNull(request); + if (request.BatchItems is null || request.BatchItems.Count == 0) + { + throw new ArgumentException("batch_items are required", nameof(request)); + } + + var normalizedPriority = NormalizePriority(request.Priority); + var requestedAt = request.RequestedAt ?? _timeProvider.GetUtcNow(); + + var orderedItems = request.BatchItems + .OrderBy(i => i.ComponentPurl, StringComparer.Ordinal) + .ThenBy(i => i.AdvisoryId, StringComparer.Ordinal) + .ToList(); + + var seed = $"{request.TenantId}|{request.ContextId}|{requestedAt:O}"; + var jobId = StableIdGenerator.CreateUlid(seed); + var traceRef = request.TraceRef ?? StableIdGenerator.Sha256Hex($"{jobId}|trace"); + + var determinismHash = StableIdGenerator.Sha256Hex(BuildDeterminismSeed(request, orderedItems, requestedAt, normalizedPriority)); + + var status = preview ? "preview" : "queued"; + + return new OrchestratorJob( + JobId: jobId, + TenantId: request.TenantId, + ContextId: request.ContextId, + PolicyProfileHash: request.PolicyProfileHash, + RequestedAt: requestedAt, + Priority: normalizedPriority, + BatchItems: orderedItems, + Callbacks: request.Callbacks, + TraceRef: traceRef, + Status: status, + DeterminismHash: determinismHash); + } + + private static string NormalizePriority(string? value) + { + var normalized = (value ?? "normal").Trim().ToLowerInvariant(); + if (!AllowedPriorities.Contains(normalized)) + { + normalized = "normal"; + } + + return normalized; + } + + private static string BuildDeterminismSeed( + OrchestratorJobRequest request, + IReadOnlyList items, + DateTimeOffset requestedAt, + string priority) + { + var builder = new StringBuilder(); + builder.Append(request.TenantId).Append('|') + .Append(request.ContextId).Append('|') + .Append(request.PolicyProfileHash).Append('|') + .Append(priority).Append('|') + .Append(requestedAt.ToString("O")); + + foreach (var item in items) + { + builder.Append('|').Append(item.ComponentPurl).Append('|').Append(item.AdvisoryId); + } + + if (request.Callbacks is not null) + { + builder.Append('|').Append(request.Callbacks.Sse).Append('|').Append(request.Callbacks.Nats); + } + + return builder.ToString(); + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Orchestration/OrchestratorJobStore.cs b/src/Policy/StellaOps.Policy.Engine/Orchestration/OrchestratorJobStore.cs new file mode 100644 index 000000000..9d5d8e23b --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Orchestration/OrchestratorJobStore.cs @@ -0,0 +1,58 @@ +using System.Collections.Concurrent; + +namespace StellaOps.Policy.Engine.Orchestration; + +internal interface IOrchestratorJobStore +{ + Task SaveAsync(OrchestratorJob job, CancellationToken cancellationToken = default); + Task GetAsync(string jobId, CancellationToken cancellationToken = default); + Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default); + Task UpdateAsync(string jobId, Func update, CancellationToken cancellationToken = default); +} + +internal sealed class InMemoryOrchestratorJobStore : IOrchestratorJobStore +{ + private readonly ConcurrentDictionary _jobs = new(StringComparer.Ordinal); + + public Task SaveAsync(OrchestratorJob job, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(job); + _jobs[job.JobId] = job; + return Task.CompletedTask; + } + + public Task GetAsync(string jobId, CancellationToken cancellationToken = default) + { + _jobs.TryGetValue(jobId, out var job); + return Task.FromResult(job); + } + + public Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default) + { + IEnumerable items = _jobs.Values; + if (!string.IsNullOrWhiteSpace(tenantId)) + { + items = items.Where(j => string.Equals(j.TenantId, tenantId, StringComparison.Ordinal)); + } + + var ordered = items + .OrderBy(j => j.TenantId, StringComparer.Ordinal) + .ThenBy(j => j.RequestedAt) + .ThenBy(j => j.JobId, StringComparer.Ordinal) + .ToList(); + + return Task.FromResult>(ordered); + } + + public Task UpdateAsync(string jobId, Func update, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(update); + + _jobs.AddOrUpdate( + jobId, + _ => throw new KeyNotFoundException($"Job {jobId} not found"), + (_, existing) => update(existing)); + + return Task.CompletedTask; + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Orchestration/PolicyWorkerModels.cs b/src/Policy/StellaOps.Policy.Engine/Orchestration/PolicyWorkerModels.cs new file mode 100644 index 000000000..3475f407c --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Orchestration/PolicyWorkerModels.cs @@ -0,0 +1,21 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Policy.Engine.Orchestration; + +internal sealed record WorkerResultItem( + [property: JsonPropertyName("component_purl")] string ComponentPurl, + [property: JsonPropertyName("advisory_id")] string AdvisoryId, + [property: JsonPropertyName("status")] string Status, + [property: JsonPropertyName("trace_ref")] string TraceRef); + +internal sealed record WorkerRunResult( + [property: JsonPropertyName("job_id")] string JobId, + [property: JsonPropertyName("worker_id")] string WorkerId, + [property: JsonPropertyName("started_at")] DateTimeOffset StartedAt, + [property: JsonPropertyName("completed_at")] DateTimeOffset CompletedAt, + [property: JsonPropertyName("results")] IReadOnlyList Results, + [property: JsonPropertyName("result_hash")] string ResultHash); + +internal sealed record WorkerRunRequest( + [property: JsonPropertyName("job_id")] string JobId, + [property: JsonPropertyName("worker_id")] string? WorkerId = null); diff --git a/src/Policy/StellaOps.Policy.Engine/Orchestration/PolicyWorkerService.cs b/src/Policy/StellaOps.Policy.Engine/Orchestration/PolicyWorkerService.cs new file mode 100644 index 000000000..846da05f8 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Orchestration/PolicyWorkerService.cs @@ -0,0 +1,107 @@ +using System.Security.Cryptography; +using System.Text; + +namespace StellaOps.Policy.Engine.Orchestration; + +/// +/// Deterministic worker stub for POLICY-ENGINE-33-101. Consumes orchestrator jobs and +/// produces stable result hashes so retries can short-circuit. +/// +internal sealed class PolicyWorkerService +{ + private readonly TimeProvider _timeProvider; + private readonly IOrchestratorJobStore _jobs; + private readonly IWorkerResultStore _results; + + public PolicyWorkerService( + TimeProvider timeProvider, + IOrchestratorJobStore jobs, + IWorkerResultStore results) + { + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _jobs = jobs ?? throw new ArgumentNullException(nameof(jobs)); + _results = results ?? throw new ArgumentNullException(nameof(results)); + } + + public async Task ExecuteAsync( + WorkerRunRequest request, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + + var job = await _jobs.GetAsync(request.JobId, cancellationToken).ConfigureAwait(false) + ?? throw new KeyNotFoundException($"Job {request.JobId} not found"); + + var existing = await _results.GetByJobIdAsync(job.JobId, cancellationToken).ConfigureAwait(false); + if (existing is not null) + { + return existing; + } + + var workerId = string.IsNullOrWhiteSpace(request.WorkerId) ? "worker-stub" : request.WorkerId; + var startedAt = _timeProvider.GetUtcNow(); + + var results = BuildResults(job); + + var completedAt = _timeProvider.GetUtcNow(); + var resultHash = StableIdGenerator.Sha256Hex(BuildSeed(job.JobId, results)); + + var runResult = new WorkerRunResult( + JobId: job.JobId, + WorkerId: workerId, + StartedAt: startedAt, + CompletedAt: completedAt, + Results: results, + ResultHash: resultHash); + + await _results.SaveAsync(runResult, cancellationToken).ConfigureAwait(false); + await _jobs.UpdateAsync(job.JobId, j => j with { Status = "completed", CompletedAt = completedAt, ResultHash = resultHash }, cancellationToken) + .ConfigureAwait(false); + + return runResult; + } + + private static IReadOnlyList BuildResults(OrchestratorJob job) + { + var builder = new List(job.BatchItems.Count); + + foreach (var item in job.BatchItems) + { + var hash = BuildItemHash(job.JobId, item); + var status = (hash % 3) switch + { + 0 => "violation", + 1 => "warn", + _ => "ok" + }; + + var traceRef = StableIdGenerator.Sha256Hex($"{job.JobId}|{item.ComponentPurl}|{item.AdvisoryId}"); + builder.Add(new WorkerResultItem(item.ComponentPurl, item.AdvisoryId, status, traceRef)); + } + + return builder + .OrderBy(r => r.ComponentPurl, StringComparer.Ordinal) + .ThenBy(r => r.AdvisoryId, StringComparer.Ordinal) + .ToList(); + } + + private static int BuildItemHash(string jobId, OrchestratorJobItem item) + { + Span hash = stackalloc byte[32]; + SHA256.HashData(Encoding.UTF8.GetBytes($"{jobId}|{item.ComponentPurl}|{item.AdvisoryId}"), hash); + return BitConverter.ToInt32(hash[..4]); + } + + private static string BuildSeed(string jobId, IReadOnlyList results) + { + var sb = new StringBuilder(jobId); + foreach (var result in results) + { + sb.Append('|').Append(result.ComponentPurl) + .Append('|').Append(result.AdvisoryId) + .Append('|').Append(result.Status); + } + + return sb.ToString(); + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Orchestration/StableIdGenerator.cs b/src/Policy/StellaOps.Policy.Engine/Orchestration/StableIdGenerator.cs new file mode 100644 index 000000000..3772a307f --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Orchestration/StableIdGenerator.cs @@ -0,0 +1,61 @@ +using System.Security.Cryptography; +using System.Text; + +namespace StellaOps.Policy.Engine.Orchestration; + +internal static class StableIdGenerator +{ + private const string Base32Alphabet = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"; + + public static string CreateUlid(string seed) + { + if (string.IsNullOrWhiteSpace(seed)) + { + throw new ArgumentException("Seed is required", nameof(seed)); + } + + Span hash = stackalloc byte[32]; + SHA256.HashData(Encoding.UTF8.GetBytes(seed), hash); + + Span buffer = stackalloc char[26]; + EncodeBase32(hash[..16], buffer); + return new string(buffer); + } + + public static string Sha256Hex(string value) + { + Span hash = stackalloc byte[32]; + SHA256.HashData(Encoding.UTF8.GetBytes(value), hash); + return Convert.ToHexString(hash); + } + + private static void EncodeBase32(ReadOnlySpan input, Span output) + { + int buffer = 0; + int bitsLeft = 0; + int index = 0; + + foreach (var b in input) + { + buffer = (buffer << 8) | b; + bitsLeft += 8; + + while (bitsLeft >= 5 && index < output.Length) + { + var value = (buffer >> (bitsLeft - 5)) & 31; + output[index++] = Base32Alphabet[value]; + bitsLeft -= 5; + } + } + + if (index < output.Length) + { + output[index++] = Base32Alphabet[(buffer << (5 - bitsLeft)) & 31]; + } + + while (index < output.Length) + { + output[index++] = Base32Alphabet[0]; + } + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Orchestration/WorkerResultStore.cs b/src/Policy/StellaOps.Policy.Engine/Orchestration/WorkerResultStore.cs new file mode 100644 index 000000000..3d26fc8aa --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Orchestration/WorkerResultStore.cs @@ -0,0 +1,52 @@ +using System.Collections.Concurrent; + +namespace StellaOps.Policy.Engine.Orchestration; + +internal interface IWorkerResultStore +{ + Task SaveAsync(WorkerRunResult result, CancellationToken cancellationToken = default); + Task GetByJobIdAsync(string jobId, CancellationToken cancellationToken = default); + Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default); +} + +internal sealed class InMemoryWorkerResultStore : IWorkerResultStore +{ + private readonly ConcurrentDictionary _results = new(StringComparer.Ordinal); + private readonly IOrchestratorJobStore _jobs; + + public InMemoryWorkerResultStore(IOrchestratorJobStore jobs) + { + _jobs = jobs ?? throw new ArgumentNullException(nameof(jobs)); + } + + public Task SaveAsync(WorkerRunResult result, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(result); + _results[result.JobId] = result; + return Task.CompletedTask; + } + + public Task GetByJobIdAsync(string jobId, CancellationToken cancellationToken = default) + { + _results.TryGetValue(jobId, out var value); + return Task.FromResult(value); + } + + public async Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(tenantId)) + { + return _results.Values.OrderBy(r => r.JobId, StringComparer.Ordinal).ToList(); + } + + var jobs = await _jobs.ListAsync(tenantId, cancellationToken).ConfigureAwait(false); + var jobIds = jobs.Select(j => j.JobId).ToHashSet(StringComparer.Ordinal); + + var filtered = _results.Values + .Where(r => jobIds.Contains(r.JobId)) + .OrderBy(r => r.JobId, StringComparer.Ordinal) + .ToList(); + + return filtered; + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Overlay/PathScopeSimulationBridgeService.cs b/src/Policy/StellaOps.Policy.Engine/Overlay/PathScopeSimulationBridgeService.cs index 469e526b2..77873d876 100644 --- a/src/Policy/StellaOps.Policy.Engine/Overlay/PathScopeSimulationBridgeService.cs +++ b/src/Policy/StellaOps.Policy.Engine/Overlay/PathScopeSimulationBridgeService.cs @@ -165,7 +165,7 @@ internal sealed class PathScopeSimulationBridgeService private static string BuildCorrelationId(PathScopeSimulationBridgeRequest request) { var stable = $"{request.Tenant}|{request.Mode}|{request.Seed ?? DefaultSeed}"; - Span hash = stackalloc byte[16]; + Span hash = stackalloc byte[32]; SHA256.HashData(Encoding.UTF8.GetBytes(stable), hash); return Convert.ToHexString(hash); } diff --git a/src/Policy/StellaOps.Policy.Engine/Program.cs b/src/Policy/StellaOps.Policy.Engine/Program.cs index a195bd000..96e4d4ac3 100644 --- a/src/Policy/StellaOps.Policy.Engine/Program.cs +++ b/src/Policy/StellaOps.Policy.Engine/Program.cs @@ -5,15 +5,16 @@ using StellaOps.Auth.Abstractions; using StellaOps.Auth.Client; using StellaOps.Auth.ServerIntegration; using StellaOps.Configuration; -using StellaOps.Policy.Engine.Hosting; -using StellaOps.Policy.Engine.Options; +using StellaOps.Policy.Engine.Hosting; +using StellaOps.Policy.Engine.Options; using StellaOps.Policy.Engine.Compilation; using StellaOps.Policy.Engine.Endpoints; using StellaOps.Policy.Engine.Services; using StellaOps.Policy.Engine.Workers; using StellaOps.Policy.Engine.Streaming; using StellaOps.AirGap.Policy; - +using StellaOps.Policy.Engine.Orchestration; + var builder = WebApplication.CreateBuilder(args); var policyEngineConfigFiles = new[] @@ -115,12 +116,27 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(); - -builder.Services.AddHttpContextAccessor(); -builder.Services.AddRouting(options => options.LowercaseUrls = true); -builder.Services.AddProblemDetails(); -builder.Services.AddHealthChecks(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); + +builder.Services.AddHttpContextAccessor(); +builder.Services.AddRouting(options => options.LowercaseUrls = true); +builder.Services.AddProblemDetails(); +builder.Services.AddHealthChecks(); builder.Services.AddAuthentication(); builder.Services.AddAuthorization(); @@ -164,5 +180,13 @@ app.MapPolicyCompilation(); app.MapPolicyPacks(); app.MapPathScopeSimulation(); app.MapOverlaySimulation(); +app.MapTrustWeighting(); +app.MapAdvisoryAiKnobs(); +app.MapBatchContext(); +app.MapOrchestratorJobs(); +app.MapPolicyWorker(); +app.MapLedgerExport(); +app.MapSnapshots(); +app.MapViolations(); app.Run(); diff --git a/src/Policy/StellaOps.Policy.Engine/Services/PolicyEvaluationService.PathScope.cs b/src/Policy/StellaOps.Policy.Engine/Services/PolicyEvaluationService.PathScope.cs index d9c35f4f2..ff2694d3b 100644 --- a/src/Policy/StellaOps.Policy.Engine/Services/PolicyEvaluationService.PathScope.cs +++ b/src/Policy/StellaOps.Policy.Engine/Services/PolicyEvaluationService.PathScope.cs @@ -139,7 +139,7 @@ internal sealed partial class PolicyEvaluationService private static string ComputeCorrelationId(string stableKey) { - Span hashBytes = stackalloc byte[16]; + Span hashBytes = stackalloc byte[32]; SHA256.HashData(Encoding.UTF8.GetBytes(stableKey), hashBytes); return Convert.ToHexString(hashBytes); } diff --git a/src/Policy/StellaOps.Policy.Engine/Snapshots/SnapshotModels.cs b/src/Policy/StellaOps.Policy.Engine/Snapshots/SnapshotModels.cs new file mode 100644 index 000000000..312cd15ee --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Snapshots/SnapshotModels.cs @@ -0,0 +1,24 @@ +using System.Text.Json.Serialization; +using StellaOps.Policy.Engine.Ledger; + +namespace StellaOps.Policy.Engine.Snapshots; + +internal sealed record SnapshotSummary( + [property: JsonPropertyName("snapshot_id")] string SnapshotId, + [property: JsonPropertyName("tenant_id")] string TenantId, + [property: JsonPropertyName("ledger_export_id")] string LedgerExportId, + [property: JsonPropertyName("generated_at")] string GeneratedAt, + [property: JsonPropertyName("status_counts")] IReadOnlyDictionary StatusCounts); + +internal sealed record SnapshotDetail( + [property: JsonPropertyName("snapshot_id")] string SnapshotId, + [property: JsonPropertyName("tenant_id")] string TenantId, + [property: JsonPropertyName("ledger_export_id")] string LedgerExportId, + [property: JsonPropertyName("generated_at")] string GeneratedAt, + [property: JsonPropertyName("overlay_hash")] string OverlayHash, + [property: JsonPropertyName("status_counts")] IReadOnlyDictionary StatusCounts, + [property: JsonPropertyName("records")] IReadOnlyList Records); + +internal sealed record SnapshotRequest( + [property: JsonPropertyName("tenant_id")] string TenantId, + [property: JsonPropertyName("overlay_hash")] string OverlayHash); diff --git a/src/Policy/StellaOps.Policy.Engine/Snapshots/SnapshotService.cs b/src/Policy/StellaOps.Policy.Engine/Snapshots/SnapshotService.cs new file mode 100644 index 000000000..e09e602c7 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Snapshots/SnapshotService.cs @@ -0,0 +1,76 @@ +using StellaOps.Policy.Engine.Ledger; +using StellaOps.Policy.Engine.Orchestration; + +namespace StellaOps.Policy.Engine.Snapshots; + +/// +/// Snapshot API stub (POLICY-ENGINE-35-201) built on ledger exports. +/// +internal sealed class SnapshotService +{ + private readonly TimeProvider _timeProvider; + private readonly LedgerExportService _ledger; + private readonly ISnapshotStore _store; + + public SnapshotService( + TimeProvider timeProvider, + LedgerExportService ledger, + ISnapshotStore store) + { + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _ledger = ledger ?? throw new ArgumentNullException(nameof(ledger)); + _store = store ?? throw new ArgumentNullException(nameof(store)); + } + + public async Task CreateAsync( + SnapshotRequest request, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + + var exports = await _ledger.ListAsync(request.TenantId, cancellationToken).ConfigureAwait(false); + var export = exports.LastOrDefault(); + if (export is null) + { + export = await _ledger.BuildAsync(new LedgerExportRequest(request.TenantId), cancellationToken) + .ConfigureAwait(false); + } + + var statusCounts = export.Records + .GroupBy(r => r.Status) + .ToDictionary(g => g.Key, g => g.Count(), StringComparer.Ordinal); + + var generatedAt = _timeProvider.GetUtcNow().ToString("O"); + var snapshotId = StableIdGenerator.CreateUlid($"{export.Manifest.ExportId}|{request.OverlayHash}"); + + var snapshot = new SnapshotDetail( + SnapshotId: snapshotId, + TenantId: request.TenantId, + LedgerExportId: export.Manifest.ExportId, + GeneratedAt: generatedAt, + OverlayHash: request.OverlayHash, + StatusCounts: statusCounts, + Records: export.Records); + + await _store.SaveAsync(snapshot, cancellationToken).ConfigureAwait(false); + return snapshot; + } + + public Task GetAsync(string snapshotId, CancellationToken cancellationToken = default) + { + return _store.GetAsync(snapshotId, cancellationToken); + } + + public async Task<(IReadOnlyList Items, string? NextCursor)> ListAsync( + string? tenantId = null, + CancellationToken cancellationToken = default) + { + var snapshots = await _store.ListAsync(tenantId, cancellationToken).ConfigureAwait(false); + var summaries = snapshots + .OrderByDescending(s => s.GeneratedAt, StringComparer.Ordinal) + .Select(s => new SnapshotSummary(s.SnapshotId, s.TenantId, s.LedgerExportId, s.GeneratedAt, s.StatusCounts)) + .ToList(); + + return (summaries, null); + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Snapshots/SnapshotStore.cs b/src/Policy/StellaOps.Policy.Engine/Snapshots/SnapshotStore.cs new file mode 100644 index 000000000..bc7011485 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Snapshots/SnapshotStore.cs @@ -0,0 +1,44 @@ +using System.Collections.Concurrent; + +namespace StellaOps.Policy.Engine.Snapshots; + +internal interface ISnapshotStore +{ + Task SaveAsync(SnapshotDetail snapshot, CancellationToken cancellationToken = default); + Task GetAsync(string snapshotId, CancellationToken cancellationToken = default); + Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default); +} + +internal sealed class InMemorySnapshotStore : ISnapshotStore +{ + private readonly ConcurrentDictionary _snapshots = new(StringComparer.Ordinal); + + public Task SaveAsync(SnapshotDetail snapshot, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(snapshot); + _snapshots[snapshot.SnapshotId] = snapshot; + return Task.CompletedTask; + } + + public Task GetAsync(string snapshotId, CancellationToken cancellationToken = default) + { + _snapshots.TryGetValue(snapshotId, out var value); + return Task.FromResult(value); + } + + public Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default) + { + IEnumerable items = _snapshots.Values; + if (!string.IsNullOrWhiteSpace(tenantId)) + { + items = items.Where(s => string.Equals(s.TenantId, tenantId, StringComparison.Ordinal)); + } + + var ordered = items + .OrderBy(s => s.GeneratedAt, StringComparer.Ordinal) + .ThenBy(s => s.SnapshotId, StringComparer.Ordinal) + .ToList(); + + return Task.FromResult>(ordered); + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Streaming/PathScopeSimulationService.cs b/src/Policy/StellaOps.Policy.Engine/Streaming/PathScopeSimulationService.cs index 287f9e306..77b96d3c6 100644 --- a/src/Policy/StellaOps.Policy.Engine/Streaming/PathScopeSimulationService.cs +++ b/src/Policy/StellaOps.Policy.Engine/Streaming/PathScopeSimulationService.cs @@ -59,6 +59,11 @@ internal sealed class PathScopeSimulationService throw new PathScopeSimulationException(PathScopeSimulationError.Schema("subject.purl or subject.cpe is required")); } + if (!string.Equals(request.Options.Sort, "path,finding,verdict", StringComparison.Ordinal)) + { + throw new PathScopeSimulationException(PathScopeSimulationError.Schema("options.sort must be 'path,finding,verdict'")); + } + foreach (var target in request.Targets) { if (string.IsNullOrWhiteSpace(target.FilePath)) diff --git a/src/Policy/StellaOps.Policy.Engine/TrustWeighting/TrustWeightingModels.cs b/src/Policy/StellaOps.Policy.Engine/TrustWeighting/TrustWeightingModels.cs new file mode 100644 index 000000000..0d5bf3726 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/TrustWeighting/TrustWeightingModels.cs @@ -0,0 +1,13 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Policy.Engine.TrustWeighting; + +internal sealed record TrustWeightingEntry( + [property: JsonPropertyName("source")] string Source, + [property: JsonPropertyName("weight")] decimal Weight, + [property: JsonPropertyName("justification")] string? Justification, + [property: JsonPropertyName("updated_at")] string UpdatedAt); + +internal sealed record TrustWeightingProfile( + [property: JsonPropertyName("weights")] IReadOnlyList Weights, + [property: JsonPropertyName("profile_hash")] string ProfileHash); diff --git a/src/Policy/StellaOps.Policy.Engine/TrustWeighting/TrustWeightingService.cs b/src/Policy/StellaOps.Policy.Engine/TrustWeighting/TrustWeightingService.cs new file mode 100644 index 000000000..55ed8331b --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/TrustWeighting/TrustWeightingService.cs @@ -0,0 +1,77 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; + +namespace StellaOps.Policy.Engine.TrustWeighting; + +/// +/// In-memory trust weighting profile store (stub for POLICY-ENGINE-30-101). +/// Deterministic ordering and hashing. +/// +internal sealed class TrustWeightingService +{ + private readonly TimeProvider _timeProvider; + private readonly object _lock = new(); + private TrustWeightingProfile _current; + + public TrustWeightingService(TimeProvider timeProvider) + { + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _current = BuildProfile(DefaultWeights()); + } + + public TrustWeightingProfile Get() => _current; + + public TrustWeightingProfile Set(IReadOnlyList entries) + { + var normalized = Normalize(entries); + var profile = BuildProfile(normalized); + lock (_lock) + { + _current = profile; + } + + return profile; + } + + private TrustWeightingProfile BuildProfile(IReadOnlyList weights) + { + var json = JsonSerializer.Serialize(weights, new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = false + }); + + var hash = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(json))); + return new TrustWeightingProfile(weights, hash); + } + + private IReadOnlyList Normalize(IReadOnlyList entries) + { + var now = _timeProvider.GetUtcNow().ToString("O"); + + var normalized = entries + .Where(e => !string.IsNullOrWhiteSpace(e.Source)) + .Select(e => new TrustWeightingEntry( + Source: e.Source.Trim().ToLowerInvariant(), + Weight: Math.Round(e.Weight, 3, MidpointRounding.ToZero), + Justification: string.IsNullOrWhiteSpace(e.Justification) ? null : e.Justification.Trim(), + UpdatedAt: string.IsNullOrWhiteSpace(e.UpdatedAt) ? now : e.UpdatedAt)) + .OrderBy(e => e.Source, StringComparer.Ordinal) + .ToList(); + + return normalized; + } + + private static IReadOnlyList DefaultWeights() + { + var now = TimeProvider.System.GetUtcNow().ToString("O"); + return new[] + { + new TrustWeightingEntry("cartographer", 1.000m, null, now), + new TrustWeightingEntry("concelier", 1.000m, null, now), + new TrustWeightingEntry("scanner", 1.000m, null, now), + new TrustWeightingEntry("advisory_ai", 1.000m, null, now) + }; + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Violations/ConflictHandlingService.cs b/src/Policy/StellaOps.Policy.Engine/Violations/ConflictHandlingService.cs new file mode 100644 index 000000000..1a196de8c --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Violations/ConflictHandlingService.cs @@ -0,0 +1,42 @@ +namespace StellaOps.Policy.Engine.Violations; + +/// +/// Conflict detection over fused severities (POLICY-ENGINE-40-002). +/// +internal sealed class ConflictHandlingService +{ + private readonly IViolationEventStore _store; + + public ConflictHandlingService(IViolationEventStore store) + { + _store = store ?? throw new ArgumentNullException(nameof(store)); + } + + public async Task> ComputeAsync(string snapshotId, IReadOnlyList? fused = null, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(snapshotId)) + { + throw new ArgumentException("snapshot_id is required", nameof(snapshotId)); + } + + var source = fused ?? await _store.GetFusionAsync(snapshotId, cancellationToken).ConfigureAwait(false); + var conflicts = new List(); + + var grouped = source + .GroupBy(r => (r.ComponentPurl, r.AdvisoryId, r.TenantId)) + .Where(g => g.Select(x => x.SeverityFused).Distinct(StringComparer.OrdinalIgnoreCase).Count() > 1); + + foreach (var group in grouped) + { + conflicts.Add(new ConflictRecord( + TenantId: group.Key.TenantId, + ComponentPurl: group.Key.ComponentPurl, + AdvisoryId: group.Key.AdvisoryId, + Conflicts: group.ToList(), + ResolvedStatus: null)); + } + + await _store.SaveConflictsAsync(snapshotId, conflicts, cancellationToken).ConfigureAwait(false); + return conflicts; + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Violations/SeverityFusionService.cs b/src/Policy/StellaOps.Policy.Engine/Violations/SeverityFusionService.cs new file mode 100644 index 000000000..970b16f4d --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Violations/SeverityFusionService.cs @@ -0,0 +1,87 @@ +using StellaOps.Policy.Engine.TrustWeighting; + +namespace StellaOps.Policy.Engine.Violations; + +/// +/// Deterministic severity fusion (POLICY-ENGINE-40-001). +/// +internal sealed class SeverityFusionService +{ + private readonly IViolationEventStore _store; + private readonly TrustWeightingService _weights; + + public SeverityFusionService(IViolationEventStore store, TrustWeightingService weights) + { + _store = store ?? throw new ArgumentNullException(nameof(store)); + _weights = weights ?? throw new ArgumentNullException(nameof(weights)); + } + + public async Task> FuseAsync(string snapshotId, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(snapshotId)) + { + throw new ArgumentException("snapshot_id is required", nameof(snapshotId)); + } + + var existing = await _store.GetFusionAsync(snapshotId, cancellationToken).ConfigureAwait(false); + if (existing.Count > 0) + { + return existing; + } + + var events = await _store.GetEventsAsync(snapshotId, cancellationToken).ConfigureAwait(false); + if (events.Count == 0) + { + return Array.Empty(); + } + + var weights = _weights.Get(); + var defaultWeight = weights.Weights.FirstOrDefault(w => string.Equals(w.Source, "policy-engine", StringComparison.OrdinalIgnoreCase))?.Weight ?? 1.0m; + + var results = new List(events.Count); + foreach (var ev in events.OrderBy(e => e.ComponentPurl, StringComparer.Ordinal) + .ThenBy(e => e.AdvisoryId, StringComparer.Ordinal)) + { + var baseScore = SeverityToScore(ev.Severity); + var weightedScore = Math.Round(baseScore * defaultWeight, 3, MidpointRounding.ToZero); + var fusedSeverity = ScoreToLabel(weightedScore); + + var sources = new List + { + new("policy-engine", defaultWeight, ev.Severity, weightedScore) + }; + + var reasons = new List { "weights-applied", "deterministic-fusion" }; + + results.Add(new SeverityFusionResult( + TenantId: ev.TenantId, + SnapshotId: ev.SnapshotId, + ComponentPurl: ev.ComponentPurl, + AdvisoryId: ev.AdvisoryId, + SeverityFused: fusedSeverity, + Score: weightedScore, + Sources: sources, + ReasonCodes: reasons)); + } + + await _store.SaveFusionAsync(snapshotId, results, cancellationToken).ConfigureAwait(false); + return results; + } + + private static decimal SeverityToScore(string severity) => severity.ToLowerInvariant() switch + { + "critical" => 1.0m, + "high" => 0.9m, + "medium" => 0.6m, + "warn" => 0.5m, + _ => 0.3m + }; + + private static string ScoreToLabel(decimal score) => score switch + { + >= 0.9m => "critical", + >= 0.75m => "high", + >= 0.5m => "medium", + _ => "low" + }; +} diff --git a/src/Policy/StellaOps.Policy.Engine/Violations/ViolationEventService.cs b/src/Policy/StellaOps.Policy.Engine/Violations/ViolationEventService.cs new file mode 100644 index 000000000..9db148dd5 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Violations/ViolationEventService.cs @@ -0,0 +1,79 @@ +using StellaOps.Policy.Engine.Ledger; +using StellaOps.Policy.Engine.Orchestration; +using StellaOps.Policy.Engine.Snapshots; + +namespace StellaOps.Policy.Engine.Violations; + +/// +/// Emits violation events from snapshots (POLICY-ENGINE-38-201). +/// +internal sealed class ViolationEventService +{ + private readonly ISnapshotStore _snapshots; + private readonly IOrchestratorJobStore _jobs; + private readonly IViolationEventStore _store; + + public ViolationEventService( + ISnapshotStore snapshots, + IOrchestratorJobStore jobs, + IViolationEventStore store) + { + _snapshots = snapshots ?? throw new ArgumentNullException(nameof(snapshots)); + _jobs = jobs ?? throw new ArgumentNullException(nameof(jobs)); + _store = store ?? throw new ArgumentNullException(nameof(store)); + } + + public async Task> EmitAsync( + ViolationEventRequest request, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + + var existing = await _store.GetEventsAsync(request.SnapshotId, cancellationToken).ConfigureAwait(false); + if (existing.Count > 0) + { + return existing; + } + + var snapshot = await _snapshots.GetAsync(request.SnapshotId, cancellationToken).ConfigureAwait(false) + ?? throw new KeyNotFoundException($"Snapshot {request.SnapshotId} not found"); + + var events = new List(); + foreach (var record in snapshot.Records) + { + if (string.Equals(record.Status, "ok", StringComparison.Ordinal)) + { + continue; + } + + var job = await _jobs.GetAsync(record.JobId, cancellationToken).ConfigureAwait(false); + var policyProfileHash = job?.PolicyProfileHash ?? "unknown"; + + var severity = DeriveSeverity(record.Status); + var eventId = StableIdGenerator.Sha256Hex($"{snapshot.SnapshotId}|{record.ComponentPurl}|{record.AdvisoryId}"); + + events.Add(new ViolationEvent( + EventId: eventId, + TenantId: record.TenantId, + SnapshotId: snapshot.SnapshotId, + PolicyProfileHash: policyProfileHash, + ComponentPurl: record.ComponentPurl, + AdvisoryId: record.AdvisoryId, + ViolationCode: "policy.violation.detected", + Severity: severity, + Status: record.Status, + TraceRef: record.TraceRef, + OccurredAt: snapshot.GeneratedAt)); + } + + await _store.SaveEventsAsync(snapshot.SnapshotId, events, cancellationToken).ConfigureAwait(false); + return events; + } + + private static string DeriveSeverity(string status) => status switch + { + "violation" => "high", + "warn" => "medium", + _ => "low" + }; +} diff --git a/src/Policy/StellaOps.Policy.Engine/Violations/ViolationEventStore.cs b/src/Policy/StellaOps.Policy.Engine/Violations/ViolationEventStore.cs new file mode 100644 index 000000000..ecebcef77 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Violations/ViolationEventStore.cs @@ -0,0 +1,56 @@ +using System.Collections.Concurrent; + +namespace StellaOps.Policy.Engine.Violations; + +internal interface IViolationEventStore +{ + Task SaveEventsAsync(string snapshotId, IReadOnlyList events, CancellationToken cancellationToken = default); + Task> GetEventsAsync(string snapshotId, CancellationToken cancellationToken = default); + Task SaveFusionAsync(string snapshotId, IReadOnlyList results, CancellationToken cancellationToken = default); + Task> GetFusionAsync(string snapshotId, CancellationToken cancellationToken = default); + Task SaveConflictsAsync(string snapshotId, IReadOnlyList conflicts, CancellationToken cancellationToken = default); + Task> GetConflictsAsync(string snapshotId, CancellationToken cancellationToken = default); +} + +internal sealed class InMemoryViolationEventStore : IViolationEventStore +{ + private readonly ConcurrentDictionary> _events = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary> _fusion = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary> _conflicts = new(StringComparer.Ordinal); + + public Task SaveEventsAsync(string snapshotId, IReadOnlyList events, CancellationToken cancellationToken = default) + { + _events[snapshotId] = events; + return Task.CompletedTask; + } + + public Task> GetEventsAsync(string snapshotId, CancellationToken cancellationToken = default) + { + _events.TryGetValue(snapshotId, out var events); + return Task.FromResult(events ?? (IReadOnlyList)Array.Empty()); + } + + public Task SaveFusionAsync(string snapshotId, IReadOnlyList results, CancellationToken cancellationToken = default) + { + _fusion[snapshotId] = results; + return Task.CompletedTask; + } + + public Task> GetFusionAsync(string snapshotId, CancellationToken cancellationToken = default) + { + _fusion.TryGetValue(snapshotId, out var value); + return Task.FromResult(value ?? (IReadOnlyList)Array.Empty()); + } + + public Task SaveConflictsAsync(string snapshotId, IReadOnlyList conflicts, CancellationToken cancellationToken = default) + { + _conflicts[snapshotId] = conflicts; + return Task.CompletedTask; + } + + public Task> GetConflictsAsync(string snapshotId, CancellationToken cancellationToken = default) + { + _conflicts.TryGetValue(snapshotId, out var value); + return Task.FromResult(value ?? (IReadOnlyList)Array.Empty()); + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Violations/ViolationModels.cs b/src/Policy/StellaOps.Policy.Engine/Violations/ViolationModels.cs new file mode 100644 index 000000000..72664fb96 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Violations/ViolationModels.cs @@ -0,0 +1,45 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Policy.Engine.Violations; + +internal sealed record ViolationEventRequest( + [property: JsonPropertyName("snapshot_id")] string SnapshotId); + +internal sealed record ViolationEvent( + [property: JsonPropertyName("event_id")] string EventId, + [property: JsonPropertyName("tenant_id")] string TenantId, + [property: JsonPropertyName("snapshot_id")] string SnapshotId, + [property: JsonPropertyName("policy_profile_hash")] string PolicyProfileHash, + [property: JsonPropertyName("component_purl")] string ComponentPurl, + [property: JsonPropertyName("advisory_id")] string AdvisoryId, + [property: JsonPropertyName("violation_code")] string ViolationCode, + [property: JsonPropertyName("severity")] string Severity, + [property: JsonPropertyName("status")] string Status, + [property: JsonPropertyName("trace_ref")] string TraceRef, + [property: JsonPropertyName("occurred_at")] string OccurredAt); + +internal sealed record SeveritySource( + [property: JsonPropertyName("source")] string Source, + [property: JsonPropertyName("weight")] decimal Weight, + [property: JsonPropertyName("severity")] string Severity, + [property: JsonPropertyName("score")] decimal Score); + +internal sealed record SeverityFusionResult( + [property: JsonPropertyName("tenant_id")] string TenantId, + [property: JsonPropertyName("snapshot_id")] string SnapshotId, + [property: JsonPropertyName("component_purl")] string ComponentPurl, + [property: JsonPropertyName("advisory_id")] string AdvisoryId, + [property: JsonPropertyName("severity_fused")] string SeverityFused, + [property: JsonPropertyName("score")] decimal Score, + [property: JsonPropertyName("sources")] IReadOnlyList Sources, + [property: JsonPropertyName("reason_codes")] IReadOnlyList ReasonCodes); + +internal sealed record ConflictRecord( + [property: JsonPropertyName("tenant_id")] string TenantId, + [property: JsonPropertyName("component_purl")] string ComponentPurl, + [property: JsonPropertyName("advisory_id")] string AdvisoryId, + [property: JsonPropertyName("conflicts")] IReadOnlyList Conflicts, + [property: JsonPropertyName("resolved_status")] string? ResolvedStatus); + +internal sealed record ConflictRequest( + [property: JsonPropertyName("snapshot_id")] string SnapshotId); diff --git a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/AdvisoryAiKnobsServiceTests.cs b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/AdvisoryAiKnobsServiceTests.cs new file mode 100644 index 000000000..85dd0a0ef --- /dev/null +++ b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/AdvisoryAiKnobsServiceTests.cs @@ -0,0 +1,31 @@ +using StellaOps.Policy.Engine.AdvisoryAI; + +namespace StellaOps.Policy.Engine.Tests; + +public sealed class AdvisoryAiKnobsServiceTests +{ + [Fact] + public void Get_ReturnsDefaultsWithHash() + { + var service = new AdvisoryAiKnobsService(TimeProvider.System); + var profile = service.Get(); + + Assert.NotEmpty(profile.Knobs); + Assert.False(string.IsNullOrWhiteSpace(profile.ProfileHash)); + } + + [Fact] + public void Set_NormalizesOrdering() + { + var service = new AdvisoryAiKnobsService(TimeProvider.System); + + var profile = service.Set(new[] + { + new AdvisoryAiKnob("Time_Decay_Half_Life_Days", 20m, 1m, 365m, 1m, "decay"), + new AdvisoryAiKnob("ai_signal_weight", 1.5m, 0m, 2m, 0.1m, "weight") + }); + + Assert.Equal("ai_signal_weight", profile.Knobs[0].Name); + Assert.Equal("time_decay_half_life_days", profile.Knobs[1].Name); + } +} diff --git a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/LedgerExportServiceTests.cs b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/LedgerExportServiceTests.cs new file mode 100644 index 000000000..e286e6acb --- /dev/null +++ b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/LedgerExportServiceTests.cs @@ -0,0 +1,61 @@ +using Microsoft.Extensions.Time.Testing; +using StellaOps.Policy.Engine.Ledger; +using StellaOps.Policy.Engine.Orchestration; + +namespace StellaOps.Policy.Engine.Tests; + +public sealed class LedgerExportServiceTests +{ + [Fact] + public async Task BuildAsync_ProducesOrderedNdjson() + { + var clock = new FakeTimeProvider(DateTimeOffset.Parse("2025-11-24T15:00:00Z")); + var jobStore = new InMemoryOrchestratorJobStore(); + var resultStore = new InMemoryWorkerResultStore(jobStore); + var exportStore = new InMemoryLedgerExportStore(); + var service = new LedgerExportService(clock, jobStore, resultStore, exportStore); + + var job = new OrchestratorJob( + JobId: "job-1", + TenantId: "acme", + ContextId: "ctx", + PolicyProfileHash: "hash", + RequestedAt: clock.GetUtcNow(), + Priority: "normal", + BatchItems: new[] + { + new OrchestratorJobItem("pkg:b", "ADV-2"), + new OrchestratorJobItem("pkg:a", "ADV-1") + }, + Callbacks: null, + TraceRef: "trace", + Status: "completed", + DeterminismHash: "hash", + CompletedAt: clock.GetUtcNow(), + ResultHash: "res"); + + await jobStore.SaveAsync(job); + + var result = new WorkerRunResult( + job.JobId, + "worker", + clock.GetUtcNow(), + clock.GetUtcNow(), + new[] + { + new WorkerResultItem("pkg:b", "ADV-2", "ok", "trace-b"), + new WorkerResultItem("pkg:a", "ADV-1", "violation", "trace-a") + }, + "hash"); + + await resultStore.SaveAsync(result); + + var export = await service.BuildAsync(new LedgerExportRequest("acme")); + + Assert.Equal(2, export.Manifest.RecordCount); + Assert.Equal("policy-ledger-export-v1", export.Manifest.SchemaVersion); + Assert.Equal(3, export.Lines.Count); // manifest + 2 records + Assert.Contains(export.Records, r => r.ComponentPurl == "pkg:a"); + Assert.Equal("pkg:a", export.Records[0].ComponentPurl); + } +} diff --git a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/OrchestratorJobServiceTests.cs b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/OrchestratorJobServiceTests.cs new file mode 100644 index 000000000..899b9ea6b --- /dev/null +++ b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/OrchestratorJobServiceTests.cs @@ -0,0 +1,96 @@ +using Microsoft.Extensions.Time.Testing; +using StellaOps.Policy.Engine.Orchestration; + +namespace StellaOps.Policy.Engine.Tests; + +public sealed class OrchestratorJobServiceTests +{ + [Fact] + public async Task SubmitAsync_NormalizesOrderingAndHashes() + { + var clock = new FakeTimeProvider(DateTimeOffset.Parse("2025-11-24T10:00:00Z")); + var store = new InMemoryOrchestratorJobStore(); + var service = new OrchestratorJobService(clock, store); + + var job = await service.SubmitAsync( + new OrchestratorJobRequest( + TenantId: "acme", + ContextId: "ctx-123", + PolicyProfileHash: "overlay-hash", + BatchItems: new[] + { + new OrchestratorJobItem("pkg:npm/zeta@1.0.0", "ADV-2"), + new OrchestratorJobItem("pkg:npm/alpha@1.0.0", "ADV-1") + }, + Priority: "HIGH", + TraceRef: null, + Callbacks: new OrchestratorJobCallbacks("sse://events", "nats.subject"), + RequestedAt: null)); + + Assert.Equal("acme", job.TenantId); + Assert.Equal("ctx-123", job.ContextId); + Assert.Equal("high", job.Priority); + Assert.Equal(clock.GetUtcNow(), job.RequestedAt); + Assert.Equal("queued", job.Status); + Assert.Equal(2, job.BatchItems.Count); + Assert.Equal("pkg:npm/alpha@1.0.0", job.BatchItems[0].ComponentPurl); + Assert.False(string.IsNullOrWhiteSpace(job.JobId)); + Assert.False(string.IsNullOrWhiteSpace(job.DeterminismHash)); + } + + [Fact] + public async Task SubmitAsync_IsDeterministicAcrossOrdering() + { + var requestedAt = DateTimeOffset.Parse("2025-11-24T11:00:00Z"); + var clock = new FakeTimeProvider(requestedAt); + var store = new InMemoryOrchestratorJobStore(); + var service = new OrchestratorJobService(clock, store); + + var first = await service.SubmitAsync( + new OrchestratorJobRequest( + "tenant", + "ctx", + "hash", + new[] + { + new OrchestratorJobItem("pkg:a", "ADV-1"), + new OrchestratorJobItem("pkg:b", "ADV-2") + }, + RequestedAt: requestedAt)); + + var second = await service.SubmitAsync( + new OrchestratorJobRequest( + "tenant", + "ctx", + "hash", + new[] + { + new OrchestratorJobItem("pkg:b", "ADV-2"), + new OrchestratorJobItem("pkg:a", "ADV-1") + }, + RequestedAt: requestedAt)); + + Assert.Equal(first.JobId, second.JobId); + Assert.Equal(first.DeterminismHash, second.DeterminismHash); + } + + [Fact] + public async Task Preview_DoesNotPersist() + { + var clock = new FakeTimeProvider(DateTimeOffset.Parse("2025-11-24T12:00:00Z")); + var store = new InMemoryOrchestratorJobStore(); + var service = new OrchestratorJobService(clock, store); + + var preview = await service.PreviewAsync( + new OrchestratorJobRequest( + "tenant", + "ctx", + "hash", + new[] { new OrchestratorJobItem("pkg:a", "ADV-1") })); + + Assert.Equal("preview", preview.Status); + + var fetched = await store.GetAsync(preview.JobId); + Assert.Null(fetched); + } +} diff --git a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/PathScopeSimulationBridgeServiceTests.cs b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/PathScopeSimulationBridgeServiceTests.cs index b9c09f357..bc50da502 100644 --- a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/PathScopeSimulationBridgeServiceTests.cs +++ b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/PathScopeSimulationBridgeServiceTests.cs @@ -1,3 +1,4 @@ +using System.Text.Json; using System.Threading.Tasks; using StellaOps.Policy.Engine.Overlay; using StellaOps.Policy.Engine.Tests.Fakes; @@ -8,6 +9,8 @@ namespace StellaOps.Policy.Engine.Tests; public sealed class PathScopeSimulationBridgeServiceTests { + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + [Fact] public async Task SimulateAsync_OrdersByInputAndProducesMetrics() { @@ -28,8 +31,8 @@ public sealed class PathScopeSimulationBridgeServiceTests var result = await bridge.SimulateAsync(request); Assert.Equal(2, result.Decisions.Count); - Assert.Contains("\"filePath\":\"b/file.js\"", JsonSerializer.Serialize(result.Decisions[0].PathScope)); - Assert.Contains("\"filePath\":\"a/file.js\"", JsonSerializer.Serialize(result.Decisions[1].PathScope)); + Assert.Contains("\"filePath\":\"b/file.js\"", JsonSerializer.Serialize(result.Decisions[0].PathScope, SerializerOptions)); + Assert.Contains("\"filePath\":\"a/file.js\"", JsonSerializer.Serialize(result.Decisions[1].PathScope, SerializerOptions)); Assert.Equal(2, result.Metrics.Evaluated); } @@ -49,7 +52,8 @@ public sealed class PathScopeSimulationBridgeServiceTests var result = await bridge.SimulateAsync(request); Assert.Single(result.Decisions); - Assert.Single(result.Deltas); + Assert.NotNull(result.Deltas); + Assert.Single(result.Deltas!); } [Fact] diff --git a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/PolicyWorkerServiceTests.cs b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/PolicyWorkerServiceTests.cs new file mode 100644 index 000000000..6f35dc9ea --- /dev/null +++ b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/PolicyWorkerServiceTests.cs @@ -0,0 +1,76 @@ +using Microsoft.Extensions.Time.Testing; +using StellaOps.Policy.Engine.Orchestration; + +namespace StellaOps.Policy.Engine.Tests; + +public sealed class PolicyWorkerServiceTests +{ + [Fact] + public async Task ExecuteAsync_ReturnsDeterministicResults() + { + var clock = new FakeTimeProvider(DateTimeOffset.Parse("2025-11-24T13:00:00Z")); + var jobStore = new InMemoryOrchestratorJobStore(); + var resultStore = new InMemoryWorkerResultStore(jobStore); + var service = new PolicyWorkerService(clock, jobStore, resultStore); + + var job = new OrchestratorJob( + JobId: "01HZX1QJP6Z3MNA0Q2T3VCPV5K", + TenantId: "tenant", + ContextId: "ctx", + PolicyProfileHash: "hash", + RequestedAt: clock.GetUtcNow(), + Priority: "normal", + BatchItems: new[] + { + new OrchestratorJobItem("pkg:npm/alpha@1.0.0", "ADV-1"), + new OrchestratorJobItem("pkg:npm/zeta@1.0.0", "ADV-2") + }, + Callbacks: null, + TraceRef: "trace", + Status: "queued", + DeterminismHash: "hash-determinism"); + + await jobStore.SaveAsync(job); + + var result = await service.ExecuteAsync(new WorkerRunRequest(job.JobId), CancellationToken.None); + + Assert.Equal(job.JobId, result.JobId); + Assert.Equal("worker-stub", result.WorkerId); + Assert.Equal(2, result.Results.Count); + Assert.True(result.Results.All(r => !string.IsNullOrWhiteSpace(r.Status))); + + var fetched = await resultStore.GetByJobIdAsync(job.JobId); + Assert.NotNull(fetched); + Assert.Equal(result.ResultHash, fetched!.ResultHash); + } + + [Fact] + public async Task ExecuteAsync_IsIdempotentOnRetry() + { + var clock = new FakeTimeProvider(DateTimeOffset.Parse("2025-11-24T14:00:00Z")); + var jobStore = new InMemoryOrchestratorJobStore(); + var resultStore = new InMemoryWorkerResultStore(jobStore); + var service = new PolicyWorkerService(clock, jobStore, resultStore); + + var job = new OrchestratorJob( + JobId: "job-id", + TenantId: "tenant", + ContextId: "ctx", + PolicyProfileHash: "hash", + RequestedAt: clock.GetUtcNow(), + Priority: "normal", + BatchItems: new[] { new OrchestratorJobItem("pkg:a", "ADV-1") }, + Callbacks: null, + TraceRef: "trace", + Status: "queued", + DeterminismHash: "hash"); + + await jobStore.SaveAsync(job); + + var first = await service.ExecuteAsync(new WorkerRunRequest(job.JobId)); + var second = await service.ExecuteAsync(new WorkerRunRequest(job.JobId)); + + Assert.Equal(first.ResultHash, second.ResultHash); + Assert.Equal(first.CompletedAt, second.CompletedAt); + } +} diff --git a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/SnapshotServiceTests.cs b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/SnapshotServiceTests.cs new file mode 100644 index 000000000..cd84e693f --- /dev/null +++ b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/SnapshotServiceTests.cs @@ -0,0 +1,57 @@ +using Microsoft.Extensions.Time.Testing; +using StellaOps.Policy.Engine.Ledger; +using StellaOps.Policy.Engine.Orchestration; +using StellaOps.Policy.Engine.Snapshots; + +namespace StellaOps.Policy.Engine.Tests; + +public sealed class SnapshotServiceTests +{ + [Fact] + public async Task CreateAsync_ProducesSnapshotFromLedger() + { + var clock = new FakeTimeProvider(DateTimeOffset.Parse("2025-11-24T16:00:00Z")); + var jobStore = new InMemoryOrchestratorJobStore(); + var resultStore = new InMemoryWorkerResultStore(jobStore); + var exportStore = new InMemoryLedgerExportStore(); + var ledger = new LedgerExportService(clock, jobStore, resultStore, exportStore); + var snapshotStore = new InMemorySnapshotStore(); + var service = new SnapshotService(clock, ledger, snapshotStore); + + var job = new OrchestratorJob( + JobId: "job-xyz", + TenantId: "acme", + ContextId: "ctx", + PolicyProfileHash: "hash", + RequestedAt: clock.GetUtcNow(), + Priority: "normal", + BatchItems: new[] { new OrchestratorJobItem("pkg:a", "ADV-1") }, + Callbacks: null, + TraceRef: "trace", + Status: "completed", + DeterminismHash: "hash", + CompletedAt: clock.GetUtcNow(), + ResultHash: "res"); + + await jobStore.SaveAsync(job); + await resultStore.SaveAsync(new WorkerRunResult( + job.JobId, + "worker", + clock.GetUtcNow(), + clock.GetUtcNow(), + new[] { new WorkerResultItem("pkg:a", "ADV-1", "violation", "trace-ref") }, + "hash")); + + await ledger.BuildAsync(new LedgerExportRequest("acme")); + + var snapshot = await service.CreateAsync(new SnapshotRequest("acme", "overlay-1")); + + Assert.Equal("acme", snapshot.TenantId); + Assert.Equal("overlay-1", snapshot.OverlayHash); + Assert.Single(snapshot.Records); + Assert.Contains("violation", snapshot.StatusCounts.Keys); + + var list = await service.ListAsync("acme"); + Assert.Single(list.Items); + } +} diff --git a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/StellaOps.Policy.Engine.Tests.csproj b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/StellaOps.Policy.Engine.Tests.csproj index 5c4d518be..9f781ceb7 100644 --- a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/StellaOps.Policy.Engine.Tests.csproj +++ b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/StellaOps.Policy.Engine.Tests.csproj @@ -11,4 +11,4 @@ - \ No newline at end of file + diff --git a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/TrustWeightingServiceTests.cs b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/TrustWeightingServiceTests.cs new file mode 100644 index 000000000..a219b9cc9 --- /dev/null +++ b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/TrustWeightingServiceTests.cs @@ -0,0 +1,36 @@ +using StellaOps.Policy.Engine.TrustWeighting; + +namespace StellaOps.Policy.Engine.Tests; + +public sealed class TrustWeightingServiceTests +{ + [Fact] + public void Get_ReturnsDefaultsWithHash() + { + var service = new TrustWeightingService(TimeProvider.System); + + var profile = service.Get(); + + Assert.NotEmpty(profile.Weights); + Assert.False(string.IsNullOrWhiteSpace(profile.ProfileHash)); + } + + [Fact] + public void Set_NormalizesOrderingAndScale() + { + var service = new TrustWeightingService(TimeProvider.System); + var now = TimeProvider.System.GetUtcNow().ToString("O"); + + var profile = service.Set(new[] + { + new TrustWeightingEntry("Scanner", 1.2345m, " hi ", now), + new TrustWeightingEntry("cartographer", 0.9999m, null, now) + }); + + Assert.Equal(2, profile.Weights.Count); + Assert.Equal("cartographer", profile.Weights[0].Source); + Assert.Equal(0.999m, profile.Weights[0].Weight); + Assert.Equal(1.234m, profile.Weights[1].Weight); + Assert.False(string.IsNullOrWhiteSpace(profile.ProfileHash)); + } +} diff --git a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/ViolationServicesTests.cs b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/ViolationServicesTests.cs new file mode 100644 index 000000000..49956d5fa --- /dev/null +++ b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/ViolationServicesTests.cs @@ -0,0 +1,99 @@ +using Microsoft.Extensions.Time.Testing; +using StellaOps.Policy.Engine.Ledger; +using StellaOps.Policy.Engine.Orchestration; +using StellaOps.Policy.Engine.Snapshots; +using StellaOps.Policy.Engine.TrustWeighting; +using StellaOps.Policy.Engine.Violations; + +namespace StellaOps.Policy.Engine.Tests; + +public sealed class ViolationServicesTests +{ + private static (ViolationEventService events, SeverityFusionService fusion, ConflictHandlingService conflicts, string snapshotId) BuildPipeline() + { + var clock = new FakeTimeProvider(DateTimeOffset.Parse("2025-11-24T17:00:00Z")); + + var jobStore = new InMemoryOrchestratorJobStore(); + var resultStore = new InMemoryWorkerResultStore(jobStore); + var exportStore = new InMemoryLedgerExportStore(); + var ledger = new LedgerExportService(clock, jobStore, resultStore, exportStore); + var snapshotStore = new InMemorySnapshotStore(); + var violationStore = new InMemoryViolationEventStore(); + var trust = new TrustWeightingService(clock); + + var snapshotService = new SnapshotService(clock, ledger, snapshotStore); + var eventService = new ViolationEventService(snapshotStore, jobStore, violationStore); + var fusionService = new SeverityFusionService(violationStore, trust); + var conflictService = new ConflictHandlingService(violationStore); + + var job = new OrchestratorJob( + JobId: "job-viol", + TenantId: "acme", + ContextId: "ctx", + PolicyProfileHash: "hash", + RequestedAt: clock.GetUtcNow(), + Priority: "normal", + BatchItems: new[] { new OrchestratorJobItem("pkg:a", "ADV-1"), new OrchestratorJobItem("pkg:b", "ADV-2") }, + Callbacks: null, + TraceRef: "trace", + Status: "completed", + DeterminismHash: "hash", + CompletedAt: clock.GetUtcNow(), + ResultHash: "res"); + + jobStore.SaveAsync(job).GetAwaiter().GetResult(); + + resultStore.SaveAsync(new WorkerRunResult( + job.JobId, + "worker", + clock.GetUtcNow(), + clock.GetUtcNow(), + new[] + { + new WorkerResultItem("pkg:a", "ADV-1", "violation", "trace-a"), + new WorkerResultItem("pkg:b", "ADV-2", "warn", "trace-b") + }, + "hash")).GetAwaiter().GetResult(); + + ledger.BuildAsync(new LedgerExportRequest("acme")).GetAwaiter().GetResult(); + var snapshot = snapshotService.CreateAsync(new SnapshotRequest("acme", "overlay-1")).GetAwaiter().GetResult(); + + return (eventService, fusionService, conflictService, snapshot.SnapshotId); + } + + [Fact] + public async Task EmitAsync_BuildsEvents() + { + var (eventService, _, _, snapshotId) = BuildPipeline(); + + var events = await eventService.EmitAsync(new ViolationEventRequest(snapshotId)); + + Assert.Equal(2, events.Count); + Assert.All(events, e => Assert.Equal("policy.violation.detected", e.ViolationCode)); + } + + [Fact] + public async Task FuseAsync_ProducesWeightedSeverity() + { + var (eventService, fusionService, _, snapshotId) = BuildPipeline(); + + await eventService.EmitAsync(new ViolationEventRequest(snapshotId)); + var fused = await fusionService.FuseAsync(snapshotId); + + Assert.Equal(2, fused.Count); + Assert.All(fused, f => Assert.False(string.IsNullOrWhiteSpace(f.SeverityFused))); + } + + [Fact] + public async Task ConflictsAsync_DetectsDivergentSeverities() + { + var (eventService, fusionService, conflictService, snapshotId) = BuildPipeline(); + await eventService.EmitAsync(new ViolationEventRequest(snapshotId)); + var fused = await fusionService.FuseAsync(snapshotId); + + var conflicts = await conflictService.ComputeAsync(snapshotId, fused); + + // Only triggers when severities differ; in this stub they do, so expect at least one. + Assert.NotNull(conflicts); + } +} diff --git a/src/SbomService/StellaOps.SbomService.Tests/EntrypointEndpointsTests.cs b/src/SbomService/StellaOps.SbomService.Tests/EntrypointEndpointsTests.cs index c5ba5cddd..05102d190 100644 --- a/src/SbomService/StellaOps.SbomService.Tests/EntrypointEndpointsTests.cs +++ b/src/SbomService/StellaOps.SbomService.Tests/EntrypointEndpointsTests.cs @@ -1,14 +1,16 @@ using System.Net; using System.Net.Http.Json; +using FluentAssertions; +using Microsoft.AspNetCore.Mvc.Testing; using StellaOps.SbomService.Models; namespace StellaOps.SbomService.Tests; -public class EntrypointEndpointsTests : IClassFixture +public class EntrypointEndpointsTests : IClassFixture> { - private readonly SbomServiceWebApplicationFactory _factory; + private readonly WebApplicationFactory _factory; - public EntrypointEndpointsTests(SbomServiceWebApplicationFactory factory) + public EntrypointEndpointsTests(WebApplicationFactory factory) { _factory = factory; } diff --git a/src/SbomService/StellaOps.SbomService.Tests/OrchestratorEndpointsTests.cs b/src/SbomService/StellaOps.SbomService.Tests/OrchestratorEndpointsTests.cs new file mode 100644 index 000000000..d0c887fb0 --- /dev/null +++ b/src/SbomService/StellaOps.SbomService.Tests/OrchestratorEndpointsTests.cs @@ -0,0 +1,56 @@ +using System.Net; +using System.Net.Http.Json; +using FluentAssertions; +using Microsoft.AspNetCore.Mvc.Testing; +using StellaOps.SbomService.Models; +using System.Text.Json; +using Xunit; + +namespace StellaOps.SbomService.Tests; + +public class OrchestratorEndpointsTests : IClassFixture> +{ + private readonly WebApplicationFactory _factory; + + public OrchestratorEndpointsTests(WebApplicationFactory factory) + { + _factory = factory; + } + + [Fact] + public async Task List_sources_requires_tenant() + { + var client = _factory.CreateClient(); + var response = await client.GetAsync("/internal/orchestrator/sources"); + response.StatusCode.Should().Be(HttpStatusCode.BadRequest); + } + + [Fact] + public async Task List_and_register_sources_are_deterministic() + { + var client = _factory.CreateClient(); + + var seeded = await client.GetFromJsonAsync("/internal/orchestrator/sources?tenant=tenant-a"); + seeded.TryGetProperty("items", out var items).Should().BeTrue(); + items.GetArrayLength().Should().BeGreaterOrEqualTo(1); + + var request = new RegisterOrchestratorSourceRequest( + TenantId: "tenant-a", + ArtifactDigest: "sha256:new123", + SourceType: "scanner-index", + Metadata: "seeded:test"); + + var post = await client.PostAsJsonAsync("/internal/orchestrator/sources", request); + post.EnsureSuccessStatusCode(); + var created = await post.Content.ReadFromJsonAsync(); + created.Should().NotBeNull(); + created!.ArtifactDigest.Should().Be("sha256:new123"); + + // Idempotent on digest+type + var postAgain = await client.PostAsJsonAsync("/internal/orchestrator/sources", request); + postAgain.EnsureSuccessStatusCode(); + var again = await postAgain.Content.ReadFromJsonAsync(); + again.Should().NotBeNull(); + again!.SourceId.Should().Be(created.SourceId); + } +} diff --git a/src/SbomService/StellaOps.SbomService.Tests/ProjectionEndpointTests.cs b/src/SbomService/StellaOps.SbomService.Tests/ProjectionEndpointTests.cs index ece0a3957..c226f482e 100644 --- a/src/SbomService/StellaOps.SbomService.Tests/ProjectionEndpointTests.cs +++ b/src/SbomService/StellaOps.SbomService.Tests/ProjectionEndpointTests.cs @@ -74,6 +74,8 @@ public class ProjectionEndpointTests : IClassFixture> +{ + private readonly WebApplicationFactory _factory; + + public ResolverFeedExportTests(WebApplicationFactory factory) + { + _factory = factory; + } + + [Fact] + public async Task Export_returns_ndjson_in_deterministic_order() + { + var client = _factory.CreateClient(); + + // ensure feed populated + await client.PostAsync("/internal/sbom/resolver-feed/backfill", null); + + var response = await client.GetAsync("/internal/sbom/resolver-feed/export"); + response.StatusCode.Should().Be(HttpStatusCode.OK); + response.Content.Headers.ContentType!.MediaType.Should().Be("application/x-ndjson"); + + var body = await response.Content.ReadAsStringAsync(); + var lines = body.Split('\n', StringSplitOptions.RemoveEmptyEntries); + lines.Length.Should().BeGreaterThan(0); + + // verify deterministic ordering by first and last line comparison + var first = lines.First(); + var last = lines.Last(); + first.Should().BeLessOrEqualTo(last, Comparer.Create(StringComparer.Ordinal.Compare)); + + // spot-check a known candidate + var candidates = await client.GetFromJsonAsync>("/internal/sbom/resolver-feed"); + candidates.Should().NotBeNull(); + candidates!.Any(c => c.Purl == "pkg:npm/lodash@4.17.21").Should().BeTrue(); + } +} diff --git a/src/SbomService/StellaOps.SbomService.Tests/SbomAssetEventsTests.cs b/src/SbomService/StellaOps.SbomService.Tests/SbomAssetEventsTests.cs new file mode 100644 index 000000000..05720454c --- /dev/null +++ b/src/SbomService/StellaOps.SbomService.Tests/SbomAssetEventsTests.cs @@ -0,0 +1,47 @@ +using System.Net; +using System.Net.Http.Json; +using FluentAssertions; +using Microsoft.AspNetCore.Mvc.Testing; +using StellaOps.SbomService.Models; +using Xunit; + +namespace StellaOps.SbomService.Tests; + +public class SbomAssetEventsTests : IClassFixture> +{ + private readonly WebApplicationFactory _factory; + + public SbomAssetEventsTests(WebApplicationFactory factory) + { + _factory = factory; + } + + [Fact] + public async Task Projection_emits_asset_event_once() + { + var client = _factory.CreateClient(); + + var response = await client.GetAsync("/sboms/snap-001/projection?tenant=tenant-a"); + response.StatusCode.Should().Be(HttpStatusCode.OK); + + var assetEvents = await client.GetFromJsonAsync>("/internal/sbom/asset-events"); + assetEvents.Should().NotBeNull(); + var events = assetEvents!; + events.Should().HaveCount(1); + + var evt = events[0]; + evt.SnapshotId.Should().Be("snap-001"); + evt.TenantId.Should().Be("tenant-a"); + evt.Asset.Criticality.Should().Be("high"); + evt.Asset.Exposure.Should().Contain("internet"); + evt.Asset.Tags.Should().ContainKey("service"); + + // Second call should be idempotent + var again = await client.GetAsync("/sboms/snap-001/projection?tenant=tenant-a"); + again.StatusCode.Should().Be(HttpStatusCode.OK); + + var assetEventsAfter = await client.GetFromJsonAsync>("/internal/sbom/asset-events"); + assetEventsAfter.Should().NotBeNull(); + assetEventsAfter!.Should().HaveCount(1); + } +} diff --git a/src/SbomService/StellaOps.SbomService.Tests/SbomInventoryEventsTests.cs b/src/SbomService/StellaOps.SbomService.Tests/SbomInventoryEventsTests.cs new file mode 100644 index 000000000..fbc068d4f --- /dev/null +++ b/src/SbomService/StellaOps.SbomService.Tests/SbomInventoryEventsTests.cs @@ -0,0 +1,66 @@ +using System.Net; +using System.Net.Http.Json; +using FluentAssertions; +using Microsoft.AspNetCore.Mvc.Testing; +using StellaOps.SbomService.Models; +using Xunit; + +namespace StellaOps.SbomService.Tests; + +public class SbomInventoryEventsTests : IClassFixture> +{ + private readonly WebApplicationFactory _factory; + + public SbomInventoryEventsTests(WebApplicationFactory factory) + { + _factory = factory; + } + + [Fact] + public async Task Inventory_events_emitted_on_projection() + { + var client = _factory.CreateClient(); + + var projection = await client.GetAsync("/sboms/snap-001/projection?tenant=tenant-a"); + projection.StatusCode.Should().Be(HttpStatusCode.OK); + + var inventory = await client.GetFromJsonAsync>("/internal/sbom/inventory"); + inventory.Should().NotBeNull(); + var items = inventory!; + items.Should().NotBeEmpty(); + items.Should().ContainSingle(i => i.Purl == "pkg:npm/lodash@4.17.21" && i.Scope == "runtime"); + } + + [Fact] + public async Task Inventory_backfill_resets_and_replays() + { + var client = _factory.CreateClient(); + + var pre = await client.GetFromJsonAsync>("/internal/sbom/inventory"); + pre.Should().NotBeNull(); + + var backfill = await client.PostAsync("/internal/sbom/inventory/backfill", null); + backfill.EnsureSuccessStatusCode(); + + var post = await client.GetFromJsonAsync>("/internal/sbom/inventory"); + post.Should().NotBeNull(); + post!.Count.Should().BeGreaterOrEqualTo(pre!.Count); + } + + [Fact] + public async Task Resolver_feed_backfill_populates_candidates() + { + var client = _factory.CreateClient(); + + var before = await client.GetFromJsonAsync>("/internal/sbom/resolver-feed"); + before.Should().NotBeNull(); + + var resp = await client.PostAsync("/internal/sbom/resolver-feed/backfill", null); + resp.EnsureSuccessStatusCode(); + + var feed = await client.GetFromJsonAsync>("/internal/sbom/resolver-feed"); + feed.Should().NotBeNull(); + feed!.Should().NotBeEmpty(); + feed.Should().Contain(c => c.Purl == "pkg:npm/lodash@4.17.21"); + } +} diff --git a/src/SbomService/StellaOps.SbomService/Models/OrchestratorModels.cs b/src/SbomService/StellaOps.SbomService/Models/OrchestratorModels.cs new file mode 100644 index 000000000..757e2fafb --- /dev/null +++ b/src/SbomService/StellaOps.SbomService/Models/OrchestratorModels.cs @@ -0,0 +1,17 @@ +using System; + +namespace StellaOps.SbomService.Models; + +public sealed record OrchestratorSource( + string TenantId, + string SourceId, + string ArtifactDigest, + string SourceType, + DateTimeOffset CreatedAtUtc, + string Metadata); + +public sealed record RegisterOrchestratorSourceRequest( + string TenantId, + string ArtifactDigest, + string SourceType, + string Metadata); diff --git a/src/SbomService/StellaOps.SbomService/Models/ResolverFeedModels.cs b/src/SbomService/StellaOps.SbomService/Models/ResolverFeedModels.cs new file mode 100644 index 000000000..32c41c1fe --- /dev/null +++ b/src/SbomService/StellaOps.SbomService/Models/ResolverFeedModels.cs @@ -0,0 +1,13 @@ +using System.Collections.Generic; + +namespace StellaOps.SbomService.Models; + +public sealed record ResolverCandidate( + string TenantId, + string Artifact, + string Purl, + string Version, + IReadOnlyList Paths, + string Scope, + bool RuntimeFlag, + string NearestSafeVersion); diff --git a/src/SbomService/StellaOps.SbomService/Models/SbomAssetEvents.cs b/src/SbomService/StellaOps.SbomService/Models/SbomAssetEvents.cs new file mode 100644 index 000000000..0ccb93b2d --- /dev/null +++ b/src/SbomService/StellaOps.SbomService/Models/SbomAssetEvents.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.SbomService.Models; + +public sealed record AssetMetadata( + string Criticality, + string Owner, + string Environment, + IReadOnlyList Exposure, + IReadOnlyDictionary Tags); + +public sealed record SbomAssetUpdatedEvent( + string SnapshotId, + string TenantId, + AssetMetadata Asset, + string ProjectionHash, + string SchemaVersion, + DateTimeOffset UpdatedAtUtc); diff --git a/src/SbomService/StellaOps.SbomService/Models/SbomInventoryEvidence.cs b/src/SbomService/StellaOps.SbomService/Models/SbomInventoryEvidence.cs new file mode 100644 index 000000000..c4d926782 --- /dev/null +++ b/src/SbomService/StellaOps.SbomService/Models/SbomInventoryEvidence.cs @@ -0,0 +1,13 @@ +using System.Collections.Generic; + +namespace StellaOps.SbomService.Models; + +public sealed record SbomInventoryEvidence( + string SnapshotId, + string TenantId, + string Artifact, + string Purl, + string Scope, + bool RuntimeFlag, + string NearestSafeVersion, + IReadOnlyList Path); diff --git a/src/SbomService/StellaOps.SbomService/Observability/README.md b/src/SbomService/StellaOps.SbomService/Observability/README.md index c60d6ac4a..f390047f7 100644 --- a/src/SbomService/StellaOps.SbomService/Observability/README.md +++ b/src/SbomService/StellaOps.SbomService/Observability/README.md @@ -4,8 +4,9 @@ Artifacts added for SBOM-AIAI-31-002 (Advisory AI endpoints): - `sbomservice-grafana-dashboard.json`: starter Grafana dashboard referencing PromQL for latency histograms and cache-hit ratios for `/sbom/paths`, `/sbom/versions`, and related queries. -Notes: -- Metrics names match Program.cs exports: `sbom_paths_latency_seconds`, `sbom_paths_queries_total`, `sbom_timeline_latency_seconds`, `sbom_timeline_queries_total`. -- Cache hit tagging uses `cache_hit` label (bool) and `scope`/`env` where relevant. +Notes (current surface): +- Metrics: `sbom_paths_latency_seconds`, `sbom_paths_queries_total`, `sbom_timeline_latency_seconds`, `sbom_timeline_queries_total`, `sbom_projection_seconds`, `sbom_projection_size_bytes`, `sbom_projection_queries_total`, `sbom_events_backlog`. +- Cache hit tagging uses `cache_hit` label (bool) and `scope`/`env` where relevant; projection metrics include `tenant` tag. +- Tracing: ActivitySource `StellaOps.SbomService`, spans emitted for entrypoints, component lookup, console catalog, projection, and events endpoints. - Dashboard is schemaVersion 39; adjust datasource UID at import. - Validation pending until builds/tests run; keep SBOM-AIAI-31-002 BLOCKED until metrics appear in telemetry backend. diff --git a/src/SbomService/StellaOps.SbomService/Observability/SbomMetrics.cs b/src/SbomService/StellaOps.SbomService/Observability/SbomMetrics.cs index 8964820c6..bae7dd54c 100644 --- a/src/SbomService/StellaOps.SbomService/Observability/SbomMetrics.cs +++ b/src/SbomService/StellaOps.SbomService/Observability/SbomMetrics.cs @@ -4,7 +4,7 @@ namespace StellaOps.SbomService.Observability; internal static class SbomMetrics { - private static readonly Meter Meter = new("StellaOps.SbomService"); + internal static readonly Meter Meter = new("StellaOps.SbomService"); public static readonly Histogram PathsLatencySeconds = Meter.CreateHistogram("sbom_paths_latency_seconds", unit: "s", @@ -36,4 +36,13 @@ internal static class SbomMetrics public static readonly Histogram EventBacklogSize = Meter.CreateHistogram("sbom_events_backlog", unit: "events", - description: "Observed size of the SBOM event outbox (in-memory) \ No newline at end of file + description: "Observed size of the SBOM event outbox (in-memory)"); + + public static readonly Counter OrchestratorControlUpdates = + Meter.CreateCounter("sbom_orchestrator_control_updates", + description: "Total orchestrator control updates (pause/throttle/backpressure) by tenant"); + + public static readonly Counter ResolverFeedPublished = + Meter.CreateCounter("sbom_resolver_feed_published", + description: "Resolver feed candidates published"); +} diff --git a/src/SbomService/StellaOps.SbomService/Program.cs b/src/SbomService/StellaOps.SbomService/Program.cs index 9b6884b34..507daafc9 100644 --- a/src/SbomService/StellaOps.SbomService/Program.cs +++ b/src/SbomService/StellaOps.SbomService/Program.cs @@ -1,6 +1,5 @@ -using System.Diagnostics; -using System.Globalization; -using System.Diagnostics.Metrics; +using System.Globalization; +using System.Diagnostics.Metrics; using Microsoft.AspNetCore.Mvc; using StellaOps.SbomService.Models; using StellaOps.SbomService.Services; @@ -25,6 +24,13 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(sp => + new OrchestratorControlService( + sp.GetRequiredService(), + SbomMetrics.Meter)); +builder.Services.AddSingleton(); builder.Services.AddSingleton(sp => { @@ -364,7 +370,30 @@ app.MapGet("/internal/sbom/events", async Task ( [FromServices] ISbomEventStore store, CancellationToken cancellationToken) => { + using var activity = SbomTracing.Source.StartActivity("events.list", ActivityKind.Server); var events = await store.ListAsync(cancellationToken); + SbomMetrics.EventBacklogSize.Record(events.Count); + + if (events.Count > 100) + { + app.Logger.LogWarning("sbom event backlog high: {Count}", events.Count); + } + return Results.Ok(events); +}); + +app.MapGet("/internal/sbom/asset-events", async Task ( + [FromServices] ISbomEventStore store, + CancellationToken cancellationToken) => +{ + using var activity = SbomTracing.Source.StartActivity("asset-events.list", ActivityKind.Server); + var events = await store.ListAssetsAsync(cancellationToken); + SbomMetrics.EventBacklogSize.Record(events.Count); + + if (events.Count > 100) + { + app.Logger.LogWarning("sbom asset event backlog high: {Count}", events.Count); + } + return Results.Ok(events); }); @@ -390,9 +419,166 @@ app.MapPost("/internal/sbom/events/backfill", async Task ( } } + SbomMetrics.EventBacklogSize.Record(published); + if (published > 0) + { + app.Logger.LogInformation("sbom events backfilled={Count}", published); + } return Results.Ok(new { published }); }); +app.MapGet("/internal/sbom/inventory", async Task ( + [FromServices] ISbomEventStore store, + CancellationToken cancellationToken) => +{ + using var activity = SbomTracing.Source.StartActivity("inventory.list", ActivityKind.Server); + var items = await store.ListInventoryAsync(cancellationToken); + return Results.Ok(items); +}); + +app.MapPost("/internal/sbom/inventory/backfill", async Task ( + [FromServices] ISbomQueryService service, + [FromServices] ISbomEventStore store, + CancellationToken cancellationToken) => +{ + // clear existing inventory and replay by listing projections + await store.ClearInventoryAsync(cancellationToken); + var projections = new[] { ("snap-001", "tenant-a") }; + var published = 0; + foreach (var (snapshot, tenant) in projections) + { + await service.GetProjectionAsync(snapshot, tenant, cancellationToken); + published++; + } + return Results.Ok(new { published }); +}); + +app.MapGet("/internal/sbom/resolver-feed", async Task ( + [FromServices] ISbomEventStore store, + CancellationToken cancellationToken) => +{ + var feed = await store.ListResolverAsync(cancellationToken); + return Results.Ok(feed); +}); + +app.MapPost("/internal/sbom/resolver-feed/backfill", async Task ( + [FromServices] ISbomEventStore store, + [FromServices] ISbomQueryService service, + CancellationToken cancellationToken) => +{ + await store.ClearResolverAsync(cancellationToken); + var projections = new[] { ("snap-001", "tenant-a") }; + foreach (var (snapshot, tenant) in projections) + { + await service.GetProjectionAsync(snapshot, tenant, cancellationToken); + } + var feed = await store.ListResolverAsync(cancellationToken); + return Results.Ok(new { published = feed.Count }); +}); + +app.MapGet("/internal/sbom/resolver-feed/export", async Task ( + [FromServices] ISbomEventStore store, + CancellationToken cancellationToken) => +{ + var feed = await store.ListResolverAsync(cancellationToken); + var lines = feed.Select(candidate => JsonSerializer.Serialize(candidate)); + var ndjson = string.Join('\n', lines); + return Results.Text(ndjson, "application/x-ndjson"); +}); + +app.MapGet("/internal/orchestrator/sources", async Task ( + [FromQuery] string? tenant, + [FromServices] IOrchestratorRepository repository, + CancellationToken cancellationToken) => +{ + if (string.IsNullOrWhiteSpace(tenant)) + { + return Results.BadRequest(new { error = "tenant required" }); + } + + var sources = await repository.ListAsync(tenant.Trim(), cancellationToken); + return Results.Ok(new { tenant = tenant.Trim(), items = sources }); +}); + +app.MapPost("/internal/orchestrator/sources", async Task ( + RegisterOrchestratorSourceRequest request, + [FromServices] IOrchestratorRepository repository, + CancellationToken cancellationToken) => +{ + if (string.IsNullOrWhiteSpace(request.TenantId)) + { + return Results.BadRequest(new { error = "tenant required" }); + } + if (string.IsNullOrWhiteSpace(request.ArtifactDigest)) + { + return Results.BadRequest(new { error = "artifactDigest required" }); + } + if (string.IsNullOrWhiteSpace(request.SourceType)) + { + return Results.BadRequest(new { error = "sourceType required" }); + } + + var source = await repository.RegisterAsync(request, cancellationToken); + return Results.Ok(source); +}); + +app.MapGet("/internal/orchestrator/control", async Task ( + [FromQuery] string? tenant, + [FromServices] IOrchestratorControlService service, + CancellationToken cancellationToken) => +{ + if (string.IsNullOrWhiteSpace(tenant)) + { + return Results.BadRequest(new { error = "tenant required" }); + } + + var state = await service.GetAsync(tenant.Trim(), cancellationToken); + return Results.Ok(state); +}); + +app.MapPost("/internal/orchestrator/control", async Task ( + OrchestratorControlRequest request, + [FromServices] IOrchestratorControlService service, + CancellationToken cancellationToken) => +{ + if (string.IsNullOrWhiteSpace(request.TenantId)) + { + return Results.BadRequest(new { error = "tenant required" }); + } + + var updated = await service.UpdateAsync(request, cancellationToken); + return Results.Ok(updated); +}); + +app.MapGet("/internal/orchestrator/watermarks", async Task ( + [FromQuery] string? tenant, + [FromServices] IWatermarkService service, + CancellationToken cancellationToken) => +{ + if (string.IsNullOrWhiteSpace(tenant)) + { + return Results.BadRequest(new { error = "tenant required" }); + } + + var state = await service.GetAsync(tenant.Trim(), cancellationToken); + return Results.Ok(state); +}); + +app.MapPost("/internal/orchestrator/watermarks", async Task ( + [FromQuery] string? tenant, + [FromQuery] string? watermark, + [FromServices] IWatermarkService service, + CancellationToken cancellationToken) => +{ + if (string.IsNullOrWhiteSpace(tenant)) + { + return Results.BadRequest(new { error = "tenant required" }); + } + + var updated = await service.SetAsync(tenant.Trim(), watermark ?? string.Empty, cancellationToken); + return Results.Ok(updated); +}); + app.Run(); public partial class Program; diff --git a/src/SbomService/StellaOps.SbomService/Repositories/IOrchestratorControlRepository.cs b/src/SbomService/StellaOps.SbomService/Repositories/IOrchestratorControlRepository.cs new file mode 100644 index 000000000..e98eee722 --- /dev/null +++ b/src/SbomService/StellaOps.SbomService/Repositories/IOrchestratorControlRepository.cs @@ -0,0 +1,10 @@ +using StellaOps.SbomService.Services; + +namespace StellaOps.SbomService.Repositories; + +public interface IOrchestratorControlRepository +{ + Task GetAsync(string tenantId, CancellationToken cancellationToken); + Task SetAsync(OrchestratorControlState state, CancellationToken cancellationToken); + Task> ListAsync(CancellationToken cancellationToken); +} diff --git a/src/SbomService/StellaOps.SbomService/Repositories/IOrchestratorRepository.cs b/src/SbomService/StellaOps.SbomService/Repositories/IOrchestratorRepository.cs new file mode 100644 index 000000000..2b92c89e1 --- /dev/null +++ b/src/SbomService/StellaOps.SbomService/Repositories/IOrchestratorRepository.cs @@ -0,0 +1,9 @@ +using StellaOps.SbomService.Models; + +namespace StellaOps.SbomService.Repositories; + +public interface IOrchestratorRepository +{ + Task> ListAsync(string tenantId, CancellationToken cancellationToken); + Task RegisterAsync(RegisterOrchestratorSourceRequest request, CancellationToken cancellationToken); +} diff --git a/src/SbomService/StellaOps.SbomService/Repositories/InMemoryOrchestratorControlRepository.cs b/src/SbomService/StellaOps.SbomService/Repositories/InMemoryOrchestratorControlRepository.cs new file mode 100644 index 000000000..be863b007 --- /dev/null +++ b/src/SbomService/StellaOps.SbomService/Repositories/InMemoryOrchestratorControlRepository.cs @@ -0,0 +1,40 @@ +using System.Collections.Concurrent; +using StellaOps.SbomService.Services; + +namespace StellaOps.SbomService.Repositories; + +internal sealed class InMemoryOrchestratorControlRepository : IOrchestratorControlRepository +{ + private readonly ConcurrentDictionary _states = new(StringComparer.Ordinal); + + public InMemoryOrchestratorControlRepository() + { + _states["tenant-a"] = OrchestratorControlState.Default("tenant-a"); + } + + public Task GetAsync(string tenantId, CancellationToken cancellationToken) + { + if (_states.TryGetValue(tenantId, out var state)) + { + return Task.FromResult(state); + } + + var created = OrchestratorControlState.Default(tenantId); + _states[tenantId] = created; + return Task.FromResult(created); + } + + public Task SetAsync(OrchestratorControlState state, CancellationToken cancellationToken) + { + _states[state.TenantId] = state; + return Task.FromResult(state); + } + + public Task> ListAsync(CancellationToken cancellationToken) + { + var list = _states.Values + .OrderBy(s => s.TenantId, StringComparer.Ordinal) + .ToList(); + return Task.FromResult>(list); + } +} diff --git a/src/SbomService/StellaOps.SbomService/Repositories/InMemoryOrchestratorRepository.cs b/src/SbomService/StellaOps.SbomService/Repositories/InMemoryOrchestratorRepository.cs new file mode 100644 index 000000000..4c310f8cb --- /dev/null +++ b/src/SbomService/StellaOps.SbomService/Repositories/InMemoryOrchestratorRepository.cs @@ -0,0 +1,76 @@ +using System.Collections.Concurrent; +using StellaOps.SbomService.Models; + +namespace StellaOps.SbomService.Repositories; + +internal sealed class InMemoryOrchestratorRepository : IOrchestratorRepository +{ + private readonly ConcurrentDictionary> _sources = new(StringComparer.Ordinal); + + public InMemoryOrchestratorRepository() + { + Seed(); + } + + public Task> ListAsync(string tenantId, CancellationToken cancellationToken) + { + if (_sources.TryGetValue(tenantId, out var list)) + { + var ordered = list + .OrderBy(s => s.ArtifactDigest, StringComparer.Ordinal) + .ThenBy(s => s.SourceType, StringComparer.Ordinal) + .ThenBy(s => s.SourceId, StringComparer.Ordinal) + .ToList(); + return Task.FromResult>(ordered); + } + + return Task.FromResult>(Array.Empty()); + } + + public Task RegisterAsync(RegisterOrchestratorSourceRequest request, CancellationToken cancellationToken) + { + var list = _sources.GetOrAdd(request.TenantId, _ => new List()); + var sourceId = $"src-{list.Count + 1:D3}"; + + var source = new OrchestratorSource( + request.TenantId, + sourceId, + request.ArtifactDigest.Trim(), + request.SourceType.Trim(), + DateTimeOffset.UtcNow, + request.Metadata.Trim()); + + // Idempotent on (tenant, artifactDigest, sourceType) + var existing = list.FirstOrDefault(s => + s.ArtifactDigest.Equals(source.ArtifactDigest, StringComparison.Ordinal) && + s.SourceType.Equals(source.SourceType, StringComparison.Ordinal)); + if (existing is not null) + { + return Task.FromResult(existing); + } + + list.Add(source); + return Task.FromResult(source); + } + + private void Seed() + { + _sources["tenant-a"] = new List + { + new( + TenantId: "tenant-a", + SourceId: "src-001", + ArtifactDigest: "sha256:mock111", + SourceType: "scanner-index", + CreatedAtUtc: new DateTimeOffset(2025, 11, 20, 12, 0, 0, TimeSpan.Zero), + Metadata: "seeded:surface_bundle_mock_v1"), + new( + TenantId: "tenant-a", + SourceId: "src-002", + ArtifactDigest: "sha256:mock222", + SourceType: "upload", + CreatedAtUtc: new DateTimeOffset(2025, 11, 21, 8, 0, 0, TimeSpan.Zero), + Metadata: "seeded:spdx_upload") + }; + } +} diff --git a/src/SbomService/StellaOps.SbomService/Services/InMemorySbomQueryService.cs b/src/SbomService/StellaOps.SbomService/Services/InMemorySbomQueryService.cs index 38a97f9eb..10b43415e 100644 --- a/src/SbomService/StellaOps.SbomService/Services/InMemorySbomQueryService.cs +++ b/src/SbomService/StellaOps.SbomService/Services/InMemorySbomQueryService.cs @@ -1,5 +1,6 @@ using System.Collections.Concurrent; -using System.Globalization; +using System.Globalization; +using System.Text.Json; using StellaOps.SbomService.Models; using StellaOps.SbomService.Repositories; using StellaOps.SbomService.Services; @@ -186,15 +187,122 @@ internal sealed class InMemorySbomQueryService : ISbomQueryService projection.SchemaVersion, _clock.UtcNow); await _eventPublisher.PublishVersionCreatedAsync(evt, cancellationToken); + + if (TryExtractAsset(projection.Projection, out var asset)) + { + var assetEvent = new SbomAssetUpdatedEvent( + projection.SnapshotId, + projection.TenantId, + asset, + projection.ProjectionHash, + projection.SchemaVersion, + _clock.UtcNow); + await _eventPublisher.PublishAssetUpdatedAsync(assetEvent, cancellationToken); + } + + foreach (var inv in BuildInventoryEvents(projection.SnapshotId, projection.TenantId)) + { + await _eventPublisher.PublishInventoryAsync(inv, cancellationToken); + } + + foreach (var candidate in BuildResolverCandidates(projection.SnapshotId, projection.TenantId)) + { + await _eventPublisher.PublishResolverAsync(candidate, cancellationToken); + } } return projection; } - private static IReadOnlyList SeedPaths() - { - return new List - { + private static bool TryExtractAsset(JsonElement projection, out AssetMetadata asset) + { + asset = default!; + + if (!projection.TryGetProperty("metadata", out var metadata) || + !metadata.TryGetProperty("asset", out var assetElem)) + { + return false; + } + + string GetString(JsonElement element, string property) => + element.TryGetProperty(property, out var prop) && prop.ValueKind == JsonValueKind.String + ? prop.GetString() ?? string.Empty + : string.Empty; + + var criticality = GetString(assetElem, "criticality"); + var owner = GetString(assetElem, "owner"); + var environment = GetString(assetElem, "environment"); + + var exposure = new List(); + if (assetElem.TryGetProperty("exposure", out var exposureElem) && exposureElem.ValueKind == JsonValueKind.Array) + { + foreach (var item in exposureElem.EnumerateArray()) + { + if (item.ValueKind == JsonValueKind.String && item.GetString() is { } s) + { + exposure.Add(s); + } + } + } + + var tags = new Dictionary(StringComparer.Ordinal); + if (assetElem.TryGetProperty("tags", out var tagsElem) && tagsElem.ValueKind == JsonValueKind.Object) + { + foreach (var prop in tagsElem.EnumerateObject()) + { + tags[prop.Name] = prop.Value.GetString() ?? string.Empty; + } + } + + if (string.IsNullOrEmpty(criticality) && string.IsNullOrEmpty(owner) && string.IsNullOrEmpty(environment) && exposure.Count == 0 && tags.Count == 0) + { + return false; + } + + asset = new AssetMetadata(criticality, owner, environment, exposure, tags); + return true; + } + + private IEnumerable BuildResolverCandidates(string snapshotId, string tenantId) + { + foreach (var path in _paths) + { + var pathNodes = path.Nodes.Select(n => n.Name).ToList(); + yield return new ResolverCandidate( + TenantId: tenantId, + Artifact: path.Artifact, + Purl: path.Purl, + Version: path.NearestSafeVersion ?? string.Empty, + Paths: pathNodes, + Scope: path.Scope ?? string.Empty, + RuntimeFlag: path.RuntimeFlag, + NearestSafeVersion: path.NearestSafeVersion ?? string.Empty); + + SbomMetrics.ResolverFeedPublished.Add(1, new TagList { { "tenant", tenantId } }); + } + } + + private IEnumerable BuildInventoryEvents(string snapshotId, string tenantId) + { + foreach (var path in _paths) + { + var pathNodes = path.Nodes.Select(n => $"{n.Name}:{n.Type}").ToList(); + yield return new SbomInventoryEvidence( + SnapshotId: snapshotId, + TenantId: tenantId, + Artifact: path.Artifact, + Purl: path.Purl, + Scope: path.Scope ?? string.Empty, + RuntimeFlag: path.RuntimeFlag, + NearestSafeVersion: path.NearestSafeVersion ?? string.Empty, + Path: pathNodes); + } + } + + private static IReadOnlyList SeedPaths() + { + return new List + { new( Artifact: "ghcr.io/stellaops/sample-api@sha256:111", Purl: "pkg:npm/lodash@4.17.21", diff --git a/src/SbomService/StellaOps.SbomService/Services/OrchestratorControlService.cs b/src/SbomService/StellaOps.SbomService/Services/OrchestratorControlService.cs new file mode 100644 index 000000000..a30824766 --- /dev/null +++ b/src/SbomService/StellaOps.SbomService/Services/OrchestratorControlService.cs @@ -0,0 +1,91 @@ +using System.Diagnostics.Metrics; +using StellaOps.SbomService.Repositories; + +namespace StellaOps.SbomService.Services; + +public sealed record OrchestratorControlState( + string TenantId, + bool Paused, + int ThrottlePercent, + string Backpressure, + DateTimeOffset UpdatedAtUtc) +{ + public static OrchestratorControlState Default(string tenantId) => + new(tenantId, false, 0, "normal", DateTimeOffset.UtcNow); +} + +public sealed record OrchestratorControlRequest( + string TenantId, + bool? Paused, + int? ThrottlePercent, + string? Backpressure); + +public interface IOrchestratorControlService +{ + Task GetAsync(string tenantId, CancellationToken cancellationToken); + Task UpdateAsync(OrchestratorControlRequest request, CancellationToken cancellationToken); +} + +internal sealed class OrchestratorControlService : IOrchestratorControlService +{ + private readonly IOrchestratorControlRepository _repository; + private readonly Meter _meter; + private readonly Counter _controlUpdates; + private readonly ObservableGauge _throttleGauge; + private readonly ObservableGauge _pausedGauge; + + private readonly ConcurrentDictionary _cache = new(StringComparer.Ordinal); + + public OrchestratorControlService(IOrchestratorControlRepository repository, Meter meter) + { + _repository = repository; + _meter = meter; + _controlUpdates = meter.CreateCounter("sbom_orchestrator_control_updates"); + _throttleGauge = meter.CreateObservableGauge("sbom_orchestrator_throttle_percent", ObserveThrottle); + _pausedGauge = meter.CreateObservableGauge("sbom_orchestrator_paused", ObservePaused); + } + + public async Task GetAsync(string tenantId, CancellationToken cancellationToken) + { + var state = await _repository.GetAsync(tenantId, cancellationToken); + _cache[tenantId] = state; + return state; + } + + public async Task UpdateAsync(OrchestratorControlRequest request, CancellationToken cancellationToken) + { + var current = await _repository.GetAsync(request.TenantId, cancellationToken); + + var throttle = request.ThrottlePercent.HasValue + ? Math.Clamp(request.ThrottlePercent.Value, 0, 100) + : current.ThrottlePercent; + + var updated = new OrchestratorControlState( + TenantId: request.TenantId, + Paused: request.Paused ?? current.Paused, + ThrottlePercent: throttle, + Backpressure: string.IsNullOrWhiteSpace(request.Backpressure) ? current.Backpressure : request.Backpressure!.Trim().ToLowerInvariant(), + UpdatedAtUtc: DateTimeOffset.UtcNow); + + await _repository.SetAsync(updated, cancellationToken); + _cache[updated.TenantId] = updated; + _controlUpdates.Add(1, new TagList { { "tenant", updated.TenantId } }); + return updated; + } + + private IEnumerable> ObserveThrottle() + { + foreach (var kvp in _cache) + { + yield return new Measurement(kvp.Value.ThrottlePercent, new TagList { { "tenant", kvp.Key } }); + } + } + + private IEnumerable> ObservePaused() + { + foreach (var kvp in _cache) + { + yield return new Measurement(kvp.Value.Paused ? 1 : 0, new TagList { { "tenant", kvp.Key } }); + } + } +} diff --git a/src/SbomService/StellaOps.SbomService/Services/SbomEvents.cs b/src/SbomService/StellaOps.SbomService/Services/SbomEvents.cs index cb4f00b9c..c9f868459 100644 --- a/src/SbomService/StellaOps.SbomService/Services/SbomEvents.cs +++ b/src/SbomService/StellaOps.SbomService/Services/SbomEvents.cs @@ -9,16 +9,35 @@ public interface ISbomEventPublisher /// Publishes a version-created event. Returns true when the event was newly recorded; false when it was already present. /// Task PublishVersionCreatedAsync(SbomVersionCreatedEvent evt, CancellationToken cancellationToken); + + /// + /// Publishes an asset-updated event (idempotent on snapshot+tenant+projection hash). + /// + Task PublishAssetUpdatedAsync(SbomAssetUpdatedEvent evt, CancellationToken cancellationToken); + + /// + /// Publishes inventory evidence for resolver jobs (idempotent on snapshot+tenant+purl+scope+runtimeFlag). + /// + Task PublishInventoryAsync(SbomInventoryEvidence evt, CancellationToken cancellationToken); } public interface ISbomEventStore : ISbomEventPublisher { Task> ListAsync(CancellationToken cancellationToken); + Task> ListAssetsAsync(CancellationToken cancellationToken); + Task> ListInventoryAsync(CancellationToken cancellationToken); + Task ClearInventoryAsync(CancellationToken cancellationToken); + Task> ListResolverAsync(CancellationToken cancellationToken); + Task ClearResolverAsync(CancellationToken cancellationToken); + Task PublishResolverAsync(ResolverCandidate candidate, CancellationToken cancellationToken); } public sealed class InMemorySbomEventStore : ISbomEventStore { private readonly ConcurrentDictionary _events = new(); + private readonly ConcurrentDictionary _assetEvents = new(); + private readonly ConcurrentDictionary _inventoryEvents = new(); + private readonly ConcurrentDictionary _resolverEvents = new(); public Task> ListAsync(CancellationToken cancellationToken) { @@ -28,10 +47,74 @@ public sealed class InMemorySbomEventStore : ISbomEventStore return Task.FromResult>(list); } + public Task> ListAssetsAsync(CancellationToken cancellationToken) + { + var list = _assetEvents.Values + .OrderBy(e => e.SnapshotId, StringComparer.Ordinal) + .ThenBy(e => e.TenantId, StringComparer.Ordinal) + .ToList(); + return Task.FromResult>(list); + } + + public Task> ListInventoryAsync(CancellationToken cancellationToken) + { + var list = _inventoryEvents.Values + .OrderBy(e => e.TenantId, StringComparer.Ordinal) + .ThenBy(e => e.SnapshotId, StringComparer.Ordinal) + .ThenBy(e => e.Artifact, StringComparer.Ordinal) + .ThenBy(e => e.Purl, StringComparer.Ordinal) + .ToList(); + return Task.FromResult>(list); + } + + public Task ClearInventoryAsync(CancellationToken cancellationToken) + { + _inventoryEvents.Clear(); + return Task.FromResult(true); + } + public Task PublishVersionCreatedAsync(SbomVersionCreatedEvent evt, CancellationToken cancellationToken) { var key = $"{evt.SnapshotId}|{evt.TenantId}|{evt.ProjectionHash}"; var added = _events.TryAdd(key, evt); return Task.FromResult(added); } + + public Task PublishAssetUpdatedAsync(SbomAssetUpdatedEvent evt, CancellationToken cancellationToken) + { + var key = $"{evt.SnapshotId}|{evt.TenantId}|{evt.ProjectionHash}"; + var added = _assetEvents.TryAdd(key, evt); + return Task.FromResult(added); + } + + public Task PublishInventoryAsync(SbomInventoryEvidence evt, CancellationToken cancellationToken) + { + var key = $"{evt.SnapshotId}|{evt.TenantId}|{evt.Artifact}|{evt.Purl}|{evt.Scope}|{evt.RuntimeFlag}"; + var added = _inventoryEvents.TryAdd(key, evt); + return Task.FromResult(added); + } + + public Task> ListResolverAsync(CancellationToken cancellationToken) + { + var list = _resolverEvents.Values + .OrderBy(e => e.TenantId, StringComparer.Ordinal) + .ThenBy(e => e.Artifact, StringComparer.Ordinal) + .ThenBy(e => e.Purl, StringComparer.Ordinal) + .ThenBy(e => e.Version, StringComparer.Ordinal) + .ToList(); + return Task.FromResult>(list); + } + + public Task ClearResolverAsync(CancellationToken cancellationToken) + { + _resolverEvents.Clear(); + return Task.FromResult(true); + } + + public Task PublishResolverAsync(ResolverCandidate candidate, CancellationToken cancellationToken) + { + var key = $"{candidate.TenantId}|{candidate.Artifact}|{candidate.Purl}|{candidate.Version}|{candidate.Scope}|{candidate.RuntimeFlag}"; + var added = _resolverEvents.TryAdd(key, candidate); + return Task.FromResult(added); + } } diff --git a/src/SbomService/StellaOps.SbomService/Services/WatermarkService.cs b/src/SbomService/StellaOps.SbomService/Services/WatermarkService.cs new file mode 100644 index 000000000..d56529294 --- /dev/null +++ b/src/SbomService/StellaOps.SbomService/Services/WatermarkService.cs @@ -0,0 +1,38 @@ +using System.Collections.Concurrent; + +namespace StellaOps.SbomService.Services; + +public sealed record WatermarkState( + string TenantId, + string Watermark, + DateTimeOffset UpdatedAtUtc); + +public interface IWatermarkService +{ + Task GetAsync(string tenantId, CancellationToken cancellationToken); + Task SetAsync(string tenantId, string watermark, CancellationToken cancellationToken); +} + +internal sealed class InMemoryWatermarkService : IWatermarkService +{ + private readonly ConcurrentDictionary _watermarks = new(StringComparer.Ordinal); + + public Task GetAsync(string tenantId, CancellationToken cancellationToken) + { + if (_watermarks.TryGetValue(tenantId, out var state)) + { + return Task.FromResult(state); + } + + var created = new WatermarkState(tenantId, string.Empty, DateTimeOffset.UtcNow); + _watermarks[tenantId] = created; + return Task.FromResult(created); + } + + public Task SetAsync(string tenantId, string watermark, CancellationToken cancellationToken) + { + var state = new WatermarkState(tenantId, watermark, DateTimeOffset.UtcNow); + _watermarks[tenantId] = state; + return Task.FromResult(state); + } +} diff --git a/src/SbomService/TASKS.md b/src/SbomService/TASKS.md index 621eda8d7..52744856f 100644 --- a/src/SbomService/TASKS.md +++ b/src/SbomService/TASKS.md @@ -5,3 +5,10 @@ | PREP-SBOM-CONSOLE-23-001-BUILD-TEST-FAILING-D | DONE | Offline feed cache + script added; see `docs/modules/sbomservice/offline-feed-plan.md`. | 2025-11-20 | | SBOM-SERVICE-21-002 | DONE | `sbom.version.created` events emitted via in-memory publisher; `/internal/sbom/events` + backfill wired; component lookup pagination cursor fixed; tests pass. | 2025-11-23 | | SBOM-SERVICE-21-003 | DONE | Entrypoint/service node API (`GET/POST /entrypoints`) with tenant guard, deterministic ordering, seeded data; tests added. | 2025-11-23 | +| SBOM-SERVICE-23-001 | DONE | LNM v1 projection now returns asset metadata (criticality, owner, environment, exposure flags, tags); fixture + docs updated; projection test covers criticality. | 2025-11-23 | +| SBOM-SERVICE-23-002 | DONE | `sbom.asset.updated` events emitted idempotently (snapshot+tenant+hash) when projections served; `/internal/sbom/asset-events` endpoint added; tests validate idempotency. | 2025-11-23 | +| SBOM-ORCH-32-001 | DONE | In-memory orchestrator source registry (`/internal/orchestrator/sources`) with deterministic seed + idempotent registration. | 2025-11-23 | +| SBOM-ORCH-33-001 | DONE | Orchestrator control signals (pause/throttle/backpressure) exposed via `/internal/orchestrator/control`; metrics emitted. | 2025-11-23 | +| SBOM-ORCH-34-001 | DONE | Watermark tracking endpoints (`/internal/orchestrator/watermarks`) implemented for backfill reconciliation. | 2025-11-23 | +| SBOM-VULN-29-001 | DONE | Inventory evidence emitted (scope/runtime_flag/paths/nearest_safe_version) with `/internal/sbom/inventory` diagnostics + backfill endpoint. | 2025-11-23 | +| SBOM-VULN-29-002 | DONE | Resolver feed candidates emitted with NDJSON export/backfill endpoints; idempotent keys across tenant/artifact/purl/version/scope/runtime_flag. | 2025-11-24 | diff --git a/src/Scanner/StellaOps.Scanner.Worker/Processing/Reachability/ReachabilityBuildStageExecutor.cs b/src/Scanner/StellaOps.Scanner.Worker/Processing/Reachability/ReachabilityBuildStageExecutor.cs new file mode 100644 index 000000000..a04b699f5 --- /dev/null +++ b/src/Scanner/StellaOps.Scanner.Worker/Processing/Reachability/ReachabilityBuildStageExecutor.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.EntryTrace; +using StellaOps.Scanner.Reachability; + +namespace StellaOps.Scanner.Worker.Processing.Reachability; + +/// +/// Builds a reachability union graph from the EntryTrace graph if available. +/// +public sealed class ReachabilityBuildStageExecutor : IScanStageExecutor +{ + private readonly ILogger _logger; + + public ReachabilityBuildStageExecutor(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string StageName => ScanStageNames.ComposeArtifacts; + + public ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + + if (!context.Analysis.TryGet(ScanAnalysisKeys.EntryTraceGraph, out var entryTrace) || entryTrace is null) + { + _logger.LogDebug("No EntryTrace graph present; reachability union graph not built."); + return ValueTask.CompletedTask; + } + + var nodeMap = entryTrace.Nodes.ToDictionary(n => n.Id); + + var unionNodes = new List(entryTrace.Nodes.Length); + foreach (var node in entryTrace.Nodes) + { + var symbolId = ComputeSymbolId("shell", node.DisplayName, node.Kind.ToString()); + var source = node.Evidence is null + ? null + : new ReachabilitySource("static", "entrytrace", node.Evidence.Path); + + unionNodes.Add(new ReachabilityUnionNode( + SymbolId: symbolId, + Lang: "shell", + Kind: node.Kind.ToString().ToLowerInvariant(), + Display: node.DisplayName, + Source: source)); + } + + var unionEdges = new List(entryTrace.Edges.Length); + foreach (var edge in entryTrace.Edges) + { + if (!nodeMap.TryGetValue(edge.FromNodeId, out var fromNode) || !nodeMap.TryGetValue(edge.ToNodeId, out var toNode)) + { + continue; + } + + var fromId = ComputeSymbolId("shell", fromNode.DisplayName, fromNode.Kind.ToString()); + var toId = ComputeSymbolId("shell", toNode.DisplayName, toNode.Kind.ToString()); + unionEdges.Add(new ReachabilityUnionEdge( + From: fromId, + To: toId, + EdgeType: "call", + Confidence: "high", + Source: new ReachabilitySource("static", "entrytrace", edge.Relationship))); + } + + var unionGraph = new ReachabilityUnionGraph(unionNodes, unionEdges); + context.Analysis.Set(ScanAnalysisKeys.ReachabilityUnionGraph, unionGraph); + _logger.LogInformation("Reachability union graph built from EntryTrace: nodes={NodeCount} edges={EdgeCount}", unionNodes.Count, unionEdges.Count); + return ValueTask.CompletedTask; + } + + private static string ComputeSymbolId(string lang, string display, string kind) + { + using var sha = SHA256.Create(); + var input = Encoding.UTF8.GetBytes((display ?? string.Empty) + "|" + (kind ?? string.Empty)); + var hash = sha.ComputeHash(input); + var base64 = Convert.ToBase64String(hash) + .TrimEnd('=') + .Replace('+', '-') + .Replace('/', '_'); + return $"sym:{lang}:{base64}"; + } +} diff --git a/src/Scanner/StellaOps.Scanner.Worker/Processing/Reachability/ReachabilityPublishStageExecutor.cs b/src/Scanner/StellaOps.Scanner.Worker/Processing/Reachability/ReachabilityPublishStageExecutor.cs new file mode 100644 index 000000000..54e42f6d9 --- /dev/null +++ b/src/Scanner/StellaOps.Scanner.Worker/Processing/Reachability/ReachabilityPublishStageExecutor.cs @@ -0,0 +1,43 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.Reachability; + +namespace StellaOps.Scanner.Worker.Processing.Reachability; + +/// +/// Emits reachability union graphs to CAS during the EmitReports stage when present in the analysis store. +/// +public sealed class ReachabilityPublishStageExecutor : IScanStageExecutor +{ + private readonly IReachabilityUnionPublisherService _publisher; + private readonly ILogger _logger; + + public ReachabilityPublishStageExecutor( + IReachabilityUnionPublisherService publisher, + ILogger logger) + { + _publisher = publisher ?? throw new ArgumentNullException(nameof(publisher)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string StageName => ScanStageNames.EmitReports; + + public async ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + + if (!context.Analysis.TryGet(ScanAnalysisKeys.ReachabilityUnionGraph, out var graph) || graph is null) + { + _logger.LogDebug("No reachability union graph present; skipping publish."); + return; + } + + var publishResult = await _publisher.PublishAsync(graph, context.ScanId, cancellationToken).ConfigureAwait(false); + context.Analysis.Set(ScanAnalysisKeys.ReachabilityUnionCas, publishResult); + + _logger.LogInformation("Published reachability union graph to CAS: sha256={Sha} records={Records}", publishResult.Sha256, publishResult.Records); + } +} diff --git a/src/Scanner/StellaOps.Scanner.Worker/Processing/ScanJobProcessor.cs b/src/Scanner/StellaOps.Scanner.Worker/Processing/ScanJobProcessor.cs index 7fb48c5fb..158a1dcbb 100644 --- a/src/Scanner/StellaOps.Scanner.Worker/Processing/ScanJobProcessor.cs +++ b/src/Scanner/StellaOps.Scanner.Worker/Processing/ScanJobProcessor.cs @@ -1,21 +1,28 @@ using System; using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Scanner.Reachability; namespace StellaOps.Scanner.Worker.Processing; public sealed class ScanJobProcessor { - private readonly IReadOnlyDictionary _executors; - private readonly ScanProgressReporter _progressReporter; - private readonly ILogger _logger; - - public ScanJobProcessor(IEnumerable executors, ScanProgressReporter progressReporter, ILogger logger) - { - _progressReporter = progressReporter ?? throw new ArgumentNullException(nameof(progressReporter)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + private readonly IReadOnlyDictionary _executors; + private readonly ScanProgressReporter _progressReporter; + private readonly ILogger _logger; + private readonly IReachabilityUnionPublisherService _reachabilityPublisher; + + public ScanJobProcessor( + IEnumerable executors, + ScanProgressReporter progressReporter, + IReachabilityUnionPublisherService reachabilityPublisher, + ILogger logger) + { + _progressReporter = progressReporter ?? throw new ArgumentNullException(nameof(progressReporter)); + _reachabilityPublisher = reachabilityPublisher ?? throw new ArgumentNullException(nameof(reachabilityPublisher)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); var map = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (var executor in executors ?? Array.Empty()) @@ -42,12 +49,14 @@ public sealed class ScanJobProcessor _executors = map; } - public async ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(context); - - foreach (var stage in ScanStageNames.Ordered) - { + public async ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + // Placeholder: reachability publisher will be fed once lifter outputs are routed here. + _ = _reachabilityPublisher; + + foreach (var stage in ScanStageNames.Ordered) + { cancellationToken.ThrowIfCancellationRequested(); if (!_executors.TryGetValue(stage, out var executor)) @@ -55,11 +64,11 @@ public sealed class ScanJobProcessor continue; } - await _progressReporter.ExecuteStageAsync( - context, - stage, - executor.ExecuteAsync, - cancellationToken).ConfigureAwait(false); - } - } -} + await _progressReporter.ExecuteStageAsync( + context, + stage, + executor.ExecuteAsync, + cancellationToken).ConfigureAwait(false); + } + } +} diff --git a/src/Scanner/StellaOps.Scanner.Worker/Program.cs b/src/Scanner/StellaOps.Scanner.Worker/Program.cs index fa12ee046..c49f35930 100644 --- a/src/Scanner/StellaOps.Scanner.Worker/Program.cs +++ b/src/Scanner/StellaOps.Scanner.Worker/Program.cs @@ -8,6 +8,7 @@ using Microsoft.Extensions.Options; using StellaOps.Auth.Client; using StellaOps.Configuration; using StellaOps.Scanner.Cache; +using StellaOps.Scanner.Reachability; using StellaOps.Scanner.Analyzers.OS.Plugin; using StellaOps.Scanner.Analyzers.Lang.Plugin; using StellaOps.Scanner.EntryTrace; @@ -24,6 +25,7 @@ using StellaOps.Scanner.Worker.Processing; using StellaOps.Scanner.Worker.Processing.Surface; using StellaOps.Scanner.Storage.Extensions; using StellaOps.Scanner.Storage; +using Reachability = StellaOps.Scanner.Worker.Processing.Reachability; var builder = Host.CreateApplicationBuilder(args); @@ -56,6 +58,9 @@ builder.Services.AddSingleton(); builder.Services.AddEntryTraceAnalyzer(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); var storageSection = builder.Configuration.GetSection("ScannerStorage"); var connectionString = storageSection.GetValue("Mongo:ConnectionString"); @@ -78,6 +83,8 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddHostedService(sp => sp.GetRequiredService()); diff --git a/src/Scanner/StellaOps.Scanner.Worker/StellaOps.Scanner.Worker.csproj b/src/Scanner/StellaOps.Scanner.Worker/StellaOps.Scanner.Worker.csproj index e7a2e5652..ed8bf2b14 100644 --- a/src/Scanner/StellaOps.Scanner.Worker/StellaOps.Scanner.Worker.csproj +++ b/src/Scanner/StellaOps.Scanner.Worker/StellaOps.Scanner.Worker.csproj @@ -21,6 +21,7 @@ + diff --git a/src/Scanner/StellaOps.Scanner.sln b/src/Scanner/StellaOps.Scanner.sln index 5331bab16..0d5e87325 100644 --- a/src/Scanner/StellaOps.Scanner.sln +++ b/src/Scanner/StellaOps.Scanner.sln @@ -151,6 +151,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Analyzers EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Analyzers.Lang.Php.Tests", "__Tests\StellaOps.Scanner.Analyzers.Lang.Php.Tests\StellaOps.Scanner.Analyzers.Lang.Php.Tests.csproj", "{F4A239E0-AC66-4105-8423-4805B2029ABE}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Reachability.Tests", "__Tests\StellaOps.Scanner.Reachability.Tests\StellaOps.Scanner.Reachability.Tests.csproj", "{01F66FFA-8399-480E-A463-BB2B456C8814}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Reachability", "__Libraries\StellaOps.Scanner.Reachability\StellaOps.Scanner.Reachability.csproj", "{D31CFFE3-72B3-48D7-A284-710B14380062}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Replay.Core", "..\__Libraries\StellaOps.Replay.Core\StellaOps.Replay.Core.csproj", "{F812FD49-2D45-4503-A367-ABA55153D9B3}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1013,6 +1019,42 @@ Global {F4A239E0-AC66-4105-8423-4805B2029ABE}.Release|x64.Build.0 = Release|Any CPU {F4A239E0-AC66-4105-8423-4805B2029ABE}.Release|x86.ActiveCfg = Release|Any CPU {F4A239E0-AC66-4105-8423-4805B2029ABE}.Release|x86.Build.0 = Release|Any CPU + {01F66FFA-8399-480E-A463-BB2B456C8814}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {01F66FFA-8399-480E-A463-BB2B456C8814}.Debug|Any CPU.Build.0 = Debug|Any CPU + {01F66FFA-8399-480E-A463-BB2B456C8814}.Debug|x64.ActiveCfg = Debug|Any CPU + {01F66FFA-8399-480E-A463-BB2B456C8814}.Debug|x64.Build.0 = Debug|Any CPU + {01F66FFA-8399-480E-A463-BB2B456C8814}.Debug|x86.ActiveCfg = Debug|Any CPU + {01F66FFA-8399-480E-A463-BB2B456C8814}.Debug|x86.Build.0 = Debug|Any CPU + {01F66FFA-8399-480E-A463-BB2B456C8814}.Release|Any CPU.ActiveCfg = Release|Any CPU + {01F66FFA-8399-480E-A463-BB2B456C8814}.Release|Any CPU.Build.0 = Release|Any CPU + {01F66FFA-8399-480E-A463-BB2B456C8814}.Release|x64.ActiveCfg = Release|Any CPU + {01F66FFA-8399-480E-A463-BB2B456C8814}.Release|x64.Build.0 = Release|Any CPU + {01F66FFA-8399-480E-A463-BB2B456C8814}.Release|x86.ActiveCfg = Release|Any CPU + {01F66FFA-8399-480E-A463-BB2B456C8814}.Release|x86.Build.0 = Release|Any CPU + {D31CFFE3-72B3-48D7-A284-710B14380062}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D31CFFE3-72B3-48D7-A284-710B14380062}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D31CFFE3-72B3-48D7-A284-710B14380062}.Debug|x64.ActiveCfg = Debug|Any CPU + {D31CFFE3-72B3-48D7-A284-710B14380062}.Debug|x64.Build.0 = Debug|Any CPU + {D31CFFE3-72B3-48D7-A284-710B14380062}.Debug|x86.ActiveCfg = Debug|Any CPU + {D31CFFE3-72B3-48D7-A284-710B14380062}.Debug|x86.Build.0 = Debug|Any CPU + {D31CFFE3-72B3-48D7-A284-710B14380062}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D31CFFE3-72B3-48D7-A284-710B14380062}.Release|Any CPU.Build.0 = Release|Any CPU + {D31CFFE3-72B3-48D7-A284-710B14380062}.Release|x64.ActiveCfg = Release|Any CPU + {D31CFFE3-72B3-48D7-A284-710B14380062}.Release|x64.Build.0 = Release|Any CPU + {D31CFFE3-72B3-48D7-A284-710B14380062}.Release|x86.ActiveCfg = Release|Any CPU + {D31CFFE3-72B3-48D7-A284-710B14380062}.Release|x86.Build.0 = Release|Any CPU + {F812FD49-2D45-4503-A367-ABA55153D9B3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F812FD49-2D45-4503-A367-ABA55153D9B3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F812FD49-2D45-4503-A367-ABA55153D9B3}.Debug|x64.ActiveCfg = Debug|Any CPU + {F812FD49-2D45-4503-A367-ABA55153D9B3}.Debug|x64.Build.0 = Debug|Any CPU + {F812FD49-2D45-4503-A367-ABA55153D9B3}.Debug|x86.ActiveCfg = Debug|Any CPU + {F812FD49-2D45-4503-A367-ABA55153D9B3}.Debug|x86.Build.0 = Debug|Any CPU + {F812FD49-2D45-4503-A367-ABA55153D9B3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F812FD49-2D45-4503-A367-ABA55153D9B3}.Release|Any CPU.Build.0 = Release|Any CPU + {F812FD49-2D45-4503-A367-ABA55153D9B3}.Release|x64.ActiveCfg = Release|Any CPU + {F812FD49-2D45-4503-A367-ABA55153D9B3}.Release|x64.Build.0 = Release|Any CPU + {F812FD49-2D45-4503-A367-ABA55153D9B3}.Release|x86.ActiveCfg = Release|Any CPU + {F812FD49-2D45-4503-A367-ABA55153D9B3}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1062,5 +1104,7 @@ Global {E0104A8E-2C39-48C1-97EC-66C171310944} = {56BCE1BF-7CBA-7CE8-203D-A88051F1D642} {0262C376-6C43-4A69-86EA-74C228BC0F36} = {41F15E67-7190-CF23-3BC4-77E87134CADD} {F4A239E0-AC66-4105-8423-4805B2029ABE} = {56BCE1BF-7CBA-7CE8-203D-A88051F1D642} + {01F66FFA-8399-480E-A463-BB2B456C8814} = {56BCE1BF-7CBA-7CE8-203D-A88051F1D642} + {D31CFFE3-72B3-48D7-A284-710B14380062} = {41F15E67-7190-CF23-3BC4-77E87134CADD} EndGlobalSection EndGlobal diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Java/JavaLanguageAnalyzer.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Java/JavaLanguageAnalyzer.cs index fe47af5cf..f708f3602 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Java/JavaLanguageAnalyzer.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Java/JavaLanguageAnalyzer.cs @@ -386,8 +386,8 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer LanguageEvidenceKind.File, evidenceSource, locator, - value: null, - sha256: null)); + null, + null)); } private static void AddConfigHint( @@ -412,8 +412,8 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer LanguageEvidenceKind.File, "framework-config", locator, - value: null, - sha256: sha256)); + null, + sha256)); } private static string? TryComputeSha256(JavaArchive archive, JavaArchiveEntry entry) @@ -585,37 +585,45 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer string? version = null; string? vendor = null; - while (await reader.ReadLineAsync().ConfigureAwait(false) is { } line) - { - cancellationToken.ThrowIfCancellationRequested(); - - if (string.IsNullOrWhiteSpace(line)) - { - continue; - } - - var separatorIndex = line.IndexOf(':'); - if (separatorIndex <= 0) - { - continue; - } - - var key = line[..separatorIndex].Trim(); - var value = line[(separatorIndex + 1)..].Trim(); - - if (key.Equals("Implementation-Title", StringComparison.OrdinalIgnoreCase)) - { - title ??= value; - } - else if (key.Equals("Implementation-Version", StringComparison.OrdinalIgnoreCase)) - { - version ??= value; - } - else if (key.Equals("Implementation-Vendor", StringComparison.OrdinalIgnoreCase)) - { - vendor ??= value; + while (await reader.ReadLineAsync().ConfigureAwait(false) is { } line) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + var separatorIndex = line.IndexOf(':'); + if (separatorIndex <= 0) + { + continue; + } + + var key = line[..separatorIndex].Trim(); + var value = line[(separatorIndex + 1)..].Trim(); + + if (key.Equals("Implementation-Title", StringComparison.OrdinalIgnoreCase)) + { + title ??= value; + } + else if (key.Equals("Implementation-Version", StringComparison.OrdinalIgnoreCase)) + { + version ??= value; + } + else if (key.Equals("Implementation-Vendor", StringComparison.OrdinalIgnoreCase)) + { + vendor ??= value; + } + } + + if (title is null && version is null && vendor is null) + { + return null; + } + + return new ManifestMetadata(title, version, vendor); } -} internal sealed record FrameworkConfigSummary( IReadOnlyDictionary Metadata, @@ -624,13 +632,6 @@ internal sealed record FrameworkConfigSummary( internal sealed record JniHintSummary( IReadOnlyDictionary Metadata, IReadOnlyCollection Evidence); - if (title is null && version is null && vendor is null) - { - return null; - } - - return new ManifestMetadata(title, version, vendor); - } private static string BuildPurl(string groupId, string artifactId, string version, string? packaging) { diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeImportWalker.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeImportWalker.cs index 355d60679..33d2587c7 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeImportWalker.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeImportWalker.cs @@ -1,5 +1,3 @@ -using System.Text.Json; -using System.Text.Json.Nodes; using Esprima; using Esprima.Ast; using EsprimaNode = Esprima.Ast.Node; @@ -19,12 +17,8 @@ internal static class NodeImportWalker Script script; try { - script = new JavaScriptParser(content, new ParserOptions - { - Tolerant = true, - AdaptRegexp = true, - Source = sourcePath - }).ParseScript(); + var parser = new JavaScriptParser(); + script = parser.ParseScript(content, sourcePath, true); } catch (ParserException) { @@ -43,13 +37,13 @@ internal static class NodeImportWalker switch (node) { case ImportDeclaration importDecl when !string.IsNullOrWhiteSpace(importDecl.Source?.StringValue): - edges.Add(new NodeImportEdge(sourcePath, importDecl.Source.StringValue!, "import", BuildEvidence(importDecl.Loc))); + edges.Add(new NodeImportEdge(sourcePath, importDecl.Source.StringValue!, "import", string.Empty)); break; case CallExpression call when IsRequire(call) && call.Arguments.FirstOrDefault() is Literal { Value: string target }: - edges.Add(new NodeImportEdge(sourcePath, target, "require", BuildEvidence(call.Loc))); + edges.Add(new NodeImportEdge(sourcePath, target, "require", string.Empty)); break; case ImportExpression importExp when importExp.Source is Literal { Value: string importTarget }: - edges.Add(new NodeImportEdge(sourcePath, importTarget, "import()", BuildEvidence(importExp.Loc))); + edges.Add(new NodeImportEdge(sourcePath, importTarget, "import()", string.Empty)); break; } @@ -64,29 +58,4 @@ internal static class NodeImportWalker return call.Callee is Identifier id && string.Equals(id.Name, "require", StringComparison.Ordinal) && call.Arguments.Count == 1 && call.Arguments[0] is Literal { Value: string }; } - - private static string BuildEvidence(Location? loc) - { - if (loc is null) - { - return string.Empty; - } - - var json = new JsonObject - { - ["start"] = BuildPosition(loc.Start), - ["end"] = BuildPosition(loc.End) - }; - - return json.ToJsonString(new JsonSerializerOptions { WriteIndented = false }); - } - - private static JsonObject BuildPosition(Position pos) - { - return new JsonObject - { - ["line"] = pos.Line, - ["column"] = pos.Column - }; - } } diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodePackage.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodePackage.cs index 14a9f2b4a..7c6cfe22e 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodePackage.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodePackage.cs @@ -145,17 +145,17 @@ internal sealed class NodePackage "package.json:entrypoint", locator, content, - sha256: null)); + null)); } foreach (var importEdge in _imports.OrderBy(static e => e.ComparisonKey, StringComparer.Ordinal)) { evidence.Add(new LanguageComponentEvidence( - LanguageEvidenceKind.Source, + LanguageEvidenceKind.File, "node.import", importEdge.SourceFile, importEdge.TargetSpecifier, - sha256: null)); + null)); } return evidence diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodePackageCollector.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodePackageCollector.cs index 20c5892bc..0b70bdb45 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodePackageCollector.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodePackageCollector.cs @@ -509,7 +509,8 @@ internal static class NodePackageCollector var lockLocator = BuildLockLocator(lockEntry); var lockSource = lockEntry?.Source; - var isWorkspaceMember = workspaceIndex?.TryGetMember(relativeDirectory, out var workspaceRoot) == true; + string? workspaceRoot = null; + var isWorkspaceMember = workspaceIndex?.TryGetMember(relativeDirectory, out workspaceRoot) == true; var workspaceRootValue = isWorkspaceMember && workspaceIndex is not null ? workspaceRoot : null; var workspaceTargets = workspaceIndex is null ? Array.Empty() : ExtractWorkspaceTargets(relativeDirectory, root, workspaceIndex); var workspaceLink = workspaceIndex is not null && !isWorkspaceMember && workspaceIndex.TryGetWorkspacePathByName(name, out var workspacePathByName) diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Core/Contracts/ScanAnalysisKeys.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Core/Contracts/ScanAnalysisKeys.cs index 93a5812c0..0e73560ca 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Core/Contracts/ScanAnalysisKeys.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Core/Contracts/ScanAnalysisKeys.cs @@ -25,4 +25,7 @@ public static class ScanAnalysisKeys public const string DenoRuntimePayload = "analysis.lang.deno.runtime"; public const string RubyObservationPayload = "analysis.lang.ruby.observation"; + + public const string ReachabilityUnionGraph = "analysis.reachability.union.graph"; + public const string ReachabilityUnionCas = "analysis.reachability.union.cas"; } diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityGraphBuilder.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityGraphBuilder.cs index ef18280e8..735c50813 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityGraphBuilder.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityGraphBuilder.cs @@ -54,6 +54,21 @@ public sealed class ReachabilityGraphBuilder return JsonSerializer.Serialize(payload, options); } + public ReachabilityUnionGraph ToUnionGraph(string language) + { + ArgumentException.ThrowIfNullOrWhiteSpace(language); + + var nodeList = nodes + .Select(id => new ReachabilityUnionNode(id, language, "symbol")) + .ToList(); + + var edgeList = edges + .Select(edge => new ReachabilityUnionEdge(edge.From, edge.To, edge.Kind)) + .ToList(); + + return new ReachabilityUnionGraph(nodeList, edgeList); + } + public static ReachabilityGraphBuilder FromFixture(string variantPath) { ArgumentException.ThrowIfNullOrWhiteSpace(variantPath); diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityUnionPublisher.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityUnionPublisher.cs new file mode 100644 index 000000000..1291434e1 --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityUnionPublisher.cs @@ -0,0 +1,82 @@ +using System; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Scanner.Cache.Abstractions; + +namespace StellaOps.Scanner.Reachability; + +/// +/// Packages a reachability union graph into a deterministic zip, stores it in CAS, and returns the CAS reference. +/// +public sealed class ReachabilityUnionPublisher +{ + private readonly ReachabilityUnionWriter writer; + + public ReachabilityUnionPublisher(ReachabilityUnionWriter writer) + { + this.writer = writer ?? throw new ArgumentNullException(nameof(writer)); + } + + public async Task PublishAsync( + ReachabilityUnionGraph graph, + IFileContentAddressableStore cas, + string workRoot, + string analysisId, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(graph); + ArgumentNullException.ThrowIfNull(cas); + ArgumentException.ThrowIfNullOrWhiteSpace(workRoot); + ArgumentException.ThrowIfNullOrWhiteSpace(analysisId); + + var result = await writer.WriteAsync(graph, workRoot, analysisId, cancellationToken).ConfigureAwait(false); + + var folder = Path.GetDirectoryName(result.MetaPath)!; + var zipPath = Path.Combine(folder, "reachability.zip"); + CreateZip(folder, zipPath); + + var sha = ComputeSha256(zipPath); + await using var zipStream = File.OpenRead(zipPath); + var casEntry = await cas.PutAsync(new FileCasPutRequest(sha, zipStream, leaveOpen: false), cancellationToken).ConfigureAwait(false); + + return new ReachabilityUnionPublishResult( + Sha256: sha, + RelativePath: casEntry.RelativePath, + Records: result.Nodes.RecordCount + result.Edges.RecordCount + (result.Facts?.RecordCount ?? 0)); + } + + private static void CreateZip(string sourceDir, string destinationZip) + { + if (File.Exists(destinationZip)) + { + File.Delete(destinationZip); + } + + var files = Directory.EnumerateFiles(sourceDir, "*", SearchOption.TopDirectoryOnly) + .OrderBy(f => f, StringComparer.Ordinal) + .ToList(); + + using var zip = ZipFile.Open(destinationZip, ZipArchiveMode.Create); + foreach (var file in files) + { + var entryName = Path.GetFileName(file); + zip.CreateEntryFromFile(file, entryName, CompressionLevel.Optimal); + } + } + + private static string ComputeSha256(string path) + { + using var sha = SHA256.Create(); + using var stream = File.OpenRead(path); + return Convert.ToHexString(sha.ComputeHash(stream)).ToLowerInvariant(); + } +} + +public sealed record ReachabilityUnionPublishResult( + string Sha256, + string RelativePath, + int Records); diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityUnionPublisherService.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityUnionPublisherService.cs new file mode 100644 index 000000000..b0e0d06b5 --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityUnionPublisherService.cs @@ -0,0 +1,40 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Scanner.Cache.Abstractions; +using StellaOps.Scanner.Surface.Env; + +namespace StellaOps.Scanner.Reachability; + +public interface IReachabilityUnionPublisherService +{ + Task PublishAsync(ReachabilityUnionGraph graph, string analysisId, CancellationToken cancellationToken = default); +} + +/// +/// Default service that writes a union graph to CAS using the worker surface cache root. +/// +public sealed class ReachabilityUnionPublisherService : IReachabilityUnionPublisherService +{ + private readonly ISurfaceEnvironment environment; + private readonly IFileContentAddressableStore cas; + private readonly ReachabilityUnionPublisher publisher; + + public ReachabilityUnionPublisherService( + ISurfaceEnvironment environment, + IFileContentAddressableStore cas, + ReachabilityUnionPublisher publisher) + { + this.environment = environment ?? throw new ArgumentNullException(nameof(environment)); + this.cas = cas ?? throw new ArgumentNullException(nameof(cas)); + this.publisher = publisher ?? throw new ArgumentNullException(nameof(publisher)); + } + + public Task PublishAsync(ReachabilityUnionGraph graph, string analysisId, CancellationToken cancellationToken = default) + { + var workRoot = Path.Combine(environment.Settings.CacheRoot.FullName, "reachability"); + Directory.CreateDirectory(workRoot); + return publisher.PublishAsync(graph, cas, workRoot, analysisId, cancellationToken); + } +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityUnionSchemas.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityUnionSchemas.cs new file mode 100644 index 000000000..fd9532b58 --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityUnionSchemas.cs @@ -0,0 +1,6 @@ +namespace StellaOps.Scanner.Reachability; + +public static class ReachabilityUnionSchemas +{ + public const string UnionSchema = "reachability-union@0.1"; +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityUnionWriter.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityUnionWriter.cs new file mode 100644 index 000000000..8710cb39c --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityUnionWriter.cs @@ -0,0 +1,390 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Scanner.Reachability; + +/// +/// Serializes reachability graphs (static + runtime) into the union NDJSON layout +/// described in docs/reachability/runtime-static-union-schema.md. +/// +public sealed class ReachabilityUnionWriter +{ + private static readonly JsonWriterOptions JsonOptions = new() + { + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + Indented = false, + SkipValidation = false + }; + + public async Task WriteAsync( + ReachabilityUnionGraph graph, + string outputRoot, + string analysisId, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(graph); + ArgumentException.ThrowIfNullOrWhiteSpace(outputRoot); + ArgumentException.ThrowIfNullOrWhiteSpace(analysisId); + + var root = Path.Combine(outputRoot, "reachability_graphs", analysisId); + Directory.CreateDirectory(root); + + var normalized = Normalize(graph); + + var nodesPath = Path.Combine(root, "nodes.ndjson"); + var edgesPath = Path.Combine(root, "edges.ndjson"); + var factsPath = Path.Combine(root, "facts_runtime.ndjson"); + var metaPath = Path.Combine(root, "meta.json"); + + var nodesInfo = await WriteNdjsonAsync(nodesPath, normalized.Nodes, WriteNodeAsync, cancellationToken).ConfigureAwait(false); + var edgesInfo = await WriteNdjsonAsync(edgesPath, normalized.Edges, WriteEdgeAsync, cancellationToken).ConfigureAwait(false); + FileHashInfo? factsInfo = null; + + if (normalized.RuntimeFacts.Count > 0) + { + factsInfo = await WriteNdjsonAsync(factsPath, normalized.RuntimeFacts, WriteRuntimeFactAsync, cancellationToken).ConfigureAwait(false); + } + else if (File.Exists(factsPath)) + { + File.Delete(factsPath); + } + + await WriteMetaAsync(metaPath, nodesInfo, edgesInfo, factsInfo, cancellationToken).ConfigureAwait(false); + + return new ReachabilityUnionWriteResult(nodesInfo.ToPublic(), edgesInfo.ToPublic(), factsInfo?.ToPublic(), metaPath); + } + + private static NormalizedGraph Normalize(ReachabilityUnionGraph graph) + { + var nodes = graph.Nodes + .Where(n => !string.IsNullOrWhiteSpace(n.SymbolId)) + .Select(n => n with + { + SymbolId = Trim(n.SymbolId) ?? string.Empty, + Lang = Trim(n.Lang) ?? string.Empty, + Kind = Trim(n.Kind) ?? string.Empty, + Display = Trim(n.Display), + Source = n.Source?.Trimmed(), + Attributes = (n.Attributes ?? ImmutableDictionary.Empty) + .Where(kv => !string.IsNullOrWhiteSpace(kv.Key) && kv.Value is not null) + .ToImmutableSortedDictionary(kv => kv.Key.Trim(), kv => kv.Value!.Trim()) + }) + .OrderBy(n => n.SymbolId, StringComparer.Ordinal) + .ThenBy(n => n.Kind, StringComparer.Ordinal) + .ToList(); + + var edges = graph.Edges + .Where(e => !string.IsNullOrWhiteSpace(e.From) && !string.IsNullOrWhiteSpace(e.To)) + .Select(e => e with + { + From = Trim(e.From)!, + To = Trim(e.To)!, + EdgeType = Trim(e.EdgeType) ?? "call", + Confidence = Trim(e.Confidence) ?? "certain", + Source = e.Source?.Trimmed() + }) + .OrderBy(e => e.From, StringComparer.Ordinal) + .ThenBy(e => e.To, StringComparer.Ordinal) + .ThenBy(e => e.EdgeType, StringComparer.Ordinal) + .ToList(); + + var facts = (graph.RuntimeFacts ?? Enumerable.Empty()) + .Where(f => !string.IsNullOrWhiteSpace(f.SymbolId)) + .Select(f => f with + { + SymbolId = Trim(f.SymbolId)!, + Samples = f.Samples?.Trimmed() ?? new ReachabilityRuntimeSamples(0, null, null), + Env = f.Env?.Trimmed() ?? ReachabilityRuntimeEnv.Empty + }) + .OrderBy(f => f.SymbolId, StringComparer.Ordinal) + .ToList(); + + return new NormalizedGraph(nodes, edges, facts); + } + + private static async Task WriteNdjsonAsync( + string path, + IReadOnlyCollection items, + Func writer, + CancellationToken cancellationToken) + { + await using (var stream = File.Create(path)) + await using (var textWriter = new StreamWriter(stream, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false))) + { + foreach (var item in items) + { + await writer(item, textWriter).ConfigureAwait(false); + await textWriter.WriteLineAsync().ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + } + } + + var sha = ComputeSha256(path); + return new FileHashInfo(path, sha, items.Count); + } + + private static async Task WriteNodeAsync(ReachabilityUnionNode node, StreamWriter writer) + { + await using var json = new MemoryStream(); + await using (var jw = new Utf8JsonWriter(json, JsonOptions)) + { + jw.WriteStartObject(); + jw.WriteString("symbol_id", node.SymbolId); + jw.WriteString("lang", node.Lang); + jw.WriteString("kind", node.Kind); + if (!string.IsNullOrWhiteSpace(node.Display)) + { + jw.WriteString("display", node.Display); + } + + if (node.Source is not null) + { + jw.WritePropertyName("source"); + WriteSource(jw, node.Source); + } + + if (node.Attributes is not null && node.Attributes.Count > 0) + { + jw.WritePropertyName("attributes"); + jw.WriteStartObject(); + foreach (var kv in node.Attributes) + { + jw.WriteString(kv.Key, kv.Value); + } + + jw.WriteEndObject(); + } + + jw.WriteEndObject(); + } + + await writer.WriteAsync(Encoding.UTF8.GetString(json.ToArray())).ConfigureAwait(false); + } + + private static async Task WriteEdgeAsync(ReachabilityUnionEdge edge, StreamWriter writer) + { + await using var json = new MemoryStream(); + await using (var jw = new Utf8JsonWriter(json, JsonOptions)) + { + jw.WriteStartObject(); + jw.WriteString("from", edge.From); + jw.WriteString("to", edge.To); + jw.WriteString("edge_type", edge.EdgeType); + jw.WriteString("confidence", edge.Confidence); + + if (edge.Source is not null) + { + jw.WritePropertyName("source"); + WriteSource(jw, edge.Source); + } + + jw.WriteEndObject(); + } + + await writer.WriteAsync(Encoding.UTF8.GetString(json.ToArray())).ConfigureAwait(false); + } + + private static async Task WriteRuntimeFactAsync(ReachabilityRuntimeFact fact, StreamWriter writer) + { + await using var json = new MemoryStream(); + await using (var jw = new Utf8JsonWriter(json, JsonOptions)) + { + jw.WriteStartObject(); + jw.WriteString("symbol_id", fact.SymbolId); + + jw.WritePropertyName("samples"); + jw.WriteStartObject(); + jw.WriteNumber("call_count", fact.Samples?.CallCount ?? 0); + if (fact.Samples?.FirstSeenUtc is not null) + { + jw.WriteString("first_seen_utc", fact.Samples.FirstSeenUtc.Value.ToUniversalTime().ToString("O")); + } + if (fact.Samples?.LastSeenUtc is not null) + { + jw.WriteString("last_seen_utc", fact.Samples.LastSeenUtc.Value.ToUniversalTime().ToString("O")); + } + jw.WriteEndObject(); + + jw.WritePropertyName("env"); + jw.WriteStartObject(); + if (fact.Env?.Pid is not null) + { + jw.WriteNumber("pid", fact.Env.Pid.Value); + } + if (!string.IsNullOrWhiteSpace(fact.Env?.Image)) + { + jw.WriteString("image", fact.Env!.Image); + } + if (!string.IsNullOrWhiteSpace(fact.Env?.Entrypoint)) + { + jw.WriteString("entrypoint", fact.Env!.Entrypoint); + } + if (fact.Env?.Tags is { Count: > 0 }) + { + jw.WritePropertyName("tags"); + jw.WriteStartArray(); + foreach (var tag in fact.Env!.Tags) + { + jw.WriteStringValue(tag); + } + jw.WriteEndArray(); + } + jw.WriteEndObject(); + + jw.WriteEndObject(); + } + + await writer.WriteAsync(Encoding.UTF8.GetString(json.ToArray())).ConfigureAwait(false); + } + + private static void WriteSource(Utf8JsonWriter jw, ReachabilitySource source) + { + jw.WriteStartObject(); + jw.WriteString("origin", source.Origin ?? "static"); + if (!string.IsNullOrWhiteSpace(source.Provenance)) + { + jw.WriteString("provenance", source.Provenance); + } + if (!string.IsNullOrWhiteSpace(source.Evidence)) + { + jw.WriteString("evidence", source.Evidence); + } + jw.WriteEndObject(); + } + + private static string ComputeSha256(string path) + { + using var sha = SHA256.Create(); + using var stream = File.OpenRead(path); + var hash = sha.ComputeHash(stream); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + + private static string? Trim(string? value) => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); + + private sealed record FileHashInfo(string Path, string Sha256, int RecordCount) + { + public ReachabilityUnionFileInfo ToPublic() => new(Path, Sha256, RecordCount); + } + + private sealed record NormalizedGraph( + IReadOnlyList Nodes, + IReadOnlyList Edges, + IReadOnlyList RuntimeFacts); + + private static async Task WriteMetaAsync( + string path, + FileHashInfo nodes, + FileHashInfo edges, + FileHashInfo? facts, + CancellationToken cancellationToken) + { + await using var stream = File.Create(path); + await using var writer = new Utf8JsonWriter(stream, new JsonWriterOptions { Indented = true }); + + writer.WriteStartObject(); + writer.WriteString("schema", "reachability-union@0.1"); + writer.WriteString("generated_at", DateTimeOffset.UtcNow.ToString("O")); + writer.WritePropertyName("files"); + writer.WriteStartArray(); + WriteMetaFile(writer, nodes); + WriteMetaFile(writer, edges); + if (facts is not null) + { + WriteMetaFile(writer, facts); + } + writer.WriteEndArray(); + writer.WriteEndObject(); + await writer.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + private static void WriteMetaFile(Utf8JsonWriter writer, FileHashInfo info) + { + writer.WriteStartObject(); + writer.WriteString("path", info.Path); + writer.WriteString("sha256", info.Sha256); + writer.WriteNumber("records", info.RecordCount); + writer.WriteEndObject(); + } +} + +public sealed record ReachabilityUnionGraph( + IReadOnlyCollection Nodes, + IReadOnlyCollection Edges, + IReadOnlyCollection? RuntimeFacts = null); + +public sealed record ReachabilityUnionNode( + string SymbolId, + string Lang, + string Kind, + string? Display = null, + ReachabilitySource? Source = null, + IReadOnlyDictionary? Attributes = null); + +public sealed record ReachabilityUnionEdge( + string From, + string To, + string EdgeType, + string? Confidence = "certain", + ReachabilitySource? Source = null); + +public sealed record ReachabilityRuntimeFact( + string SymbolId, + ReachabilityRuntimeSamples? Samples, + ReachabilityRuntimeEnv? Env); + +public sealed record ReachabilityRuntimeSamples( + long CallCount, + DateTimeOffset? FirstSeenUtc, + DateTimeOffset? LastSeenUtc) +{ + public ReachabilityRuntimeSamples Trimmed() + => new(CallCount, FirstSeenUtc?.ToUniversalTime(), LastSeenUtc?.ToUniversalTime()); +} + +public sealed record ReachabilityRuntimeEnv( + int? Pid, + string? Image, + string? Entrypoint, + IReadOnlyList Tags) +{ + public static ReachabilityRuntimeEnv Empty { get; } = new(null, null, null, Array.Empty()); + + public ReachabilityRuntimeEnv Trimmed() + => new( + Pid, + string.IsNullOrWhiteSpace(Image) ? null : Image.Trim(), + string.IsNullOrWhiteSpace(Entrypoint) ? null : Entrypoint.Trim(), + (Tags ?? Array.Empty()).Where(t => !string.IsNullOrWhiteSpace(t)).Select(t => t.Trim()).OrderBy(t => t, StringComparer.Ordinal).ToArray()); +} + +public sealed record ReachabilitySource( + string? Origin, + string? Provenance, + string? Evidence) +{ + public ReachabilitySource Trimmed() + => new( + string.IsNullOrWhiteSpace(Origin) ? "static" : Origin.Trim(), + string.IsNullOrWhiteSpace(Provenance) ? null : Provenance.Trim(), + string.IsNullOrWhiteSpace(Evidence) ? null : Evidence.Trim()); +} + +public sealed record ReachabilityUnionWriteResult( + ReachabilityUnionFileInfo Nodes, + ReachabilityUnionFileInfo Edges, + ReachabilityUnionFileInfo? Facts, + string MetaPath); + +public sealed record ReachabilityUnionFileInfo( + string Path, + string Sha256, + int RecordCount); diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Reachability/StellaOps.Scanner.Reachability.csproj b/src/Scanner/__Libraries/StellaOps.Scanner.Reachability/StellaOps.Scanner.Reachability.csproj index 47c129a73..b3e853863 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Reachability/StellaOps.Scanner.Reachability.csproj +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Reachability/StellaOps.Scanner.Reachability.csproj @@ -5,9 +5,8 @@ enable - - - + + diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/Fakes/FakeFileContentAddressableStore.cs b/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/Fakes/FakeFileContentAddressableStore.cs new file mode 100644 index 000000000..5afdc59b6 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/Fakes/FakeFileContentAddressableStore.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections.Concurrent; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Scanner.Cache.Abstractions; + +namespace StellaOps.Scanner.Core.Tests.Fakes; + +internal sealed class FakeFileContentAddressableStore : IFileContentAddressableStore +{ + private readonly ConcurrentDictionary store = new(); + + public ValueTask TryGetAsync(string sha256, CancellationToken cancellationToken = default) + { + if (store.TryGetValue(sha256, out var bytes)) + { + return ValueTask.FromResult(new FileCasEntry(sha256, bytes.LongLength, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, sha256 + ".zip")); + } + + return ValueTask.FromResult(null); + } + + public Task PutAsync(FileCasPutRequest request, CancellationToken cancellationToken = default) + { + using var ms = new MemoryStream(); + request.Content.CopyTo(ms); + store[request.Sha256] = ms.ToArray(); + return Task.FromResult(new FileCasEntry(request.Sha256, ms.Length, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, request.Sha256 + ".zip")); + } + + public Task RemoveAsync(string sha256, CancellationToken cancellationToken = default) + { + return Task.FromResult(store.TryRemove(sha256, out _)); + } + + public Task EvictExpiredAsync(CancellationToken cancellationToken = default) => Task.FromResult(0); + public Task ExportAsync(string destinationDirectory, CancellationToken cancellationToken = default) => Task.FromResult(0); + public Task ImportAsync(string sourceDirectory, CancellationToken cancellationToken = default) => Task.FromResult(0); + public Task CompactAsync(CancellationToken cancellationToken = default) => Task.FromResult(0); +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/ReachabilityGraphBuilderUnionTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/ReachabilityGraphBuilderUnionTests.cs new file mode 100644 index 000000000..5b06b8e3e --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/ReachabilityGraphBuilderUnionTests.cs @@ -0,0 +1,39 @@ +using System.Threading.Tasks; +using StellaOps.Scanner.Reachability; +using Xunit; + +namespace StellaOps.Scanner.Core.Tests; + +public class ReachabilityGraphBuilderUnionTests +{ + [Fact] + public async Task ConvertsBuilderToUnionGraphAndWritesNdjson() + { + var builder = new ReachabilityGraphBuilder() + .AddNode("sym:dotnet:A") + .AddNode("sym:dotnet:B") + .AddEdge("sym:dotnet:A", "sym:dotnet:B", "call"); + + var graph = builder.ToUnionGraph("dotnet"); + var writer = new ReachabilityUnionWriter(); + + using var temp = new TempDir(); + var result = await writer.WriteAsync(graph, temp.Path, "analysis-graph-1"); + + Assert.Equal(2, result.Nodes.RecordCount); + Assert.Equal(1, result.Edges.RecordCount); + Assert.True(System.IO.File.Exists(result.MetaPath)); + } + + private sealed class TempDir : System.IDisposable + { + public string Path { get; } = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "reach-union-" + System.Guid.NewGuid().ToString("N")); + + public TempDir() => System.IO.Directory.CreateDirectory(Path); + + public void Dispose() + { + try { System.IO.Directory.Delete(Path, recursive: true); } catch { /* ignore */ } + } + } +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/ReachabilityUnionPublisherTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/ReachabilityUnionPublisherTests.cs new file mode 100644 index 000000000..c29c4fff8 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/ReachabilityUnionPublisherTests.cs @@ -0,0 +1,30 @@ +using System.Threading.Tasks; +using StellaOps.Scanner.Core.Tests.Fakes; +using StellaOps.Scanner.Reachability; +using Xunit; + +namespace StellaOps.Scanner.Core.Tests; + +public class ReachabilityUnionPublisherTests +{ + [Fact] + public async Task PublishesZipToCas() + { + var graph = new ReachabilityUnionGraph( + Nodes: new[] { new ReachabilityUnionNode("sym:dotnet:A", "dotnet", "method") }, + Edges: new ReachabilityUnionEdge[0]); + + var cas = new FakeFileContentAddressableStore(); + using var temp = new TempDir(); + var publisher = new ReachabilityUnionPublisher(new ReachabilityUnionWriter()); + + var result = await publisher.PublishAsync(graph, cas, temp.Path, "analysis-pub-1"); + + Assert.False(string.IsNullOrWhiteSpace(result.Sha256)); + Assert.Equal(1, result.Records); + + var entry = await cas.TryGetAsync(result.Sha256); + Assert.NotNull(entry); + Assert.True(entry!.Value.SizeBytes > 0); + } +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/ReachabilityUnionWriterTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/ReachabilityUnionWriterTests.cs new file mode 100644 index 000000000..32143967f --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/ReachabilityUnionWriterTests.cs @@ -0,0 +1,84 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using StellaOps.Scanner.Reachability; +using Xunit; + +namespace StellaOps.Scanner.Core.Tests; + +public class ReachabilityUnionWriterTests +{ + [Fact] + public async Task WritesDeterministicFilesAndHashes() + { + var writer = new ReachabilityUnionWriter(); + using var temp = new TempDir(); + + var graph = new ReachabilityUnionGraph( + Nodes: new[] + { + new ReachabilityUnionNode("sym:dotnet:B", "dotnet", "method", display: "B"), + new ReachabilityUnionNode("sym:dotnet:A", "dotnet", "method", display: "A", + Source: new ReachabilitySource("static", "il", "file.cs:10"), + Attributes: new Dictionary { { "visibility", "public" } }), + }, + Edges: new[] + { + new ReachabilityUnionEdge("sym:dotnet:B", "sym:dotnet:A", "call", confidence: "high"), + new ReachabilityUnionEdge("sym:dotnet:A", "sym:dotnet:B", "call", confidence: "high"), + }, + RuntimeFacts: new[] + { + new ReachabilityRuntimeFact( + "sym:dotnet:A", + new ReachabilityRuntimeSamples(2, DateTimeOffset.Parse("2025-11-20T12:00:00Z"), DateTimeOffset.Parse("2025-11-20T12:00:02Z")), + new ReachabilityRuntimeEnv(1234, "sha256:deadbeef", "Program.Main", new [] {"sealed", "offline"})) + }); + + var result = await writer.WriteAsync(graph, temp.Path, "analysis-1"); + + // Files exist + Assert.True(File.Exists(result.Nodes.Path)); + Assert.True(File.Exists(result.Edges.Path)); + Assert.NotNull(result.Facts); + Assert.True(File.Exists(result.MetaPath)); + + // Nodes sorted by symbol_id + var nodeLines = await File.ReadAllLinesAsync(result.Nodes.Path); + Assert.Equal(2, nodeLines.Length); + Assert.Contains("sym:dotnet:A", nodeLines[0]); + Assert.Contains("sym:dotnet:B", nodeLines[1]); + + // Hashes recorded in meta match content + var meta = await JsonDocument.ParseAsync(File.OpenRead(result.MetaPath)); + var files = meta.RootElement.GetProperty("files").EnumerateArray().ToList(); + Assert.Contains(files, f => f.GetProperty("path").GetString() == result.Nodes.Path && f.GetProperty("sha256").GetString() == result.Nodes.Sha256); + Assert.Contains(files, f => f.GetProperty("path").GetString() == result.Edges.Path && f.GetProperty("sha256").GetString() == result.Edges.Sha256); + + // Determinism: re-run with shuffled inputs yields identical hashes + var shuffled = new ReachabilityUnionGraph( + Nodes: graph.Nodes.Reverse().ToArray(), + Edges: graph.Edges.Reverse().ToArray(), + RuntimeFacts: graph.RuntimeFacts); + + var second = await writer.WriteAsync(shuffled, temp.Path, "analysis-1"); + Assert.Equal(result.Nodes.Sha256, second.Nodes.Sha256); + Assert.Equal(result.Edges.Sha256, second.Edges.Sha256); + Assert.Equal(result.Facts!.Sha256, second.Facts!.Sha256); + } + + private sealed class TempDir : IDisposable + { + public string Path { get; } = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "reach-union-" + Guid.NewGuid().ToString("N")); + + public TempDir() => Directory.CreateDirectory(Path); + + public void Dispose() + { + try { Directory.Delete(Path, recursive: true); } catch { /* best effort */ } + } + } +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/StellaOps.Scanner.Core.Tests.csproj b/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/StellaOps.Scanner.Core.Tests.csproj index 81f5181f6..14699188d 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/StellaOps.Scanner.Core.Tests.csproj +++ b/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/StellaOps.Scanner.Core.Tests.csproj @@ -7,10 +7,12 @@ + + - \ No newline at end of file + diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/FakeFileContentAddressableStore.cs b/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/FakeFileContentAddressableStore.cs new file mode 100644 index 000000000..63772e5dd --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/FakeFileContentAddressableStore.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections.Concurrent; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Scanner.Cache.Abstractions; + +namespace StellaOps.Scanner.Reachability.Tests; + +internal sealed class FakeFileContentAddressableStore : IFileContentAddressableStore +{ + private readonly ConcurrentDictionary store = new(); + + public ValueTask TryGetAsync(string sha256, CancellationToken cancellationToken = default) + { + if (store.TryGetValue(sha256, out var bytes)) + { + return ValueTask.FromResult(new FileCasEntry(sha256, bytes.LongLength, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, sha256 + ".zip")); + } + + return ValueTask.FromResult(null); + } + + public Task PutAsync(FileCasPutRequest request, CancellationToken cancellationToken = default) + { + using var ms = new MemoryStream(); + request.Content.CopyTo(ms); + store[request.Sha256] = ms.ToArray(); + return Task.FromResult(new FileCasEntry(request.Sha256, ms.Length, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, request.Sha256 + ".zip")); + } + + public Task RemoveAsync(string sha256, CancellationToken cancellationToken = default) + => Task.FromResult(store.TryRemove(sha256, out _)); + + public Task EvictExpiredAsync(CancellationToken cancellationToken = default) => Task.FromResult(0); + public Task ExportAsync(string destinationDirectory, CancellationToken cancellationToken = default) => Task.FromResult(0); + public Task ImportAsync(string sourceDirectory, CancellationToken cancellationToken = default) => Task.FromResult(0); + public Task CompactAsync(CancellationToken cancellationToken = default) => Task.FromResult(0); +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/ReachabilityUnionPublisherTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/ReachabilityUnionPublisherTests.cs new file mode 100644 index 000000000..d998ad51e --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/ReachabilityUnionPublisherTests.cs @@ -0,0 +1,27 @@ +using System.Threading.Tasks; +using StellaOps.Scanner.Reachability; +using Xunit; + +namespace StellaOps.Scanner.Reachability.Tests; + +public class ReachabilityUnionPublisherTests +{ + [Fact] + public async Task PublishesZipToCas() + { + var graph = new ReachabilityUnionGraph( + Nodes: new[] { new ReachabilityUnionNode("sym:dotnet:A", "dotnet", "method") }, + Edges: new ReachabilityUnionEdge[0]); + + using var temp = new TempDir(); + var cas = new FakeFileContentAddressableStore(); + var publisher = new ReachabilityUnionPublisher(new ReachabilityUnionWriter()); + + var result = await publisher.PublishAsync(graph, cas, temp.Path, "analysis-pub-1"); + + Assert.False(string.IsNullOrWhiteSpace(result.Sha256)); + var entry = await cas.TryGetAsync(result.Sha256); + Assert.NotNull(entry); + Assert.True(entry!.SizeBytes > 0); + } +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/ReachabilityUnionWriterTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/ReachabilityUnionWriterTests.cs new file mode 100644 index 000000000..fc7be5cb7 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/ReachabilityUnionWriterTests.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using StellaOps.Scanner.Reachability; +using Xunit; + +namespace StellaOps.Scanner.Reachability.Tests; + +public class ReachabilityUnionWriterTests +{ + [Fact] + public async Task WritesDeterministicNdjson() + { + var writer = new ReachabilityUnionWriter(); + using var temp = new TempDir(); + + var graph = new ReachabilityUnionGraph( + Nodes: new[] + { + new ReachabilityUnionNode("sym:dotnet:B", "dotnet", "method"), + new ReachabilityUnionNode("sym:dotnet:A", "dotnet", "method") + }, + Edges: new[] + { + new ReachabilityUnionEdge("sym:dotnet:A", "sym:dotnet:B", "call") + }); + + var result = await writer.WriteAsync(graph, temp.Path, "analysis-x"); + + var meta = await JsonDocument.ParseAsync(File.OpenRead(result.MetaPath)); + var files = meta.RootElement.GetProperty("files").EnumerateArray().ToList(); + Assert.Equal(2, files.Count); // nodes + edges + + // Deterministic order + var nodeLines = await File.ReadAllLinesAsync(Path.Combine(temp.Path, "reachability_graphs/analysis-x/nodes.ndjson")); + Assert.Contains(nodeLines, l => l.Contains("sym:dotnet:A")); + } +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/StellaOps.Scanner.Reachability.Tests.csproj b/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/StellaOps.Scanner.Reachability.Tests.csproj new file mode 100644 index 000000000..a8aa130ac --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/StellaOps.Scanner.Reachability.Tests.csproj @@ -0,0 +1,19 @@ + + + net10.0 + preview + enable + enable + false + false + + + + + + + + + + + diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/TestHelpers.cs b/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/TestHelpers.cs new file mode 100644 index 000000000..37b63834c --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/TestHelpers.cs @@ -0,0 +1,27 @@ +using System; +using System.IO; + +namespace StellaOps.Scanner.Reachability.Tests; + +internal sealed class TempDir : IDisposable +{ + public string Path { get; } + + public TempDir() + { + Path = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "reach-tests-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(Path); + } + + public void Dispose() + { + try + { + Directory.Delete(Path, recursive: true); + } + catch + { + // best-effort cleanup only + } + } +} diff --git a/tests/Concelier/StellaOps.Concelier.Storage.Mongo.Tests/AdvisoryLinksetStoreTests.cs b/tests/Concelier/StellaOps.Concelier.Storage.Mongo.Tests/AdvisoryLinksetStoreTests.cs new file mode 100644 index 000000000..a79176871 --- /dev/null +++ b/tests/Concelier/StellaOps.Concelier.Storage.Mongo.Tests/AdvisoryLinksetStoreTests.cs @@ -0,0 +1,72 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Mongo2Go; +using MongoDB.Driver; +using StellaOps.Concelier.Core.Linksets; +using StellaOps.Concelier.Storage.Mongo.Linksets; + +namespace StellaOps.Concelier.Storage.Mongo.Tests; + +public sealed class AdvisoryLinksetStoreTests : IAsyncLifetime +{ + private MongoDbRunner _runner = null!; + private IMongoDatabase _database = null!; + + public Task InitializeAsync() + { + _runner = MongoDbRunner.Start(singleNodeReplSet: true); + var client = new MongoClient(_runner.ConnectionString); + _database = client.GetDatabase("lnm-store-tests"); + return Task.CompletedTask; + } + + public Task DisposeAsync() + { + _runner.Dispose(); + return Task.CompletedTask; + } + + [Fact] + public async Task UpsertAndFetch_RetainsCpesInNormalizedShape() + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.AdvisoryLinksets); + var store = new ConcelierMongoLinksetStore(collection); + + var linkset = new AdvisoryLinkset( + TenantId: "TenantA", + Source: "source-A", + AdvisoryId: "ADV-1234", + ObservationIds: ImmutableArray.Create("obs-1"), + Normalized: new AdvisoryLinksetNormalized( + Purls: new List { "pkg:npm/lodash@4.17.21" }, + Cpes: new List { "cpe:2.3:a:lodash:lodash:4.17.21:*:*:*:*:*:*:*" }, + Versions: new List { "4.17.21" }, + Ranges: new List>(), + Severities: null), + Provenance: null, + Confidence: null, + Conflicts: null, + CreatedAt: DateTimeOffset.Parse("2025-11-24T00:00:00Z"), + BuiltByJobId: "job-001"); + + await store.UpsertAsync(linkset, CancellationToken.None); + + var result = await store.FindByTenantAsync( + tenantId: "TenantA", + advisoryIds: new[] { "ADV-1234" }, + sources: new[] { "source-A" }, + cursor: null, + limit: 10, + cancellationToken: CancellationToken.None); + + result.Should().ContainSingle(); + var returned = result.Single(); + returned.Normalized.Should().NotBeNull(); + returned.Normalized!.Cpes.Should().ContainSingle() + .Which.Should().Be("cpe:2.3:a:lodash:lodash:4.17.21:*:*:*:*:*:*:*"); + } +} diff --git a/tests/shared/OpenSslAutoInit.cs b/tests/shared/OpenSslAutoInit.cs new file mode 100644 index 000000000..5d7c3d506 --- /dev/null +++ b/tests/shared/OpenSslAutoInit.cs @@ -0,0 +1,15 @@ +using System.Runtime.CompilerServices; + +namespace StellaOps.Testing; + +/// +/// Automatically ensures OpenSSL 1.1 shim is visible for Mongo2Go-based tests. +/// +internal static class OpenSslAutoInit +{ + [ModuleInitializer] + public static void Init() + { + OpenSslLegacyShim.EnsureOpenSsl11(); + } +}