From 48f3071e2a0d63a1548f6b0e64b9e2290d5a8bf3 Mon Sep 17 00:00:00 2001 From: master <> Date: Tue, 21 Oct 2025 09:37:07 +0300 Subject: [PATCH] Add tests and implement StubBearer authentication for Signer endpoints - Created SignerEndpointsTests to validate the SignDsse and VerifyReferrers endpoints. - Implemented StubBearerAuthenticationDefaults and StubBearerAuthenticationHandler for token-based authentication. - Developed ConcelierExporterClient for managing Trivy DB settings and export operations. - Added TrivyDbSettingsPageComponent for UI interactions with Trivy DB settings, including form handling and export triggering. - Implemented styles and HTML structure for Trivy DB settings page. - Created NotifySmokeCheck tool for validating Redis event streams and Notify deliveries. --- .gitea/workflows/build-test-deploy.yml | 56 +- EXECPLAN.md | 193 +++-- NuGet.config | 8 +- SPRINTS.md | 257 +------ SPRINTS.updated.tmp | 427 ----------- SPRINTS_PRIOR_20251019.md | 385 +++++----- SPRINTS_VEXER.md | 2 - bench/Scanner.Analyzers/README.md | 70 +- .../BenchmarkConfig.cs | 104 +++ .../Program.cs | 302 ++++++++ .../ScenarioRunners.cs | 279 +++++++ .../StellaOps.Bench.ScannerAnalyzers.csproj | 16 + bench/Scanner.Analyzers/baseline.csv | 5 +- bench/Scanner.Analyzers/config.json | 31 +- bench/Scanner.Analyzers/run-bench.js | 249 ------ bench/TASKS.md | 2 +- deploy/README.md | 19 +- deploy/compose/README.md | 27 +- deploy/compose/docker-compose.airgap.yaml | 14 +- deploy/compose/docker-compose.dev.yaml | 14 +- deploy/compose/docker-compose.stage.yaml | 14 +- deploy/compose/env/airgap.env.example | 17 +- deploy/compose/env/dev.env.example | 15 +- deploy/compose/env/stage.env.example | 15 +- deploy/helm/stellaops/values-airgap.yaml | 36 +- deploy/helm/stellaops/values-dev.yaml | 36 +- deploy/helm/stellaops/values-stage.yaml | 36 +- docs/09_API_CLI_REFERENCE.md | 74 ++ docs/11_AUTHORITY.md | 38 +- docs/12_PERFORMANCE_WORKBOOK.md | 9 +- docs/24_OFFLINE_KIT.md | 25 +- docs/ARCHITECTURE_AUTHORITY.md | 69 +- docs/ARCHITECTURE_EXCITITOR.md | 64 +- docs/ARCHITECTURE_NOTIFY.md | 4 +- docs/ARCHITECTURE_SCANNER.md | 2 +- docs/ARCHITECTURE_VEXER.md | 10 +- docs/TASKS.md | 1 + docs/dev/authority-dpop-mtls-plan.md | 4 +- docs/dev/authority-plugin-di-coordination.md | 51 +- docs/dev/normalized_versions_rollout.md | 42 +- docs/ops/concelier-mirror-operations.md | 14 + .../2025-10-20-authority-identity-registry.md | 14 + docs/updates/2025-10-20-scanner-events.md | 5 + etc/authority.yaml.sample | 52 +- ops/devops/TASKS.md | 12 +- plugins/notify/email/notify-plugin.json | 18 + plugins/notify/slack/notify-plugin.json | 19 + plugins/notify/teams/notify-plugin.json | 19 + plugins/notify/webhook/notify-plugin.json | 18 + samples/runtime/README.md | 9 +- samples/runtime/java-demo/README.md | 5 + samples/runtime/java-demo/libs/demo.jar | Bin 0 -> 481 bytes scripts/verify-notify-plugins.ps1 | 57 ++ scripts/verify-notify-plugins.sh | 56 ++ src/Directory.Build.props | 4 + src/Directory.Build.targets | 18 + .../StandardPluginRegistrarTests.cs | 14 +- .../Bootstrap/StandardPluginBootstrapper.cs | 14 +- .../StandardPluginRegistrar.cs | 11 +- .../TASKS.md | 6 +- .../AuthorityPluginContracts.cs | 100 ++- .../AuthorityIdentityProviderRegistryTests.cs | 101 ++- .../AuthorityIdentityProviderSelectorTests.cs | 11 +- .../ClientCredentialsAndTokenHandlersTests.cs | 21 +- .../OpenIddict/PasswordGrantHandlersTests.cs | 9 +- .../TokenPersistenceIntegrationTests.cs | 4 +- .../Plugins/AuthorityPluginLoaderTests.cs | 76 ++ .../AuthorityIdentityProviderRegistry.cs | 103 ++- .../AuthorityIdentityProviderSelector.cs | 5 +- .../Handlers/ClientCredentialsHandlers.cs | 306 ++++---- .../OpenIddict/Handlers/DpopHandlers.cs | 133 ++-- .../Handlers/PasswordGrantHandlers.cs | 34 +- .../Handlers/TokenValidationHandlers.cs | 5 +- .../Plugins/AuthorityPluginLoader.cs | 199 ++++- .../StellaOps.Authority/Program.cs | 56 +- src/StellaOps.Authority/TASKS.md | 8 +- .../Commands/CommandHandlersTests.cs | 9 + .../Services/BackendOperationsClientTests.cs | 384 +++++++++- src/StellaOps.Cli/Commands/CommandFactory.cs | 102 ++- src/StellaOps.Cli/Commands/CommandHandlers.cs | 420 +++++++++- .../Configuration/CliBootstrapper.cs | 34 + .../Configuration/StellaOpsCliOptions.cs | 11 +- .../Services/BackendOperationsClient.cs | 682 ++++++++++++++++- .../Services/IBackendOperationsClient.cs | 6 + .../Services/Models/OfflineKitModels.cs | 111 +++ .../Models/Transport/OfflineKitTransport.cs | 103 +++ src/StellaOps.Cli/TASKS.md | 2 +- src/StellaOps.Cli/Telemetry/CliMetrics.cs | 51 +- .../FixtureLoader.cs | 33 + .../Fixtures/mirror-advisory.expected.json | 212 ++++++ .../Fixtures/mirror-bundle.sample.json | 202 +++++ .../MirrorAdvisoryMapperTests.cs | 47 ++ .../MirrorSignatureVerifierTests.cs | 56 +- .../SampleData.cs | 265 +++++++ ...ier.Connector.StellaOpsMirror.Tests.csproj | 13 +- .../StellaOpsMirrorConnectorTests.cs | 105 +++ .../Internal/MirrorAdvisoryMapper.cs | 203 +++++ .../Internal/MirrorBundleDocument.cs | 14 + .../Internal/StellaOpsMirrorCursor.cs | 172 +++-- .../Properties/AssemblyInfo.cs | 3 + .../Security/MirrorSignatureVerifier.cs | 139 +++- .../StellaOpsMirrorConnector.cs | 270 ++++++- ...ellaOpsMirrorDependencyInjectionRoutine.cs | 27 +- .../TASKS.md | 6 +- .../Noise/NoisePriorServiceTests.cs | 320 ++++++++ .../Noise/INoisePriorRepository.cs | 26 + .../Noise/INoisePriorService.cs | 25 + .../Noise/NoisePriorComputationRequest.cs | 10 + .../Noise/NoisePriorComputationResult.cs | 10 + .../Noise/NoisePriorService.cs | 400 ++++++++++ .../NoisePriorServiceCollectionExtensions.cs | 24 + .../Noise/NoisePriorSummary.cs | 24 + src/StellaOps.Concelier.Core/TASKS.md | 2 +- .../RANGE_PRIMITIVES_COORDINATION.md | 48 +- src/StellaOps.Concelier.Merge/TASKS.md | 4 + src/StellaOps.Concelier.WebService/TASKS.md | 2 +- .../StellaOpsAuthorityOptions.cs | 30 +- ...ellaOps.Excititor.ArtifactStores.S3.csproj | 34 +- .../StellaOps.Excititor.Attestation.csproj | 34 +- ...s.Excititor.Connectors.Abstractions.csproj | 34 +- .../Connectors/CiscoCsafConnectorTests.cs | 392 +++++----- ...Ops.Excititor.Connectors.Cisco.CSAF.csproj | 40 +- .../Connectors/MsrcCsafConnectorTests.cs | 703 ++++++++--------- ...xcititor.Connectors.MSRC.CSAF.Tests.csproj | 36 +- ...aOps.Excititor.Connectors.MSRC.CSAF.csproj | 38 +- .../OciOpenVexAttestationConnectorTests.cs | 428 +++++------ ...Connectors.OCI.OpenVEX.Attest.Tests.csproj | 35 +- ...titor.Connectors.OCI.OpenVEX.Attest.csproj | 39 +- .../Connectors/OracleCsafConnectorTests.cs | 598 +++++++-------- ...ititor.Connectors.Oracle.CSAF.Tests.csproj | 34 +- ...ps.Excititor.Connectors.Oracle.CSAF.csproj | 40 +- .../Connectors/RedHatCsafConnectorTests.cs | 30 +- ...ps.Excititor.Connectors.RedHat.CSAF.csproj | 38 +- ...titor.Connectors.SUSE.RancherVEXHub.csproj | 38 +- .../Connectors/UbuntuCsafConnectorTests.cs | 4 +- ...ititor.Connectors.Ubuntu.CSAF.Tests.csproj | 34 +- ...ps.Excititor.Connectors.Ubuntu.CSAF.csproj | 40 +- .../VexConnectorAbstractions.cs | 21 +- .../VexConsensusHold.cs | 47 ++ .../StellaOps.Excititor.Export.csproj | 38 +- .../StellaOps.Excititor.Formats.CSAF.csproj | 32 +- ...ellaOps.Excititor.Formats.CycloneDX.csproj | 32 +- ...StellaOps.Excititor.Formats.OpenVEX.csproj | 32 +- .../StellaOps.Excititor.Policy.csproj | 34 +- .../IVexStorageContracts.cs | 42 +- .../Migrations/VexConsensusHoldMigration.cs | 29 + .../MongoVexConsensusHoldStore.cs | 88 +++ .../MongoVexConsensusStore.cs | 56 +- .../ServiceCollectionExtensions.cs | 32 +- .../StellaOps.Excititor.Storage.Mongo.csproj | 36 +- .../VexMongoMappingRegistry.cs | 14 +- .../VexMongoModels.cs | 114 ++- .../IngestEndpointsTests.cs | 274 +++++++ .../MirrorEndpointsTests.cs | 438 ++++++----- .../ResolveEndpointTests.cs | 717 +++++++++--------- .../StatusEndpointTests.cs | 204 +++-- ...tellaOps.Excititor.WebService.Tests.csproj | 41 +- .../TestAuthentication.cs | 61 ++ .../TestServiceOverrides.cs | 203 +++++ .../TestWebApplicationFactory.cs | 42 + .../Endpoints/IngestEndpoints.cs | 347 ++++----- .../Endpoints/ResolveEndpoint.cs | 42 +- .../Properties/AssemblyInfo.cs | 3 + .../Services/VexIngestOrchestrator.cs | 88 ++- src/StellaOps.Excititor.WebService/TASKS.md | 4 +- .../DefaultVexProviderRunnerTests.cs | 436 +++++++++++ .../VexWorkerOptionsTests.cs | 47 +- .../Options/VexWorkerOptions.cs | 8 +- .../Options/VexWorkerOptionsValidator.cs | 67 +- .../Options/VexWorkerRefreshOptions.cs | 90 +++ src/StellaOps.Excititor.Worker/Program.cs | 27 +- .../Scheduling/DefaultVexProviderRunner.cs | 222 +++++- .../IVexConsensusRefreshScheduler.cs | 6 + .../Scheduling/VexConsensusRefreshService.cs | 622 +++++++++++++++ .../StellaOps.Excititor.Worker.csproj | 45 +- src/StellaOps.Excititor.Worker/TASKS.md | 7 +- .../EmailChannelHealthProviderTests.cs | 100 +++ ...laOps.Notify.Connectors.Email.Tests.csproj | 21 + .../EmailChannelHealthProvider.cs | 59 ++ .../EmailChannelTestProvider.cs | 13 +- .../EmailMetadataBuilder.cs | 54 ++ .../StellaOps.Notify.Connectors.Email.csproj | 19 +- .../TASKS.md | 4 +- .../notify-plugin.json | 18 + .../ConnectorHashing.cs | 31 + .../ConnectorMetadataBuilder.cs | 147 ++++ .../ConnectorValueRedactor.cs | 75 ++ .../StellaOps.Notify.Connectors.Shared.csproj | 12 + .../SlackChannelHealthProviderTests.cs | 96 +++ .../SlackChannelTestProviderTests.cs | 113 +++ ...laOps.Notify.Connectors.Slack.Tests.csproj | 21 + .../SlackChannelHealthProvider.cs | 56 ++ .../SlackChannelTestProvider.cs | 120 +-- .../SlackMetadataBuilder.cs | 77 ++ .../StellaOps.Notify.Connectors.Slack.csproj | 19 +- .../TASKS.md | 4 +- .../notify-plugin.json | 19 + ...laOps.Notify.Connectors.Teams.Tests.csproj | 21 + .../TeamsChannelHealthProviderTests.cs | 98 +++ .../TeamsChannelTestProviderTests.cs | 135 ++++ .../StellaOps.Notify.Connectors.Teams.csproj | 19 +- .../TASKS.md | 9 +- .../TeamsChannelHealthProvider.cs | 57 ++ .../TeamsChannelTestProvider.cs | 182 +++-- .../TeamsMetadataBuilder.cs | 89 +++ .../notify-plugin.json | 19 + ...StellaOps.Notify.Connectors.Webhook.csproj | 19 +- .../TASKS.md | 4 +- .../WebhookChannelTestProvider.cs | 13 +- .../WebhookMetadataBuilder.cs | 53 ++ .../notify-plugin.json | 18 + .../ChannelHealthContracts.cs | 51 ++ .../Contracts/ChannelHealthResponse.cs | 17 + src/StellaOps.Notify.WebService/Program.cs | 41 +- .../Services/NotifyChannelHealthService.cs | 182 +++++ src/StellaOps.Plugin/TASKS.md | 9 +- src/StellaOps.Policy/TASKS.md | 18 +- .../Catalog/RuntimeEventDocument.cs | 83 ++ .../Extensions/ServiceCollectionExtensions.cs | 7 +- .../Mongo/MongoBootstrapper.cs | 56 +- .../Mongo/MongoCollectionProvider.cs | 11 +- .../Repositories/RuntimeEventRepository.cs | 56 ++ .../ScannerStorageDefaults.cs | 17 +- .../ReportsEndpointsTests.cs | 156 +++- .../RuntimeEndpointsTests.cs | 266 +++++++ .../ScansEndpointsTests.cs | 97 ++- .../StellaOps.Scanner.WebService.Tests.csproj | 12 +- .../Constants/ProblemTypes.cs | 11 +- .../Contracts/RuntimeEventsContracts.cs | 22 + .../Contracts/RuntimePolicyContracts.cs | 75 ++ .../Endpoints/PolicyEndpoints.cs | 198 ++++- .../Endpoints/RuntimeEndpoints.cs | 253 ++++++ .../Options/ScannerWebServiceOptions.cs | 86 ++- .../ScannerWebServiceOptionsPostConfigure.cs | 18 +- .../ScannerWebServiceOptionsValidator.cs | 79 +- src/StellaOps.Scanner.WebService/Program.cs | 177 +++-- .../Security/ScannerAuthorityScopes.cs | 9 +- .../Security/ScannerPolicies.cs | 9 +- .../Services/IRedisConnectionFactory.cs | 13 + .../Services/RedisConnectionFactory.cs | 19 + .../Services/RedisPlatformEventPublisher.cs | 45 +- .../Services/RuntimeEventIngestionService.cs | 151 ++++ .../Services/RuntimeEventRateLimiter.cs | 173 +++++ .../Services/RuntimePolicyService.cs | 211 ++++++ .../Services/ScanProgressStream.cs | 48 +- .../StellaOps.Scanner.WebService.csproj | 34 +- src/StellaOps.Scanner.WebService/TASKS.md | 19 +- .../FixtureImpactIndexTests.cs | 142 ++++ ...ellaOps.Scheduler.ImpactIndex.Tests.csproj | 25 + .../FixtureImpactIndex.cs | 615 +++++++++++++++ .../IImpactIndex.cs | 46 ++ .../ImpactIndexServiceCollectionExtensions.cs | 26 + .../ImpactIndexStubOptions.cs | 19 + .../REMOVAL_NOTE.md | 15 + .../StellaOps.Scheduler.ImpactIndex.csproj | 14 +- src/StellaOps.Scheduler.ImpactIndex/TASKS.md | 8 +- .../SchedulerSchemaMigrationTests.cs | 119 ++- .../SchedulerSchemaMigration.cs | 448 +++++++++-- src/StellaOps.Scheduler.Models/TASKS.md | 2 +- .../docs/SCHED-MODELS-16-103-DESIGN.md | 14 +- .../RedisSchedulerQueueTests.cs | 107 ++- ...erQueueServiceCollectionExtensionsTests.cs | 115 +++ .../StellaOps.Scheduler.Queue.Tests.csproj | 10 +- .../ISchedulerQueueTransportDiagnostics.cs | 9 + .../Nats/INatsSchedulerQueuePayload.cs | 26 + .../Nats/NatsSchedulerPlannerQueue.cs | 66 ++ .../Nats/NatsSchedulerQueueBase.cs | 692 +++++++++++++++++ .../Nats/NatsSchedulerQueueLease.cs | 101 +++ .../Nats/NatsSchedulerRunnerQueue.cs | 74 ++ src/StellaOps.Scheduler.Queue/README.md | 37 +- .../Redis/RedisSchedulerQueueBase.cs | 357 +++++---- .../SchedulerQueueHealthCheck.cs | 72 ++ .../SchedulerQueueMetrics.cs | 76 +- .../SchedulerQueueOptions.cs | 106 ++- ...hedulerQueueServiceCollectionExtensions.cs | 108 ++- .../StellaOps.Scheduler.Queue.csproj | 18 +- src/StellaOps.Scheduler.Queue/TASKS.md | 6 +- .../InMemoryProofOfEntitlementIntrospector.cs | 7 +- .../SignerEndpointsTests.cs | 127 ++++ .../Contracts/SignDsseContracts.cs | 8 +- .../Endpoints/SignerEndpoints.cs | 144 ++-- .../StellaOps.Signer.WebService/Program.cs | 38 +- .../StubBearerAuthenticationDefaults.cs | 6 + .../StubBearerAuthenticationHandler.cs | 55 ++ src/StellaOps.Signer/TASKS.md | 6 +- src/StellaOps.Web/TASKS.md | 3 +- src/StellaOps.Web/src/app/app.component.html | 350 +-------- src/StellaOps.Web/src/app/app.component.scss | 59 ++ src/StellaOps.Web/src/app/app.component.ts | 24 +- src/StellaOps.Web/src/app/app.config.ts | 25 +- src/StellaOps.Web/src/app/app.routes.ts | 23 +- .../app/core/api/concelier-exporter.client.ts | 51 ++ .../trivy-db-settings-page.component.html | 108 +++ .../trivy-db-settings-page.component.scss | 230 ++++++ .../trivy-db-settings-page.component.spec.ts | 94 +++ .../trivy-db-settings-page.component.ts | 135 ++++ .../NotifySmokeCheck/NotifySmokeCheck.csproj | 12 + tools/NotifySmokeCheck/Program.cs | 198 +++++ 298 files changed, 20490 insertions(+), 5751 deletions(-) delete mode 100644 SPRINTS.updated.tmp delete mode 100644 SPRINTS_VEXER.md create mode 100644 bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/BenchmarkConfig.cs create mode 100644 bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Program.cs create mode 100644 bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/ScenarioRunners.cs create mode 100644 bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/StellaOps.Bench.ScannerAnalyzers.csproj delete mode 100644 bench/Scanner.Analyzers/run-bench.js create mode 100644 docs/updates/2025-10-20-authority-identity-registry.md create mode 100644 docs/updates/2025-10-20-scanner-events.md create mode 100644 plugins/notify/email/notify-plugin.json create mode 100644 plugins/notify/slack/notify-plugin.json create mode 100644 plugins/notify/teams/notify-plugin.json create mode 100644 plugins/notify/webhook/notify-plugin.json create mode 100644 samples/runtime/java-demo/README.md create mode 100644 samples/runtime/java-demo/libs/demo.jar create mode 100644 scripts/verify-notify-plugins.ps1 create mode 100644 scripts/verify-notify-plugins.sh create mode 100644 src/StellaOps.Cli/Services/Models/OfflineKitModels.cs create mode 100644 src/StellaOps.Cli/Services/Models/Transport/OfflineKitTransport.cs create mode 100644 src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/FixtureLoader.cs create mode 100644 src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/Fixtures/mirror-advisory.expected.json create mode 100644 src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/Fixtures/mirror-bundle.sample.json create mode 100644 src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/MirrorAdvisoryMapperTests.cs create mode 100644 src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/SampleData.cs create mode 100644 src/StellaOps.Concelier.Connector.StellaOpsMirror/Internal/MirrorAdvisoryMapper.cs create mode 100644 src/StellaOps.Concelier.Connector.StellaOpsMirror/Internal/MirrorBundleDocument.cs create mode 100644 src/StellaOps.Concelier.Connector.StellaOpsMirror/Properties/AssemblyInfo.cs create mode 100644 src/StellaOps.Concelier.Core.Tests/Noise/NoisePriorServiceTests.cs create mode 100644 src/StellaOps.Concelier.Core/Noise/INoisePriorRepository.cs create mode 100644 src/StellaOps.Concelier.Core/Noise/INoisePriorService.cs create mode 100644 src/StellaOps.Concelier.Core/Noise/NoisePriorComputationRequest.cs create mode 100644 src/StellaOps.Concelier.Core/Noise/NoisePriorComputationResult.cs create mode 100644 src/StellaOps.Concelier.Core/Noise/NoisePriorService.cs create mode 100644 src/StellaOps.Concelier.Core/Noise/NoisePriorServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Concelier.Core/Noise/NoisePriorSummary.cs create mode 100644 src/StellaOps.Excititor.Core/VexConsensusHold.cs create mode 100644 src/StellaOps.Excititor.Storage.Mongo/Migrations/VexConsensusHoldMigration.cs create mode 100644 src/StellaOps.Excititor.Storage.Mongo/MongoVexConsensusHoldStore.cs create mode 100644 src/StellaOps.Excititor.WebService.Tests/IngestEndpointsTests.cs create mode 100644 src/StellaOps.Excititor.WebService.Tests/TestAuthentication.cs create mode 100644 src/StellaOps.Excititor.WebService.Tests/TestServiceOverrides.cs create mode 100644 src/StellaOps.Excititor.WebService.Tests/TestWebApplicationFactory.cs create mode 100644 src/StellaOps.Excititor.WebService/Properties/AssemblyInfo.cs create mode 100644 src/StellaOps.Excititor.Worker.Tests/DefaultVexProviderRunnerTests.cs create mode 100644 src/StellaOps.Excititor.Worker/Options/VexWorkerRefreshOptions.cs create mode 100644 src/StellaOps.Excititor.Worker/Scheduling/IVexConsensusRefreshScheduler.cs create mode 100644 src/StellaOps.Excititor.Worker/Scheduling/VexConsensusRefreshService.cs create mode 100644 src/StellaOps.Notify.Connectors.Email.Tests/EmailChannelHealthProviderTests.cs create mode 100644 src/StellaOps.Notify.Connectors.Email.Tests/StellaOps.Notify.Connectors.Email.Tests.csproj create mode 100644 src/StellaOps.Notify.Connectors.Email/EmailChannelHealthProvider.cs create mode 100644 src/StellaOps.Notify.Connectors.Email/EmailMetadataBuilder.cs create mode 100644 src/StellaOps.Notify.Connectors.Email/notify-plugin.json create mode 100644 src/StellaOps.Notify.Connectors.Shared/ConnectorHashing.cs create mode 100644 src/StellaOps.Notify.Connectors.Shared/ConnectorMetadataBuilder.cs create mode 100644 src/StellaOps.Notify.Connectors.Shared/ConnectorValueRedactor.cs create mode 100644 src/StellaOps.Notify.Connectors.Shared/StellaOps.Notify.Connectors.Shared.csproj create mode 100644 src/StellaOps.Notify.Connectors.Slack.Tests/SlackChannelHealthProviderTests.cs create mode 100644 src/StellaOps.Notify.Connectors.Slack.Tests/SlackChannelTestProviderTests.cs create mode 100644 src/StellaOps.Notify.Connectors.Slack.Tests/StellaOps.Notify.Connectors.Slack.Tests.csproj create mode 100644 src/StellaOps.Notify.Connectors.Slack/SlackChannelHealthProvider.cs create mode 100644 src/StellaOps.Notify.Connectors.Slack/SlackMetadataBuilder.cs create mode 100644 src/StellaOps.Notify.Connectors.Slack/notify-plugin.json create mode 100644 src/StellaOps.Notify.Connectors.Teams.Tests/StellaOps.Notify.Connectors.Teams.Tests.csproj create mode 100644 src/StellaOps.Notify.Connectors.Teams.Tests/TeamsChannelHealthProviderTests.cs create mode 100644 src/StellaOps.Notify.Connectors.Teams.Tests/TeamsChannelTestProviderTests.cs create mode 100644 src/StellaOps.Notify.Connectors.Teams/TeamsChannelHealthProvider.cs create mode 100644 src/StellaOps.Notify.Connectors.Teams/TeamsMetadataBuilder.cs create mode 100644 src/StellaOps.Notify.Connectors.Teams/notify-plugin.json create mode 100644 src/StellaOps.Notify.Connectors.Webhook/WebhookMetadataBuilder.cs create mode 100644 src/StellaOps.Notify.Connectors.Webhook/notify-plugin.json create mode 100644 src/StellaOps.Notify.Engine/ChannelHealthContracts.cs create mode 100644 src/StellaOps.Notify.WebService/Contracts/ChannelHealthResponse.cs create mode 100644 src/StellaOps.Notify.WebService/Services/NotifyChannelHealthService.cs create mode 100644 src/StellaOps.Scanner.Storage/Catalog/RuntimeEventDocument.cs create mode 100644 src/StellaOps.Scanner.Storage/Repositories/RuntimeEventRepository.cs create mode 100644 src/StellaOps.Scanner.WebService.Tests/RuntimeEndpointsTests.cs create mode 100644 src/StellaOps.Scanner.WebService/Contracts/RuntimeEventsContracts.cs create mode 100644 src/StellaOps.Scanner.WebService/Contracts/RuntimePolicyContracts.cs create mode 100644 src/StellaOps.Scanner.WebService/Endpoints/RuntimeEndpoints.cs create mode 100644 src/StellaOps.Scanner.WebService/Services/IRedisConnectionFactory.cs create mode 100644 src/StellaOps.Scanner.WebService/Services/RedisConnectionFactory.cs create mode 100644 src/StellaOps.Scanner.WebService/Services/RuntimeEventIngestionService.cs create mode 100644 src/StellaOps.Scanner.WebService/Services/RuntimeEventRateLimiter.cs create mode 100644 src/StellaOps.Scanner.WebService/Services/RuntimePolicyService.cs create mode 100644 src/StellaOps.Scheduler.ImpactIndex.Tests/FixtureImpactIndexTests.cs create mode 100644 src/StellaOps.Scheduler.ImpactIndex.Tests/StellaOps.Scheduler.ImpactIndex.Tests.csproj create mode 100644 src/StellaOps.Scheduler.ImpactIndex/FixtureImpactIndex.cs create mode 100644 src/StellaOps.Scheduler.ImpactIndex/IImpactIndex.cs create mode 100644 src/StellaOps.Scheduler.ImpactIndex/ImpactIndexServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Scheduler.ImpactIndex/ImpactIndexStubOptions.cs create mode 100644 src/StellaOps.Scheduler.ImpactIndex/REMOVAL_NOTE.md create mode 100644 src/StellaOps.Scheduler.Queue.Tests/SchedulerQueueServiceCollectionExtensionsTests.cs create mode 100644 src/StellaOps.Scheduler.Queue/ISchedulerQueueTransportDiagnostics.cs create mode 100644 src/StellaOps.Scheduler.Queue/Nats/INatsSchedulerQueuePayload.cs create mode 100644 src/StellaOps.Scheduler.Queue/Nats/NatsSchedulerPlannerQueue.cs create mode 100644 src/StellaOps.Scheduler.Queue/Nats/NatsSchedulerQueueBase.cs create mode 100644 src/StellaOps.Scheduler.Queue/Nats/NatsSchedulerQueueLease.cs create mode 100644 src/StellaOps.Scheduler.Queue/Nats/NatsSchedulerRunnerQueue.cs create mode 100644 src/StellaOps.Scheduler.Queue/SchedulerQueueHealthCheck.cs create mode 100644 src/StellaOps.Signer/StellaOps.Signer.Tests/SignerEndpointsTests.cs create mode 100644 src/StellaOps.Signer/StellaOps.Signer.WebService/Security/StubBearerAuthenticationDefaults.cs create mode 100644 src/StellaOps.Signer/StellaOps.Signer.WebService/Security/StubBearerAuthenticationHandler.cs create mode 100644 src/StellaOps.Web/src/app/core/api/concelier-exporter.client.ts create mode 100644 src/StellaOps.Web/src/app/features/trivy-db-settings/trivy-db-settings-page.component.html create mode 100644 src/StellaOps.Web/src/app/features/trivy-db-settings/trivy-db-settings-page.component.scss create mode 100644 src/StellaOps.Web/src/app/features/trivy-db-settings/trivy-db-settings-page.component.spec.ts create mode 100644 src/StellaOps.Web/src/app/features/trivy-db-settings/trivy-db-settings-page.component.ts create mode 100644 tools/NotifySmokeCheck/NotifySmokeCheck.csproj create mode 100644 tools/NotifySmokeCheck/Program.cs diff --git a/.gitea/workflows/build-test-deploy.yml b/.gitea/workflows/build-test-deploy.yml index e5b90aff..90b850e0 100644 --- a/.gitea/workflows/build-test-deploy.yml +++ b/.gitea/workflows/build-test-deploy.yml @@ -539,9 +539,53 @@ PY echo " Service path: ${{ steps.params.outputs.path || '(skipped)' }}" echo " Docs path: ${{ steps.params.outputs['docs-path'] || '(skipped)' }}" - - name: Deployment skipped summary - if: steps.check-deploy.outputs.should-deploy != 'true' - run: | - echo "ℹ️ Deployment stage skipped" - echo " Event: ${{ github.event_name }}" - echo " Ref: ${{ github.ref }}" + - name: Deployment skipped summary + if: steps.check-deploy.outputs.should-deploy != 'true' + run: | + echo "ℹ️ Deployment stage skipped" + echo " Event: ${{ github.event_name }}" + echo " Ref: ${{ github.ref }}" + + notify-smoke: + runs-on: ubuntu-22.04 + needs: deploy + if: needs.deploy.result == 'success' + env: + DOTNET_VERSION: ${{ env.DOTNET_VERSION }} + NOTIFY_SMOKE_REDIS_DSN: ${{ secrets.NOTIFY_SMOKE_REDIS_DSN }} + NOTIFY_SMOKE_NOTIFY_BASEURL: ${{ secrets.NOTIFY_SMOKE_NOTIFY_BASEURL }} + NOTIFY_SMOKE_NOTIFY_TOKEN: ${{ secrets.NOTIFY_SMOKE_NOTIFY_TOKEN }} + NOTIFY_SMOKE_NOTIFY_TENANT: ${{ secrets.NOTIFY_SMOKE_NOTIFY_TENANT }} + NOTIFY_SMOKE_NOTIFY_TENANT_HEADER: ${{ secrets.NOTIFY_SMOKE_NOTIFY_TENANT_HEADER }} + NOTIFY_SMOKE_EXPECT_KINDS: ${{ vars.NOTIFY_SMOKE_EXPECT_KINDS || secrets.NOTIFY_SMOKE_EXPECT_KINDS }} + NOTIFY_SMOKE_LOOKBACK_MINUTES: ${{ vars.NOTIFY_SMOKE_LOOKBACK_MINUTES || secrets.NOTIFY_SMOKE_LOOKBACK_MINUTES }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup .NET ${{ env.DOTNET_VERSION }} + uses: actions/setup-dotnet@v4 + with: + dotnet-version: ${{ env.DOTNET_VERSION }} + include-prerelease: true + + - name: Validate Notify smoke configuration + run: | + missing=() + for name in NOTIFY_SMOKE_REDIS_DSN NOTIFY_SMOKE_NOTIFY_BASEURL NOTIFY_SMOKE_NOTIFY_TOKEN NOTIFY_SMOKE_NOTIFY_TENANT NOTIFY_SMOKE_EXPECT_KINDS NOTIFY_SMOKE_LOOKBACK_MINUTES + do + value="${!name}" + if [ -z "$value" ]; then + missing+=("$name") + fi + done + if [ ${#missing[@]} -gt 0 ]; then + echo "❌ Missing Notify smoke configuration: ${missing[*]}" + exit 1 + fi + + - name: Restore Notify smoke checker + run: dotnet restore tools/NotifySmokeCheck/NotifySmokeCheck.csproj + + - name: Run Notify smoke validation + run: dotnet run --project tools/NotifySmokeCheck/NotifySmokeCheck.csproj --configuration Release diff --git a/EXECPLAN.md b/EXECPLAN.md index 11bb4a5b..87538f1c 100644 --- a/EXECPLAN.md +++ b/EXECPLAN.md @@ -4,7 +4,7 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster ## Wave Instructions ### Wave 0 - Team Attestor Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Attestor/TASKS.md`. Focus on ATTESTOR-API-11-201 (TODO), ATTESTOR-VERIFY-11-202 (TODO), ATTESTOR-OBS-11-203 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. -- Team Authority Core & Security Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Authority/TASKS.md`. Focus on AUTH-DPOP-11-001 (DOING 2025-10-19), AUTH-MTLS-11-002 (DOING 2025-10-19). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team Authority Core & Security Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Authority/TASKS.md`. Focus on AUTH-DPOP-11-001 (DONE 2025-10-20), AUTH-MTLS-11-002 (DOING 2025-10-19). Confirm prerequisites (none) before starting and report status in module TASKS.md. - Team Authority Core & Storage Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Authority/TASKS.md`. Focus on AUTHSTORAGE-MONGO-08-001 (DONE 2025-10-19). Confirm prerequisites (none) before starting and report status in module TASKS.md. - Team DevEx/CLI: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Cli/TASKS.md`. Focus on EXCITITOR-CLI-01-002 (TODO), CLI-RUNTIME-13-005 (TODO). Confirm prerequisites (external: EXCITITOR-CLI-01-001, EXCITITOR-EXPORT-01-001) before starting and report status in module TASKS.md. - Team DevOps Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `ops/devops/TASKS.md`. Focus on DEVOPS-SEC-10-301 (DONE 2025-10-20); Wave 0A prerequisites reconfirmed so remediation work may proceed. Keep module TASKS.md/Sprints in sync as patches land. @@ -18,17 +18,18 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster - Team Notify Storage Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Notify.Storage.Mongo/TASKS.md`. Focus on NOTIFY-STORAGE-15-201 (TODO), NOTIFY-STORAGE-15-202 (TODO), NOTIFY-STORAGE-15-203 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. - Team Notify WebService Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Notify.WebService/TASKS.md`. Focus on NOTIFY-WEB-15-101 (TODO), NOTIFY-WEB-15-102 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. - Team Platform Events Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `docs/TASKS.md`. Focus on PLATFORM-EVENTS-09-401 (TODO). Confirm prerequisites (external: DOCS-EVENTS-09-003) before starting and report status in module TASKS.md. -- Team Plugin Platform Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Plugin/TASKS.md`. Focus on PLUGIN-DI-08-001 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. -- Team Plugin Platform Guild, Authority Core: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Plugin/TASKS.md`. Focus on PLUGIN-DI-08-002 (TODO); coordination session booked for 2025-10-20 to unblock implementation. Confirm prerequisites (none) before starting and report status in module TASKS.md. -- Team Policy Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Policy/TASKS.md`. Focus on POLICY-CORE-09-004 (TODO), POLICY-CORE-09-005 (TODO), POLICY-CORE-09-006 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team Plugin Platform Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Plugin/TASKS.md`. Focus on PLUGIN-DI-08-002.COORD (DONE 2025-10-20), PLUGIN-DI-08-002 (DONE 2025-10-20), PLUGIN-DI-08-003 (DONE 2025-10-20), PLUGIN-DI-08-004 (DONE 2025-10-20), and PLUGIN-DI-08-005 (DONE 2025-10-20). Confirm prerequisites (PLUGIN-DI-08-001) before starting and report status in module TASKS.md. +- Team Plugin Platform Guild, Authority Core: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Plugin/TASKS.md`. Coordination session for PLUGIN-DI-08-002 implementation completed on 2025-10-20 15:00–16:05 UTC and scoped-service changes have shipped with regression coverage; subsequent tasks (PLUGIN-DI-08-003/004/005) remain green. +- Team Policy Guild: Sprint 9 core tasks (POLICY-CORE-09-004/005/006) closed on 2025-10-19; ensure downstream consumers refresh against the published scoring config + quiet/unknown outputs and raise follow-up tasks if additional polish is required. - Team Runtime Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `docs/TASKS.md`. Focus on RUNTIME-GUILD-09-402 (TODO). Confirm prerequisites (external: SCANNER-POLICY-09-107) before starting and report status in module TASKS.md. -- Team Scanner WebService Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scanner.WebService/TASKS.md`. Focus on SCANNER-EVENTS-15-201 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. -- Team Scheduler ImpactIndex Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scheduler.ImpactIndex/TASKS.md`. Focus on SCHED-IMPACT-16-300 (DOING). Confirm prerequisites (external: SAMPLES-10-001) before starting and report status in module TASKS.md. -- Team Scheduler Models Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scheduler.Models/TASKS.md`. Focus on SCHED-MODELS-16-103 (TODO). Confirm prerequisites (external: SCHED-MODELS-16-101) before starting and report status in module TASKS.md. -- Team Scheduler Queue Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scheduler.Queue/TASKS.md`. Focus on SCHED-QUEUE-16-401 (TODO). Confirm prerequisites (external: SCHED-MODELS-16-101) before starting and report status in module TASKS.md. +- Team Scanner WebService Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scanner.WebService/TASKS.md`. Focus on SCANNER-EVENTS-15-201 (DONE 2025-10-20). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team Scanner WebService Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Scanner.WebService/TASKS.md`. Focus on SCANNER-EVENTS-16-301 (BLOCKED 2025-10-20). Wait for NOTIFY-QUEUE-15-401 before attempting integration. +- Team Scheduler ImpactIndex Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scheduler.ImpactIndex/TASKS.md`. Focus on SCHED-IMPACT-16-300 (DONE 2025-10-20) and ensure the temporary stub removal note stays tracked. Confirm prerequisites (external: SAMPLES-10-001) before starting and report status in module TASKS.md. +- Team Scheduler Models Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scheduler.Models/TASKS.md`. SCHED-MODELS-16-103 completed (2025-10-20); ensure downstream teams consume the migration helpers and log upgrade warnings. +- Team Scheduler Queue Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scheduler.Queue/TASKS.md`. SCHED-QUEUE-16-401 completed (2025-10-20); proceed with Wave 1 queue enhancements. - Team Scheduler Storage Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scheduler.Storage.Mongo/TASKS.md`. Focus on SCHED-STORAGE-16-201 (TODO). Confirm prerequisites (external: SCHED-MODELS-16-101) before starting and report status in module TASKS.md. - Team Scheduler WebService Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scheduler.WebService/TASKS.md`. Focus on SCHED-WEB-16-101 (TODO). Confirm prerequisites (external: SCHED-MODELS-16-101) before starting and report status in module TASKS.md. -- Team Signer Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Signer/TASKS.md`. Focus on SIGNER-API-11-101 (TODO), SIGNER-REF-11-102 (TODO), SIGNER-QUOTA-11-103 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team Signer Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Signer/TASKS.md`. Focus on SIGNER-API-11-101 (DONE 2025-10-21), SIGNER-REF-11-102 (DONE 2025-10-21), SIGNER-QUOTA-11-103 (DONE 2025-10-21). Confirm prerequisites (none) before starting and report status in module TASKS.md. - Team TBD: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md`. Focus on SCANNER-ANALYZERS-LANG-10-302C (TODO). Confirm prerequisites (external: SCANNER-ANALYZERS-LANG-10-302B) before starting and report status in module TASKS.md. - Team Team Connector Resumption – CERT/RedHat: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Concelier.Connector.Distro.RedHat/TASKS.md`. Focus on FEEDCONN-REDHAT-02-001 (DOING). Confirm prerequisites (none) before starting and report status in module TASKS.md. - Team Team Excititor Attestation: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Excititor.Attestation/TASKS.md`. Focus on EXCITITOR-ATTEST-01-003 (TODO). Confirm prerequisites (external: EXCITITOR-ATTEST-01-002) before starting and report status in module TASKS.md. @@ -40,13 +41,13 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster - Team Team Excititor Export: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Excititor.Export/TASKS.md`. Focus on EXCITITOR-EXPORT-01-005 (TODO). Confirm prerequisites (external: EXCITITOR-CORE-02-001, EXCITITOR-EXPORT-01-004) before starting and report status in module TASKS.md. - Team Team Excititor Formats: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Excititor.Formats.CSAF/TASKS.md`, `src/StellaOps.Excititor.Formats.CycloneDX/TASKS.md`, `src/StellaOps.Excititor.Formats.OpenVEX/TASKS.md`. Focus on EXCITITOR-FMT-CSAF-01-002 (TODO), EXCITITOR-FMT-CSAF-01-003 (TODO), EXCITITOR-FMT-CYCLONE-01-002 (TODO), EXCITITOR-FMT-CYCLONE-01-003 (TODO), EXCITITOR-FMT-OPENVEX-01-002 (TODO), EXCITITOR-FMT-OPENVEX-01-003 (TODO). Confirm prerequisites (external: EXCITITOR-EXPORT-01-001, EXCITITOR-FMT-CSAF-01-001, EXCITITOR-FMT-CYCLONE-01-001, EXCITITOR-FMT-OPENVEX-01-001, EXCITITOR-POLICY-01-001) before starting and report status in module TASKS.md. - Team Team Excititor Storage: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Excititor.Storage.Mongo/TASKS.md`. Focus on EXCITITOR-STORAGE-MONGO-08-001 (DONE 2025-10-19), EXCITITOR-STORAGE-03-001 (TODO). Confirm prerequisites (external: EXCITITOR-STORAGE-01-003, EXCITITOR-STORAGE-02-001) before starting and report status in module TASKS.md. -- Team Team Excititor WebService: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Excititor.WebService/TASKS.md`. Focus on EXCITITOR-WEB-01-002 (TODO), EXCITITOR-WEB-01-003 (TODO), EXCITITOR-WEB-01-004 (TODO). Confirm prerequisites (external: EXCITITOR-ATTEST-01-001, EXCITITOR-EXPORT-01-001, EXCITITOR-WEB-01-001) before starting and report status in module TASKS.md. -- Team Team Excititor Worker: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Excititor.Worker/TASKS.md`. Focus on EXCITITOR-WORKER-01-002 (TODO), EXCITITOR-WORKER-01-004 (TODO), EXCITITOR-WORKER-02-001 (TODO). Confirm prerequisites (external: EXCITITOR-CORE-02-001, EXCITITOR-WORKER-01-001) before starting and report status in module TASKS.md. -- Team Team Merge & QA Enforcement: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Concelier.Merge/TASKS.md`. Focus on FEEDMERGE-COORD-02-900 (DOING). Confirm prerequisites (none) before starting and report status in module TASKS.md. **2025-10-19:** Coordination refreshed; connector owners notified and TASKS.md entries updated. +- Team Team Excititor WebService: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Excititor.WebService/TASKS.md`. Focus on EXCITITOR-WEB-01-002 (DONE 2025-10-20), EXCITITOR-WEB-01-003 (TODO), EXCITITOR-WEB-01-004 (DONE 2025-10-20). Confirm prerequisites (external: EXCITITOR-ATTEST-01-001, EXCITITOR-EXPORT-01-001, EXCITITOR-WEB-01-001) before starting and report status in module TASKS.md. +- Team Team Excititor Worker: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Excititor.Worker/TASKS.md`. Focus on EXCITITOR-WORKER-01-004 (DONE 2025-10-21); EXCITITOR-WORKER-01-002 (DONE 2025-10-21) and EXCITITOR-WORKER-02-001 (DONE 2025-10-21) recorded. Confirm prerequisites (external: EXCITITOR-CORE-02-001, EXCITITOR-WORKER-01-001) before starting and report status in module TASKS.md. +- Team Team Merge & QA Enforcement: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Concelier.Merge/TASKS.md`. Focus on FEEDMERGE-COORD-02-900 (DOING). Confirm prerequisites (none) before starting and report status in module TASKS.md. **2025-10-19:** Coordination refreshed; connector owners notified and TASKS.md entries updated. **2025-10-20:** Coordination matrix + rollout dashboard refreshed with connector due dates (Cccs/Cisco 2025-10-21, CertBund 2025-10-22, ICS-CISA 2025-10-23, KISA 2025-10-24) and escalation plan logged. - Team Team Normalization & Storage Backbone: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Concelier.Storage.Mongo/TASKS.md`. Focus on FEEDSTORAGE-MONGO-08-001 (DONE 2025-10-19). Confirm prerequisites (none) before starting and report status in module TASKS.md. - Team Team WebService & Authority: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md`, `src/StellaOps.Concelier.WebService/TASKS.md`. Focus on SEC2.PLG (DOING), SEC3.PLG (DOING), SEC5.PLG (DOING), PLG4-6.CAPABILITIES (BLOCKED), PLG6.DIAGRAM (TODO), PLG7.RFC (REVIEW), FEEDWEB-DOCS-01-001 (DOING), FEEDWEB-OPS-01-006 (TODO), FEEDWEB-OPS-01-007 (BLOCKED). Confirm prerequisites (none) before starting and report status in module TASKS.md. - Team Tools Guild, BE-Conn-MSRC: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Concelier.Connector.Common/TASKS.md`. Focus on FEEDCONN-SHARED-STATE-003 (**TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. -- Team UX Specialist, Angular Eng: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Web/TASKS.md`. Focus on WEB1.TRIVY-SETTINGS (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team UX Specialist, Angular Eng: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Web/TASKS.md`. Focus on WEB1.TRIVY-SETTINGS (DONE 2025-10-21) and WEB1.TRIVY-SETTINGS-TESTS (BLOCKED 2025-10-21). Confirm prerequisites (none) before starting and report status in module TASKS.md. - Team Zastava Core Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Zastava.Core/TASKS.md`. Focus on ZASTAVA-CORE-12-201 (TODO), ZASTAVA-CORE-12-202 (TODO), ZASTAVA-CORE-12-203 (TODO), ZASTAVA-OPS-12-204 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. - Team Zastava Webhook Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Zastava.Webhook/TASKS.md`. Focus on ZASTAVA-WEBHOOK-12-101 (TODO), ZASTAVA-WEBHOOK-12-102 (TODO), ZASTAVA-WEBHOOK-12-103 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. @@ -61,9 +62,9 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster - Team Notify Engine Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Notify.Engine/TASKS.md`. Focus on NOTIFY-ENGINE-15-301 (TODO). Confirm prerequisites (internal: NOTIFY-MODELS-15-101 (Wave 0)) before starting and report status in module TASKS.md. - Team Notify Queue Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Notify.Queue/TASKS.md`. Focus on NOTIFY-QUEUE-15-401 (TODO). Confirm prerequisites (internal: NOTIFY-MODELS-15-101 (Wave 0)) before starting and report status in module TASKS.md. - Team Notify WebService Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Notify.WebService/TASKS.md`. Focus on NOTIFY-WEB-15-103 (DONE). Confirm prerequisites (internal: NOTIFY-WEB-15-102 (Wave 0)) before starting and report status in module TASKS.md. -- Team Scanner WebService Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Scanner.WebService/TASKS.md`. Focus on SCANNER-RUNTIME-12-301 (TODO). Confirm prerequisites (internal: ZASTAVA-CORE-12-201 (Wave 0)) before starting and report status in module TASKS.md. +- Team Scanner WebService Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Scanner.WebService/TASKS.md`. SCANNER-RUNTIME-12-301 closed (2025-10-20); coordinate with Zastava observer guild on batch fixtures and advance to SCANNER-RUNTIME-12-302. - Team Scheduler ImpactIndex Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Scheduler.ImpactIndex/TASKS.md`. Focus on SCHED-IMPACT-16-301 (TODO). Confirm prerequisites (internal: SCANNER-EMIT-10-605 (Wave 0)) before starting and report status in module TASKS.md. -- Team Scheduler Queue Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Scheduler.Queue/TASKS.md`. Focus on SCHED-QUEUE-16-402 (TODO), SCHED-QUEUE-16-403 (TODO). Confirm prerequisites (internal: SCHED-QUEUE-16-401 (Wave 0)) before starting and report status in module TASKS.md. +- Team Scheduler Queue Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Scheduler.Queue/TASKS.md`. SCHED-QUEUE-16-402 completed (2025-10-20); next focus is SCHED-QUEUE-16-403. - Team Scheduler Storage Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Scheduler.Storage.Mongo/TASKS.md`. Focus on SCHED-STORAGE-16-203 (TODO), SCHED-STORAGE-16-202 (TODO). Confirm prerequisites (internal: SCHED-STORAGE-16-201 (Wave 0)) before starting and report status in module TASKS.md. - Team Scheduler WebService Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Scheduler.WebService/TASKS.md`. Focus on SCHED-WEB-16-104 (TODO), SCHED-WEB-16-102 (TODO). Confirm prerequisites (internal: SCHED-QUEUE-16-401 (Wave 0), SCHED-STORAGE-16-201 (Wave 0), SCHED-WEB-16-101 (Wave 0)) before starting and report status in module TASKS.md. - Team Scheduler Worker Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Scheduler.Worker/TASKS.md`. Focus on SCHED-WORKER-16-201 (TODO). Confirm prerequisites (internal: SCHED-QUEUE-16-401 (Wave 0)) before starting and report status in module TASKS.md. @@ -99,7 +100,7 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster - Team Zastava Observer Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Zastava.Observer/TASKS.md`. Focus on ZASTAVA-OBS-12-002 (TODO). Confirm prerequisites (internal: ZASTAVA-OBS-12-001 (Wave 1)) before starting and report status in module TASKS.md. ### Wave 3 -- Team DevEx/CLI: read EXECPLAN.md Wave 3 and SPRINTS.md rows for `src/StellaOps.Cli/TASKS.md`. Focus on CLI-OFFLINE-13-006 (TODO). Confirm prerequisites (internal: DEVOPS-OFFLINE-14-002 (Wave 2)) before starting and report status in module TASKS.md. +- Team DevEx/CLI: read EXECPLAN.md Wave 3 and SPRINTS.md rows for `src/StellaOps.Cli/TASKS.md`. Focus on CLI-OFFLINE-13-006 (DONE 2025-10-21). Confirm prerequisites (internal: DEVOPS-OFFLINE-14-002 (Wave 2)) before starting and report status in module TASKS.md. - Team DevEx/CLI, Scanner WebService Guild: read EXECPLAN.md Wave 3 and SPRINTS.md rows for `src/StellaOps.Cli/TASKS.md`. Focus on CLI-RUNTIME-13-008 (TODO). Confirm prerequisites (internal: SCANNER-RUNTIME-12-302 (Wave 2)) before starting and report status in module TASKS.md. - Team Excititor Connectors – Stella: read EXECPLAN.md Wave 3 and SPRINTS.md rows for `src/StellaOps.Excititor.Connectors.StellaOpsMirror/TASKS.md`. Focus on EXCITITOR-CONN-STELLA-07-001 (TODO). Confirm prerequisites (internal: EXCITITOR-EXPORT-01-007 (Wave 2)) before starting and report status in module TASKS.md. - Team Notify Engine Guild: read EXECPLAN.md Wave 3 and SPRINTS.md rows for `src/StellaOps.Notify.Engine/TASKS.md`. Focus on NOTIFY-ENGINE-15-303 (TODO). Confirm prerequisites (internal: NOTIFY-ENGINE-15-302 (Wave 2)) before starting and report status in module TASKS.md. @@ -121,19 +122,19 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster ### Wave 5 - Team Excititor Connectors – Stella: read EXECPLAN.md Wave 5 and SPRINTS.md rows for `src/StellaOps.Excititor.Connectors.StellaOpsMirror/TASKS.md`. Focus on EXCITITOR-CONN-STELLA-07-003 (TODO). Confirm prerequisites (internal: EXCITITOR-CONN-STELLA-07-002 (Wave 4)) before starting and report status in module TASKS.md. -- Team Notify Connectors Guild: read EXECPLAN.md Wave 5 and SPRINTS.md rows for `src/StellaOps.Notify.Connectors.Email/TASKS.md`, `src/StellaOps.Notify.Connectors.Slack/TASKS.md`, `src/StellaOps.Notify.Connectors.Teams/TASKS.md`, `src/StellaOps.Notify.Connectors.Webhook/TASKS.md`. Focus on NOTIFY-CONN-SLACK-15-502 (DOING), NOTIFY-CONN-TEAMS-15-602 (DOING), NOTIFY-CONN-EMAIL-15-702 (DOING), NOTIFY-CONN-WEBHOOK-15-802 (DOING). Confirm prerequisites (internal: NOTIFY-CONN-EMAIL-15-701 (Wave 4), NOTIFY-CONN-SLACK-15-501 (Wave 4), NOTIFY-CONN-TEAMS-15-601 (Wave 4), NOTIFY-CONN-WEBHOOK-15-801 (Wave 4)) before starting and report status in module TASKS.md. +- Team Notify Connectors Guild: read EXECPLAN.md Wave 5 and SPRINTS.md rows for `src/StellaOps.Notify.Connectors.Email/TASKS.md`, `src/StellaOps.Notify.Connectors.Slack/TASKS.md`, `src/StellaOps.Notify.Connectors.Teams/TASKS.md`, `src/StellaOps.Notify.Connectors.Webhook/TASKS.md`. Focus on NOTIFY-CONN-SLACK-15-502 (DONE), NOTIFY-CONN-TEAMS-15-602 (DONE), NOTIFY-CONN-EMAIL-15-702 (BLOCKED 2025-10-20), NOTIFY-CONN-WEBHOOK-15-802 (BLOCKED 2025-10-20). Confirm prerequisites (internal: NOTIFY-CONN-EMAIL-15-701 (Wave 4), NOTIFY-CONN-SLACK-15-501 (Wave 4), NOTIFY-CONN-TEAMS-15-601 (Wave 4), NOTIFY-CONN-WEBHOOK-15-801 (Wave 4)) before starting and report status in module TASKS.md. - Team Scanner WebService Guild: read EXECPLAN.md Wave 5 and SPRINTS.md rows for `src/StellaOps.Scanner.WebService/TASKS.md`. Focus on SCANNER-RUNTIME-17-401 (TODO). Confirm prerequisites (internal: POLICY-RUNTIME-17-201 (Wave 4), SCANNER-EMIT-17-701 (Wave 1), SCANNER-RUNTIME-12-301 (Wave 1), ZASTAVA-OBS-17-005 (Wave 3)) before starting and report status in module TASKS.md. - Team TBD: read EXECPLAN.md Wave 5 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md`. Focus on SCANNER-ANALYZERS-LANG-10-308D (TODO), SCANNER-ANALYZERS-LANG-10-308G (TODO), SCANNER-ANALYZERS-LANG-10-308P (TODO), SCANNER-ANALYZERS-LANG-10-308R (TODO). Confirm prerequisites (internal: SCANNER-ANALYZERS-LANG-10-307D (Wave 4), SCANNER-ANALYZERS-LANG-10-307G (Wave 4), SCANNER-ANALYZERS-LANG-10-307P (Wave 4), SCANNER-ANALYZERS-LANG-10-307R (Wave 4)) before starting and report status in module TASKS.md. ### Wave 6 -- Team Notify Connectors Guild: read EXECPLAN.md Wave 6 and SPRINTS.md rows for `src/StellaOps.Notify.Connectors.Email/TASKS.md`, `src/StellaOps.Notify.Connectors.Slack/TASKS.md`, `src/StellaOps.Notify.Connectors.Teams/TASKS.md`, `src/StellaOps.Notify.Connectors.Webhook/TASKS.md`. Focus on NOTIFY-CONN-SLACK-15-503 (TODO), NOTIFY-CONN-TEAMS-15-603 (TODO), NOTIFY-CONN-EMAIL-15-703 (TODO), NOTIFY-CONN-WEBHOOK-15-803 (TODO). Confirm prerequisites (internal: NOTIFY-CONN-EMAIL-15-702 (Wave 5), NOTIFY-CONN-SLACK-15-502 (Wave 5), NOTIFY-CONN-TEAMS-15-602 (Wave 5), NOTIFY-CONN-WEBHOOK-15-802 (Wave 5)) before starting and report status in module TASKS.md. +- Team Notify Connectors Guild: read EXECPLAN.md Wave 6 and SPRINTS.md rows for `src/StellaOps.Notify.Connectors.Email/TASKS.md`, `src/StellaOps.Notify.Connectors.Slack/TASKS.md`, `src/StellaOps.Notify.Connectors.Teams/TASKS.md`, `src/StellaOps.Notify.Connectors.Webhook/TASKS.md`. Focus on NOTIFY-CONN-SLACK-15-503 (DONE), NOTIFY-CONN-TEAMS-15-603 (DONE), NOTIFY-CONN-EMAIL-15-703 (DONE), NOTIFY-CONN-WEBHOOK-15-803 (DONE). Confirm packaging outputs remain deterministic while upstream implementation tasks (15-702/802) stay blocked. - Team TBD: read EXECPLAN.md Wave 6 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md`. Focus on SCANNER-ANALYZERS-LANG-10-309D (TODO), SCANNER-ANALYZERS-LANG-10-309G (TODO), SCANNER-ANALYZERS-LANG-10-309P (TODO), SCANNER-ANALYZERS-LANG-10-309R (TODO). Confirm prerequisites (internal: SCANNER-ANALYZERS-LANG-10-308D (Wave 5), SCANNER-ANALYZERS-LANG-10-308G (Wave 5), SCANNER-ANALYZERS-LANG-10-308P (Wave 5), SCANNER-ANALYZERS-LANG-10-308R (Wave 5)) before starting and report status in module TASKS.md. ### Wave 7 - Team Team Core Engine & Storage Analytics: read EXECPLAN.md Wave 7 and SPRINTS.md rows for `src/StellaOps.Concelier.Core/TASKS.md`. Focus on FEEDCORE-ENGINE-07-001 (DONE 2025-10-19). Confirm prerequisites (internal: FEEDSTORAGE-DATA-07-001 (Wave 10)) before starting and report status in module TASKS.md. ### Wave 8 -- Team Team Core Engine & Data Science: read EXECPLAN.md Wave 8 and SPRINTS.md rows for `src/StellaOps.Concelier.Core/TASKS.md`. Focus on FEEDCORE-ENGINE-07-002 (TODO). Confirm prerequisites (internal: FEEDCORE-ENGINE-07-001 (Wave 7)) before starting and report status in module TASKS.md. +- Team Team Core Engine & Data Science: read EXECPLAN.md Wave 8 and SPRINTS.md rows for `src/StellaOps.Concelier.Core/TASKS.md`. Focus on FEEDCORE-ENGINE-07-002 (DONE 2025-10-21). Confirm prerequisites (internal: FEEDCORE-ENGINE-07-001 (Wave 7)) before starting and report status in module TASKS.md. ### Wave 9 - Team Team Core Engine & Storage Analytics: read EXECPLAN.md Wave 9 and SPRINTS.md rows for `src/StellaOps.Concelier.Core/TASKS.md`. Focus on FEEDCORE-ENGINE-07-003 (TODO). Confirm prerequisites (internal: FEEDCORE-ENGINE-07-001 (Wave 7)) before starting and report status in module TASKS.md. @@ -154,10 +155,10 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster - Team Concelier WebService Guild: read EXECPLAN.md Wave 14 and SPRINTS.md rows for `src/StellaOps.Concelier.WebService/TASKS.md`. CONCELIER-WEB-08-201 closed (2025-10-20); coordinate with DevOps for mirror smoke before promoting to stable. ### Wave 15 -- Team BE-Conn-Stella: read EXECPLAN.md Wave 15 and SPRINTS.md rows for `src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md`. Focus on FEEDCONN-STELLA-08-001 (TODO). Confirm prerequisites (internal: CONCELIER-EXPORT-08-201 (Wave 12)) before starting and report status in module TASKS.md. +- Team BE-Conn-Stella: read EXECPLAN.md Wave 15 and SPRINTS.md rows for `src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md`. Focus on FEEDCONN-STELLA-08-001 (DONE 2025-10-20). Confirm prerequisites (internal: CONCELIER-EXPORT-08-201 (Wave 12)) before starting and report status in module TASKS.md. ### Wave 16 -- Team BE-Conn-Stella: read EXECPLAN.md Wave 16 and SPRINTS.md rows for `src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md`. Focus on FEEDCONN-STELLA-08-002 (TODO). Confirm prerequisites (internal: FEEDCONN-STELLA-08-001 (Wave 15)) before starting and report status in module TASKS.md. +- Team BE-Conn-Stella: read EXECPLAN.md Wave 16 and SPRINTS.md rows for `src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md`. FEEDCONN-STELLA-08-002 completed (2025-10-20) with canonical DTO mapper + provenance fixtures. ### Wave 17 - Team BE-Conn-Stella: read EXECPLAN.md Wave 17 and SPRINTS.md rows for `src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md`. Focus on FEEDCONN-STELLA-08-003 (TODO). Confirm prerequisites (internal: FEEDCONN-STELLA-08-002 (Wave 16)) before starting and report status in module TASKS.md. @@ -166,9 +167,12 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster - **Sprint 1** · Backlog - Team: UX Specialist, Angular Eng - Path: `src/StellaOps.Web/TASKS.md` - 1. [TODO] WEB1.TRIVY-SETTINGS — Implement Trivy DB exporter settings panel with `publishFull`, `publishDelta`, `includeFull`, `includeDelta` toggles and “Run export now” action using future `/exporters/trivy-db/settings` API. + 1. [DONE] WEB1.TRIVY-SETTINGS — Implement Trivy DB exporter settings panel with `publishFull`, `publishDelta`, `includeFull`, `includeDelta` toggles and “Run export now” action using future `/exporters/trivy-db/settings` API. • Prereqs: — - • Current: TODO + • Current: DONE (2025-10-21) – Angular route `/concelier/trivy-db-settings` with reactive form, API client, and run-now workflow built; see `TrivyDbSettingsPageComponent`. + 2. [BLOCKED] WEB1.TRIVY-SETTINGS-TESTS — Add headless UI test run (`ng test --watch=false`) and document prerequisites once Angular tooling is chained up. + • Prereqs: WEB1.TRIVY-SETTINGS + • Current: BLOCKED (2025-10-21) – Awaiting Angular CLI/toolchain availability in CI/local dev environments before wiring Karma tests for the new screen. - **Sprint 1** · Developer Tooling - Team: DevEx/CLI - Path: `src/StellaOps.Cli/TASKS.md` @@ -226,7 +230,7 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster - Path: `src/StellaOps.Concelier.Merge/TASKS.md` 1. [DOING] FEEDMERGE-COORD-02-900 — Range primitives rollout coordination — Coordinate remaining connectors (`Acsc`, `Cccs`, `CertBund`, `CertCc`, `Cve`, `Ghsa`, `Ics.Cisa`, `Kisa`, `Ru.Bdu`, `Ru.Nkcki`, `Vndr.Apple`, `Vndr.Cisco`, `Vndr.Msrc`) to emit canonical range primitives with provenance tags; fixtures tracked in `RANGE_PRIMITIVES_COORDINATION.md`. • Prereqs: — - • Current: DOING (2025-10-12) + • Current: DOING (2025-10-20) – Coordination docs refreshed with connector due dates (Cccs/Cisco 2025-10-21, CertBund 2025-10-22, ICS-CISA 2025-10-23, KISA 2025-10-24); escalation plan defined if deadlines slip. - **Sprint 3** · Backlog - Team: Tools Guild, BE-Conn-MSRC - Path: `src/StellaOps.Concelier.Connector.Common/TASKS.md` @@ -241,9 +245,9 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster • Current: TODO – Add verification helpers for Worker/WebService, metrics/logging hooks, and negative-path regression tests. - Team: Team Excititor WebService - Path: `src/StellaOps.Excititor.WebService/TASKS.md` - 1. [TODO] EXCITITOR-WEB-01-002 — EXCITITOR-WEB-01-002 – Ingest & reconcile endpoints + 1. [DONE] EXCITITOR-WEB-01-002 — EXCITITOR-WEB-01-002 – Ingest & reconcile endpoints • Prereqs: EXCITITOR-WEB-01-001 (external/completed) - • Current: TODO – Implement `/excititor/init`, `/excititor/ingest/run`, `/excititor/ingest/resume`, `/excititor/reconcile` with token scope enforcement and structured run telemetry. + • Current: DONE (2025-10-20) – `/excititor/init`, `/excititor/ingest/run`, `/excititor/ingest/resume`, `/excititor/reconcile` enforce `vex.admin`, normalize provider inputs, and emit deterministic summaries; verified via `dotnet test src/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj --filter FullyQualifiedName~IngestEndpointsTests`. 2. [TODO] EXCITITOR-WEB-01-003 — EXCITITOR-WEB-01-003 – Export & verify endpoints • Prereqs: EXCITITOR-WEB-01-001 (external/completed), EXCITITOR-EXPORT-01-001 (external/completed), EXCITITOR-ATTEST-01-001 (external/completed) • Current: TODO – Add `/excititor/export`, `/excititor/export/{id}`, `/excititor/export/{id}/download`, `/excititor/verify`, returning artifact + attestation metadata with cache awareness. @@ -297,9 +301,9 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster • Current: TODO – Provide export serializer generating canonical OpenVEX documents with optional SBOM references and hash-stable ordering. - Team: Team Excititor Worker - Path: `src/StellaOps.Excititor.Worker/TASKS.md` - 1. [TODO] EXCITITOR-WORKER-01-002 — EXCITITOR-WORKER-01-002 – Resume tokens & retry policy + 1. [DONE 2025-10-21] EXCITITOR-WORKER-01-002 — EXCITITOR-WORKER-01-002 – Resume tokens & retry policy • Prereqs: EXCITITOR-WORKER-01-001 (external/completed) - • Current: TODO – Implement durable resume markers, exponential backoff with jitter, and quarantine for failing connectors per architecture spec. + • Current: DONE – Worker updates connector state with resume tokens + success/failure metadata and applies jittered exponential backoff with quarantine scheduling; unit coverage added for skip/backoff/resume flows. - **Sprint 7** · Contextual Truth Foundations - Team: Team Excititor Export - Path: `src/StellaOps.Excititor.Export/TASKS.md` @@ -308,12 +312,12 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster • Current: TODO – Emit consensus+score envelopes in export manifests, include policy/scoring digests, and update offline bundle/ORAS layouts to carry signed VEX responses. - Team: Team Excititor WebService - Path: `src/StellaOps.Excititor.WebService/TASKS.md` - 1. [TODO] EXCITITOR-WEB-01-004 — Resolve API & signed responses – expose `/excititor/resolve`, return signed consensus/score envelopes, document auth. + 1. [DONE 2025-10-20] EXCITITOR-WEB-01-004 — Resolve API & signed responses – expose `/excititor/resolve`, return signed consensus/score envelopes, document auth. • Prereqs: — • Current: TODO - Team: Team Excititor Worker - Path: `src/StellaOps.Excititor.Worker/TASKS.md` - 1. [TODO] EXCITITOR-WORKER-01-004 — EXCITITOR-WORKER-01-004 – TTL refresh & stability damper + 1. [DONE 2025-10-21] EXCITITOR-WORKER-01-004 — EXCITITOR-WORKER-01-004 – TTL refresh & stability damper • Prereqs: EXCITITOR-WORKER-01-001 (external/completed), EXCITITOR-CORE-02-001 (external/completed) • Current: TODO – Monitor consensus/VEX TTLs, apply 24–48h dampers before flipping published status/score, and trigger re-resolve when base image or kernel fingerprints change. - **Sprint 8** · Mongo strengthening @@ -340,9 +344,9 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster • Current: DONE – Admin backfill endpoint, CLI command (`stellaops excititor backfill-statements`), integration coverage, and operator runbook published; further automation tracked separately if needed. - Team: Team Excititor Worker - Path: `src/StellaOps.Excititor.Worker/TASKS.md` - 1. [TODO] EXCITITOR-WORKER-02-001 — EXCITITOR-WORKER-02-001 – Resolve Microsoft.Extensions.Caching.Memory advisory + 1. [DONE 2025-10-21] EXCITITOR-WORKER-02-001 — EXCITITOR-WORKER-02-001 – Resolve Microsoft.Extensions.Caching.Memory advisory • Prereqs: EXCITITOR-WORKER-01-001 (external/completed) - • Current: TODO – Bump `Microsoft.Extensions.Caching.Memory` (and related packages) to the latest .NET 10 preview, regenerate lockfiles, and re-run worker/webservice tests to clear NU1903 high severity warning. + • Current: DONE (2025-10-21) – Upgraded Excititor workers/connectors to `Microsoft.Extensions.*` 10.0.0-preview.7.25380.108, restored attestation diagnostics, and re-ran worker + webservice test suites with no NU1903 vulnerabilities. - **Sprint 8** · Plugin Infrastructure - Team: Plugin Platform Guild - Path: `src/StellaOps.Plugin/TASKS.md` @@ -351,9 +355,9 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster • Current: TODO - Team: Plugin Platform Guild, Authority Core - Path: `src/StellaOps.Plugin/TASKS.md` - 1. [TODO] PLUGIN-DI-08-002 — Update Authority plugin integration — Flow scoped services through identity-provider registrars, bootstrap flows, and background jobs; add regression coverage around scoped lifetimes. (Coordination session set for 2025-10-20 15:00–16:00 UTC; document outcomes before implementation.) + 1. [DONE] PLUGIN-DI-08-002 — Update Authority plugin integration — Flow scoped services through identity-provider registrars, bootstrap flows, and background jobs; add regression coverage around scoped lifetimes. (Implemented 2025-10-20 with scoped Standard plugin registrations and registry handles.) • Prereqs: — - • Current: TODO + • Current: DONE (2025-10-20) – Standard registrar registers scoped credential/provisioning stores and identity-provider plugins, registry Acquire returns scoped handles, and tests `dotnet test src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StellaOps.Authority.Plugin.Standard.Tests.csproj` + `dotnet test src/StellaOps.Authority/StellaOps.Authority.Tests/StellaOps.Authority.Tests.csproj` validate behaviour. - **Sprint 9** · Docs & Governance - Team: Platform Events Guild - Path: `docs/TASKS.md` @@ -368,15 +372,15 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster - **Sprint 9** · Policy Foundations - Team: Policy Guild - Path: `src/StellaOps.Policy/TASKS.md` - 1. [TODO] POLICY-CORE-09-004 — Versioned scoring config with schema validation, trust table, and golden fixtures. + 1. [DONE] POLICY-CORE-09-004 — Versioned scoring config with schema validation, trust table, and golden fixtures. (2025-10-19) • Prereqs: — - • Current: TODO - 2. [TODO] POLICY-CORE-09-005 — Scoring/quiet engine – compute score, enforce VEX-only quiet rules, emit inputs and provenance. + • Current: DONE (2025-10-19) + 2. [DONE] POLICY-CORE-09-005 — Scoring/quiet engine – compute score, enforce VEX-only quiet rules, emit inputs and provenance. (2025-10-19) • Prereqs: — - • Current: TODO - 3. [TODO] POLICY-CORE-09-006 — Unknown state & confidence decay – deterministic bands surfaced in policy outputs. + • Current: DONE (2025-10-19) + 3. [DONE] POLICY-CORE-09-006 — Unknown state & confidence decay – deterministic bands surfaced in policy outputs. (2025-10-19) • Prereqs: — - • Current: TODO + • Current: DONE (2025-10-19) - **Sprint 10** · Backlog - Team: TBD - Path: `src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md` @@ -473,7 +477,7 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster • Current: TODO - Team: Authority Core & Security Guild - Path: `src/StellaOps.Authority/TASKS.md` - 1. [DOING] AUTH-DPOP-11-001 — Implement DPoP proof validation + nonce handling for high-value audiences per architecture. + 1. [DONE] AUTH-DPOP-11-001 — Implement DPoP proof validation + nonce handling for high-value audiences per architecture. (Redis-configurable nonce store + docs landed 2025-10-20) • Prereqs: — • Current: DOING (2025-10-19) 2. [DOING] AUTH-MTLS-11-002 — Add OAuth mTLS client credential support with certificate-bound tokens and introspection updates. @@ -481,15 +485,15 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster • Current: DOING (2025-10-19) - Team: Signer Guild - Path: `src/StellaOps.Signer/TASKS.md` - 1. [TODO] SIGNER-API-11-101 — `/sign/dsse` pipeline with Authority auth, PoE introspection, release verification, DSSE signing. + 1. [DONE] SIGNER-API-11-101 — `/sign/dsse` pipeline with Authority auth, PoE introspection, release verification, DSSE signing. • Prereqs: — - • Current: TODO - 2. [TODO] SIGNER-REF-11-102 — `/verify/referrers` endpoint with OCI lookup, caching, and policy enforcement. + • Current: DONE (2025-10-21) – Minimal API host now issues DSSE bundles with PoE validation, release verification, and quota enforcement; integration tests cover success/error paths via `dotnet test src/StellaOps.Signer/StellaOps.Signer.Tests/StellaOps.Signer.Tests.csproj`. + 2. [DONE] SIGNER-REF-11-102 — `/verify/referrers` endpoint with OCI lookup, caching, and policy enforcement. • Prereqs: — - • Current: TODO - 3. [TODO] SIGNER-QUOTA-11-103 — Enforce plan quotas, concurrency/QPS limits, artifact size caps with metrics/audit logs. + • Current: DONE (2025-10-21) – Added `/api/v1/signer/verify/referrers` returning deterministic JSON responses for trusted/untrusted digests with regression coverage. + 3. [DONE] SIGNER-QUOTA-11-103 — Enforce plan quotas, concurrency/QPS limits, artifact size caps with metrics/audit logs. • Prereqs: — - • Current: TODO + • Current: DONE (2025-10-21) – In-memory quota service applies payload caps and per-tenant QPS throttles; tests cover oversize and throttled cases. - **Sprint 12** · Runtime Guardrails - Team: Zastava Core Guild - Path: `src/StellaOps.Zastava.Core/TASKS.md` @@ -555,25 +559,28 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster • Current: TODO - Team: Scanner WebService Guild - Path: `src/StellaOps.Scanner.WebService/TASKS.md` - 1. [TODO] SCANNER-EVENTS-15-201 — Emit `scanner.report.ready` + `scanner.scan.completed` events. + 1. [DONE] SCANNER-EVENTS-15-201 — Emit `scanner.report.ready` + `scanner.scan.completed` events. • Prereqs: — • Current: TODO + 2. [BLOCKED] SCANNER-EVENTS-16-301 — Redis publisher integration tests once Notify queue adapter ships. + • Prereqs: NOTIFY-QUEUE-15-401 (Wave 1) + • Current: BLOCKED – waiting on Notify queue abstraction and Redis adapter deliverables for end-to-end validation. - **Sprint 16** · Scheduler Intelligence - Team: Scheduler ImpactIndex Guild - Path: `src/StellaOps.Scheduler.ImpactIndex/TASKS.md` - 1. [DOING] SCHED-IMPACT-16-300 — **STUB** ingest/query using fixtures to unblock Scheduler planning (remove by SP16 end). + 1. [DONE (2025-10-20)] SCHED-IMPACT-16-300 — **STUB** ingest/query using fixtures to unblock Scheduler planning (remove by SP16 end). • Prereqs: SAMPLES-10-001 (external/completed) • Current: DOING - Team: Scheduler Models Guild - Path: `src/StellaOps.Scheduler.Models/TASKS.md` - 1. [TODO] SCHED-MODELS-16-103 — Versioning/migration helpers (schedule evolution, run state transitions). + 1. [DONE (2025-10-20)] SCHED-MODELS-16-103 - Versioning/migration helpers (schedule evolution, run state transitions). • Prereqs: SCHED-MODELS-16-101 (external/completed) - • Current: TODO + • Current: DONE - Team: Scheduler Queue Guild - Path: `src/StellaOps.Scheduler.Queue/TASKS.md` - 1. [TODO] SCHED-QUEUE-16-401 — Implement queue abstraction + Redis Streams adapter (planner inputs, runner segments) with ack/lease semantics. + 1. [DONE (2025-10-20)] SCHED-QUEUE-16-401 - Implement queue abstraction + Redis Streams adapter (planner inputs, runner segments) with ack/lease semantics. • Prereqs: SCHED-MODELS-16-101 (external/completed) - • Current: TODO + • Current: DONE - Team: Scheduler Storage Guild - Path: `src/StellaOps.Scheduler.Storage.Mongo/TASKS.md` 1. [TODO] SCHED-STORAGE-16-201 — Create Mongo collections (schedules, runs, impact_cursors, locks, audit) with indexes/migrations per architecture. @@ -584,6 +591,18 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster 1. [TODO] SCHED-WEB-16-101 — Bootstrap Minimal API host with Authority OpTok + DPoP, health endpoints, plug-in discovery per architecture §§1–2. • Prereqs: SCHED-MODELS-16-101 (external/completed) • Current: TODO +- **Sprint 18** · Launch Readiness + - Team: DevOps Guild + - Path: `ops/devops/TASKS.md` + 1. [TODO] DEVOPS-LAUNCH-18-100 - Finalise production environment footprint (clusters, secrets, network overlays) for full-platform go-live. + • Prereqs: — + • Current: TODO + 2. [TODO] DEVOPS-LAUNCH-18-900 - Collect "full implementation" sign-off from module owners and consolidate the launch readiness checklist. + • Prereqs: Wave 0 completion + • Current: TODO + 3. [TODO] DEVOPS-LAUNCH-18-001 - Production launch cutover rehearsal and runbook publication. + • Prereqs: DEVOPS-LAUNCH-18-100, DEVOPS-LAUNCH-18-900 + • Current: TODO ## Wave 1 — 45 task(s) ready after Wave 0 - **Sprint 6** · Excititor Ingest & Formats @@ -621,9 +640,12 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster - **Sprint 9** · DevOps Foundations - Team: DevOps Guild, Scanner WebService Guild - Path: `ops/devops/TASKS.md` - 1. [TODO] DEVOPS-SCANNER-09-204 — Surface `SCANNER__EVENTS__*` environment variables across docker-compose (dev/stage/airgap) and Helm values, defaulting to share the Redis queue DSN. + 1. [DONE] DEVOPS-SCANNER-09-204 — Surface `SCANNER__EVENTS__*` environment variables across docker-compose (dev/stage/airgap) and Helm values, defaulting to share the Redis queue DSN. (2025-10-21) • Prereqs: SCANNER-EVENTS-15-201 (Wave 0) - • Current: TODO + • Current: DONE (2025-10-21) – Compose dev/stage/airgap profiles and Helm values now expose the SCANNER__EVENTS__* toggles; docs (deploy/compose/README.md, docs/ARCHITECTURE_SCANNER.md) call out the new configuration knobs. + 2. [DONE] DEVOPS-SCANNER-09-205 — Add Notify smoke stage that tails the Redis stream and asserts `scanner.report.ready`/`scanner.scan.completed` reach Notify WebService in staging. (2025-10-21) + • Prereqs: DEVOPS-SCANNER-09-204 (Wave 0) + • Current: DONE (2025-10-21) – `notify-smoke` CI job runs the NotifySmokeCheck tool against staging Redis/Notify using configured secrets; deploy docs enumerate required configuration. - **Sprint 10** · Backlog - Team: TBD - Path: `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md` @@ -649,9 +671,9 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster - **Sprint 10** · Benchmarks - Team: Bench Guild, Language Analyzer Guild - Path: `bench/TASKS.md` - 1. [TODO] BENCH-SCANNER-10-002 — Wire real language analyzers into bench harness & refresh baselines post-implementation. + 1. [DONE] BENCH-SCANNER-10-002 — Wire real language analyzers into bench harness & refresh baselines post-implementation. (2025-10-21) • Prereqs: SCANNER-ANALYZERS-LANG-10-301 (Wave 0) - • Current: TODO + • Current: DONE (2025-10-21) – Harness now invokes language analyzers via `StellaOps.Bench.ScannerAnalyzers`, baseline refreshed against samples/runtime fixtures, and README/config updated for the new flow. - **Sprint 10** · Scanner Analyzers & SBOM - Team: Emit Guild - Path: `src/StellaOps.Scanner.Emit/TASKS.md` @@ -687,9 +709,15 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster - **Sprint 12** · Runtime Guardrails - Team: Scanner WebService Guild - Path: `src/StellaOps.Scanner.WebService/TASKS.md` - 1. [TODO] SCANNER-RUNTIME-12-301 — Implement `/runtime/events` ingestion endpoint with validation, batching, and storage hooks per Zastava contract. + 1. [DONE] SCANNER-RUNTIME-12-301 — Implement `/runtime/events` ingestion endpoint with validation, batching, and storage hooks per Zastava contract. (2025-10-20) • Prereqs: ZASTAVA-CORE-12-201 (Wave 0) - • Current: TODO + • Current: DONE (2025-10-20) — Mongo persistence + rate limiting shipped; observer fixtures can replay batches end-to-end. + 2. [DOING] SCANNER-RUNTIME-12-302 — Implement `/policy/runtime` endpoint joining SBOM baseline + policy verdict, returning admission guidance. + • Prereqs: SCANNER-RUNTIME-12-301 (Wave 1), ZASTAVA-CORE-12-201 (Wave 0) + • Current: DOING (2025-10-20) — Locking response schema with Policy/CLI guilds, wiring determinism tests. + 3. [TODO] SCANNER-RUNTIME-12-303 — Align runtime verdicts with canonical policy evaluation (Feedser/Vexer inputs) once upstream dependencies land. + 4. [TODO] SCANNER-RUNTIME-12-304 — Surface attestation/Rekor verification results via Authority/Attestor integration. + 5. [TODO] SCANNER-RUNTIME-12-305 — Finalize shared fixtures and CI automation with Zastava + CLI teams for runtime APIs. - Team: Zastava Observer Guild - Path: `src/StellaOps.Zastava.Observer/TASKS.md` 1. [TODO] ZASTAVA-OBS-12-001 — Build container lifecycle watcher that tails CRI (containerd/cri-o/docker) events and emits deterministic runtime records with buffering + backoff. @@ -759,12 +787,12 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster • Current: TODO - Team: Scheduler Queue Guild - Path: `src/StellaOps.Scheduler.Queue/TASKS.md` - 1. [TODO] SCHED-QUEUE-16-402 — Add NATS JetStream adapter with configuration binding, health probes, failover. + 1. [DONE (2025-10-20)] SCHED-QUEUE-16-402 - Add NATS JetStream adapter with configuration binding, health probes, failover. • Prereqs: SCHED-QUEUE-16-401 (Wave 0) - • Current: TODO - 2. [TODO] SCHED-QUEUE-16-403 — Dead-letter handling + metrics (queue depth, retry counts), configuration toggles. + • Current: DONE + 2. [DONE (2025-10-20)] SCHED-QUEUE-16-403 - Dead-letter handling + metrics (queue depth, retry counts), configuration toggles. • Prereqs: SCHED-QUEUE-16-401 (Wave 0) - • Current: TODO + • Current: DONE - Team: Scheduler Storage Guild - Path: `src/StellaOps.Scheduler.Storage.Mongo/TASKS.md` 1. [TODO] SCHED-STORAGE-16-203 — Audit/logging pipeline + run stats materialized views for UI. @@ -983,9 +1011,9 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster - **Sprint 13** · UX & CLI Experience - Team: DevEx/CLI - Path: `src/StellaOps.Cli/TASKS.md` - 1. [TODO] CLI-OFFLINE-13-006 — CLI-OFFLINE-13-006 – Offline kit workflows + 1. [DONE] CLI-OFFLINE-13-006 — CLI-OFFLINE-13-006 – Offline kit workflows • Prereqs: DEVOPS-OFFLINE-14-002 (Wave 2) - • Current: TODO – Implement `offline kit pull/import/status` commands with integrity checks, resumable downloads, and doc updates. + • Current: DONE (2025-10-21) – Delivered `offline kit pull/import/status` commands with resumable downloads, digest/metadata validation, CLI metrics + docs, and regression coverage (`dotnet test src/StellaOps.Cli.Tests`). - Team: DevEx/CLI, Scanner WebService Guild - Path: `src/StellaOps.Cli/TASKS.md` 1. [TODO] CLI-RUNTIME-13-008 — CLI-RUNTIME-13-008 – Runtime policy contract sync @@ -1121,17 +1149,20 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster - **Sprint 15** · Notify Foundations - Team: Notify Connectors Guild - Path: `src/StellaOps.Notify.Connectors.Email/TASKS.md` - 1. [DOING] NOTIFY-CONN-EMAIL-15-702 — Add DKIM signing optional support and health/test-send flows. + 1. [BLOCKED] NOTIFY-CONN-EMAIL-15-702 — Add DKIM signing optional support and health/test-send flows. • Prereqs: NOTIFY-CONN-EMAIL-15-701 (Wave 4) - • Current: TODO + • Current: BLOCKED – waiting on base SMTP connector implementation (NOTIFY-CONN-EMAIL-15-701). - Path: `src/StellaOps.Notify.Connectors.Slack/TASKS.md` - 1. [DOING] NOTIFY-CONN-SLACK-15-502 — Health check & test-send support with minimal scopes and redacted tokens. + 1. [DONE] NOTIFY-CONN-SLACK-15-502 — Health check & test-send support with minimal scopes and redacted tokens. • Prereqs: NOTIFY-CONN-SLACK-15-501 (Wave 4) • Current: TODO - Path: `src/StellaOps.Notify.Connectors.Teams/TASKS.md` - 1. [DOING] NOTIFY-CONN-TEAMS-15-602 — Provide health/test-send support with fallback text for legacy clients. + 1. [DONE] NOTIFY-CONN-TEAMS-15-602 — Provide health/test-send support with fallback text for legacy clients. • Prereqs: NOTIFY-CONN-TEAMS-15-601 (Wave 4) • Current: TODO + 2. [DONE] NOTIFY-CONN-TEAMS-15-604 — Align Teams health endpoint output with preview metadata redaction. + • Prereqs: NOTIFY-CONN-TEAMS-15-602 (Wave 5) + • Current: DONE - Path: `src/StellaOps.Notify.Connectors.Webhook/TASKS.md` 1. [DOING] NOTIFY-CONN-WEBHOOK-15-802 — Health/test-send support with signature validation hints and secret management. • Prereqs: NOTIFY-CONN-WEBHOOK-15-801 (Wave 4) @@ -1165,19 +1196,19 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster - **Sprint 15** · Notify Foundations - Team: Notify Connectors Guild - Path: `src/StellaOps.Notify.Connectors.Email/TASKS.md` - 1. [TODO] NOTIFY-CONN-EMAIL-15-703 — Package Email connector as restart-time plug-in (manifest + host registration). + 1. [DONE] NOTIFY-CONN-EMAIL-15-703 — Package Email connector as restart-time plug-in (manifest + host registration). • Prereqs: NOTIFY-CONN-EMAIL-15-702 (Wave 5) • Current: TODO - Path: `src/StellaOps.Notify.Connectors.Slack/TASKS.md` - 1. [TODO] NOTIFY-CONN-SLACK-15-503 — Package Slack connector as restart-time plug-in (manifest + host registration). + 1. [DONE] NOTIFY-CONN-SLACK-15-503 — Package Slack connector as restart-time plug-in (manifest + host registration). • Prereqs: NOTIFY-CONN-SLACK-15-502 (Wave 5) • Current: TODO - Path: `src/StellaOps.Notify.Connectors.Teams/TASKS.md` - 1. [TODO] NOTIFY-CONN-TEAMS-15-603 — Package Teams connector as restart-time plug-in (manifest + host registration). + 1. [DONE] NOTIFY-CONN-TEAMS-15-603 — Package Teams connector as restart-time plug-in (manifest + host registration). • Prereqs: NOTIFY-CONN-TEAMS-15-602 (Wave 5) • Current: TODO - Path: `src/StellaOps.Notify.Connectors.Webhook/TASKS.md` - 1. [TODO] NOTIFY-CONN-WEBHOOK-15-803 — Package Webhook connector as restart-time plug-in (manifest + host registration). + 1. [DONE] NOTIFY-CONN-WEBHOOK-15-803 — Package Webhook connector as restart-time plug-in (manifest + host registration). • Prereqs: NOTIFY-CONN-WEBHOOK-15-802 (Wave 5) • Current: TODO @@ -1193,9 +1224,9 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster - **Sprint 7** · Contextual Truth Foundations - Team: Team Core Engine & Data Science - Path: `src/StellaOps.Concelier.Core/TASKS.md` - 1. [TODO] FEEDCORE-ENGINE-07-002 — FEEDCORE-ENGINE-07-002 – Noise prior computation service + 1. [DONE] FEEDCORE-ENGINE-07-002 — FEEDCORE-ENGINE-07-002 – Noise prior computation service • Prereqs: FEEDCORE-ENGINE-07-001 (Wave 7) - • Current: TODO – Build rule-based learner capturing false-positive priors per package/env, persist summaries, and expose APIs for Excititor/scan suppressors with reproducible statistics. + • Current: DONE (2025-10-21) – Added NoisePriorService with rule-based aggregation of advisory statements, repository contracts for deterministic summaries, DI helper, and unit tests covering heuristics and persistence. ## Wave 9 — 1 task(s) ready after Wave 8 - **Sprint 7** · Contextual Truth Foundations @@ -1249,22 +1280,22 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster - **Sprint 8** · Mirror Distribution - Team: BE-Conn-Stella - Path: `src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md` - 1. [DOING] FEEDCONN-STELLA-08-001 — Implement Concelier mirror fetcher hitting `https://.stella-ops.org/concelier/exports/index.json`, verify signatures/digests, and persist raw documents with provenance. + 1. [DONE] FEEDCONN-STELLA-08-001 — Implement Concelier mirror fetcher hitting `https://.stella-ops.org/concelier/exports/index.json`, verify signatures/digests, and persist raw documents with provenance. • Prereqs: CONCELIER-EXPORT-08-201 (Wave 12) - • Current: DOING (2025-10-19) – Client consuming new signed mirror bundles/index, standing up verification + storage plumbing ahead of DTO mapping. + • Current: DONE (2025-10-20) – Fetch job persists manifest/bundle metadata, enforces digest and detached JWS verification (fallback PEM support), and regression coverage captured via `dotnet test src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/StellaOps.Concelier.Connector.StellaOpsMirror.Tests.csproj`. ## Wave 16 — 1 task(s) ready after Wave 15 - **Sprint 8** · Mirror Distribution - Team: BE-Conn-Stella - Path: `src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md` - 1. [TODO] FEEDCONN-STELLA-08-002 — Map mirror payloads into canonical advisory DTOs with provenance referencing mirror domain + original source metadata. + 1. [DONE] FEEDCONN-STELLA-08-002 — Map mirror payloads into canonical advisory DTOs with provenance referencing mirror domain + original source metadata. (2025-10-20) • Prereqs: FEEDCONN-STELLA-08-001 (Wave 15) - • Current: TODO + • Current: DONE (2025-10-20) – `MirrorAdvisoryMapper` emits canonical advisories and fixtures assert parity with exporter outputs. ## Wave 17 — 1 task(s) ready after Wave 16 - **Sprint 8** · Mirror Distribution - Team: BE-Conn-Stella - Path: `src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md` - 1. [TODO] FEEDCONN-STELLA-08-003 — Add incremental cursor + resume support (per-export fingerprint) and document configuration for downstream Concelier instances. + 1. [DONE] FEEDCONN-STELLA-08-003 — Add incremental cursor + resume support (per-export fingerprint) and document configuration for downstream Concelier instances. (2025-10-20) • Prereqs: FEEDCONN-STELLA-08-002 (Wave 16) - • Current: TODO + • Current: DONE (2025-10-20) – Connector records per-export fingerprints, resumes pending documents, and ops guide documents offline configuration knobs. diff --git a/NuGet.config b/NuGet.config index 0e524b46..9cfcc4e1 100644 --- a/NuGet.config +++ b/NuGet.config @@ -2,16 +2,16 @@ + - + + + - - - diff --git a/SPRINTS.md b/SPRINTS.md index 59a33462..30a9b4db 100644 --- a/SPRINTS.md +++ b/SPRINTS.md @@ -2,211 +2,33 @@ This file describe implementation of Stella Ops (docs/README.md). Implementation | Sprint | Theme | Tasks File Path | Status | Type of Specialist | Task ID | Task Description | | --- | --- | --- | --- | --- | --- | --- | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-12) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-001 | SemVer primitive range-style metadata
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Concelier.Models/AGENTS.md. This task lays the groundwork—complete the SemVer helper updates before teammates pick up FEEDMODELS-SCHEMA-01-002/003 and FEEDMODELS-SCHEMA-02-900. Use ./src/FASTER_MODELING_AND_NORMALIZATION.md for the target rule structure. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-11) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-002 | Provenance decision rationale field
Instructions to work:
AdvisoryProvenance now carries `decisionReason` and docs/tests were updated. Connectors and merge tasks should populate the field when applying precedence/freshness/tie-breaker logic; see src/StellaOps.Concelier.Models/PROVENANCE_GUIDELINES.md for usage guidance. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-11) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-003 | Normalized version rules collection
Instructions to work:
`AffectedPackage.NormalizedVersions` and supporting comparer/docs/tests shipped. Connector owners must emit rule arrays per ./src/FASTER_MODELING_AND_NORMALIZATION.md and report progress via FEEDMERGE-COORD-02-900 so merge/storage backfills can proceed. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-12) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-02-900 | Range primitives for SemVer/EVR/NEVRA metadata
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Concelier.Models/AGENTS.md before resuming this stalled effort. Confirm helpers align with the new `NormalizedVersions` representation so connectors finishing in Sprint 2 can emit consistent metadata. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Normalization/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDNORM-NORM-02-001 | SemVer normalized rule emitter
Shared `SemVerRangeRuleBuilder` now outputs primitives + normalized rules per `FASTER_MODELING_AND_NORMALIZATION.md`; CVE/GHSA connectors consuming the API have verified fixtures. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill
AdvisoryStore dual-writes flattened `normalizedVersions` when `concelier.storage.enableSemVerStyle` is set; migration `20251011-semver-style-backfill` updates historical records and docs outline the rollout. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-002 | Provenance decision reason persistence
Storage now persists `provenance.decisionReason` for advisories and merge events; tests cover round-trips. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-003 | Normalized versions indexing
Bootstrapper seeds compound/sparse indexes for flattened normalized rules and `docs/dev/mongo_indices.md` documents query guidance. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-TESTS-02-004 | Restore AdvisoryStore build after normalized versions refactor
Updated constructors/tests keep storage suites passing with the new feature flag defaults. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-ENGINE-01-002 | Plumb Authority client resilience options
WebService wires `authority.resilience.*` into `AddStellaOpsAuthClient` and adds binding coverage via `AuthorityClientResilienceOptionsAreBound`. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-003 | Author ops guidance for resilience tuning
Install/runbooks document connected vs air-gapped resilience profiles and monitoring hooks. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-004 | Document authority bypass logging patterns
Operator guides now call out `route/status/subject/clientId/scopes/bypass/remote` audit fields and SIEM triggers. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-005 | Update Concelier operator guide for enforcement cutoff
Install guide reiterates the 2025-12-31 cutoff and links audit signals to the rollout checklist. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | SEC3.HOST | Rate limiter policy binding
Authority host now applies configuration-driven fixed windows to `/token`, `/authorize`, and `/internal/*`; integration tests assert 429 + `Retry-After` headers; docs/config samples refreshed for Docs guild diagrams. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | SEC3.BUILD | Authority rate-limiter follow-through
`Security.RateLimiting` now fronts token/authorize/internal limiters; Authority + Configuration matrices (`dotnet test src/StellaOps.Authority/StellaOps.Authority.sln`, `dotnet test src/StellaOps.Configuration.Tests/StellaOps.Configuration.Tests.csproj`) passed on 2025-10-11; awaiting #authority-core broadcast. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-14) | Team Authority Platform & Security Guild | AUTHCORE-BUILD-OPENIDDICT / AUTHCORE-STORAGE-DEVICE-TOKENS / AUTHCORE-BOOTSTRAP-INVITES | Address remaining Authority compile blockers (OpenIddict transaction shim, token device document, bootstrap invite cleanup) so `dotnet build src/StellaOps.Authority.sln` returns success. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | PLG6.DOC | Plugin developer guide polish
Section 9 now documents rate limiter metadata, config keys, and lockout interplay; YAML samples updated alongside Authority config templates. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-001 | Fetch pipeline & state tracking
Summary planner now drives monthly/yearly VINCE fetches, persists pending summaries/notes, and hydrates VINCE detail queue with telemetry.
Team instructions: Read ./AGENTS.md and src/StellaOps.Concelier.Connector.CertCc/AGENTS.md. Coordinate daily with Models/Merge leads so new normalizedVersions output and provenance tags stay aligned with ./src/FASTER_MODELING_AND_NORMALIZATION.md. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-002 | VINCE note detail fetcher
Summary planner queues VINCE note detail endpoints, persists raw JSON with SHA/ETag metadata, and records retry/backoff metrics. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-003 | DTO & parser implementation
Added VINCE DTO aggregate, Markdown→text sanitizer, vendor/status/vulnerability parsers, and parser regression fixture. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-004 | Canonical mapping & range primitives
VINCE DTO aggregate flows through `CertCcMapper`, emitting vendor range primitives + normalized version rules that persist via `_advisoryStore`. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-005 | Deterministic fixtures/tests
Snapshot harness refreshed 2025-10-12; `certcc-*.snapshot.json` regenerated and regression suite green without UPDATE flag drift. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-006 | Telemetry & documentation
`CertCcDiagnostics` publishes summary/detail/parse/map metrics (meter `StellaOps.Concelier.Connector.CertCc`), README documents instruments, and log guidance captured for Ops on 2025-10-12. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-007 | Connector test harness remediation
Harness now wires `AddSourceCommon`, resets `FakeTimeProvider`, and passes canned-response regression run dated 2025-10-12. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-008 | Snapshot coverage handoff
Fixtures regenerated with normalized ranges + provenance fields on 2025-10-11; QA handoff notes published and merge backfill unblocked. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-012 | Schema sync & snapshot regen follow-up
Fixtures regenerated with normalizedVersions + provenance decision reasons; handoff notes updated for Merge backfill 2025-10-12. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-009 | Detail/map reintegration plan
Staged reintegration plan published in `src/StellaOps.Concelier.Connector.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md`; coordinates enablement with FEEDCONN-CERTCC-02-004. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-010 | Partial-detail graceful degradation
Detail fetch now tolerates 404/403/410 responses and regression tests cover mixed endpoint availability. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Distro.RedHat/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-REDHAT-02-001 | Fixture validation sweep
Instructions to work:
Fixtures regenerated post-model-helper rollout; provenance ordering and normalizedVersions scaffolding verified via tests. Conflict resolver deltas logged in src/StellaOps.Concelier.Connector.Distro.RedHat/CONFLICT_RESOLVER_NOTES.md for Sprint 3 consumers. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-12) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-001 | Canonical mapping & range primitives
Mapper emits SemVer rules (`scheme=apple:*`); fixtures regenerated with trimmed references + new RSR coverage, update tooling finalized. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-002 | Deterministic fixtures/tests
Sanitized live fixtures + regression snapshots wired into tests; normalized rule coverage asserted. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-003 | Telemetry & documentation
Apple meter metrics wired into Concelier WebService OpenTelemetry configuration; README and fixtures document normalizedVersions coverage. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-12) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-004 | Live HTML regression sweep
Sanitised HT125326/HT125328/HT106355/HT214108/HT215500 fixtures recorded and regression tests green on 2025-10-12. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-005 | Fixture regeneration tooling
`UPDATE_APPLE_FIXTURES=1` flow fetches & rewrites fixtures; README documents usage.
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Concelier.Connector.Vndr.Apple/AGENTS.md. Resume stalled tasks, ensuring normalizedVersions output and fixtures align with ./src/FASTER_MODELING_AND_NORMALIZATION.md before handing data to the conflict sprint. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-GHSA-02-001 | GHSA normalized versions & provenance
Team instructions: Read ./AGENTS.md and each module's AGENTS file. Adopt the `NormalizedVersions` array emitted by the models sprint, wiring provenance `decisionReason` where merge overrides occur. Follow ./src/FASTER_MODELING_AND_NORMALIZATION.md; report via src/StellaOps.Concelier.Merge/TASKS.md (FEEDMERGE-COORD-02-900). Progress 2025-10-11: GHSA/OSV emit normalized arrays with refreshed fixtures; CVE mapper now surfaces SemVer normalized ranges; NVD/KEV adoption pending; outstanding follow-ups include FEEDSTORAGE-DATA-02-001, FEEDMERGE-ENGINE-02-002, and rolling `tools/FixtureUpdater` updates across connectors. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-OSV-02-003 | OSV normalized versions & freshness | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-NVD-02-002 | NVD normalized versions & timestamps | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Cve/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-CVE-02-003 | CVE normalized versions uplift | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Kev/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-KEV-02-003 | KEV normalized versions propagation | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-OSV-04-003 | OSV parity fixture refresh | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-10) | Team WebService & Authority | FEEDWEB-DOCS-01-001 | Document authority toggle & scope requirements
Quickstart carries toggle/scope guidance pending docs guild review (no change this sprint). | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-ENGINE-01-002 | Plumb Authority client resilience options
WebService wires `authority.resilience.*` into `AddStellaOpsAuthClient` and adds binding coverage via `AuthorityClientResilienceOptionsAreBound`. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-003 | Author ops guidance for resilience tuning
Operator docs now outline connected vs air-gapped resilience profiles and monitoring cues. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-004 | Document authority bypass logging patterns
Audit logging guidance highlights `route/status/subject/clientId/scopes/bypass/remote` fields and SIEM alerts. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-005 | Update Concelier operator guide for enforcement cutoff
Install guide reiterates the 2025-12-31 cutoff and ties audit signals to rollout checks. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-19) | Team WebService & Authority | FEEDWEB-OPS-01-006 | Rename plugin drop directory to namespaced path
Build outputs now point at `StellaOps.Concelier.PluginBinaries`/`StellaOps.Authority.PluginBinaries`; defaults/docs/tests updated to reflect the new layout. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-OPS-01-007 | Authority resilience adoption
Deployment docs and CLI notes explain the LIB5 resilience knobs for rollout.
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Concelier.WebService/AGENTS.md. These items were mid-flight; resume implementation ensuring docs/operators receive timely updates. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHCORE-ENGINE-01-001 | CORE8.RL — Rate limiter plumbing validated; integration tests green and docs handoff recorded for middleware ordering + Retry-After headers (see `docs/dev/authority-rate-limit-tuning-outline.md` for continuing guidance). | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHCRYPTO-ENGINE-01-001 | SEC3.A — Shared metadata resolver confirmed via host test run; SEC3.B now unblocked for tuning guidance (outline captured in `docs/dev/authority-rate-limit-tuning-outline.md`). | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-13) | Team Authority Platform & Security Guild | AUTHSEC-DOCS-01-002 | SEC3.B — Published `docs/security/rate-limits.md` with tuning matrix, alert thresholds, and lockout interplay guidance; Docs guild can lift copy into plugin guide. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-14) | Team Authority Platform & Security Guild | AUTHSEC-CRYPTO-02-001 | SEC5.B1 — Introduce libsodium signing provider and parity tests to unblock CLI verification enhancements. | -| Sprint 1 | Bootstrap & Replay Hardening | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-14) | Security Guild | AUTHSEC-CRYPTO-02-004 | SEC5.D/E — Finish bootstrap invite lifecycle (API/store/cleanup) and token device heuristics; build currently red due to pending handler integration. | -| Sprint 1 | Developer Tooling | src/StellaOps.Cli/TASKS.md | DONE (2025-10-15) | DevEx/CLI | AUTHCLI-DIAG-01-001 | Surface password policy diagnostics in CLI startup/output so operators see weakened overrides immediately.
CLI now loads Authority plug-ins at startup, logs weakened password policies (length/complexity), and regression coverage lives in `StellaOps.Cli.Tests/Services/AuthorityDiagnosticsReporterTests`. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHPLUG-DOCS-01-001 | PLG6.DOC — Developer guide copy + diagrams merged 2025-10-11; limiter guidance incorporated and handed to Docs guild for asset export. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Normalization/TASKS.md | DONE (2025-10-12) | Team Normalization & Storage Backbone | FEEDNORM-NORM-02-001 | SemVer normalized rule emitter
`SemVerRangeRuleBuilder` shipped 2025-10-12 with comparator/`||` support and fixtures aligning to `FASTER_MODELING_AND_NORMALIZATION.md`. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-002 | Provenance decision reason persistence | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-003 | Normalized versions indexing
Indexes seeded + docs updated 2025-10-11 to cover flattened normalized rules for connector adoption. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDMERGE-ENGINE-02-002 | Normalized versions union & dedupe
Affected package resolver unions/dedupes normalized rules, stamps merge provenance with `decisionReason`, and tests cover the rollout. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-001 | GHSA normalized versions & provenance | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-004 | GHSA credits & ecosystem severity mapping | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-005 | GitHub quota monitoring & retries | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-006 | Production credential & scheduler rollout | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-007 | Credit parity regression fixtures | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-002 | NVD normalized versions & timestamps | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-004 | NVD CVSS & CWE precedence payloads | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-005 | NVD merge/export parity regression | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-003 | OSV normalized versions & freshness | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-004 | OSV references & credits alignment | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-005 | Fixture updater workflow
Resolved 2025-10-12: OSV mapper now derives canonical PURLs for Go + scoped npm packages when raw payloads omit `purl`; conflict fixtures unchanged for invalid npm names. Verified via `dotnet test src/StellaOps.Concelier.Connector.Osv.Tests`, `src/StellaOps.Concelier.Connector.Ghsa.Tests`, `src/StellaOps.Concelier.Connector.Nvd.Tests`, and backbone normalization/storage suites. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Acsc/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-ACSC-02-001 … 02-008 | Fetch→parse→map pipeline, fixtures, diagnostics, and README finished 2025-10-12; downstream export parity captured via FEEDEXPORT-JSON-04-001 / FEEDEXPORT-TRIVY-04-001 (completed). | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Cccs/TASKS.md | DONE (2025-10-16) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CCCS-02-001 … 02-008 | Observability meter, historical harvest plan, and DOM sanitizer refinements wrapped; ops notes live under `docs/ops/concelier-cccs-operations.md` with fixtures validating EN/FR list handling. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.CertBund/TASKS.md | DONE (2025-10-15) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CERTBUND-02-001 … 02-008 | Telemetry/docs (02-006) and history/locale sweep (02-007) completed alongside pipeline; runbook `docs/ops/concelier-certbund-operations.md` captures locale guidance and offline packaging. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Kisa/TASKS.md | DONE (2025-10-14) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-KISA-02-001 … 02-007 | Connector, tests, and telemetry/docs (02-006) finalized; localisation notes in `docs/dev/kisa_connector_notes.md` complete rollout. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ru.Bdu/TASKS.md | DONE (2025-10-14) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-RUBDU-02-001 … 02-008 | Fetch/parser/mapper refinements, regression fixtures, telemetry/docs, access options, and trusted root packaging all landed; README documents offline access strategy. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ru.Nkcki/TASKS.md | DONE (2025-10-13) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-NKCKI-02-001 … 02-008 | Listing fetch, parser, mapper, fixtures, telemetry/docs, and archive plan finished; Mongo2Go/libcrypto dependency resolved via bundled OpenSSL noted in ops guide. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ics.Cisa/TASKS.md | DONE (2025-10-16) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-ICSCISA-02-001 … 02-011 | Feed parser attachment fixes, SemVer exact values, regression suites, telemetry/docs updates, and handover complete; ops runbook now details attachment verification + proxy usage. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Vndr.Cisco/TASKS.md | DONE (2025-10-14) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CISCO-02-001 … 02-007 | OAuth fetch pipeline, DTO/mapping, tests, and telemetry/docs shipped; monitoring/export integration follow-ups recorded in Ops docs and exporter backlog (completed). | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Vndr.Msrc/TASKS.md | DONE (2025-10-15) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-MSRC-02-001 … 02-008 | Azure AD onboarding (02-008) unblocked fetch/parse/map pipeline; fixtures, telemetry/docs, and Offline Kit guidance published in `docs/ops/concelier-msrc-operations.md`. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Cve/TASKS.md | DONE (2025-10-15) | Team Connector Support & Monitoring | FEEDCONN-CVE-02-001 … 02-002 | CVE data-source selection, fetch pipeline, and docs landed 2025-10-10. 2025-10-15: smoke verified using the seeded mirror fallback; connector now logs a warning and pulls from `seed-data/cve/` until live CVE Services credentials arrive. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Kev/TASKS.md | DONE (2025-10-12) | Team Connector Support & Monitoring | FEEDCONN-KEV-02-001 … 02-002 | KEV catalog ingestion, fixtures, telemetry, and schema validation completed 2025-10-12; ops dashboard published. | -| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-01-001 | Canonical schema docs refresh
Updated canonical schema + provenance guides with SemVer style, normalized version rules, decision reason change log, and migration notes. | -| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-02-001 | Concelier-SemVer Playbook
Published merge playbook covering mapper patterns, dedupe flow, indexes, and rollout checklist. | -| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-02-002 | Normalized versions query guide
Delivered Mongo index/query addendum with `$unwind` recipes, dedupe checks, and operational checklist.
Instructions to work:
DONE Read ./AGENTS.md and docs/AGENTS.md. Document every schema/index/query change produced in Sprint 1-2 leveraging ./src/FASTER_MODELING_AND_NORMALIZATION.md. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-03-001 | Canonical merger implementation
`CanonicalMerger` ships with freshness/tie-breaker logic, provenance, and unit coverage feeding Merge. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-03-002 | Field precedence and tie-breaker map
Field precedence tables and tie-breaker metrics wired into the canonical merge flow; docs/tests updated.
Instructions to work:
Read ./AGENTS.md and core AGENTS. Implement the conflict resolver exactly as specified in ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md, coordinating with Merge and Storage teammates. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-DATA-03-001 | Merge event provenance audit prep
Merge events now persist `fieldDecisions` and analytics-ready provenance snapshots. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill
Dual-write/backfill flag delivered; migration + options validated in tests. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-TESTS-02-004 | Restore AdvisoryStore build after normalized versions refactor
Storage tests adjusted for normalized versions/decision reasons.
Instructions to work:
Read ./AGENTS.md and storage AGENTS. Extend merge events with decision reasons and analytics views to support the conflict rules, and deliver the dual-write/backfill for `NormalizedVersions` + `decisionReason` so connectors can roll out safely. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-001 | GHSA/NVD/OSV conflict rules
Merge pipeline consumes `CanonicalMerger` output prior to precedence merge. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-002 | Override metrics instrumentation
Merge events capture per-field decisions; counters/logs align with conflict rules. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-003 | Reference & credit union pipeline
Canonical merge preserves unions with updated tests. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-QA-04-001 | End-to-end conflict regression suite
Added regression tests (`AdvisoryMergeServiceTests`) covering canonical + precedence flow.
Instructions to work:
Read ./AGENTS.md and merge AGENTS. Integrate the canonical merger, instrument metrics, and deliver comprehensive regression tests following ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-GHSA-04-002 | GHSA conflict regression fixtures | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-NVD-04-002 | NVD conflict regression fixtures | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-OSV-04-002 | OSV conflict regression fixtures
Instructions to work:
Read ./AGENTS.md and module AGENTS. Produce fixture triples supporting the precedence/tie-breaker paths defined in ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md and hand them to Merge QA. | -| Sprint 3 | Conflict Resolution Integration & Communications | docs/TASKS.md | DONE (2025-10-11) | Team Documentation Guild – Conflict Guidance | FEEDDOCS-DOCS-05-001 | Concelier Conflict Rules
Runbook published at `docs/ops/concelier-conflict-resolution.md`; metrics/log guidance aligned with Sprint 3 merge counters. | -| Sprint 3 | Conflict Resolution Integration & Communications | docs/TASKS.md | DONE (2025-10-16) | Team Documentation Guild – Conflict Guidance | FEEDDOCS-DOCS-05-002 | Conflict runbook ops rollout
Ops review completed, alert thresholds applied, and change log appended in `docs/ops/concelier-conflict-resolution.md`; task closed after connector signals verified. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-15) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-04-001 | Advisory schema parity (description/CWE/canonical metric)
Extend `Advisory` and related records with description text, CWE collection, and canonical metric pointer; refresh validation + serializer determinism tests. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-15) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-04-003 | Canonical merger parity for new fields
Teach `CanonicalMerger` to populate description, CWEResults, and canonical metric pointer with provenance + regression coverage. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-15) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-04-004 | Reference normalization & freshness instrumentation cleanup
Implement URL normalization for reference dedupe, align freshness-sensitive instrumentation, and add analytics tests. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-15) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-004 | Merge pipeline parity for new advisory fields
Ensure merge service + merge events surface description/CWE/canonical metric decisions with updated metrics/tests. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-15) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-005 | Connector coordination for new advisory fields
GHSA/NVD/OSV connectors now ship description, CWE, and canonical metric data with refreshed fixtures; merge coordination log updated and exporters notified. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Exporter.Json/TASKS.md | DONE (2025-10-15) | Team Exporters – JSON | FEEDEXPORT-JSON-04-001 | Surface new advisory fields in JSON exporter
Update schemas/offline bundle + fixtures once model/core parity lands.
2025-10-15: `dotnet test src/StellaOps.Concelier.Exporter.Json.Tests` validated canonical metric/CWE emission. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Exporter.TrivyDb/TASKS.md | DONE (2025-10-15) | Team Exporters – Trivy DB | FEEDEXPORT-TRIVY-04-001 | Propagate new advisory fields into Trivy DB package
Extend Bolt builder, metadata, and regression tests for the expanded schema.
2025-10-15: `dotnet test src/StellaOps.Concelier.Exporter.TrivyDb.Tests` confirmed canonical metric/CWE propagation. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-16) | Team Connector Regression Fixtures | FEEDCONN-GHSA-04-004 | Harden CVSS fallback so canonical metric ids persist when GitHub omits vectors; extend fixtures and document severity precedence hand-off to Merge. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-16) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-04-005 | Map OSV advisories lacking CVSS vectors to canonical metric ids/notes and document CWE provenance quirks; schedule parity fixture updates. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-15) | Team Excititor Core & Policy | EXCITITOR-CORE-01-001 | Stand up canonical VEX claim/consensus records with deterministic serializers so Storage/Exports share a stable contract. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-15) | Team Excititor Core & Policy | EXCITITOR-CORE-01-002 | Implement trust-weighted consensus resolver with baseline policy weights, justification gates, telemetry output, and majority/tie handling. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-15) | Team Excititor Core & Policy | EXCITITOR-CORE-01-003 | Publish shared connector/exporter/attestation abstractions and deterministic query signature utilities for cache/attestation workflows. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-15) | Team Excititor Policy | EXCITITOR-POLICY-01-001 | Established policy options & snapshot provider covering baseline weights/overrides. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-15) | Team Excititor Policy | EXCITITOR-POLICY-01-002 | Policy evaluator now feeds consensus resolver with immutable snapshots. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-16) | Team Excititor Policy | EXCITITOR-POLICY-01-003 | Author policy diagnostics, CLI/WebService surfacing, and documentation updates. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-16) | Team Excititor Policy | EXCITITOR-POLICY-01-004 | Implement YAML/JSON schema validation and deterministic diagnostics for operator bundles. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-16) | Team Excititor Policy | EXCITITOR-POLICY-01-005 | Add policy change tracking, snapshot digests, and telemetry/logging hooks. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-15) | Team Excititor Storage | EXCITITOR-STORAGE-01-001 | Mongo mapping registry plus raw/export entities and DI extensions in place. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-16) | Team Excititor Storage | EXCITITOR-STORAGE-01-004 | Build provider/consensus/cache class maps and related collections. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Export/TASKS.md | DONE (2025-10-15) | Team Excititor Export | EXCITITOR-EXPORT-01-001 | Export engine delivers cache lookup, manifest creation, and policy integration. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Export/TASKS.md | DONE (2025-10-17) | Team Excititor Export | EXCITITOR-EXPORT-01-004 | Connect export engine to attestation client and persist Rekor metadata. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Attestation/TASKS.md | DONE (2025-10-16) | Team Excititor Attestation | EXCITITOR-ATTEST-01-001 | Implement in-toto predicate + DSSE builder providing envelopes for export attestation. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Connectors.Abstractions/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors | EXCITITOR-CONN-ABS-01-001 | Deliver shared connector context/base classes so provider plug-ins can be activated via WebService/Worker. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.WebService/TASKS.md | DONE (2025-10-17) | Team Excititor WebService | EXCITITOR-WEB-01-001 | Scaffold minimal API host, DI, and `/excititor/status` endpoint integrating policy, storage, export, and attestation services. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Worker/TASKS.md | DONE (2025-10-17) | Team Excititor Worker | EXCITITOR-WORKER-01-001 | Create Worker host with provider scheduling and logging to drive recurring pulls/reconciliation. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Formats | EXCITITOR-FMT-CSAF-01-001 | Implement CSAF normalizer foundation translating provider documents into `VexClaim` entries. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.CycloneDX/TASKS.md | DONE (2025-10-17) | Team Excititor Formats | EXCITITOR-FMT-CYCLONE-01-001 | Implement CycloneDX VEX normalizer capturing `analysis` state and component references. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.OpenVEX/TASKS.md | DONE (2025-10-17) | Team Excititor Formats | EXCITITOR-FMT-OPENVEX-01-001 | Implement OpenVEX normalizer to ingest attestations into canonical claims with provenance. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-001 | Ship Red Hat CSAF provider metadata discovery enabling incremental pulls. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-002 | Fetch CSAF windows with ETag handling, resume tokens, quarantine on schema errors, and persist raw docs. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-003 | Populate provider trust overrides (cosign issuer, identity regex) and provenance hints for policy evaluation/logging. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-004 | Persist resume cursors (last updated timestamp/document hashes) in storage and reload during fetch to avoid duplicates. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-005 | Register connector in Worker/WebService DI, add scheduled jobs, and document CLI triggers for Red Hat CSAF pulls. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-006 | Add CSAF normalization parity fixtures ensuring RHSA-specific metadata is preserved. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Cisco.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Cisco | EXCITITOR-CONN-CISCO-01-001 | Implement Cisco CSAF endpoint discovery/auth to unlock paginated pulls. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Cisco.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Cisco | EXCITITOR-CONN-CISCO-01-002 | Implement Cisco CSAF paginated fetch loop with dedupe and raw persistence support. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – SUSE | EXCITITOR-CONN-SUSE-01-001 | Build Rancher VEX Hub discovery/subscription path with offline snapshot support. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.MSRC.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – MSRC | EXCITITOR-CONN-MS-01-001 | Deliver AAD onboarding/token cache for MSRC CSAF ingestion. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Oracle.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Oracle | EXCITITOR-CONN-ORACLE-01-001 | Implement Oracle CSAF catalogue discovery with CPU calendar awareness. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Ubuntu | EXCITITOR-CONN-UBUNTU-01-001 | Implement Ubuntu CSAF discovery and channel selection for USN ingestion. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/TASKS.md | DONE (2025-10-18) | Team Excititor Connectors – OCI | EXCITITOR-CONN-OCI-01-001 | Wire OCI discovery/auth to fetch OpenVEX attestations for configured images. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/TASKS.md | DONE (2025-10-18) | Team Excititor Connectors – OCI | EXCITITOR-CONN-OCI-01-002 | Attestation fetch & verify loop – download DSSE attestations, trigger verification, handle retries/backoff, persist raw statements. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/TASKS.md | DONE (2025-10-18) | Team Excititor Connectors – OCI | EXCITITOR-CONN-OCI-01-003 | Provenance metadata & policy hooks – emit image, subject digest, issuer, and trust metadata for policy weighting/logging. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Cli/TASKS.md | DONE (2025-10-18) | DevEx/CLI | EXCITITOR-CLI-01-001 | Add `excititor` CLI verbs bridging to WebService with consistent auth and offline UX. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-19) | Team Excititor Core & Policy | EXCITITOR-CORE-02-001 | Context signal schema prep – extend consensus models with severity/KEV/EPSS fields and update canonical serializers. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-19) | Team Excititor Policy | EXCITITOR-POLICY-02-001 | Scoring coefficients & weight ceilings – add α/β options, weight boosts, and validation guidance. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-19) | Team Excititor Storage | EXCITITOR-STORAGE-02-001 | Statement events & scoring signals – immutable VEX statements store, consensus signal fields, and migration `20251019-consensus-signals-statements` with tests (`dotnet test src/StellaOps.Excititor.Core.Tests/StellaOps.Excititor.Core.Tests.csproj`, `dotnet test src/StellaOps.Excititor.Storage.Mongo.Tests/StellaOps.Excititor.Storage.Mongo.Tests.csproj`). | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Team Excititor WebService | EXCITITOR-WEB-01-004 | Resolve API & signed responses – expose `/excititor/resolve`, return signed consensus/score envelopes, document auth. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.WebService/TASKS.md | DONE (2025-10-20) | Team Excititor WebService | EXCITITOR-WEB-01-002 | Ingest & reconcile endpoints – scope-enforced `/excititor/init`, `/excititor/ingest/run`, `/excititor/ingest/resume`, `/excititor/reconcile`; regression via `dotnet test … --filter FullyQualifiedName~IngestEndpointsTests`. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.WebService/TASKS.md | DONE (2025-10-20) | Team Excititor WebService | EXCITITOR-WEB-01-004 | Resolve API & signed responses – expose `/excititor/resolve`, return signed consensus/score envelopes, document auth. | | Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Team Excititor WebService | EXCITITOR-WEB-01-005 | Mirror distribution endpoints – expose download APIs for downstream Excititor instances. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Attestation/TASKS.md | DONE (2025-10-16) | Team Excititor Attestation | EXCITITOR-ATTEST-01-002 | Rekor v2 client integration – ship transparency log client with retries and offline queue. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Worker/TASKS.md | TODO | Team Excititor Worker | EXCITITOR-WORKER-01-004 | TTL refresh & stability damper – schedule re-resolve loops and guard against status flapping. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Worker/TASKS.md | DONE (2025-10-21) | Team Excititor Worker | EXCITITOR-WORKER-01-004 | TTL refresh & stability damper – schedule re-resolve loops and guard against status flapping. | | Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Export/TASKS.md | TODO | Team Excititor Export | EXCITITOR-EXPORT-01-005 | Score & resolve envelope surfaces – include signed consensus/score artifacts in exports. | | Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Export/TASKS.md | TODO | Team Excititor Export | EXCITITOR-EXPORT-01-006 | Quiet provenance packaging – attach quieted-by statement IDs, signers, justification codes to exports and attestations. | | Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Export/TASKS.md | TODO | Team Excititor Export | EXCITITOR-EXPORT-01-007 | Mirror bundle + domain manifest – publish signed consensus bundles for mirrors. | | Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Connectors.StellaOpsMirror/TASKS.md | TODO | Excititor Connectors – Stella | EXCITITOR-CONN-STELLA-07-001 | Excititor mirror connector – ingest signed mirror bundles and map to VexClaims with resume handling. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-19) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-07-001 | Advisory event log & asOf queries – surface immutable statements and replay capability. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-19) | Concelier WebService Guild | FEEDWEB-EVENTS-07-001 | Advisory event replay API – expose `/concelier/advisories/{key}/replay` with `asOf` filter, hex hashes, and conflict data. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Core/TASKS.md | TODO | Team Core Engine & Data Science | FEEDCORE-ENGINE-07-002 | Noise prior computation service – learn false-positive priors and expose deterministic summaries. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-21) | Team Core Engine & Data Science | FEEDCORE-ENGINE-07-002 | Noise prior computation service – learn false-positive priors and expose deterministic summaries. | | Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Core/TASKS.md | TODO | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-07-003 | Unknown state ledger & confidence seeding – persist unknown flags, seed confidence bands, expose query surface. | | Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | TODO | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-07-001 | Advisory statement & conflict collections – provision Mongo schema/indexes for event-sourced merge. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-20) | BE-Merge | FEEDMERGE-ENGINE-07-001 | Conflict sets & explainers – persist conflict materialization and replay hashes for merge decisions. | -| Sprint 8 | Mongo strengthening | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-19) | Team Normalization & Storage Backbone | FEEDSTORAGE-MONGO-08-001 | Causal-consistent Concelier storage sessions
Scoped session facilitator registered, repositories accept optional session handles, and replica-set failover tests verify read-your-write + monotonic reads. | -| Sprint 8 | Mongo strengthening | src/StellaOps.Authority/TASKS.md | DONE (2025-10-19) | Authority Core & Storage Guild | AUTHSTORAGE-MONGO-08-001 | Harden Authority Mongo usage
Scoped Mongo sessions with majority read/write concerns wired through stores and GraphQL/HTTP pipelines; replica-set election regression validated. | -| Sprint 8 | Mongo strengthening | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-19) | Team Excititor Storage | EXCITITOR-STORAGE-MONGO-08-001 | Causal consistency for Excititor repositories
Session-scoped repositories shipped with new Mongo records, orchestrators/workers now share scoped sessions, and replica-set failover coverage added via `dotnet test src/StellaOps.Excititor.Storage.Mongo.Tests/StellaOps.Excititor.Storage.Mongo.Tests.csproj`. | -| Sprint 8 | Platform Maintenance | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-19) | Team Excititor Storage | EXCITITOR-STORAGE-03-001 | Statement backfill tooling – shipped admin backfill endpoint, CLI hook (`stellaops excititor backfill-statements`), integration tests, and operator runbook (`docs/dev/EXCITITOR_STATEMENT_BACKFILL.md`). | -| Sprint 8 | Mirror Distribution | src/StellaOps.Concelier.Exporter.Json/TASKS.md | DONE (2025-10-19) | Concelier Export Guild | CONCELIER-EXPORT-08-201 | Mirror bundle + domain manifest – produce signed JSON aggregates for `*.stella-ops.org` mirrors. | -| Sprint 8 | Mirror Distribution | src/StellaOps.Concelier.Exporter.TrivyDb/TASKS.md | DONE (2025-10-19) | Concelier Export Guild | CONCELIER-EXPORT-08-202 | Mirror-ready Trivy DB bundles – mirror options emit per-domain manifests/metadata/db archives with deterministic digests for downstream sync. | -| Sprint 8 | Mirror Distribution | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-20) | Concelier WebService Guild | CONCELIER-WEB-08-201 | Mirror distribution endpoints – expose domain-scoped index/download APIs with auth/quota. | -| Sprint 8 | Mirror Distribution | src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md | DOING (2025-10-19) | BE-Conn-Stella | FEEDCONN-STELLA-08-001 | Concelier mirror connector – fetch mirror manifest, verify signatures, and hydrate canonical DTOs with resume support. | -| Sprint 8 | Mirror Distribution | ops/devops/TASKS.md | DONE (2025-10-19) | DevOps Guild | DEVOPS-MIRROR-08-001 | Managed mirror deployments for `*.stella-ops.org` – Helm/Compose overlays, CDN, runbooks. | -| Sprint 8 | Plugin Infrastructure | src/StellaOps.Plugin/TASKS.md | DOING | Plugin Platform Guild, Authority Core | PLUGIN-DI-08-002.COORD | Authority scoped-service integration handshake
Session scheduled for 2025-10-20 15:00–16:00 UTC; agenda + attendees logged in `docs/dev/authority-plugin-di-coordination.md`. | -| Sprint 8 | Plugin Infrastructure | src/StellaOps.Authority/TASKS.md | DOING | Authority Core, Plugin Platform Guild | AUTH-PLUGIN-COORD-08-002 | Coordinate scoped-service adoption for Authority plug-in registrars
Workshop locked for 2025-10-20 15:00–16:00 UTC; pre-read checklist tracked in `docs/dev/authority-plugin-di-coordination.md`. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Core/TASKS.md | DONE (2025-10-18) | Team Scanner Core | SCANNER-CORE-09-501 | Define shared DTOs (ScanJob, ProgressEvent), error taxonomy, and deterministic ID/timestamp helpers aligning with `ARCHITECTURE_SCANNER.md` §3–§4. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Core/TASKS.md | DONE (2025-10-18) | Team Scanner Core | SCANNER-CORE-09-502 | Observability helpers (correlation IDs, logging scopes, metric namespacing, deterministic hashes) consumed by WebService/Worker. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Core/TASKS.md | DONE (2025-10-18) | Team Scanner Core | SCANNER-CORE-09-503 | Security utilities: Authority client factory, OpTok caching, DPoP verifier, restart-time plug-in guardrails for scanner components. | -| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-001 | Buildx driver scaffold + handshake with Scanner.Emit (local CAS). | -| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-002 | OCI annotations + provenance hand-off to Attestor. | -| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-003 | CI demo: minimal SBOM push & backend report wiring. | -| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-004 | Stabilize descriptor nonce derivation so repeated builds emit deterministic placeholders. | -| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-005 | Integrate determinism guard into GitHub/Gitea workflows and archive proof artifacts. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-18) | Team Scanner WebService | SCANNER-WEB-09-101 | Minimal API host with Authority enforcement, health/ready endpoints, and restart-time plug-in loader per architecture §1, §4. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-18) | Team Scanner WebService | SCANNER-WEB-09-102 | `/api/v1/scans` submission/status endpoints with deterministic IDs, validation, and cancellation support. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Team Scanner WebService | SCANNER-WEB-09-103 | Progress streaming (SSE/JSONL) with correlation IDs and ISO-8601 UTC timestamps, documented in API reference. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-19) | Team Scanner WebService | SCANNER-WEB-09-104 | Configuration binding for Mongo, MinIO, queue, feature flags; startup diagnostics and fail-fast policy. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Team Scanner WebService | SCANNER-POLICY-09-105 | Policy snapshot loader + schema + OpenAPI (YAML ignore rules, VEX include/exclude, vendor precedence). | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Team Scanner WebService | SCANNER-POLICY-09-106 | `/reports` verdict assembly (Feedser+Vexer+Policy) + signed response envelope. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Team Scanner WebService | SCANNER-POLICY-09-107 | Expose score inputs, config version, and quiet provenance in `/reports` JSON and signed payload. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-201 | Worker host bootstrap with Authority auth, hosted services, and graceful shutdown semantics. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-202 | Lease/heartbeat loop with retry+jitter, poison-job quarantine, structured logging. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-203 | Analyzer dispatch skeleton emitting deterministic stage progress and honoring cancellation tokens. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-204 | Worker metrics (queue latency, stage duration, failure counts) with OpenTelemetry resource wiring. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-205 | Harden heartbeat jitter so lease safety margin stays ≥3× and cover with regression tests + optional live queue smoke run. | -| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | DONE | Policy Guild | POLICY-CORE-09-001 | Policy schema + binder + diagnostics. | -| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | DONE | Policy Guild | POLICY-CORE-09-002 | Policy snapshot store + revision digests. | -| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | DONE | Policy Guild | POLICY-CORE-09-003 | `/policy/preview` API (image digest → projected verdict diff). | -| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild | POLICY-CORE-09-004 | Versioned scoring config with schema validation, trust table, and golden fixtures. | -| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild | POLICY-CORE-09-005 | Scoring/quiet engine – compute score, enforce VEX-only quiet rules, emit inputs and provenance. | -| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild | POLICY-CORE-09-006 | Unknown state & confidence decay – deterministic bands surfaced in policy outputs. | -| Sprint 9 | DevOps Foundations | ops/devops/TASKS.md | DONE (2025-10-19) | DevOps Guild | DEVOPS-HELM-09-001 | Helm/Compose environment profiles (dev/staging/airgap) with deterministic digests. | -| Sprint 9 | Docs & Governance | docs/TASKS.md | DONE (2025-10-19) | Docs Guild, DevEx | DOCS-ADR-09-001 | Establish ADR process and template. | -| Sprint 9 | Docs & Governance | docs/TASKS.md | DONE (2025-10-19) | Docs Guild, Platform Events | DOCS-EVENTS-09-002 | Publish event schema catalog (`docs/events/`) for critical envelopes. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Storage/TASKS.md | DONE (2025-10-19) | Team Scanner Storage | SCANNER-STORAGE-09-301 | Mongo catalog schemas/indexes for images, layers, artifacts, jobs, lifecycle rules plus migrations. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Storage/TASKS.md | DONE (2025-10-19) | Team Scanner Storage | SCANNER-STORAGE-09-302 | MinIO layout, immutability policies, client abstraction, and configuration binding. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Storage/TASKS.md | DONE (2025-10-19) | Team Scanner Storage | SCANNER-STORAGE-09-303 | Repositories/services with dual-write feature flag, deterministic digests, TTL enforcement tests. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Queue/TASKS.md | DONE (2025-10-19) | Team Scanner Queue | SCANNER-QUEUE-09-401 | Queue abstraction + Redis Streams adapter with ack/claim APIs and idempotency tokens. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Queue/TASKS.md | DONE (2025-10-19) | Team Scanner Queue | SCANNER-QUEUE-09-402 | Pluggable backend support (Redis, NATS) with configuration binding, health probes, failover docs. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Queue/TASKS.md | DONE (2025-10-19) | Team Scanner Queue | SCANNER-QUEUE-09-403 | Retry + dead-letter strategy with structured logs/metrics for offline deployments. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Web/TASKS.md | BLOCKED (2025-10-21) | UX Specialist, Angular Eng | WEB1.TRIVY-SETTINGS-TESTS | Add headless UI test run (`ng test --watch=false`) and document prerequisites once Angular tooling is chained up. | +| Sprint 8 | Mirror Distribution | src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md | DONE (2025-10-20) | BE-Conn-Stella | FEEDCONN-STELLA-08-001 | Concelier mirror connector – fetch mirror manifest, verify signatures, and hydrate canonical DTOs with resume support. | +| Sprint 8 | Mirror Distribution | src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md | DONE (2025-10-20) | BE-Conn-Stella | FEEDCONN-STELLA-08-002 | Map mirror payloads into canonical advisory DTOs with provenance referencing mirror domain + original source metadata. | +| Sprint 8 | Mirror Distribution | src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md | DONE (2025-10-20) | BE-Conn-Stella | FEEDCONN-STELLA-08-003 | Add incremental cursor + resume support (per-export fingerprint) and document configuration for downstream Concelier instances. | +| Sprint 8 | Plugin Infrastructure | src/StellaOps.Plugin/TASKS.md | DONE (2025-10-20) | Plugin Platform Guild, Authority Core | PLUGIN-DI-08-002.COORD | Authority scoped-service integration handshake
Workshop concluded 2025-10-20 15:00–16:05 UTC; decisions + follow-ups recorded in `docs/dev/authority-plugin-di-coordination.md`. | +| Sprint 8 | Plugin Infrastructure | src/StellaOps.Plugin/TASKS.md | DONE (2025-10-20) | Plugin Platform Guild, Authority Core | PLUGIN-DI-08-002 | Authority plugin integration updates – scoped identity-provider services with registry handles; regression coverage via scoped registrar/unit tests. | +| Sprint 8 | Plugin Infrastructure | src/StellaOps.Authority/TASKS.md | DONE (2025-10-20) | Authority Core, Plugin Platform Guild | AUTH-PLUGIN-COORD-08-002 | Coordinate scoped-service adoption for Authority plug-in registrars
Workshop notes and follow-up backlog captured 2025-10-20 in `docs/dev/authority-plugin-di-coordination.md`. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-19) | Team Scanner WebService | SCANNER-WEB-09-103 | Progress streaming (SSE/JSONL) with correlation IDs and ISO-8601 UTC timestamps, documented in API reference. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-19) | Team Scanner WebService | SCANNER-POLICY-09-105 | Policy snapshot loader + schema + OpenAPI (YAML ignore rules, VEX include/exclude, vendor precedence). | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-19) | Team Scanner WebService | SCANNER-POLICY-09-106 | `/reports` verdict assembly (Feedser+Vexer+Policy) + signed response envelope. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-19) | Team Scanner WebService | SCANNER-POLICY-09-107 | Expose score inputs, config version, and quiet provenance in `/reports` JSON and signed payload. | +| Sprint 9 | DevOps Foundations | ops/devops/TASKS.md | DONE (2025-10-21) | DevOps Guild, Scanner WebService Guild | DEVOPS-SCANNER-09-204 | Surface `SCANNER__EVENTS__*` env config across Compose/Helm and document overrides. | +| Sprint 9 | DevOps Foundations | ops/devops/TASKS.md | DONE (2025-10-21) | DevOps Guild, Notify Guild | DEVOPS-SCANNER-09-205 | Notify smoke job validates Redis stream + Notify deliveries after staging deploys. | +| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | DONE (2025-10-19) | Policy Guild | POLICY-CORE-09-004 | Versioned scoring config with schema validation, trust table, and golden fixtures. | +| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | DONE (2025-10-19) | Policy Guild | POLICY-CORE-09-005 | Scoring/quiet engine – compute score, enforce VEX-only quiet rules, emit inputs and provenance. | +| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | DONE (2025-10-19) | Policy Guild | POLICY-CORE-09-006 | Unknown state & confidence decay – deterministic bands surfaced in policy outputs. | | Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Cache/TASKS.md | TODO | Scanner Cache Guild | SCANNER-CACHE-10-101 | Implement layer cache store keyed by layer digest with metadata retention per architecture §3.3. | | Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Cache/TASKS.md | TODO | Scanner Cache Guild | SCANNER-CACHE-10-102 | Build file CAS with dedupe, TTL enforcement, and offline import/export hooks. | | Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Cache/TASKS.md | TODO | Scanner Cache Guild | SCANNER-CACHE-10-103 | Expose cache metrics/logging and configuration toggles for warm/cold thresholds. | @@ -244,15 +66,13 @@ This file describe implementation of Stella Ops (docs/README.md). Implementation | Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-10-605 | Emit BOM-Index sidecar schema/fixtures (CRITICAL PATH for SP16). | | Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-10-606 | Usage view bit flags integrated with EntryTrace. | | Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-10-607 | Embed scoring inputs, confidence band, and quiet provenance in CycloneDX/DSSE artifacts. | -| Sprint 10 | Benchmarks | bench/TASKS.md | TODO | Bench Guild, Scanner Team | BENCH-SCANNER-10-001 | Analyzer microbench harness + baseline CSV. | +| Sprint 10 | Benchmarks | bench/TASKS.md | DONE (2025-10-21) | Bench Guild, Language Analyzer Guild | BENCH-SCANNER-10-002 | Wire real language analyzers into bench harness & refresh baselines post-implementation. | | Sprint 10 | Samples | samples/TASKS.md | TODO | Samples Guild, Scanner Team | SAMPLES-10-001 | Sample images with SBOM/BOM-Index sidecars. | -| Sprint 10 | DevOps Security | ops/devops/TASKS.md | DONE (2025-10-20) | DevOps Guild | DEVOPS-SEC-10-301 | Address NU1902/NU1903 advisories for `MongoDB.Driver` 2.12.0 and `SharpCompress` 0.23.0; Wave 0A prerequisites confirmed complete before remediation work. | | Sprint 10 | DevOps Perf | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-PERF-10-001 | Perf smoke job ensuring <5 s SBOM compose. | -| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Authority/TASKS.md | DOING (2025-10-19) | Authority Core & Security Guild | AUTH-DPOP-11-001 | Implement DPoP proof validation + nonce handling for high-value audiences per architecture. | | Sprint 11 | Signing Chain Bring-up | src/StellaOps.Authority/TASKS.md | DOING (2025-10-19) | Authority Core & Security Guild | AUTH-MTLS-11-002 | Add OAuth mTLS client credential support with certificate-bound tokens and introspection updates. | -| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Signer/TASKS.md | TODO | Signer Guild | SIGNER-API-11-101 | `/sign/dsse` pipeline with Authority auth, PoE introspection, release verification, DSSE signing. | -| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Signer/TASKS.md | TODO | Signer Guild | SIGNER-REF-11-102 | `/verify/referrers` endpoint with OCI lookup, caching, and policy enforcement. | -| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Signer/TASKS.md | TODO | Signer Guild | SIGNER-QUOTA-11-103 | Enforce plan quotas, concurrency/QPS limits, artifact size caps with metrics/audit logs. | +| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Signer/TASKS.md | DONE (2025-10-21) | Signer Guild | SIGNER-API-11-101 | `/sign/dsse` pipeline with Authority auth, PoE introspection, release verification, DSSE signing. | +| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Signer/TASKS.md | DONE (2025-10-21) | Signer Guild | SIGNER-REF-11-102 | `/verify/referrers` endpoint with OCI lookup, caching, and policy enforcement. | +| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Signer/TASKS.md | DONE (2025-10-21) | Signer Guild | SIGNER-QUOTA-11-103 | Enforce plan quotas, concurrency/QPS limits, artifact size caps with metrics/audit logs. | | Sprint 11 | Signing Chain Bring-up | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Guild | ATTESTOR-API-11-201 | `/rekor/entries` submission pipeline with dedupe, proof acquisition, and persistence. | | Sprint 11 | Signing Chain Bring-up | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Guild | ATTESTOR-VERIFY-11-202 | `/rekor/verify` + retrieval endpoints validating signatures and Merkle proofs. | | Sprint 11 | Signing Chain Bring-up | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Guild | ATTESTOR-OBS-11-203 | Telemetry, alerting, mTLS hardening, and archive workflow for Attestor. | @@ -268,8 +88,11 @@ This file describe implementation of Stella Ops (docs/README.md). Implementation | Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Webhook/TASKS.md | TODO | Zastava Webhook Guild | ZASTAVA-WEBHOOK-12-101 | Admission controller host with TLS bootstrap and Authority auth. | | Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Webhook/TASKS.md | TODO | Zastava Webhook Guild | ZASTAVA-WEBHOOK-12-102 | Query Scanner `/policy/runtime`, resolve digests, enforce verdicts. | | Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Webhook/TASKS.md | TODO | Zastava Webhook Guild | ZASTAVA-WEBHOOK-12-103 | Caching, fail-open/closed toggles, metrics/logging for admission decisions. | -| Sprint 12 | Runtime Guardrails | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Scanner WebService Guild | SCANNER-RUNTIME-12-301 | `/runtime/events` ingestion endpoint with validation, batching, storage hooks. | -| Sprint 12 | Runtime Guardrails | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Scanner WebService Guild | SCANNER-RUNTIME-12-302 | `/policy/runtime` endpoint joining SBOM baseline + policy verdict with TTL guidance. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-20) | Scanner WebService Guild | SCANNER-RUNTIME-12-301 | `/runtime/events` ingestion endpoint with validation, batching, storage hooks. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Scanner.WebService/TASKS.md | DOING (2025-10-20) | Scanner WebService Guild | SCANNER-RUNTIME-12-302 | `/policy/runtime` endpoint joining SBOM baseline + policy verdict, returning admission guidance. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Scanner WebService Guild | SCANNER-RUNTIME-12-303 | Align `/policy/runtime` verdicts with canonical policy evaluation (Feedser/Vexer). | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Scanner WebService Guild | SCANNER-RUNTIME-12-304 | Integrate attestation verification into runtime policy metadata. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Scanner WebService Guild | SCANNER-RUNTIME-12-305 | Deliver shared fixtures + e2e validation with Zastava/CLI teams. | | Sprint 13 | UX & CLI Experience | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-AUTH-13-001 | Integrate Authority OIDC + DPoP flows with session management. | | Sprint 13 | UX & CLI Experience | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-SCANS-13-002 | Build scans module (list/detail/SBOM/diff/attestation) with performance + accessibility targets. | | Sprint 13 | UX & CLI Experience | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-VEX-13-003 | Implement VEX explorer + policy editor with preview integration. | @@ -277,7 +100,7 @@ This file describe implementation of Stella Ops (docs/README.md). Implementation | Sprint 13 | UX & CLI Experience | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-SCHED-13-005 | Scheduler panel: schedules CRUD, run history, dry-run preview. | | Sprint 13 | UX & CLI Experience | src/StellaOps.UI/TASKS.md | DOING (2025-10-19) | UI Guild | UI-NOTIFY-13-006 | Notify panel: channels/rules CRUD, deliveries view, test send. | | Sprint 13 | UX & CLI Experience | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI | CLI-RUNTIME-13-005 | Add runtime policy test verbs that consume `/policy/runtime` and display verdicts. | -| Sprint 13 | UX & CLI Experience | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI | CLI-OFFLINE-13-006 | Implement offline kit pull/import/status commands with integrity checks. | +| Sprint 13 | UX & CLI Experience | src/StellaOps.Cli/TASKS.md | DONE (2025-10-21) | DevEx/CLI | CLI-OFFLINE-13-006 | Implement offline kit pull/import/status commands with integrity checks. | | Sprint 13 | UX & CLI Experience | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI | CLI-PLUGIN-13-007 | Package non-core CLI verbs as restart-time plug-ins (manifest + loader tests). | | Sprint 14 | Release & Offline Ops | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-REL-14-001 | Deterministic build/release pipeline with SBOM/provenance, signing, and manifest generation. | | Sprint 14 | Release & Offline Ops | ops/offline-kit/TASKS.md | TODO | Offline Kit Guild | DEVOPS-OFFLINE-14-002 | Offline kit packaging workflow with integrity verification and documentation. | @@ -298,39 +121,28 @@ This file describe implementation of Stella Ops (docs/README.md). Implementation | Sprint 15 | Notify Foundations | src/StellaOps.Notify.Engine/TASKS.md | TODO | Notify Engine Guild | NOTIFY-ENGINE-15-304 | Test-send sandbox + preview utilities. | | Sprint 15 | Notify Foundations | src/StellaOps.Notify.WebService/TASKS.md | TODO | Notify WebService Guild | NOTIFY-WEB-15-101 | Minimal API host with Authority enforcement and plug-in loading. | | Sprint 15 | Notify Foundations | src/StellaOps.Notify.WebService/TASKS.md | TODO | Notify WebService Guild | NOTIFY-WEB-15-102 | Rules/channel/template CRUD with audit logging. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.WebService/TASKS.md | DONE (2025-10-19) | Notify WebService Guild | NOTIFY-WEB-15-103 | Delivery history & test-send endpoints. | | Sprint 15 | Notify Foundations | src/StellaOps.Notify.WebService/TASKS.md | TODO | Notify WebService Guild | NOTIFY-WEB-15-104 | Configuration binding + startup diagnostics. | | Sprint 15 | Notify Foundations | src/StellaOps.Notify.Worker/TASKS.md | TODO | Notify Worker Guild | NOTIFY-WORKER-15-201 | Bus subscription + leasing loop with backoff. | | Sprint 15 | Notify Foundations | src/StellaOps.Notify.Worker/TASKS.md | TODO | Notify Worker Guild | NOTIFY-WORKER-15-202 | Rules evaluation pipeline integration. | | Sprint 15 | Notify Foundations | src/StellaOps.Notify.Worker/TASKS.md | TODO | Notify Worker Guild | NOTIFY-WORKER-15-203 | Channel dispatch orchestration with retries. | | Sprint 15 | Notify Foundations | src/StellaOps.Notify.Worker/TASKS.md | TODO | Notify Worker Guild | NOTIFY-WORKER-15-204 | Metrics/telemetry for Notify workers. | | Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Slack/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-SLACK-15-501 | Slack connector with rate-limit aware delivery. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Slack/TASKS.md | DOING (2025-10-19) | Notify Connectors Guild | NOTIFY-CONN-SLACK-15-502 | Slack health/test-send support. | | Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Teams/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-TEAMS-15-601 | Teams connector with Adaptive Cards. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Teams/TASKS.md | DOING (2025-10-19) | Notify Connectors Guild | NOTIFY-CONN-TEAMS-15-602 | Teams health/test-send support. | | Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Email/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-EMAIL-15-701 | SMTP connector with TLS + rendering. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Email/TASKS.md | DOING (2025-10-19) | Notify Connectors Guild | NOTIFY-CONN-EMAIL-15-702 | DKIM + health/test-send flows. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Email/TASKS.md | BLOCKED (2025-10-20) | Notify Connectors Guild | NOTIFY-CONN-EMAIL-15-702 | DKIM + health/test-send flows. | | Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Webhook/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-WEBHOOK-15-801 | Webhook connector with signing/retries. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Webhook/TASKS.md | DOING (2025-10-19) | Notify Connectors Guild | NOTIFY-CONN-WEBHOOK-15-802 | Webhook health/test-send support. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Slack/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-SLACK-15-503 | Package Slack connector as restart-time plug-in (manifest + host registration). | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Teams/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-TEAMS-15-603 | Package Teams connector as restart-time plug-in (manifest + host registration). | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Email/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-EMAIL-15-703 | Package Email connector as restart-time plug-in (manifest + host registration). | -| Sprint 15 | Notify Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DOING (2025-10-19) | Scanner WebService Guild | SCANNER-EVENTS-15-201 | Emit `scanner.report.ready` + `scanner.scan.completed` events. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Webhook/TASKS.md | BLOCKED (2025-10-20) | Notify Connectors Guild | NOTIFY-CONN-WEBHOOK-15-802 | Webhook health/test-send support. | +| Sprint 16 | Notify Foundations | src/StellaOps.Scanner.WebService/TASKS.md | BLOCKED (2025-10-20) | Scanner WebService Guild | SCANNER-EVENTS-16-301 | Redis publisher integration tests once Notify queue adapter ships. | | Sprint 15 | Benchmarks | bench/TASKS.md | TODO | Bench Guild, Notify Team | BENCH-NOTIFY-15-001 | Notify dispatch throughput bench with results CSV. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Webhook/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-WEBHOOK-15-803 | Package Webhook connector as restart-time plug-in (manifest + host registration). | | Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Models/TASKS.md | TODO | Scheduler Models Guild | SCHED-MODELS-16-101 | Define Scheduler DTOs & validation. | | Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Models/TASKS.md | TODO | Scheduler Models Guild | SCHED-MODELS-16-102 | Publish schema docs/sample payloads. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Models/TASKS.md | TODO | Scheduler Models Guild | SCHED-MODELS-16-103 | Versioning/migration helpers for schedules/runs. | | Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Storage.Mongo/TASKS.md | TODO | Scheduler Storage Guild | SCHED-STORAGE-16-201 | Mongo schemas/indexes for Scheduler state. | | Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Storage.Mongo/TASKS.md | TODO | Scheduler Storage Guild | SCHED-STORAGE-16-202 | Repositories with tenant scoping, TTL, causal consistency. | | Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Storage.Mongo/TASKS.md | TODO | Scheduler Storage Guild | SCHED-STORAGE-16-203 | Audit + stats materialization for UI. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Queue/TASKS.md | TODO | Scheduler Queue Guild | SCHED-QUEUE-16-401 | Queue abstraction + Redis Streams adapter. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Queue/TASKS.md | TODO | Scheduler Queue Guild | SCHED-QUEUE-16-402 | NATS JetStream adapter with health probes. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Queue/TASKS.md | TODO | Scheduler Queue Guild | SCHED-QUEUE-16-403 | Dead-letter handling + metrics. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Queue/TASKS.md | DONE (2025-10-20) | Scheduler Queue Guild | SCHED-QUEUE-16-403 | Dead-letter handling + metrics. | | Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.ImpactIndex/TASKS.md | TODO | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-301 | Ingest BOM-Index into roaring bitmap store. | | Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.ImpactIndex/TASKS.md | TODO | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-302 | Query APIs for ResolveByPurls/ResolveByVulns/ResolveAll. | | Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.ImpactIndex/TASKS.md | TODO | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-303 | Snapshot/compaction/invalidation workflow. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.ImpactIndex/TASKS.md | DOING | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-300 | **STUB** ImpactIndex ingest/query using fixtures (to be removed by SP16 completion). | | Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-WEB-16-101 | Minimal API host with Authority enforcement. | | Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-WEB-16-102 | Schedules CRUD (cron validation, pause/resume, audit). | | Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-WEB-16-103 | Runs API (list/detail/cancel) + impact previews. | @@ -346,3 +158,4 @@ This file describe implementation of Stella Ops (docs/README.md). Implementation | Sprint 17 | Symbol Intelligence & Forensics | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Scanner WebService Guild | SCANNER-RUNTIME-17-401 | Persist runtime build-id observations and expose them for debug-symbol correlation. | | Sprint 17 | Symbol Intelligence & Forensics | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-REL-17-002 | Ship stripped debug artifacts organised by build-id within release/offline kits. | | Sprint 17 | Symbol Intelligence & Forensics | docs/TASKS.md | TODO | Docs Guild | DOCS-RUNTIME-17-004 | Document build-id workflows for SBOMs, runtime events, and debug-store usage. | +| Sprint 18 | Launch Readiness | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-LAUNCH-18-001 | Production launch cutover rehearsal and runbook publication (blocked on implementation sign-off and environment setup). | diff --git a/SPRINTS.updated.tmp b/SPRINTS.updated.tmp deleted file mode 100644 index c01fcdd3..00000000 --- a/SPRINTS.updated.tmp +++ /dev/null @@ -1,427 +0,0 @@ -This file describe implementation of Stella Ops (docs/README.md). Implementation must respect rules from AGENTS.md (read if you have not). - -| Sprint | Theme | Tasks File Path | Status | Type of Specialist | Task ID | Task Description | -| --- | --- | --- | --- | --- | --- | --- | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-12) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-001 | SemVer primitive range-style metadata
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Concelier.Models/AGENTS.md. This task lays the groundwork—complete the SemVer helper updates before teammates pick up FEEDMODELS-SCHEMA-01-002/003 and FEEDMODELS-SCHEMA-02-900. Use ./src/FASTER_MODELING_AND_NORMALIZATION.md for the target rule structure. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-11) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-002 | Provenance decision rationale field
Instructions to work:
AdvisoryProvenance now carries `decisionReason` and docs/tests were updated. Connectors and merge tasks should populate the field when applying precedence/freshness/tie-breaker logic; see src/StellaOps.Concelier.Models/PROVENANCE_GUIDELINES.md for usage guidance. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-11) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-003 | Normalized version rules collection
Instructions to work:
`AffectedPackage.NormalizedVersions` and supporting comparer/docs/tests shipped. Connector owners must emit rule arrays per ./src/FASTER_MODELING_AND_NORMALIZATION.md and report progress via FEEDMERGE-COORD-02-900 so merge/storage backfills can proceed. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-12) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-02-900 | Range primitives for SemVer/EVR/NEVRA metadata
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Concelier.Models/AGENTS.md before resuming this stalled effort. Confirm helpers align with the new `NormalizedVersions` representation so connectors finishing in Sprint 2 can emit consistent metadata. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Normalization/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDNORM-NORM-02-001 | SemVer normalized rule emitter
Shared `SemVerRangeRuleBuilder` now outputs primitives + normalized rules per `FASTER_MODELING_AND_NORMALIZATION.md`; CVE/GHSA connectors consuming the API have verified fixtures. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill
AdvisoryStore dual-writes flattened `normalizedVersions` when `concelier.storage.enableSemVerStyle` is set; migration `20251011-semver-style-backfill` updates historical records and docs outline the rollout. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-002 | Provenance decision reason persistence
Storage now persists `provenance.decisionReason` for advisories and merge events; tests cover round-trips. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-003 | Normalized versions indexing
Bootstrapper seeds compound/sparse indexes for flattened normalized rules and `docs/dev/mongo_indices.md` documents query guidance. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-TESTS-02-004 | Restore AdvisoryStore build after normalized versions refactor
Updated constructors/tests keep storage suites passing with the new feature flag defaults. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-ENGINE-01-002 | Plumb Authority client resilience options
WebService wires `authority.resilience.*` into `AddStellaOpsAuthClient` and adds binding coverage via `AuthorityClientResilienceOptionsAreBound`. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-003 | Author ops guidance for resilience tuning
Install/runbooks document connected vs air-gapped resilience profiles and monitoring hooks. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-004 | Document authority bypass logging patterns
Operator guides now call out `route/status/subject/clientId/scopes/bypass/remote` audit fields and SIEM triggers. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-005 | Update Concelier operator guide for enforcement cutoff
Install guide reiterates the 2025-12-31 cutoff and links audit signals to the rollout checklist. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | SEC3.HOST | Rate limiter policy binding
Authority host now applies configuration-driven fixed windows to `/token`, `/authorize`, and `/internal/*`; integration tests assert 429 + `Retry-After` headers; docs/config samples refreshed for Docs guild diagrams. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | SEC3.BUILD | Authority rate-limiter follow-through
`Security.RateLimiting` now fronts token/authorize/internal limiters; Authority + Configuration matrices (`dotnet test src/StellaOps.Authority/StellaOps.Authority.sln`, `dotnet test src/StellaOps.Configuration.Tests/StellaOps.Configuration.Tests.csproj`) passed on 2025-10-11; awaiting #authority-core broadcast. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-14) | Team Authority Platform & Security Guild | AUTHCORE-BUILD-OPENIDDICT / AUTHCORE-STORAGE-DEVICE-TOKENS / AUTHCORE-BOOTSTRAP-INVITES | Address remaining Authority compile blockers (OpenIddict transaction shim, token device document, bootstrap invite cleanup) so `dotnet build src/StellaOps.Authority.sln` returns success. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | PLG6.DOC | Plugin developer guide polish
Section 9 now documents rate limiter metadata, config keys, and lockout interplay; YAML samples updated alongside Authority config templates. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | DOING (2025-10-14) | Team WebService & Authority | SEC2.PLG | Emit audit events from password verification outcomes and persist via `IAuthorityLoginAttemptStore`; Serilog enrichment complete, storage durability tests in flight. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | DOING (2025-10-14) | Team WebService & Authority | SEC3.PLG | Ensure lockout responses carry rate-limit metadata through plugin logs/events; retry-after propagation and limiter tests underway. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | DOING (2025-10-14) | Team WebService & Authority | SEC5.PLG | Address plugin-specific mitigations in threat model backlog; mitigation items tracked, docs updates pending. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | BLOCKED (2025-10-12) | Team WebService & Authority | PLG4-6.CAPABILITIES | Finalise capability metadata exposure and docs once Authority rate-limiter stream (CORE8/SEC3) is stable; awaiting dependency unblock. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | TODO | Team WebService & Authority | PLG6.DIAGRAM | Export final sequence/component diagrams for the developer guide and add offline-friendly assets under `docs/assets/authority`. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | REVIEW (2025-10-13) | Team WebService & Authority | PLG7.RFC | Socialize LDAP plugin RFC and capture guild feedback; awaiting final review sign-off and follow-up issue tracking. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-001 | Fetch pipeline & state tracking
Summary planner now drives monthly/yearly VINCE fetches, persists pending summaries/notes, and hydrates VINCE detail queue with telemetry.
Team instructions: Read ./AGENTS.md and src/StellaOps.Concelier.Connector.CertCc/AGENTS.md. Coordinate daily with Models/Merge leads so new normalizedVersions output and provenance tags stay aligned with ./src/FASTER_MODELING_AND_NORMALIZATION.md. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-002 | VINCE note detail fetcher
Summary planner queues VINCE note detail endpoints, persists raw JSON with SHA/ETag metadata, and records retry/backoff metrics. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-003 | DTO & parser implementation
Added VINCE DTO aggregate, Markdown→text sanitizer, vendor/status/vulnerability parsers, and parser regression fixture. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-004 | Canonical mapping & range primitives
VINCE DTO aggregate flows through `CertCcMapper`, emitting vendor range primitives + normalized version rules that persist via `_advisoryStore`. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-005 | Deterministic fixtures/tests
Snapshot harness refreshed 2025-10-12; `certcc-*.snapshot.json` regenerated and regression suite green without UPDATE flag drift. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-006 | Telemetry & documentation
`CertCcDiagnostics` publishes summary/detail/parse/map metrics (meter `StellaOps.Concelier.Connector.CertCc`), README documents instruments, and log guidance captured for Ops on 2025-10-12. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-007 | Connector test harness remediation
Harness now wires `AddSourceCommon`, resets `FakeTimeProvider`, and passes canned-response regression run dated 2025-10-12. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-008 | Snapshot coverage handoff
Fixtures regenerated with normalized ranges + provenance fields on 2025-10-11; QA handoff notes published and merge backfill unblocked. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-012 | Schema sync & snapshot regen follow-up
Fixtures regenerated with normalizedVersions + provenance decision reasons; handoff notes updated for Merge backfill 2025-10-12. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-009 | Detail/map reintegration plan
Staged reintegration plan published in `src/StellaOps.Concelier.Connector.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md`; coordinates enablement with FEEDCONN-CERTCC-02-004. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-010 | Partial-detail graceful degradation
Detail fetch now tolerates 404/403/410 responses and regression tests cover mixed endpoint availability. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Distro.RedHat/TASKS.md | DOING (2025-10-10) | Team Connector Resumption – CERT/RedHat | FEEDCONN-REDHAT-02-001 | Fixture validation sweep
Instructions to work:
Regenerating RHSA fixtures awaits remaining range provenance patches; review snapshot diffs and update docs once upstream helpers land. Conflict resolver deltas logged in src/StellaOps.Concelier.Connector.Distro.RedHat/CONFLICT_RESOLVER_NOTES.md for Sprint 3 consumers. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-12) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-001 | Canonical mapping & range primitives
Mapper emits SemVer rules (`scheme=apple:*`); fixtures regenerated with trimmed references + new RSR coverage, update tooling finalized. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-002 | Deterministic fixtures/tests
Sanitized live fixtures + regression snapshots wired into tests; normalized rule coverage asserted. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-003 | Telemetry & documentation
Apple meter metrics wired into Concelier WebService OpenTelemetry configuration; README and fixtures document normalizedVersions coverage. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-12) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-004 | Live HTML regression sweep
Sanitised HT125326/HT125328/HT106355/HT214108/HT215500 fixtures recorded and regression tests green on 2025-10-12. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-005 | Fixture regeneration tooling
`UPDATE_APPLE_FIXTURES=1` flow fetches & rewrites fixtures; README documents usage.
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Concelier.Connector.Vndr.Apple/AGENTS.md. Resume stalled tasks, ensuring normalizedVersions output and fixtures align with ./src/FASTER_MODELING_AND_NORMALIZATION.md before handing data to the conflict sprint. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-GHSA-02-001 | GHSA normalized versions & provenance
Team instructions: Read ./AGENTS.md and each module's AGENTS file. Adopt the `NormalizedVersions` array emitted by the models sprint, wiring provenance `decisionReason` where merge overrides occur. Follow ./src/FASTER_MODELING_AND_NORMALIZATION.md; report via src/StellaOps.Concelier.Merge/TASKS.md (FEEDMERGE-COORD-02-900). Progress 2025-10-11: GHSA/OSV emit normalized arrays with refreshed fixtures; CVE mapper now surfaces SemVer normalized ranges; NVD/KEV adoption pending; outstanding follow-ups include FEEDSTORAGE-DATA-02-001, FEEDMERGE-ENGINE-02-002, and rolling `tools/FixtureUpdater` updates across connectors. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-OSV-02-003 | OSV normalized versions & freshness | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-NVD-02-002 | NVD normalized versions & timestamps | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Cve/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-CVE-02-003 | CVE normalized versions uplift | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Kev/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-KEV-02-003 | KEV normalized versions propagation | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-OSV-04-003 | OSV parity fixture refresh | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DOING (2025-10-10) | Team WebService & Authority | FEEDWEB-DOCS-01-001 | Document authority toggle & scope requirements
Quickstart updates are staged; awaiting Docs guild review before publishing operator guide refresh. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-ENGINE-01-002 | Plumb Authority client resilience options
WebService wires `authority.resilience.*` into `AddStellaOpsAuthClient` and adds binding coverage via `AuthorityClientResilienceOptionsAreBound`. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-003 | Author ops guidance for resilience tuning
Operator docs now outline connected vs air-gapped resilience profiles and monitoring cues. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-004 | Document authority bypass logging patterns
Audit logging guidance highlights `route/status/subject/clientId/scopes/bypass/remote` fields and SIEM alerts. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-005 | Update Concelier operator guide for enforcement cutoff
Install guide reiterates the 2025-12-31 cutoff and ties audit signals to rollout checks. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-19) | Team WebService & Authority | FEEDWEB-OPS-01-006 | Rename plugin drop directory to namespaced path
Build outputs now point at `StellaOps.Concelier.PluginBinaries`/`StellaOps.Authority.PluginBinaries`; defaults/docs/tests updated to reflect the new layout. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | BLOCKED (2025-10-10) | Team WebService & Authority | FEEDWEB-OPS-01-007 | Authority resilience adoption
Roll out retry/offline knobs to deployment docs and align CLI parity once LIB5 resilience options land; unblock when library release is available and docs review completes. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHCORE-ENGINE-01-001 | CORE8.RL — Rate limiter plumbing validated; integration tests green and docs handoff recorded for middleware ordering + Retry-After headers (see `docs/dev/authority-rate-limit-tuning-outline.md` for continuing guidance). | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHCRYPTO-ENGINE-01-001 | SEC3.A — Shared metadata resolver confirmed via host test run; SEC3.B now unblocked for tuning guidance (outline captured in `docs/dev/authority-rate-limit-tuning-outline.md`). | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-13) | Team Authority Platform & Security Guild | AUTHSEC-DOCS-01-002 | SEC3.B — Published `docs/security/rate-limits.md` with tuning matrix, alert thresholds, and lockout interplay guidance; Docs guild can lift copy into plugin guide. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-14) | Team Authority Platform & Security Guild | AUTHSEC-CRYPTO-02-001 | SEC5.B1 — Introduce libsodium signing provider and parity tests to unblock CLI verification enhancements. | -| Sprint 9 | Sovereign Crypto Foundations | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-19) | Security Guild | SEC6.A | BouncyCastle-backed Ed25519 signing plug-in wired via `ICryptoProviderRegistry`; Scanner WebService now resolves signing through the registry; AGENTS updated to enforce plug-in rule. | -| Sprint 1 | Bootstrap & Replay Hardening | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-14) | Security Guild | AUTHSEC-CRYPTO-02-004 | SEC5.D/E — Finish bootstrap invite lifecycle (API/store/cleanup) and token device heuristics; build currently red due to pending handler integration. | -| Sprint 1 | Developer Tooling | src/StellaOps.Cli/TASKS.md | DONE (2025-10-15) | DevEx/CLI | AUTHCLI-DIAG-01-001 | Surface password policy diagnostics in CLI startup/output so operators see weakened overrides immediately.
CLI now loads Authority plug-ins at startup, logs weakened password policies (length/complexity), and regression coverage lives in `StellaOps.Cli.Tests/Services/AuthorityDiagnosticsReporterTests`. | -| Sprint 1 | Developer Tooling | src/StellaOps.Cli/TASKS.md | TODO – Display export metadata (sha256, size, Rekor link), support optional artifact download path, and handle cache hits gracefully. | DevEx/CLI | EXCITITOR-CLI-01-002 | EXCITITOR-CLI-01-002 – Export download & attestation UX | -| Sprint 1 | Developer Tooling | src/StellaOps.Cli/TASKS.md | TODO – Update docs/09_API_CLI_REFERENCE.md and quickstart snippets to cover Excititor verbs, offline guidance, and attestation verification workflow. | Docs/CLI | EXCITITOR-CLI-01-003 | EXCITITOR-CLI-01-003 – CLI docs & examples for Excititor | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHPLUG-DOCS-01-001 | PLG6.DOC — Developer guide copy + diagrams merged 2025-10-11; limiter guidance incorporated and handed to Docs guild for asset export. | -| Sprint 1 | Backlog | src/StellaOps.Web/TASKS.md | TODO | UX Specialist, Angular Eng | WEB1.TRIVY-SETTINGS | Implement Trivy DB exporter settings panel with `publishFull`, `publishDelta`, `includeFull`, `includeDelta` toggles and “Run export now” action using future `/exporters/trivy-db/settings` API. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Normalization/TASKS.md | DONE (2025-10-12) | Team Normalization & Storage Backbone | FEEDNORM-NORM-02-001 | SemVer normalized rule emitter
`SemVerRangeRuleBuilder` shipped 2025-10-12 with comparator/` | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-002 | Provenance decision reason persistence | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-003 | Normalized versions indexing
Indexes seeded + docs updated 2025-10-11 to cover flattened normalized rules for connector adoption. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDMERGE-ENGINE-02-002 | Normalized versions union & dedupe
Affected package resolver unions/dedupes normalized rules, stamps merge provenance with `decisionReason`, and tests cover the rollout. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Merge/TASKS.md | DOING (2025-10-12) | Team Merge & QA Enforcement | FEEDMERGE-COORD-02-900 | Range primitives rollout coordination
Coordinate remaining connectors (`Acsc`, `Cccs`, `CertBund`, `CertCc`, `Cve`, `Ghsa`, `Ics.Cisa`, `Kisa`, `Ru.Bdu`, `Ru.Nkcki`, `Vndr.Apple`, `Vndr.Cisco`, `Vndr.Msrc`) to emit canonical range primitives with provenance tags; fixtures tracked in `RANGE_PRIMITIVES_COORDINATION.md`. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-001 | GHSA normalized versions & provenance | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-004 | GHSA credits & ecosystem severity mapping | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-005 | GitHub quota monitoring & retries | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-006 | Production credential & scheduler rollout | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-007 | Credit parity regression fixtures | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-002 | NVD normalized versions & timestamps | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-004 | NVD CVSS & CWE precedence payloads | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-005 | NVD merge/export parity regression | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-003 | OSV normalized versions & freshness | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-004 | OSV references & credits alignment | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-005 | Fixture updater workflow
Resolved 2025-10-12: OSV mapper now derives canonical PURLs for Go + scoped npm packages when raw payloads omit `purl`; conflict fixtures unchanged for invalid npm names. Verified via `dotnet test src/StellaOps.Concelier.Connector.Osv.Tests`, `src/StellaOps.Concelier.Connector.Ghsa.Tests`, `src/StellaOps.Concelier.Connector.Nvd.Tests`, and backbone normalization/storage suites. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Acsc/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-ACSC-02-001 … 02-008 | Fetch→parse→map pipeline, fixtures, diagnostics, and README finished 2025-10-12; downstream export parity captured via FEEDEXPORT-JSON-04-001 / FEEDEXPORT-TRIVY-04-001 (completed). | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Acsc/TASKS.md | **DONE (2025-10-11)** – Reproduced Akamai resets, drafted downgrade plan (two-stage HTTP/2 retry + relay fallback), and filed `FEEDCONN-SHARED-HTTP2-001`; module README TODO will host the per-environment knob matrix. | BE-Conn-ACSC | FEEDCONN-ACSC-02-008 | FEEDCONN-ACSC-02-008 HTTP client compatibility plan | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Cccs/TASKS.md | DONE (2025-10-16) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CCCS-02-001 … 02-008 | Observability meter, historical harvest plan, and DOM sanitizer refinements wrapped; ops notes live under `docs/ops/concelier-cccs-operations.md` with fixtures validating EN/FR list handling. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.CertBund/TASKS.md | DONE (2025-10-15) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CERTBUND-02-001 … 02-008 | Telemetry/docs (02-006) and history/locale sweep (02-007) completed alongside pipeline; runbook `docs/ops/concelier-certbund-operations.md` captures locale guidance and offline packaging. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Kisa/TASKS.md | DONE (2025-10-14) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-KISA-02-001 … 02-007 | Connector, tests, and telemetry/docs (02-006) finalized; localisation notes in `docs/dev/kisa_connector_notes.md` complete rollout. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ru.Bdu/TASKS.md | DONE (2025-10-14) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-RUBDU-02-001 … 02-008 | Fetch/parser/mapper refinements, regression fixtures, telemetry/docs, access options, and trusted root packaging all landed; README documents offline access strategy. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ru.Nkcki/TASKS.md | DONE (2025-10-13) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-NKCKI-02-001 … 02-008 | Listing fetch, parser, mapper, fixtures, telemetry/docs, and archive plan finished; Mongo2Go/libcrypto dependency resolved via bundled OpenSSL noted in ops guide. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ics.Cisa/TASKS.md | DONE (2025-10-16) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-ICSCISA-02-001 … 02-011 | Feed parser attachment fixes, SemVer exact values, regression suites, telemetry/docs updates, and handover complete; ops runbook now details attachment verification + proxy usage. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Vndr.Cisco/TASKS.md | DONE (2025-10-14) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CISCO-02-001 … 02-007 | OAuth fetch pipeline, DTO/mapping, tests, and telemetry/docs shipped; monitoring/export integration follow-ups recorded in Ops docs and exporter backlog (completed). | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Vndr.Msrc/TASKS.md | DONE (2025-10-15) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-MSRC-02-001 … 02-008 | Azure AD onboarding (02-008) unblocked fetch/parse/map pipeline; fixtures, telemetry/docs, and Offline Kit guidance published in `docs/ops/concelier-msrc-operations.md`. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Cve/TASKS.md | DONE (2025-10-15) | Team Connector Support & Monitoring | FEEDCONN-CVE-02-001 … 02-002 | CVE data-source selection, fetch pipeline, and docs landed 2025-10-10. 2025-10-15: smoke verified using the seeded mirror fallback; connector now logs a warning and pulls from `seed-data/cve/` until live CVE Services credentials arrive. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Kev/TASKS.md | DONE (2025-10-12) | Team Connector Support & Monitoring | FEEDCONN-KEV-02-001 … 02-002 | KEV catalog ingestion, fixtures, telemetry, and schema validation completed 2025-10-12; ops dashboard published. | -| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-01-001 | Canonical schema docs refresh
Updated canonical schema + provenance guides with SemVer style, normalized version rules, decision reason change log, and migration notes. | -| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-02-001 | Concelier-SemVer Playbook
Published merge playbook covering mapper patterns, dedupe flow, indexes, and rollout checklist. | -| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-02-002 | Normalized versions query guide
Delivered Mongo index/query addendum with `$unwind` recipes, dedupe checks, and operational checklist.
Instructions to work:
DONE Read ./AGENTS.md and docs/AGENTS.md. Document every schema/index/query change produced in Sprint 1-2 leveraging ./src/FASTER_MODELING_AND_NORMALIZATION.md. | -| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | REVIEW | Docs Guild, Plugin Team | DOC4.AUTH-PDG | Copy-edit `docs/dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md`, export lifecycle diagram, add LDAP RFC cross-link. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-03-001 | Canonical merger implementation
`CanonicalMerger` ships with freshness/tie-breaker logic, provenance, and unit coverage feeding Merge. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-03-002 | Field precedence and tie-breaker map
Field precedence tables and tie-breaker metrics wired into the canonical merge flow; docs/tests updated.
Instructions to work:
Read ./AGENTS.md and core AGENTS. Implement the conflict resolver exactly as specified in ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md, coordinating with Merge and Storage teammates. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-DATA-03-001 | Merge event provenance audit prep
Merge events now persist `fieldDecisions` and analytics-ready provenance snapshots. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill
Dual-write/backfill flag delivered; migration + options validated in tests. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-TESTS-02-004 | Restore AdvisoryStore build after normalized versions refactor
Storage tests adjusted for normalized versions/decision reasons.
Instructions to work:
Read ./AGENTS.md and storage AGENTS. Extend merge events with decision reasons and analytics views to support the conflict rules, and deliver the dual-write/backfill for `NormalizedVersions` + `decisionReason` so connectors can roll out safely. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-001 | GHSA/NVD/OSV conflict rules
Merge pipeline consumes `CanonicalMerger` output prior to precedence merge. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-002 | Override metrics instrumentation
Merge events capture per-field decisions; counters/logs align with conflict rules. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-003 | Reference & credit union pipeline
Canonical merge preserves unions with updated tests. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-QA-04-001 | End-to-end conflict regression suite
Added regression tests (`AdvisoryMergeServiceTests`) covering canonical + precedence flow.
Instructions to work:
Read ./AGENTS.md and merge AGENTS. Integrate the canonical merger, instrument metrics, and deliver comprehensive regression tests following ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-GHSA-04-002 | GHSA conflict regression fixtures | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-NVD-04-002 | NVD conflict regression fixtures | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-OSV-04-002 | OSV conflict regression fixtures
Instructions to work:
Read ./AGENTS.md and module AGENTS. Produce fixture triples supporting the precedence/tie-breaker paths defined in ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md and hand them to Merge QA. | -| Sprint 3 | Conflict Resolution Integration & Communications | docs/TASKS.md | DONE (2025-10-11) | Team Documentation Guild – Conflict Guidance | FEEDDOCS-DOCS-05-001 | Concelier Conflict Rules
Runbook published at `docs/ops/concelier-conflict-resolution.md`; metrics/log guidance aligned with Sprint 3 merge counters. | -| Sprint 3 | Conflict Resolution Integration & Communications | docs/TASKS.md | DONE (2025-10-16) | Team Documentation Guild – Conflict Guidance | FEEDDOCS-DOCS-05-002 | Conflict runbook ops rollout
Ops review completed, alert thresholds applied, and change log appended in `docs/ops/concelier-conflict-resolution.md`; task closed after connector signals verified. | -| Sprint 3 | Backlog | src/StellaOps.Concelier.Connector.Common/TASKS.md | **TODO (2025-10-15)** – Provide a reusable CLI/utility to seed `pendingDocuments`/`pendingMappings` for connectors (MSRC backfills require scripted CVRF + detail injection). Coordinate with MSRC team for expected JSON schema and handoff once prototype lands. | Tools Guild, BE-Conn-MSRC | FEEDCONN-SHARED-STATE-003 | FEEDCONN-SHARED-STATE-003 Source state seeding helper | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-15) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-04-001 | Advisory schema parity (description/CWE/canonical metric)
Extend `Advisory` and related records with description text, CWE collection, and canonical metric pointer; refresh validation + serializer determinism tests. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-15) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-04-003 | Canonical merger parity for new fields
Teach `CanonicalMerger` to populate description, CWEResults, and canonical metric pointer with provenance + regression coverage. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-15) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-04-004 | Reference normalization & freshness instrumentation cleanup
Implement URL normalization for reference dedupe, align freshness-sensitive instrumentation, and add analytics tests. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-15) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-004 | Merge pipeline parity for new advisory fields
Ensure merge service + merge events surface description/CWE/canonical metric decisions with updated metrics/tests. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-15) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-005 | Connector coordination for new advisory fields
GHSA/NVD/OSV connectors now ship description, CWE, and canonical metric data with refreshed fixtures; merge coordination log updated and exporters notified. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Exporter.Json/TASKS.md | DONE (2025-10-15) | Team Exporters – JSON | FEEDEXPORT-JSON-04-001 | Surface new advisory fields in JSON exporter
Update schemas/offline bundle + fixtures once model/core parity lands.
2025-10-15: `dotnet test src/StellaOps.Concelier.Exporter.Json.Tests` validated canonical metric/CWE emission. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Exporter.TrivyDb/TASKS.md | DONE (2025-10-15) | Team Exporters – Trivy DB | FEEDEXPORT-TRIVY-04-001 | Propagate new advisory fields into Trivy DB package
Extend Bolt builder, metadata, and regression tests for the expanded schema.
2025-10-15: `dotnet test src/StellaOps.Concelier.Exporter.TrivyDb.Tests` confirmed canonical metric/CWE propagation. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-16) | Team Connector Regression Fixtures | FEEDCONN-GHSA-04-004 | Harden CVSS fallback so canonical metric ids persist when GitHub omits vectors; extend fixtures and document severity precedence hand-off to Merge. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-16) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-04-005 | Map OSV advisories lacking CVSS vectors to canonical metric ids/notes and document CWE provenance quirks; schedule parity fixture updates. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-15) | Team Excititor Core & Policy | EXCITITOR-CORE-01-001 | Stand up canonical VEX claim/consensus records with deterministic serializers so Storage/Exports share a stable contract. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-15) | Team Excititor Core & Policy | EXCITITOR-CORE-01-002 | Implement trust-weighted consensus resolver with baseline policy weights, justification gates, telemetry output, and majority/tie handling. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-15) | Team Excititor Core & Policy | EXCITITOR-CORE-01-003 | Publish shared connector/exporter/attestation abstractions and deterministic query signature utilities for cache/attestation workflows. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-15) | Team Excititor Policy | EXCITITOR-POLICY-01-001 | Established policy options & snapshot provider covering baseline weights/overrides. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-15) | Team Excititor Policy | EXCITITOR-POLICY-01-002 | Policy evaluator now feeds consensus resolver with immutable snapshots. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-16) | Team Excititor Policy | EXCITITOR-POLICY-01-003 | Author policy diagnostics, CLI/WebService surfacing, and documentation updates. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-16) | Team Excititor Policy | EXCITITOR-POLICY-01-004 | Implement YAML/JSON schema validation and deterministic diagnostics for operator bundles. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-16) | Team Excititor Policy | EXCITITOR-POLICY-01-005 | Add policy change tracking, snapshot digests, and telemetry/logging hooks. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-15) | Team Excititor Storage | EXCITITOR-STORAGE-01-001 | Mongo mapping registry plus raw/export entities and DI extensions in place. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-16) | Team Excititor Storage | EXCITITOR-STORAGE-01-004 | Build provider/consensus/cache class maps and related collections. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Export/TASKS.md | DONE (2025-10-15) | Team Excititor Export | EXCITITOR-EXPORT-01-001 | Export engine delivers cache lookup, manifest creation, and policy integration. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Export/TASKS.md | DONE (2025-10-17) | Team Excititor Export | EXCITITOR-EXPORT-01-004 | Connect export engine to attestation client and persist Rekor metadata. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Attestation/TASKS.md | DONE (2025-10-16) | Team Excititor Attestation | EXCITITOR-ATTEST-01-001 | Implement in-toto predicate + DSSE builder providing envelopes for export attestation. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Attestation/TASKS.md | TODO – Add verification helpers for Worker/WebService, metrics/logging hooks, and negative-path regression tests. | Team Excititor Attestation | EXCITITOR-ATTEST-01-003 | EXCITITOR-ATTEST-01-003 – Verification suite & observability | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Connectors.Abstractions/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors | EXCITITOR-CONN-ABS-01-001 | Deliver shared connector context/base classes so provider plug-ins can be activated via WebService/Worker. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.WebService/TASKS.md | DONE (2025-10-17) | Team Excititor WebService | EXCITITOR-WEB-01-001 | Scaffold minimal API host, DI, and `/excititor/status` endpoint integrating policy, storage, export, and attestation services. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.WebService/TASKS.md | TODO – Implement `/excititor/init`, `/excititor/ingest/run`, `/excititor/ingest/resume`, `/excititor/reconcile` with token scope enforcement and structured run telemetry. | Team Excititor WebService | EXCITITOR-WEB-01-002 | EXCITITOR-WEB-01-002 – Ingest & reconcile endpoints | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.WebService/TASKS.md | TODO – Add `/excititor/export`, `/excititor/export/{id}`, `/excititor/export/{id}/download`, `/excititor/verify`, returning artifact + attestation metadata with cache awareness. | Team Excititor WebService | EXCITITOR-WEB-01-003 | EXCITITOR-WEB-01-003 – Export & verify endpoints | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Worker/TASKS.md | DONE (2025-10-17) | Team Excititor Worker | EXCITITOR-WORKER-01-001 | Create Worker host with provider scheduling and logging to drive recurring pulls/reconciliation. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Worker/TASKS.md | TODO – Implement durable resume markers, exponential backoff with jitter, and quarantine for failing connectors per architecture spec. | Team Excititor Worker | EXCITITOR-WORKER-01-002 | EXCITITOR-WORKER-01-002 – Resume tokens & retry policy | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Worker/TASKS.md | TODO – Add scheduled attestation re-verification and cache pruning routines, surfacing metrics for export reuse ratios. | Team Excititor Worker | EXCITITOR-WORKER-01-003 | EXCITITOR-WORKER-01-003 – Verification & cache GC loops | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Formats | EXCITITOR-FMT-CSAF-01-001 | Implement CSAF normalizer foundation translating provider documents into `VexClaim` entries. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.CSAF/TASKS.md | TODO – Normalize CSAF `product_status` + `justification` values into policy-aware enums with audit diagnostics for unsupported codes. | Team Excititor Formats | EXCITITOR-FMT-CSAF-01-002 | EXCITITOR-FMT-CSAF-01-002 – Status/justification mapping | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.CSAF/TASKS.md | TODO – Provide CSAF export writer producing deterministic documents (per vuln/product) and manifest metadata for attestation. | Team Excititor Formats | EXCITITOR-FMT-CSAF-01-003 | EXCITITOR-FMT-CSAF-01-003 – CSAF export adapter | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.CycloneDX/TASKS.md | DONE (2025-10-17) | Team Excititor Formats | EXCITITOR-FMT-CYCLONE-01-001 | Implement CycloneDX VEX normalizer capturing `analysis` state and component references. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.CycloneDX/TASKS.md | TODO – Implement helpers to reconcile component/service references against policy expectations and emit diagnostics for missing SBOM links. | Team Excititor Formats | EXCITITOR-FMT-CYCLONE-01-002 | EXCITITOR-FMT-CYCLONE-01-002 – Component reference reconciliation | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.CycloneDX/TASKS.md | TODO – Provide exporters producing CycloneDX VEX output with canonical ordering and hash-stable manifests. | Team Excititor Formats | EXCITITOR-FMT-CYCLONE-01-003 | EXCITITOR-FMT-CYCLONE-01-003 – CycloneDX export serializer | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.OpenVEX/TASKS.md | DONE (2025-10-17) | Team Excititor Formats | EXCITITOR-FMT-OPENVEX-01-001 | Implement OpenVEX normalizer to ingest attestations into canonical claims with provenance. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.OpenVEX/TASKS.md | TODO – Add reducers merging multiple OpenVEX statements, resolving conflicts deterministically, and emitting policy diagnostics. | Team Excititor Formats | EXCITITOR-FMT-OPENVEX-01-002 | EXCITITOR-FMT-OPENVEX-01-002 – Statement merge utilities | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.OpenVEX/TASKS.md | TODO – Provide export serializer generating canonical OpenVEX documents with optional SBOM references and hash-stable ordering. | Team Excititor Formats | EXCITITOR-FMT-OPENVEX-01-003 | EXCITITOR-FMT-OPENVEX-01-003 – OpenVEX export writer | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-001 | Ship Red Hat CSAF provider metadata discovery enabling incremental pulls. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-002 | Fetch CSAF windows with ETag handling, resume tokens, quarantine on schema errors, and persist raw docs. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-003 | Populate provider trust overrides (cosign issuer, identity regex) and provenance hints for policy evaluation/logging. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-004 | Persist resume cursors (last updated timestamp/document hashes) in storage and reload during fetch to avoid duplicates. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-005 | Register connector in Worker/WebService DI, add scheduled jobs, and document CLI triggers for Red Hat CSAF pulls. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-006 | Add CSAF normalization parity fixtures ensuring RHSA-specific metadata is preserved. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Cisco.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Cisco | EXCITITOR-CONN-CISCO-01-001 | Implement Cisco CSAF endpoint discovery/auth to unlock paginated pulls. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Cisco.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Cisco | EXCITITOR-CONN-CISCO-01-002 | Implement Cisco CSAF paginated fetch loop with dedupe and raw persistence support. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Cisco.CSAF/TASKS.md | TODO – Emit cosign/PGP trust metadata and advisory provenance hints for policy weighting. | Team Excititor Connectors – Cisco | EXCITITOR-CONN-CISCO-01-003 | EXCITITOR-CONN-CISCO-01-003 – Provider trust metadata | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – SUSE | EXCITITOR-CONN-SUSE-01-001 | Build Rancher VEX Hub discovery/subscription path with offline snapshot support. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/TASKS.md | TODO – Process hub events with resume checkpoints, deduplication, and quarantine path for malformed payloads. | Team Excititor Connectors – SUSE | EXCITITOR-CONN-SUSE-01-002 | EXCITITOR-CONN-SUSE-01-002 – Checkpointed event ingestion | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/TASKS.md | TODO – Emit provider trust configuration (signers, weight overrides) and attach provenance hints for consensus engine. | Team Excititor Connectors – SUSE | EXCITITOR-CONN-SUSE-01-003 | EXCITITOR-CONN-SUSE-01-003 – Trust metadata & policy hints | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.MSRC.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – MSRC | EXCITITOR-CONN-MS-01-001 | Deliver AAD onboarding/token cache for MSRC CSAF ingestion. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.MSRC.CSAF/TASKS.md | TODO – Fetch CSAF packages with retry/backoff, checksum verification, and raw document persistence plus quarantine for schema failures. | Team Excititor Connectors – MSRC | EXCITITOR-CONN-MS-01-002 | EXCITITOR-CONN-MS-01-002 – CSAF download pipeline | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.MSRC.CSAF/TASKS.md | TODO – Emit cosign/AAD issuer metadata, attach provenance details, and document policy integration. | Team Excititor Connectors – MSRC | EXCITITOR-CONN-MS-01-003 | EXCITITOR-CONN-MS-01-003 – Trust metadata & provenance hints | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Oracle.CSAF/TASKS.md | DOING (2025-10-17) | Team Excititor Connectors – Oracle | EXCITITOR-CONN-ORACLE-01-001 | Implement Oracle CSAF catalogue discovery with CPU calendar awareness and offline snapshot import; connector wiring and fixtures underway. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Oracle.CSAF/TASKS.md | TODO – Fetch CSAF documents with retry/backoff, checksum validation, revision deduplication, and raw persistence. | Team Excititor Connectors – Oracle | EXCITITOR-CONN-ORACLE-01-002 | EXCITITOR-CONN-ORACLE-01-002 – CSAF download & dedupe pipeline | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Oracle.CSAF/TASKS.md | TODO – Emit Oracle signing metadata (PGP/cosign) and provenance hints for consensus weighting. | Team Excititor Connectors – Oracle | EXCITITOR-CONN-ORACLE-01-003 | EXCITITOR-CONN-ORACLE-01-003 – Trust metadata + provenance | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Ubuntu | EXCITITOR-CONN-UBUNTU-01-001 | Implement Ubuntu CSAF discovery and channel selection for USN ingestion. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/TASKS.md | TODO – Fetch CSAF bundles with ETag handling, checksum validation, deduplication, and raw persistence. | Team Excititor Connectors – Ubuntu | EXCITITOR-CONN-UBUNTU-01-002 | EXCITITOR-CONN-UBUNTU-01-002 – Incremental fetch & deduplication | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/TASKS.md | TODO – Emit Ubuntu signing metadata (GPG fingerprints) plus provenance hints for policy weighting and diagnostics. | Team Excititor Connectors – Ubuntu | EXCITITOR-CONN-UBUNTU-01-003 | EXCITITOR-CONN-UBUNTU-01-003 – Trust metadata & provenance | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/TASKS.md | DONE (2025-10-18) | Team Excititor Connectors – OCI | EXCITITOR-CONN-OCI-01-001 | Wire OCI discovery/auth to fetch OpenVEX attestations for configured images. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/TASKS.md | DONE (2025-10-18) | Team Excititor Connectors – OCI | EXCITITOR-CONN-OCI-01-002 | Attestation fetch & verify loop – download DSSE attestations, trigger verification, handle retries/backoff, persist raw statements. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/TASKS.md | DONE (2025-10-18) | Team Excititor Connectors – OCI | EXCITITOR-CONN-OCI-01-003 | Provenance metadata & policy hooks – emit image, subject digest, issuer, and trust metadata for policy weighting/logging. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Cli/TASKS.md | DONE (2025-10-18) | DevEx/CLI | EXCITITOR-CLI-01-001 | Add `excititor` CLI verbs bridging to WebService with consistent auth and offline UX. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-19) | Team Excititor Core & Policy | EXCITITOR-CORE-02-001 | Context signal schema prep – extend consensus models with severity/KEV/EPSS fields and update canonical serializers. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-19) | Team Excititor Policy | EXCITITOR-POLICY-02-001 | Scoring coefficients & weight ceilings – add α/β options, weight boosts, and validation guidance. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-19) | Team Excititor Storage | EXCITITOR-STORAGE-02-001 | Statement events & scoring signals – create immutable VEX statement store plus consensus extensions with indexes/migrations. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Team Excititor WebService | EXCITITOR-WEB-01-004 | Resolve API & signed responses – expose `/excititor/resolve`, return signed consensus/score envelopes, document auth. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.WebService/TASKS.md | DONE (2025-10-19) | Team Excititor WebService | EXCITITOR-WEB-01-005 | Mirror distribution endpoints – expose download APIs for downstream Excititor instances. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Attestation/TASKS.md | DONE (2025-10-16) | Team Excititor Attestation | EXCITITOR-ATTEST-01-002 | Rekor v2 client integration – ship transparency log client with retries and offline queue. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Worker/TASKS.md | TODO | Team Excititor Worker | EXCITITOR-WORKER-01-004 | TTL refresh & stability damper – schedule re-resolve loops and guard against status flapping. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Export/TASKS.md | TODO | Team Excititor Export | EXCITITOR-EXPORT-01-005 | Score & resolve envelope surfaces – include signed consensus/score artifacts in exports. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Export/TASKS.md | TODO | Team Excititor Export | EXCITITOR-EXPORT-01-006 | Quiet provenance packaging – attach quieted-by statement IDs, signers, justification codes to exports and attestations. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Export/TASKS.md | TODO | Team Excititor Export | EXCITITOR-EXPORT-01-007 | Mirror bundle + domain manifest – publish signed consensus bundles for mirrors. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Connectors.StellaOpsMirror/TASKS.md | TODO | Excititor Connectors – Stella | EXCITITOR-CONN-STELLA-07-001 | Excititor mirror connector – ingest signed mirror bundles and map to VexClaims with resume handling. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Connectors.StellaOpsMirror/TASKS.md | TODO | Excititor Connectors – Stella | EXCITITOR-CONN-STELLA-07-002 | Normalize mirror bundles into VexClaim sets referencing original provider metadata and mirror provenance. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Connectors.StellaOpsMirror/TASKS.md | TODO | Excititor Connectors – Stella | EXCITITOR-CONN-STELLA-07-003 | Implement incremental cursor handling per-export digest, support resume, and document configuration for downstream Excititor mirrors. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-19) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-07-001 | Advisory event log & asOf queries – surface immutable statements and replay capability. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-19) | Concelier WebService Guild | FEEDWEB-EVENTS-07-001 | Advisory event replay API – expose `/concelier/advisories/{key}/replay` with `asOf` filter, hex hashes, and conflict data. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Core/TASKS.md | TODO | Team Core Engine & Data Science | FEEDCORE-ENGINE-07-002 | Noise prior computation service – learn false-positive priors and expose deterministic summaries. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Core/TASKS.md | TODO | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-07-003 | Unknown state ledger & confidence seeding – persist unknown flags, seed confidence bands, expose query surface. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | TODO | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-07-001 | Advisory statement & conflict collections – provision Mongo schema/indexes for event-sourced merge. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Merge/TASKS.md | TODO | BE-Merge | FEEDMERGE-ENGINE-07-001 | Conflict sets & explainers – persist conflict materialization and replay hashes for merge decisions. | -| Sprint 8 | Plugin Infrastructure | src/StellaOps.Plugin/TASKS.md | TODO | Plugin Platform Guild | PLUGIN-DI-08-001 | Scoped service support in plugin bootstrap
Teach the plugin loader/registrar to surface services with scoped lifetimes, honour `StellaOps.DependencyInjection` metadata, and document the new contract. | -| Sprint 8 | Plugin Infrastructure | src/StellaOps.Plugin/TASKS.md | TODO | Plugin Platform Guild, Authority Core | PLUGIN-DI-08-002 | Update Authority plugin integration
Flow scoped services through identity-provider registrars, bootstrap flows, and background jobs; add regression coverage around scoped lifetimes. | -| Sprint 8 | Mongo strengthening | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-19) | Team Normalization & Storage Backbone | FEEDSTORAGE-MONGO-08-001 | Causal-consistent Concelier storage sessions
Scoped session facilitator registered, repositories accept optional session handles, and replica-set failover tests verify read-your-write + monotonic reads. | -| Sprint 8 | Mongo strengthening | src/StellaOps.Authority/TASKS.md | DONE (2025-10-19) | Authority Core & Storage Guild | AUTHSTORAGE-MONGO-08-001 | Harden Authority Mongo usage
Scoped Mongo sessions with majority read/write concerns wired through stores and GraphQL/HTTP pipelines; replica-set election regression validated. | -| Sprint 8 | Mongo strengthening | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | TODO | Team Excititor Storage | EXCITITOR-STORAGE-MONGO-08-001 | Causal consistency for Excititor repositories
Register Mongo options with majority defaults, push session-aware overloads through raw/export/consensus/cache stores, and extend migration/tests to validate causal reads after writes (including GridFS-backed content) under replica-set failover. | -| Sprint 8 | Platform Maintenance | src/StellaOps.Excititor.Worker/TASKS.md | TODO | Team Excititor Worker | EXCITITOR-WORKER-02-001 | Resolve Microsoft.Extensions.Caching.Memory advisory – bump to latest .NET 10 preview, regenerate lockfiles, and rerun worker/webservice tests to clear NU1903. | -| Sprint 8 | Platform Maintenance | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | TODO | Team Excititor Storage | EXCITITOR-STORAGE-03-001 | Statement backfill tooling – provide CLI/backfill scripts that populate the `vex.statements` log via WebService ingestion and validate severity/KEV/EPSS signal replay. | -| Sprint 8 | Mirror Distribution | src/StellaOps.Concelier.Exporter.Json/TASKS.md | TODO | Concelier Export Guild | CONCELIER-EXPORT-08-201 | Mirror bundle + domain manifest – produce signed JSON aggregates for `*.stella-ops.org` mirrors. | -| Sprint 8 | Mirror Distribution | src/StellaOps.Concelier.Exporter.TrivyDb/TASKS.md | DONE (2025-10-19) | Concelier Export Guild | CONCELIER-EXPORT-08-202 | Mirror-ready Trivy DB bundles – mirror options emit per-domain manifests/metadata/db archives with deterministic digests for downstream sync. | -| Sprint 8 | Mirror Distribution | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-WEB-08-201 | Mirror distribution endpoints – expose domain-scoped index/download APIs with auth/quota. | -| Sprint 8 | Mirror Distribution | src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md | TODO | BE-Conn-Stella | FEEDCONN-STELLA-08-001 | Concelier mirror connector – fetch mirror manifest, verify signatures, and hydrate canonical DTOs with resume support. | -| Sprint 8 | Mirror Distribution | src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md | TODO | BE-Conn-Stella | FEEDCONN-STELLA-08-002 | Map mirror payloads into canonical advisory DTOs with provenance referencing mirror domain + original source metadata. | -| Sprint 8 | Mirror Distribution | src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md | TODO | BE-Conn-Stella | FEEDCONN-STELLA-08-003 | Add incremental cursor + resume support (per-export fingerprint) and document configuration for downstream Concelier instances. | -| Sprint 8 | Mirror Distribution | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-MIRROR-08-001 | Managed mirror deployments for `*.stella-ops.org` – Helm/Compose overlays, CDN, runbooks. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Core/TASKS.md | DONE (2025-10-19) | Team Scanner Core | SCANNER-CORE-09-501 | Define shared DTOs (ScanJob, ProgressEvent), error taxonomy, and deterministic ID/timestamp helpers aligning with `ARCHITECTURE_SCANNER.md` §3–§4. `docs/scanner-core-contracts.md` now carries the canonical JSON snippet + acceptance notes. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Core/TASKS.md | DONE (2025-10-19) | Team Scanner Core | SCANNER-CORE-09-502 | Observability helpers (correlation IDs, logging scopes, metric namespacing, deterministic hashes) consumed by WebService/Worker. Added `ScannerLogExtensionsPerformanceTests` to lock ≤5 µs overhead. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Core/TASKS.md | DONE (2025-10-18) | Team Scanner Core | SCANNER-CORE-09-503 | Security utilities: Authority client factory, OpTok caching, DPoP verifier, restart-time plug-in guardrails for scanner components. | -| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-001 | Buildx driver scaffold + handshake with Scanner.Emit (local CAS). | -| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-002 | OCI annotations + provenance hand-off to Attestor. | -| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-003 | CI demo: minimal SBOM push & backend report wiring. | -| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-004 | Stabilize descriptor nonce derivation so repeated builds emit deterministic placeholders. | -| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-005 | Integrate determinism guard into GitHub/Gitea workflows and archive proof artifacts. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-18) | Team Scanner WebService | SCANNER-WEB-09-101 | Minimal API host with Authority enforcement, health/ready endpoints, and restart-time plug-in loader per architecture §1, §4. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-18) | Team Scanner WebService | SCANNER-WEB-09-102 | `/api/v1/scans` submission/status endpoints with deterministic IDs, validation, and cancellation support. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-19) | Team Scanner WebService | SCANNER-WEB-09-103 | Progress streaming (SSE/JSONL) with correlation IDs and ISO-8601 UTC timestamps, documented in API reference. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-19) | Team Scanner WebService | SCANNER-WEB-09-104 | Configuration binding for Mongo, MinIO, queue, feature flags; startup diagnostics and fail-fast policy. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-19) | Team Scanner WebService | SCANNER-POLICY-09-105 | Policy snapshot loader + schema + OpenAPI (YAML ignore rules, VEX include/exclude, vendor precedence). | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-19) | Team Scanner WebService | SCANNER-POLICY-09-106 | `/reports` verdict assembly (Feedser+Vexer+Policy) + signed response envelope. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-19) | Team Scanner WebService | SCANNER-POLICY-09-107 | Expose score inputs, config version, and quiet provenance in `/reports` JSON and signed payload. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-201 | Worker host bootstrap with Authority auth, hosted services, and graceful shutdown semantics. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-202 | Lease/heartbeat loop with retry+jitter, poison-job quarantine, structured logging. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-203 | Analyzer dispatch skeleton emitting deterministic stage progress and honoring cancellation tokens. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-204 | Worker metrics (queue latency, stage duration, failure counts) with OpenTelemetry resource wiring. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-205 | Harden heartbeat jitter so lease safety margin stays ≥3× and cover with regression tests + optional live queue smoke run. | -| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | DONE | Policy Guild | POLICY-CORE-09-001 | Policy schema + binder + diagnostics. | -| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | DONE | Policy Guild | POLICY-CORE-09-002 | Policy snapshot store + revision digests. | -| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | DONE | Policy Guild | POLICY-CORE-09-003 | `/policy/preview` API (image digest → projected verdict diff). | -| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild | POLICY-CORE-09-004 | Versioned scoring config with schema validation, trust table, and golden fixtures. | -| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild | POLICY-CORE-09-005 | Scoring/quiet engine – compute score, enforce VEX-only quiet rules, emit inputs and provenance. | -| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild | POLICY-CORE-09-006 | Unknown state & confidence decay – deterministic bands surfaced in policy outputs. | -| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild, Scanner WebService Guild | POLICY-RUNTIME-17-201 | Define runtime reachability feed contract and alignment plan for `SCANNER-RUNTIME-17-401` once Zastava endpoints land; document policy expectations for reachability tags. | -| Sprint 9 | DevOps Foundations | ops/devops/TASKS.md | DONE (2025-10-19) | DevOps Guild | DEVOPS-HELM-09-001 | Helm/Compose environment profiles (dev/staging/airgap) with deterministic digests. | -| Sprint 9 | DevOps Foundations | ops/devops/TASKS.md | TODO | DevOps Guild, Scanner WebService Guild | DEVOPS-SCANNER-09-204 | Surface `SCANNER__EVENTS__*` environment variables across docker-compose (dev/stage/airgap) and Helm values, defaulting to share the Redis queue DSN. | -| Sprint 9 | DevOps Foundations | ops/devops/TASKS.md | TODO | DevOps Guild, Notify Guild | DEVOPS-SCANNER-09-205 | Add Notify smoke stage that tails the Redis stream and asserts `scanner.report.ready`/`scanner.scan.completed` reach Notify WebService in staging. | -| Sprint 9 | Docs & Governance | docs/TASKS.md | DONE (2025-10-19) | Docs Guild, DevEx | DOCS-ADR-09-001 | Establish ADR process and template. | -| Sprint 9 | Docs & Governance | docs/TASKS.md | DONE (2025-10-19) | Docs Guild, Platform Events | DOCS-EVENTS-09-002 | Publish event schema catalog (`docs/events/`) for critical envelopes. | -| Sprint 9 | Docs & Governance | docs/TASKS.md | TODO | Platform Events Guild | PLATFORM-EVENTS-09-401 | Embed canonical event samples into contract/integration tests and ensure CI validates payloads against published schemas. | -| Sprint 9 | Docs & Governance | docs/TASKS.md | TODO | Runtime Guild | RUNTIME-GUILD-09-402 | Confirm Scanner WebService surfaces `quietedFindingCount` and progress hints to runtime consumers; document readiness checklist. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Storage/TASKS.md | DONE (2025-10-19) | Team Scanner Storage | SCANNER-STORAGE-09-301 | Mongo catalog schemas/indexes for images, layers, artifacts, jobs, lifecycle rules plus migrations. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Storage/TASKS.md | DONE (2025-10-19) | Team Scanner Storage | SCANNER-STORAGE-09-302 | MinIO layout, immutability policies, client abstraction, and configuration binding. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Storage/TASKS.md | DONE (2025-10-19) | Team Scanner Storage | SCANNER-STORAGE-09-303 | Repositories/services with dual-write feature flag, deterministic digests, TTL enforcement tests. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Queue/TASKS.md | DONE (2025-10-19) | Team Scanner Queue | SCANNER-QUEUE-09-401 | Queue abstraction + Redis Streams adapter with ack/claim APIs and idempotency tokens. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Queue/TASKS.md | DONE (2025-10-19) | Team Scanner Queue | SCANNER-QUEUE-09-402 | Pluggable backend support (Redis, NATS) with configuration binding, health probes, failover docs. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Queue/TASKS.md | DONE (2025-10-19) | Team Scanner Queue | SCANNER-QUEUE-09-403 | Retry + dead-letter strategy with structured logs/metrics for offline deployments. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Cache/TASKS.md | DONE (2025-10-19) | Scanner Cache Guild | SCANNER-CACHE-10-101 | Implement layer cache store keyed by layer digest with metadata retention per architecture §3.3. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Cache/TASKS.md | DONE (2025-10-19) | Scanner Cache Guild | SCANNER-CACHE-10-102 | Build file CAS with dedupe, TTL enforcement, and offline import/export hooks. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Cache/TASKS.md | DONE (2025-10-19) | Scanner Cache Guild | SCANNER-CACHE-10-103 | Expose cache metrics/logging and configuration toggles for warm/cold thresholds. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Cache/TASKS.md | DONE (2025-10-19) | Scanner Cache Guild | SCANNER-CACHE-10-104 | Implement cache invalidation workflows (layer delete, TTL expiry, diff invalidation). | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | DONE (2025-10-19) | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-201 | Alpine/apk analyzer emitting deterministic components with provenance. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | DONE (2025-10-19) | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-202 | Debian/dpkg analyzer mapping packages to purl identity with evidence. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | DONE (2025-10-19) | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-203 | RPM analyzer capturing EVR, file listings, provenance. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | DONE (2025-10-19) | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-204 | Shared OS evidence helpers for package identity + provenance. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | DONE (2025-10-19) | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-205 | Vendor metadata enrichment (source packages, license, CVE hints). | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | DONE (2025-10-19) | QA + OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-206 | Determinism harness + fixtures for OS analyzers. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | DONE (2025-10-19) | OS Analyzer Guild + DevOps | SCANNER-ANALYZERS-OS-10-207 | Package OS analyzers as restart-time plug-ins (manifest + host registration). | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-301 | Java analyzer emitting `pkg:maven` with provenance. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | DOING (2025-10-19) | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-302 | Node analyzer handling workspaces/symlinks emitting `pkg:npm`; workspace/symlink coverage and determinism harness in progress. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-303 | Python analyzer reading `*.dist-info`, RECORD hashes, entry points. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-304 | Go analyzer leveraging buildinfo for `pkg:golang` components. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-305 | .NET analyzer parsing `*.deps.json`, assembly metadata, RID variants. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-306 | Rust analyzer detecting crates or falling back to `bin:{sha256}`. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-307 | Shared language evidence helpers + usage flag propagation. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-308 | Determinism + fixture harness for language analyzers. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | DOING (2025-10-19) | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-309 | Package language analyzers as restart-time plug-ins (manifest + host registration); manifests and DI wiring under development. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/SPRINTS_LANG_IMPLEMENTATION_PLAN.md | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-302..309 | Detailed per-language sprint plan (Node, Python, Go, .NET, Rust) with gates and benchmarks. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-401 | POSIX shell AST parser with deterministic output. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-402 | Command resolution across layered rootfs with evidence attribution. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-403 | Interpreter tracing for shell wrappers to Python/Node/Java launchers. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-404 | Python entry analyzer (venv shebang, module invocation, usage flag). | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-405 | Node/Java launcher analyzer capturing script/jar targets. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-406 | Explainability + diagnostics for unresolved constructs with metrics. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-407 | Package EntryTrace analyzers as restart-time plug-ins (manifest + host registration). | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Diff/TASKS.md | TODO | Diff Guild | SCANNER-DIFF-10-501 | Build component differ tracking add/remove/version changes with deterministic ordering. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Diff/TASKS.md | TODO | Diff Guild | SCANNER-DIFF-10-502 | Attribute diffs to introducing/removing layers including provenance evidence. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Diff/TASKS.md | TODO | Diff Guild | SCANNER-DIFF-10-503 | Produce JSON diff output for inventory vs usage views aligned with API contract. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-10-601 | Compose inventory SBOM (CycloneDX JSON/Protobuf) from layer fragments. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-10-602 | Compose usage SBOM leveraging EntryTrace to flag actual usage. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-10-603 | Generate BOM index sidecar (purl table + roaring bitmap + usage flag). | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-10-604 | Package artifacts for export + attestation with deterministic manifests. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-10-605 | Emit BOM-Index sidecar schema/fixtures (CRITICAL PATH for SP16). | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-10-606 | Usage view bit flags integrated with EntryTrace. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-10-607 | Embed scoring inputs, confidence band, and quiet provenance in CycloneDX/DSSE artifacts. | -| Sprint 10 | Benchmarks | bench/TASKS.md | DONE (2025-10-19) | Bench Guild, Scanner Team | BENCH-SCANNER-10-001 | Analyzer microbench harness committed with baseline CSV + CLI hook. | -| Sprint 10 | Benchmarks | bench/TASKS.md | TODO | Bench Guild, Language Analyzer Guild | BENCH-SCANNER-10-002 | Wire real language analyzers into bench harness & refresh baselines post-implementation. | -| Sprint 10 | Samples | samples/TASKS.md | DONE (2025-10-19) | Samples Guild, Scanner Team | SAMPLES-10-001 | Sample images, SBOMs, and BOM-Index fixtures published under `samples/`. | -| Sprint 10 | Samples | samples/TASKS.md | TODO | Samples Guild, Policy Guild | SAMPLES-13-004 | Add policy preview/report fixtures showing confidence bands and unknown-age tags. | -| Sprint 10 | DevOps Perf | ops/devops/TASKS.md | DONE (2025-10-19) | DevOps Guild | DEVOPS-PERF-10-001 | Perf smoke job added to CI enforcing <5 s compose budget with regression guard. | -| Sprint 10 | DevOps Perf | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-PERF-10-002 | Publish analyzer bench metrics to Grafana/perf workbook and alarm on ≥20 % regressions. | -| Sprint 10 | DevOps Perf | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-SEC-10-301 | Address NU1902/NU1903 advisories for `MongoDB.Driver` 2.12.0 and `SharpCompress` 0.23.0 surfaced during scanner cache and worker test runs. | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-305A | Parse `*.deps.json` + `runtimeconfig.json`, build RID graph, and normalize to `pkg:nuget` components. | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-305B | Extract assembly metadata (strong name, file/product info) and optional Authenticode details when offline cert bundle provided. | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-305C | Handle self-contained apps and native assets; merge with EntryTrace usage hints. | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-307D | Integrate shared helpers (license mapping, quiet provenance) and concurrency-safe caches. | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-308D | Determinism fixtures + benchmark harness; compare to competitor scanners for accuracy/perf. | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-309D | Package plug-in (manifest, DI registration) and update Offline Kit instructions. | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-304A | Parse Go build info blob (`runtime/debug` format) and `.note.go.buildid`; map to module/version and evidence. | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-304B | Implement DWARF-lite reader for VCS metadata + dirty flag; add cache to avoid re-reading identical binaries. | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-304C | Fallback heuristics for stripped binaries with deterministic `bin:{sha256}` labeling and quiet provenance. | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-307G | Wire shared helpers (license mapping, usage flags) and ensure concurrency-safe buffer reuse. | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-308G | Determinism fixtures + benchmark harness (Vs competitor). | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-309G | Package plug-in manifest + Offline Kit notes; ensure Worker DI registration. | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-302C | Surface script metadata (postinstall/preinstall) and policy hints; emit telemetry counters and evidence records. | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-307N | Integrate shared helpers for license/licence evidence, canonical JSON serialization, and usage flag propagation. | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-308N | Author determinism harness + fixtures for Node analyzer; add benchmark suite. | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-309N | Package Node analyzer as restart-time plug-in (manifest, DI registration, Offline Kit notes). | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-303A | STREAM-based parser for `*.dist-info` (`METADATA`, `WHEEL`, `entry_points.txt`) with normalization + evidence capture. | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-303B | RECORD hash verifier with chunked hashing, Zip64 support, and mismatch diagnostics. | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-303C | Editable install + pip cache detection; integrate EntryTrace hints for runtime usage flags. | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-307P | Shared helper integration (license metadata, quiet provenance, component merging). | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-308P | Golden fixtures + determinism harness for Python analyzer; add benchmark and hash throughput reporting. | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-309P | Package plug-in (manifest, DI registration) and document Offline Kit bundling of Python stdlib metadata if needed. | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-306A | Parse Cargo metadata (`Cargo.lock`, `.fingerprint`, `.metadata`) and map crates to components with evidence. | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-306B | Implement heuristic classifier using ELF section names, symbol mangling, and `.comment` data for stripped binaries. | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-306C | Integrate binary hash fallback (`bin:{sha256}`) and tie into shared quiet provenance helpers. | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-307R | Finalize shared helper usage (license, usage flags) and concurrency-safe caches. | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-308R | Determinism fixtures + performance benchmarks; compare against competitor heuristic coverage. | -| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-309R | Package plug-in manifest + Offline Kit documentation; ensure Worker integration. | -| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-DPOP-11-001 | Implement DPoP proof validation + nonce handling for high-value audiences per architecture. | -| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-MTLS-11-002 | Add OAuth mTLS client credential support with certificate-bound tokens and introspection updates. | -| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Signer/TASKS.md | TODO | Signer Guild | SIGNER-API-11-101 | `/sign/dsse` pipeline with Authority auth, PoE introspection, release verification, DSSE signing. | -| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Signer/TASKS.md | TODO | Signer Guild | SIGNER-REF-11-102 | `/verify/referrers` endpoint with OCI lookup, caching, and policy enforcement. | -| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Signer/TASKS.md | TODO | Signer Guild | SIGNER-QUOTA-11-103 | Enforce plan quotas, concurrency/QPS limits, artifact size caps with metrics/audit logs. | -| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Guild | ATTESTOR-API-11-201 | `/rekor/entries` submission pipeline with dedupe, proof acquisition, and persistence. | -| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Guild | ATTESTOR-VERIFY-11-202 | `/rekor/verify` + retrieval endpoints validating signatures and Merkle proofs. | -| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Guild | ATTESTOR-OBS-11-203 | Telemetry, alerting, mTLS hardening, and archive workflow for Attestor. | -| Sprint 11 | UI Integration | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-ATTEST-11-005 | Attestation visibility (Rekor id, status) on Scan Detail. | -| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Core/TASKS.md | TODO | Zastava Core Guild | ZASTAVA-CORE-12-201 | Define runtime event/admission DTOs, hashing helpers, and versioning strategy. | -| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Core/TASKS.md | TODO | Zastava Core Guild | ZASTAVA-CORE-12-202 | Provide configuration/logging/metrics utilities shared by Observer/Webhook. | -| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Core/TASKS.md | TODO | Zastava Core Guild | ZASTAVA-CORE-12-203 | Authority client helpers, OpTok caching, and security guardrails for runtime services. | -| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Core/TASKS.md | TODO | Zastava Core Guild | ZASTAVA-OPS-12-204 | Operational runbooks, alert rules, and dashboard exports for runtime plane. | -| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Observer/TASKS.md | TODO | Zastava Observer Guild | ZASTAVA-OBS-12-001 | Container lifecycle watcher emitting deterministic runtime events with buffering. | -| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Observer/TASKS.md | TODO | Zastava Observer Guild | ZASTAVA-OBS-12-002 | Capture entrypoint traces + loaded libraries, hashing binaries and linking to baseline SBOM. | -| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Observer/TASKS.md | TODO | Zastava Observer Guild | ZASTAVA-OBS-12-003 | Posture checks for signatures/SBOM/attestation with offline caching. | -| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Observer/TASKS.md | TODO | Zastava Observer Guild | ZASTAVA-OBS-12-004 | Batch `/runtime/events` submissions with disk-backed buffer and rate limits. | -| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Webhook/TASKS.md | TODO | Zastava Webhook Guild | ZASTAVA-WEBHOOK-12-101 | Admission controller host with TLS bootstrap and Authority auth. | -| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Webhook/TASKS.md | TODO | Zastava Webhook Guild | ZASTAVA-WEBHOOK-12-102 | Query Scanner `/policy/runtime`, resolve digests, enforce verdicts. | -| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Webhook/TASKS.md | TODO | Zastava Webhook Guild | ZASTAVA-WEBHOOK-12-103 | Caching, fail-open/closed toggles, metrics/logging for admission decisions. | -| Sprint 12 | Runtime Guardrails | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Scanner WebService Guild | SCANNER-RUNTIME-12-301 | `/runtime/events` ingestion endpoint with validation, batching, storage hooks. | -| Sprint 12 | Runtime Guardrails | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Scanner WebService Guild | SCANNER-RUNTIME-12-302 | `/policy/runtime` endpoint joining SBOM baseline + policy verdict with TTL guidance. | -| Sprint 13 | UX & CLI Experience | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-AUTH-13-001 | Integrate Authority OIDC + DPoP flows with session management. | -| Sprint 13 | UX & CLI Experience | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-SCANS-13-002 | Build scans module (list/detail/SBOM/diff/attestation) with performance + accessibility targets. | -| Sprint 13 | UX & CLI Experience | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-VEX-13-003 | Implement VEX explorer + policy editor with preview integration. | -| Sprint 13 | UX & CLI Experience | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-ADMIN-13-004 | Deliver admin area (tenants/clients/quotas/licensing) with RBAC + audit hooks. | -| Sprint 13 | UX & CLI Experience | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-SCHED-13-005 | Scheduler panel: schedules CRUD, run history, dry-run preview. | -| Sprint 13 | UX & CLI Experience | src/StellaOps.UI/TASKS.md | DOING (2025-10-19) | UI Guild | UI-NOTIFY-13-006 | Notify panel: channels/rules CRUD, deliveries view, test send. | -| Sprint 13 | UX & CLI Experience | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-POLICY-13-007 | Surface policy confidence metadata (band, age, quiet provenance) on preview and report views. | -| Sprint 13 | UX & CLI Experience | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI | CLI-RUNTIME-13-005 | Add runtime policy test verbs that consume `/policy/runtime` and display verdicts. | -| Sprint 13 | UX & CLI Experience | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI | CLI-OFFLINE-13-006 | Implement offline kit pull/import/status commands with integrity checks. | -| Sprint 13 | UX & CLI Experience | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI | CLI-PLUGIN-13-007 | Package non-core CLI verbs as restart-time plug-ins (manifest + loader tests). | -| Sprint 13 | UX & CLI Experience | src/StellaOps.Cli/TASKS.md | TODO – Once `/api/v1/scanner/policy/runtime` exits TODO, verify CLI output against final schema (field names, metadata) and update formatter/tests if the contract moves. Capture joint review notes in docs/09 and link Scanner task sign-off. | DevEx/CLI, Scanner WebService Guild | CLI-RUNTIME-13-008 | CLI-RUNTIME-13-008 – Runtime policy contract sync | -| Sprint 13 | UX & CLI Experience | src/StellaOps.Cli/TASKS.md | TODO – Build Spectre test harness exercising `runtime policy test` against a stubbed backend to lock output shape (table + `--json`) and guard regressions. Integrate into `dotnet test` suite. | DevEx/CLI, QA Guild | CLI-RUNTIME-13-009 | CLI-RUNTIME-13-009 – Runtime policy smoke fixture | -| Sprint 14 | Release & Offline Ops | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-REL-14-001 | Deterministic build/release pipeline with SBOM/provenance, signing, and manifest generation. | -| Sprint 14 | Release & Offline Ops | ops/offline-kit/TASKS.md | TODO | Offline Kit Guild | DEVOPS-OFFLINE-14-002 | Offline kit packaging workflow with integrity verification and documentation. | -| Sprint 14 | Release & Offline Ops | ops/deployment/TASKS.md | TODO | Deployment Guild | DEVOPS-OPS-14-003 | Deployment/update/rollback automation and channel management documentation. | -| Sprint 14 | Release & Offline Ops | ops/licensing/TASKS.md | TODO | Licensing Guild | DEVOPS-LIC-14-004 | Registry token service tied to Authority, plan gating, revocation handling, monitoring. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Models/TASKS.md | TODO | Notify Models Guild | NOTIFY-MODELS-15-101 | Define core Notify DTOs, validation helpers, canonical serialization. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Models/TASKS.md | TODO | Notify Models Guild | NOTIFY-MODELS-15-102 | Publish schema docs and sample payloads for Notify. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Models/TASKS.md | TODO | Notify Models Guild | NOTIFY-MODELS-15-103 | Versioning/migration helpers for rules/templates/deliveries. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Storage.Mongo/TASKS.md | TODO | Notify Storage Guild | NOTIFY-STORAGE-15-201 | Mongo schemas/indexes for rules, channels, deliveries, digests, locks, audit. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Storage.Mongo/TASKS.md | TODO | Notify Storage Guild | NOTIFY-STORAGE-15-202 | Repositories with tenant scoping, soft delete, TTL, causal consistency options. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Storage.Mongo/TASKS.md | TODO | Notify Storage Guild | NOTIFY-STORAGE-15-203 | Delivery history retention and query APIs. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Queue/TASKS.md | TODO | Notify Queue Guild | NOTIFY-QUEUE-15-401 | Bus abstraction + Redis Streams adapter with ordering/idempotency. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Queue/TASKS.md | TODO | Notify Queue Guild | NOTIFY-QUEUE-15-402 | NATS JetStream adapter with health probes and failover. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Queue/TASKS.md | TODO | Notify Queue Guild | NOTIFY-QUEUE-15-403 | Delivery queue with retry/dead-letter + metrics. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Engine/TASKS.md | TODO | Notify Engine Guild | NOTIFY-ENGINE-15-301 | Rules evaluation core (filters, throttles, idempotency). | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Engine/TASKS.md | TODO | Notify Engine Guild | NOTIFY-ENGINE-15-302 | Action planner + digest coalescer. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Engine/TASKS.md | TODO | Notify Engine Guild | NOTIFY-ENGINE-15-303 | Template rendering engine (Slack/Teams/Email/Webhook). | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Engine/TASKS.md | TODO | Notify Engine Guild | NOTIFY-ENGINE-15-304 | Test-send sandbox + preview utilities. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.WebService/TASKS.md | TODO | Notify WebService Guild | NOTIFY-WEB-15-101 | Minimal API host with Authority enforcement and plug-in loading. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.WebService/TASKS.md | TODO | Notify WebService Guild | NOTIFY-WEB-15-102 | Rules/channel/template CRUD with audit logging. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.WebService/TASKS.md | DONE (2025-10-19) | Notify WebService Guild | NOTIFY-WEB-15-103 | Delivery history & test-send endpoints. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.WebService/TASKS.md | TODO | Notify WebService Guild | NOTIFY-WEB-15-104 | Configuration binding + startup diagnostics. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Worker/TASKS.md | TODO | Notify Worker Guild | NOTIFY-WORKER-15-201 | Bus subscription + leasing loop with backoff. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Worker/TASKS.md | TODO | Notify Worker Guild | NOTIFY-WORKER-15-202 | Rules evaluation pipeline integration. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Worker/TASKS.md | TODO | Notify Worker Guild | NOTIFY-WORKER-15-203 | Channel dispatch orchestration with retries. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Worker/TASKS.md | TODO | Notify Worker Guild | NOTIFY-WORKER-15-204 | Metrics/telemetry for Notify workers. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Slack/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-SLACK-15-501 | Slack connector with rate-limit aware delivery. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Slack/TASKS.md | DOING (2025-10-19) | Notify Connectors Guild | NOTIFY-CONN-SLACK-15-502 | Slack health/test-send support. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Teams/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-TEAMS-15-601 | Teams connector with Adaptive Cards. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Teams/TASKS.md | DOING (2025-10-19) | Notify Connectors Guild | NOTIFY-CONN-TEAMS-15-602 | Teams health/test-send support. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Email/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-EMAIL-15-701 | SMTP connector with TLS + rendering. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Email/TASKS.md | DOING (2025-10-19) | Notify Connectors Guild | NOTIFY-CONN-EMAIL-15-702 | DKIM + health/test-send flows. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Webhook/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-WEBHOOK-15-801 | Webhook connector with signing/retries. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Webhook/TASKS.md | DOING (2025-10-19) | Notify Connectors Guild | NOTIFY-CONN-WEBHOOK-15-802 | Webhook health/test-send support. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Slack/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-SLACK-15-503 | Package Slack connector as restart-time plug-in (manifest + host registration). | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Teams/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-TEAMS-15-603 | Package Teams connector as restart-time plug-in (manifest + host registration). | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Email/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-EMAIL-15-703 | Package Email connector as restart-time plug-in (manifest + host registration). | -| Sprint 15 | Notify Foundations | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Scanner WebService Guild | SCANNER-EVENTS-15-201 | Emit `scanner.report.ready` + `scanner.scan.completed` events. | -| Sprint 15 | Benchmarks | bench/TASKS.md | TODO | Bench Guild, Notify Team | BENCH-NOTIFY-15-001 | Notify dispatch throughput bench with results CSV. | -| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Webhook/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-WEBHOOK-15-803 | Package Webhook connector as restart-time plug-in (manifest + host registration). | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Models/TASKS.md | DONE (2025-10-19) | Scheduler Models Guild | SCHED-MODELS-16-101 | Define Scheduler DTOs & validation. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Models/TASKS.md | DONE (2025-10-19) | Scheduler Models Guild | SCHED-MODELS-16-102 | Publish schema docs/sample payloads. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Models/TASKS.md | TODO | Scheduler Models Guild | SCHED-MODELS-16-103 | Versioning/migration helpers for schedules/runs. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Storage.Mongo/TASKS.md | TODO | Scheduler Storage Guild | SCHED-STORAGE-16-201 | Mongo schemas/indexes for Scheduler state (models ready 2025-10-19). | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Storage.Mongo/TASKS.md | TODO | Scheduler Storage Guild | SCHED-STORAGE-16-202 | Repositories with tenant scoping, TTL, causal consistency (models ready 2025-10-19). | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Storage.Mongo/TASKS.md | TODO | Scheduler Storage Guild | SCHED-STORAGE-16-203 | Audit + stats materialization for UI (models ready 2025-10-19). | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Queue/TASKS.md | TODO | Scheduler Queue Guild | SCHED-QUEUE-16-401 | Queue abstraction + Redis Streams adapter (samples available 2025-10-19). | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Queue/TASKS.md | TODO | Scheduler Queue Guild | SCHED-QUEUE-16-402 | NATS JetStream adapter with health probes (samples available 2025-10-19). | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Queue/TASKS.md | TODO | Scheduler Queue Guild | SCHED-QUEUE-16-403 | Dead-letter handling + metrics (samples available 2025-10-19). | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.ImpactIndex/TASKS.md | TODO | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-301 | Ingest BOM-Index into roaring bitmap store. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.ImpactIndex/TASKS.md | TODO | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-302 | Query APIs for ResolveByPurls/ResolveByVulns/ResolveAll. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.ImpactIndex/TASKS.md | TODO | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-303 | Snapshot/compaction/invalidation workflow. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.ImpactIndex/TASKS.md | DOING | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-300 | **STUB** ImpactIndex ingest/query using fixtures (to be removed by SP16 completion). | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-WEB-16-101 | Minimal API host with Authority enforcement. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-WEB-16-102 | Schedules CRUD (cron validation, pause/resume, audit). | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-WEB-16-103 | Runs API (list/detail/cancel) + impact previews. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-WEB-16-104 | Feedser/Vexer webhook handlers with security enforcement. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-16-201 | Planner loop (cron/event triggers, leases, fairness). | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-16-202 | ImpactIndex targeting and shard planning. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-16-203 | Runner execution invoking Scanner analysis/content refresh. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-16-204 | Emit rescan/report events for Notify/UI. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-16-205 | Metrics/telemetry for Scheduler planners/runners. | -| Sprint 16 | Benchmarks | bench/TASKS.md | TODO | Bench Guild, Scheduler Team | BENCH-IMPACT-16-001 | ImpactIndex throughput bench + RAM profile. | -| Sprint 17 | Symbol Intelligence & Forensics | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-17-701 | Record GNU build-id for ELF components and surface it in SBOM/diff outputs. | -| Sprint 17 | Symbol Intelligence & Forensics | src/StellaOps.Zastava.Observer/TASKS.md | TODO | Zastava Observer Guild | ZASTAVA-OBS-17-005 | Collect GNU build-id during runtime observation and attach it to emitted events. | -| Sprint 17 | Symbol Intelligence & Forensics | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Scanner WebService Guild | SCANNER-RUNTIME-17-401 | Persist runtime build-id observations and expose them for debug-symbol correlation. | -| Sprint 17 | Symbol Intelligence & Forensics | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-REL-17-002 | Ship stripped debug artifacts organised by build-id within release/offline kits. | -| Sprint 17 | Symbol Intelligence & Forensics | docs/TASKS.md | TODO | Docs Guild | DOCS-RUNTIME-17-004 | Document build-id workflows for SBOMs, runtime events, and debug-store usage. | diff --git a/SPRINTS_PRIOR_20251019.md b/SPRINTS_PRIOR_20251019.md index c9017860..f7b3cd43 100644 --- a/SPRINTS_PRIOR_20251019.md +++ b/SPRINTS_PRIOR_20251019.md @@ -1,177 +1,208 @@ -Closed sprint tasks archived from SPRINTS.md on 2025-10-19. - -| Sprint | Theme | Tasks File Path | Status | Type of Specialist | Task ID | Task Description | -| --- | --- | --- | --- | --- | --- | --- | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-12) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-001 | SemVer primitive range-style metadata
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Concelier.Models/AGENTS.md. This task lays the groundwork—complete the SemVer helper updates before teammates pick up FEEDMODELS-SCHEMA-01-002/003 and FEEDMODELS-SCHEMA-02-900. Use ./src/FASTER_MODELING_AND_NORMALIZATION.md for the target rule structure. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-11) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-002 | Provenance decision rationale field
Instructions to work:
AdvisoryProvenance now carries `decisionReason` and docs/tests were updated. Connectors and merge tasks should populate the field when applying precedence/freshness/tie-breaker logic; see src/StellaOps.Concelier.Models/PROVENANCE_GUIDELINES.md for usage guidance. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-11) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-003 | Normalized version rules collection
Instructions to work:
`AffectedPackage.NormalizedVersions` and supporting comparer/docs/tests shipped. Connector owners must emit rule arrays per ./src/FASTER_MODELING_AND_NORMALIZATION.md and report progress via FEEDMERGE-COORD-02-900 so merge/storage backfills can proceed. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-12) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-02-900 | Range primitives for SemVer/EVR/NEVRA metadata
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Concelier.Models/AGENTS.md before resuming this stalled effort. Confirm helpers align with the new `NormalizedVersions` representation so connectors finishing in Sprint 2 can emit consistent metadata. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Normalization/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDNORM-NORM-02-001 | SemVer normalized rule emitter
Shared `SemVerRangeRuleBuilder` now outputs primitives + normalized rules per `FASTER_MODELING_AND_NORMALIZATION.md`; CVE/GHSA connectors consuming the API have verified fixtures. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill
AdvisoryStore dual-writes flattened `normalizedVersions` when `concelier.storage.enableSemVerStyle` is set; migration `20251011-semver-style-backfill` updates historical records and docs outline the rollout. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-002 | Provenance decision reason persistence
Storage now persists `provenance.decisionReason` for advisories and merge events; tests cover round-trips. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-003 | Normalized versions indexing
Bootstrapper seeds compound/sparse indexes for flattened normalized rules and `docs/dev/mongo_indices.md` documents query guidance. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-TESTS-02-004 | Restore AdvisoryStore build after normalized versions refactor
Updated constructors/tests keep storage suites passing with the new feature flag defaults. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-ENGINE-01-002 | Plumb Authority client resilience options
WebService wires `authority.resilience.*` into `AddStellaOpsAuthClient` and adds binding coverage via `AuthorityClientResilienceOptionsAreBound`. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-003 | Author ops guidance for resilience tuning
Install/runbooks document connected vs air-gapped resilience profiles and monitoring hooks. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-004 | Document authority bypass logging patterns
Operator guides now call out `route/status/subject/clientId/scopes/bypass/remote` audit fields and SIEM triggers. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-005 | Update Concelier operator guide for enforcement cutoff
Install guide reiterates the 2025-12-31 cutoff and links audit signals to the rollout checklist. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | SEC3.HOST | Rate limiter policy binding
Authority host now applies configuration-driven fixed windows to `/token`, `/authorize`, and `/internal/*`; integration tests assert 429 + `Retry-After` headers; docs/config samples refreshed for Docs guild diagrams. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | SEC3.BUILD | Authority rate-limiter follow-through
`Security.RateLimiting` now fronts token/authorize/internal limiters; Authority + Configuration matrices (`dotnet test src/StellaOps.Authority/StellaOps.Authority.sln`, `dotnet test src/StellaOps.Configuration.Tests/StellaOps.Configuration.Tests.csproj`) passed on 2025-10-11; awaiting #authority-core broadcast. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-14) | Team Authority Platform & Security Guild | AUTHCORE-BUILD-OPENIDDICT / AUTHCORE-STORAGE-DEVICE-TOKENS / AUTHCORE-BOOTSTRAP-INVITES | Address remaining Authority compile blockers (OpenIddict transaction shim, token device document, bootstrap invite cleanup) so `dotnet build src/StellaOps.Authority.sln` returns success. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | PLG6.DOC | Plugin developer guide polish
Section 9 now documents rate limiter metadata, config keys, and lockout interplay; YAML samples updated alongside Authority config templates. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-001 | Fetch pipeline & state tracking
Summary planner now drives monthly/yearly VINCE fetches, persists pending summaries/notes, and hydrates VINCE detail queue with telemetry.
Team instructions: Read ./AGENTS.md and src/StellaOps.Concelier.Connector.CertCc/AGENTS.md. Coordinate daily with Models/Merge leads so new normalizedVersions output and provenance tags stay aligned with ./src/FASTER_MODELING_AND_NORMALIZATION.md. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-002 | VINCE note detail fetcher
Summary planner queues VINCE note detail endpoints, persists raw JSON with SHA/ETag metadata, and records retry/backoff metrics. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-003 | DTO & parser implementation
Added VINCE DTO aggregate, Markdown→text sanitizer, vendor/status/vulnerability parsers, and parser regression fixture. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-004 | Canonical mapping & range primitives
VINCE DTO aggregate flows through `CertCcMapper`, emitting vendor range primitives + normalized version rules that persist via `_advisoryStore`. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-005 | Deterministic fixtures/tests
Snapshot harness refreshed 2025-10-12; `certcc-*.snapshot.json` regenerated and regression suite green without UPDATE flag drift. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-006 | Telemetry & documentation
`CertCcDiagnostics` publishes summary/detail/parse/map metrics (meter `StellaOps.Concelier.Connector.CertCc`), README documents instruments, and log guidance captured for Ops on 2025-10-12. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-007 | Connector test harness remediation
Harness now wires `AddSourceCommon`, resets `FakeTimeProvider`, and passes canned-response regression run dated 2025-10-12. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-008 | Snapshot coverage handoff
Fixtures regenerated with normalized ranges + provenance fields on 2025-10-11; QA handoff notes published and merge backfill unblocked. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-012 | Schema sync & snapshot regen follow-up
Fixtures regenerated with normalizedVersions + provenance decision reasons; handoff notes updated for Merge backfill 2025-10-12. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-009 | Detail/map reintegration plan
Staged reintegration plan published in `src/StellaOps.Concelier.Connector.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md`; coordinates enablement with FEEDCONN-CERTCC-02-004. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-010 | Partial-detail graceful degradation
Detail fetch now tolerates 404/403/410 responses and regression tests cover mixed endpoint availability. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Distro.RedHat/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-REDHAT-02-001 | Fixture validation sweep
Instructions to work:
Fixtures regenerated post-model-helper rollout; provenance ordering and normalizedVersions scaffolding verified via tests. Conflict resolver deltas logged in src/StellaOps.Concelier.Connector.Distro.RedHat/CONFLICT_RESOLVER_NOTES.md for Sprint 3 consumers. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-12) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-001 | Canonical mapping & range primitives
Mapper emits SemVer rules (`scheme=apple:*`); fixtures regenerated with trimmed references + new RSR coverage, update tooling finalized. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-002 | Deterministic fixtures/tests
Sanitized live fixtures + regression snapshots wired into tests; normalized rule coverage asserted. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-003 | Telemetry & documentation
Apple meter metrics wired into Concelier WebService OpenTelemetry configuration; README and fixtures document normalizedVersions coverage. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-12) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-004 | Live HTML regression sweep
Sanitised HT125326/HT125328/HT106355/HT214108/HT215500 fixtures recorded and regression tests green on 2025-10-12. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-005 | Fixture regeneration tooling
`UPDATE_APPLE_FIXTURES=1` flow fetches & rewrites fixtures; README documents usage.
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Concelier.Connector.Vndr.Apple/AGENTS.md. Resume stalled tasks, ensuring normalizedVersions output and fixtures align with ./src/FASTER_MODELING_AND_NORMALIZATION.md before handing data to the conflict sprint. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-GHSA-02-001 | GHSA normalized versions & provenance
Team instructions: Read ./AGENTS.md and each module's AGENTS file. Adopt the `NormalizedVersions` array emitted by the models sprint, wiring provenance `decisionReason` where merge overrides occur. Follow ./src/FASTER_MODELING_AND_NORMALIZATION.md; report via src/StellaOps.Concelier.Merge/TASKS.md (FEEDMERGE-COORD-02-900). Progress 2025-10-11: GHSA/OSV emit normalized arrays with refreshed fixtures; CVE mapper now surfaces SemVer normalized ranges; NVD/KEV adoption pending; outstanding follow-ups include FEEDSTORAGE-DATA-02-001, FEEDMERGE-ENGINE-02-002, and rolling `tools/FixtureUpdater` updates across connectors. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-OSV-02-003 | OSV normalized versions & freshness | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-NVD-02-002 | NVD normalized versions & timestamps | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Cve/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-CVE-02-003 | CVE normalized versions uplift | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Kev/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-KEV-02-003 | KEV normalized versions propagation | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-OSV-04-003 | OSV parity fixture refresh | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-10) | Team WebService & Authority | FEEDWEB-DOCS-01-001 | Document authority toggle & scope requirements
Quickstart carries toggle/scope guidance pending docs guild review (no change this sprint). | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-ENGINE-01-002 | Plumb Authority client resilience options
WebService wires `authority.resilience.*` into `AddStellaOpsAuthClient` and adds binding coverage via `AuthorityClientResilienceOptionsAreBound`. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-003 | Author ops guidance for resilience tuning
Operator docs now outline connected vs air-gapped resilience profiles and monitoring cues. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-004 | Document authority bypass logging patterns
Audit logging guidance highlights `route/status/subject/clientId/scopes/bypass/remote` fields and SIEM alerts. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-005 | Update Concelier operator guide for enforcement cutoff
Install guide reiterates the 2025-12-31 cutoff and ties audit signals to rollout checks. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-OPS-01-006 | Rename plugin drop directory to namespaced path
Build outputs, tests, and docs now target `StellaOps.Concelier.PluginBinaries`/`StellaOps.Authority.PluginBinaries`. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-OPS-01-007 | Authority resilience adoption
Deployment docs and CLI notes explain the LIB5 resilience knobs for rollout.
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Concelier.WebService/AGENTS.md. These items were mid-flight; resume implementation ensuring docs/operators receive timely updates. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHCORE-ENGINE-01-001 | CORE8.RL — Rate limiter plumbing validated; integration tests green and docs handoff recorded for middleware ordering + Retry-After headers (see `docs/dev/authority-rate-limit-tuning-outline.md` for continuing guidance). | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHCRYPTO-ENGINE-01-001 | SEC3.A — Shared metadata resolver confirmed via host test run; SEC3.B now unblocked for tuning guidance (outline captured in `docs/dev/authority-rate-limit-tuning-outline.md`). | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-13) | Team Authority Platform & Security Guild | AUTHSEC-DOCS-01-002 | SEC3.B — Published `docs/security/rate-limits.md` with tuning matrix, alert thresholds, and lockout interplay guidance; Docs guild can lift copy into plugin guide. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-14) | Team Authority Platform & Security Guild | AUTHSEC-CRYPTO-02-001 | SEC5.B1 — Introduce libsodium signing provider and parity tests to unblock CLI verification enhancements. | -| Sprint 1 | Bootstrap & Replay Hardening | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-14) | Security Guild | AUTHSEC-CRYPTO-02-004 | SEC5.D/E — Finish bootstrap invite lifecycle (API/store/cleanup) and token device heuristics; build currently red due to pending handler integration. | -| Sprint 1 | Developer Tooling | src/StellaOps.Cli/TASKS.md | DONE (2025-10-15) | DevEx/CLI | AUTHCLI-DIAG-01-001 | Surface password policy diagnostics in CLI startup/output so operators see weakened overrides immediately.
CLI now loads Authority plug-ins at startup, logs weakened password policies (length/complexity), and regression coverage lives in `StellaOps.Cli.Tests/Services/AuthorityDiagnosticsReporterTests`. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHPLUG-DOCS-01-001 | PLG6.DOC — Developer guide copy + diagrams merged 2025-10-11; limiter guidance incorporated and handed to Docs guild for asset export. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Normalization/TASKS.md | DONE (2025-10-12) | Team Normalization & Storage Backbone | FEEDNORM-NORM-02-001 | SemVer normalized rule emitter
`SemVerRangeRuleBuilder` shipped 2025-10-12 with comparator/`||` support and fixtures aligning to `FASTER_MODELING_AND_NORMALIZATION.md`. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-002 | Provenance decision reason persistence | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-003 | Normalized versions indexing
Indexes seeded + docs updated 2025-10-11 to cover flattened normalized rules for connector adoption. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDMERGE-ENGINE-02-002 | Normalized versions union & dedupe
Affected package resolver unions/dedupes normalized rules, stamps merge provenance with `decisionReason`, and tests cover the rollout. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-001 | GHSA normalized versions & provenance | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-004 | GHSA credits & ecosystem severity mapping | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-005 | GitHub quota monitoring & retries | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-006 | Production credential & scheduler rollout | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-007 | Credit parity regression fixtures | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-002 | NVD normalized versions & timestamps | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-004 | NVD CVSS & CWE precedence payloads | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-005 | NVD merge/export parity regression | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-003 | OSV normalized versions & freshness | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-004 | OSV references & credits alignment | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-005 | Fixture updater workflow
Resolved 2025-10-12: OSV mapper now derives canonical PURLs for Go + scoped npm packages when raw payloads omit `purl`; conflict fixtures unchanged for invalid npm names. Verified via `dotnet test src/StellaOps.Concelier.Connector.Osv.Tests`, `src/StellaOps.Concelier.Connector.Ghsa.Tests`, `src/StellaOps.Concelier.Connector.Nvd.Tests`, and backbone normalization/storage suites. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Acsc/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-ACSC-02-001 … 02-008 | Fetch→parse→map pipeline, fixtures, diagnostics, and README finished 2025-10-12; downstream export parity captured via FEEDEXPORT-JSON-04-001 / FEEDEXPORT-TRIVY-04-001 (completed). | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Cccs/TASKS.md | DONE (2025-10-16) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CCCS-02-001 … 02-008 | Observability meter, historical harvest plan, and DOM sanitizer refinements wrapped; ops notes live under `docs/ops/concelier-cccs-operations.md` with fixtures validating EN/FR list handling. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.CertBund/TASKS.md | DONE (2025-10-15) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CERTBUND-02-001 … 02-008 | Telemetry/docs (02-006) and history/locale sweep (02-007) completed alongside pipeline; runbook `docs/ops/concelier-certbund-operations.md` captures locale guidance and offline packaging. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Kisa/TASKS.md | DONE (2025-10-14) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-KISA-02-001 … 02-007 | Connector, tests, and telemetry/docs (02-006) finalized; localisation notes in `docs/dev/kisa_connector_notes.md` complete rollout. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ru.Bdu/TASKS.md | DONE (2025-10-14) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-RUBDU-02-001 … 02-008 | Fetch/parser/mapper refinements, regression fixtures, telemetry/docs, access options, and trusted root packaging all landed; README documents offline access strategy. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ru.Nkcki/TASKS.md | DONE (2025-10-13) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-NKCKI-02-001 … 02-008 | Listing fetch, parser, mapper, fixtures, telemetry/docs, and archive plan finished; Mongo2Go/libcrypto dependency resolved via bundled OpenSSL noted in ops guide. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ics.Cisa/TASKS.md | DONE (2025-10-16) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-ICSCISA-02-001 … 02-011 | Feed parser attachment fixes, SemVer exact values, regression suites, telemetry/docs updates, and handover complete; ops runbook now details attachment verification + proxy usage. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Vndr.Cisco/TASKS.md | DONE (2025-10-14) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CISCO-02-001 … 02-007 | OAuth fetch pipeline, DTO/mapping, tests, and telemetry/docs shipped; monitoring/export integration follow-ups recorded in Ops docs and exporter backlog (completed). | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Vndr.Msrc/TASKS.md | DONE (2025-10-15) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-MSRC-02-001 … 02-008 | Azure AD onboarding (02-008) unblocked fetch/parse/map pipeline; fixtures, telemetry/docs, and Offline Kit guidance published in `docs/ops/concelier-msrc-operations.md`. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Cve/TASKS.md | DONE (2025-10-15) | Team Connector Support & Monitoring | FEEDCONN-CVE-02-001 … 02-002 | CVE data-source selection, fetch pipeline, and docs landed 2025-10-10. 2025-10-15: smoke verified using the seeded mirror fallback; connector now logs a warning and pulls from `seed-data/cve/` until live CVE Services credentials arrive. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Kev/TASKS.md | DONE (2025-10-12) | Team Connector Support & Monitoring | FEEDCONN-KEV-02-001 … 02-002 | KEV catalog ingestion, fixtures, telemetry, and schema validation completed 2025-10-12; ops dashboard published. | -| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-01-001 | Canonical schema docs refresh
Updated canonical schema + provenance guides with SemVer style, normalized version rules, decision reason change log, and migration notes. | -| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-02-001 | Concelier-SemVer Playbook
Published merge playbook covering mapper patterns, dedupe flow, indexes, and rollout checklist. | -| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-02-002 | Normalized versions query guide
Delivered Mongo index/query addendum with `$unwind` recipes, dedupe checks, and operational checklist.
Instructions to work:
DONE Read ./AGENTS.md and docs/AGENTS.md. Document every schema/index/query change produced in Sprint 1-2 leveraging ./src/FASTER_MODELING_AND_NORMALIZATION.md. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-03-001 | Canonical merger implementation
`CanonicalMerger` ships with freshness/tie-breaker logic, provenance, and unit coverage feeding Merge. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-03-002 | Field precedence and tie-breaker map
Field precedence tables and tie-breaker metrics wired into the canonical merge flow; docs/tests updated.
Instructions to work:
Read ./AGENTS.md and core AGENTS. Implement the conflict resolver exactly as specified in ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md, coordinating with Merge and Storage teammates. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-DATA-03-001 | Merge event provenance audit prep
Merge events now persist `fieldDecisions` and analytics-ready provenance snapshots. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill
Dual-write/backfill flag delivered; migration + options validated in tests. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-TESTS-02-004 | Restore AdvisoryStore build after normalized versions refactor
Storage tests adjusted for normalized versions/decision reasons.
Instructions to work:
Read ./AGENTS.md and storage AGENTS. Extend merge events with decision reasons and analytics views to support the conflict rules, and deliver the dual-write/backfill for `NormalizedVersions` + `decisionReason` so connectors can roll out safely. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-001 | GHSA/NVD/OSV conflict rules
Merge pipeline consumes `CanonicalMerger` output prior to precedence merge. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-002 | Override metrics instrumentation
Merge events capture per-field decisions; counters/logs align with conflict rules. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-003 | Reference & credit union pipeline
Canonical merge preserves unions with updated tests. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-QA-04-001 | End-to-end conflict regression suite
Added regression tests (`AdvisoryMergeServiceTests`) covering canonical + precedence flow.
Instructions to work:
Read ./AGENTS.md and merge AGENTS. Integrate the canonical merger, instrument metrics, and deliver comprehensive regression tests following ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-GHSA-04-002 | GHSA conflict regression fixtures | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-NVD-04-002 | NVD conflict regression fixtures | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-OSV-04-002 | OSV conflict regression fixtures
Instructions to work:
Read ./AGENTS.md and module AGENTS. Produce fixture triples supporting the precedence/tie-breaker paths defined in ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md and hand them to Merge QA. | -| Sprint 3 | Conflict Resolution Integration & Communications | docs/TASKS.md | DONE (2025-10-11) | Team Documentation Guild – Conflict Guidance | FEEDDOCS-DOCS-05-001 | Concelier Conflict Rules
Runbook published at `docs/ops/concelier-conflict-resolution.md`; metrics/log guidance aligned with Sprint 3 merge counters. | -| Sprint 3 | Conflict Resolution Integration & Communications | docs/TASKS.md | DONE (2025-10-16) | Team Documentation Guild – Conflict Guidance | FEEDDOCS-DOCS-05-002 | Conflict runbook ops rollout
Ops review completed, alert thresholds applied, and change log appended in `docs/ops/concelier-conflict-resolution.md`; task closed after connector signals verified. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-15) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-04-001 | Advisory schema parity (description/CWE/canonical metric)
Extend `Advisory` and related records with description text, CWE collection, and canonical metric pointer; refresh validation + serializer determinism tests. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-15) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-04-003 | Canonical merger parity for new fields
Teach `CanonicalMerger` to populate description, CWEResults, and canonical metric pointer with provenance + regression coverage. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-15) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-04-004 | Reference normalization & freshness instrumentation cleanup
Implement URL normalization for reference dedupe, align freshness-sensitive instrumentation, and add analytics tests. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-15) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-004 | Merge pipeline parity for new advisory fields
Ensure merge service + merge events surface description/CWE/canonical metric decisions with updated metrics/tests. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-15) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-005 | Connector coordination for new advisory fields
GHSA/NVD/OSV connectors now ship description, CWE, and canonical metric data with refreshed fixtures; merge coordination log updated and exporters notified. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Exporter.Json/TASKS.md | DONE (2025-10-15) | Team Exporters – JSON | FEEDEXPORT-JSON-04-001 | Surface new advisory fields in JSON exporter
Update schemas/offline bundle + fixtures once model/core parity lands.
2025-10-15: `dotnet test src/StellaOps.Concelier.Exporter.Json.Tests` validated canonical metric/CWE emission. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Exporter.TrivyDb/TASKS.md | DONE (2025-10-15) | Team Exporters – Trivy DB | FEEDEXPORT-TRIVY-04-001 | Propagate new advisory fields into Trivy DB package
Extend Bolt builder, metadata, and regression tests for the expanded schema.
2025-10-15: `dotnet test src/StellaOps.Concelier.Exporter.TrivyDb.Tests` confirmed canonical metric/CWE propagation. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-16) | Team Connector Regression Fixtures | FEEDCONN-GHSA-04-004 | Harden CVSS fallback so canonical metric ids persist when GitHub omits vectors; extend fixtures and document severity precedence hand-off to Merge. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-16) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-04-005 | Map OSV advisories lacking CVSS vectors to canonical metric ids/notes and document CWE provenance quirks; schedule parity fixture updates. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-15) | Team Excititor Core & Policy | EXCITITOR-CORE-01-001 | Stand up canonical VEX claim/consensus records with deterministic serializers so Storage/Exports share a stable contract. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-15) | Team Excititor Core & Policy | EXCITITOR-CORE-01-002 | Implement trust-weighted consensus resolver with baseline policy weights, justification gates, telemetry output, and majority/tie handling. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-15) | Team Excititor Core & Policy | EXCITITOR-CORE-01-003 | Publish shared connector/exporter/attestation abstractions and deterministic query signature utilities for cache/attestation workflows. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-15) | Team Excititor Policy | EXCITITOR-POLICY-01-001 | Established policy options & snapshot provider covering baseline weights/overrides. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-15) | Team Excititor Policy | EXCITITOR-POLICY-01-002 | Policy evaluator now feeds consensus resolver with immutable snapshots. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-16) | Team Excititor Policy | EXCITITOR-POLICY-01-003 | Author policy diagnostics, CLI/WebService surfacing, and documentation updates. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-16) | Team Excititor Policy | EXCITITOR-POLICY-01-004 | Implement YAML/JSON schema validation and deterministic diagnostics for operator bundles. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-16) | Team Excititor Policy | EXCITITOR-POLICY-01-005 | Add policy change tracking, snapshot digests, and telemetry/logging hooks. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-15) | Team Excititor Storage | EXCITITOR-STORAGE-01-001 | Mongo mapping registry plus raw/export entities and DI extensions in place. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-16) | Team Excititor Storage | EXCITITOR-STORAGE-01-004 | Build provider/consensus/cache class maps and related collections. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Export/TASKS.md | DONE (2025-10-15) | Team Excititor Export | EXCITITOR-EXPORT-01-001 | Export engine delivers cache lookup, manifest creation, and policy integration. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Export/TASKS.md | DONE (2025-10-17) | Team Excititor Export | EXCITITOR-EXPORT-01-004 | Connect export engine to attestation client and persist Rekor metadata. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Attestation/TASKS.md | DONE (2025-10-16) | Team Excititor Attestation | EXCITITOR-ATTEST-01-001 | Implement in-toto predicate + DSSE builder providing envelopes for export attestation. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Connectors.Abstractions/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors | EXCITITOR-CONN-ABS-01-001 | Deliver shared connector context/base classes so provider plug-ins can be activated via WebService/Worker. | -| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.WebService/TASKS.md | DONE (2025-10-17) | Team Excititor WebService | EXCITITOR-WEB-01-001 | Scaffold minimal API host, DI, and `/excititor/status` endpoint integrating policy, storage, export, and attestation services. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Worker/TASKS.md | DONE (2025-10-17) | Team Excititor Worker | EXCITITOR-WORKER-01-001 | Create Worker host with provider scheduling and logging to drive recurring pulls/reconciliation. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Formats | EXCITITOR-FMT-CSAF-01-001 | Implement CSAF normalizer foundation translating provider documents into `VexClaim` entries. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.CycloneDX/TASKS.md | DONE (2025-10-17) | Team Excititor Formats | EXCITITOR-FMT-CYCLONE-01-001 | Implement CycloneDX VEX normalizer capturing `analysis` state and component references. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.OpenVEX/TASKS.md | DONE (2025-10-17) | Team Excititor Formats | EXCITITOR-FMT-OPENVEX-01-001 | Implement OpenVEX normalizer to ingest attestations into canonical claims with provenance. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-001 | Ship Red Hat CSAF provider metadata discovery enabling incremental pulls. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-002 | Fetch CSAF windows with ETag handling, resume tokens, quarantine on schema errors, and persist raw docs. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-003 | Populate provider trust overrides (cosign issuer, identity regex) and provenance hints for policy evaluation/logging. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-004 | Persist resume cursors (last updated timestamp/document hashes) in storage and reload during fetch to avoid duplicates. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-005 | Register connector in Worker/WebService DI, add scheduled jobs, and document CLI triggers for Red Hat CSAF pulls. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-006 | Add CSAF normalization parity fixtures ensuring RHSA-specific metadata is preserved. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Cisco.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Cisco | EXCITITOR-CONN-CISCO-01-001 | Implement Cisco CSAF endpoint discovery/auth to unlock paginated pulls. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Cisco.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Cisco | EXCITITOR-CONN-CISCO-01-002 | Implement Cisco CSAF paginated fetch loop with dedupe and raw persistence support. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – SUSE | EXCITITOR-CONN-SUSE-01-001 | Build Rancher VEX Hub discovery/subscription path with offline snapshot support. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.MSRC.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – MSRC | EXCITITOR-CONN-MS-01-001 | Deliver AAD onboarding/token cache for MSRC CSAF ingestion. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Oracle.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Oracle | EXCITITOR-CONN-ORACLE-01-001 | Implement Oracle CSAF catalogue discovery with CPU calendar awareness. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Ubuntu | EXCITITOR-CONN-UBUNTU-01-001 | Implement Ubuntu CSAF discovery and channel selection for USN ingestion. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/TASKS.md | DONE (2025-10-18) | Team Excititor Connectors – OCI | EXCITITOR-CONN-OCI-01-001 | Wire OCI discovery/auth to fetch OpenVEX attestations for configured images. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/TASKS.md | DONE (2025-10-18) | Team Excititor Connectors – OCI | EXCITITOR-CONN-OCI-01-002 | Attestation fetch & verify loop – download DSSE attestations, trigger verification, handle retries/backoff, persist raw statements. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/TASKS.md | DONE (2025-10-18) | Team Excititor Connectors – OCI | EXCITITOR-CONN-OCI-01-003 | Provenance metadata & policy hooks – emit image, subject digest, issuer, and trust metadata for policy weighting/logging. | -| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Cli/TASKS.md | DONE (2025-10-18) | DevEx/CLI | EXCITITOR-CLI-01-001 | Add `excititor` CLI verbs bridging to WebService with consistent auth and offline UX. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-19) | Team Excititor Core & Policy | EXCITITOR-CORE-02-001 | Context signal schema prep – extend consensus models with severity/KEV/EPSS fields and update canonical serializers. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-19) | Team Excititor Policy | EXCITITOR-POLICY-02-001 | Scoring coefficients & weight ceilings – add α/β options, weight boosts, and validation guidance. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Attestation/TASKS.md | DONE (2025-10-16) | Team Excititor Attestation | EXCITITOR-ATTEST-01-002 | Rekor v2 client integration – ship transparency log client with retries and offline queue. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Core/TASKS.md | DONE (2025-10-18) | Team Scanner Core | SCANNER-CORE-09-501 | Define shared DTOs (ScanJob, ProgressEvent), error taxonomy, and deterministic ID/timestamp helpers aligning with `ARCHITECTURE_SCANNER.md` §3–§4. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Core/TASKS.md | DONE (2025-10-18) | Team Scanner Core | SCANNER-CORE-09-502 | Observability helpers (correlation IDs, logging scopes, metric namespacing, deterministic hashes) consumed by WebService/Worker. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Core/TASKS.md | DONE (2025-10-18) | Team Scanner Core | SCANNER-CORE-09-503 | Security utilities: Authority client factory, OpTok caching, DPoP verifier, restart-time plug-in guardrails for scanner components. | -| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-001 | Buildx driver scaffold + handshake with Scanner.Emit (local CAS). | -| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-002 | OCI annotations + provenance hand-off to Attestor. | -| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-003 | CI demo: minimal SBOM push & backend report wiring. | -| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-004 | Stabilize descriptor nonce derivation so repeated builds emit deterministic placeholders. | -| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-005 | Integrate determinism guard into GitHub/Gitea workflows and archive proof artifacts. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-18) | Team Scanner WebService | SCANNER-WEB-09-101 | Minimal API host with Authority enforcement, health/ready endpoints, and restart-time plug-in loader per architecture §1, §4. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-18) | Team Scanner WebService | SCANNER-WEB-09-102 | `/api/v1/scans` submission/status endpoints with deterministic IDs, validation, and cancellation support. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-19) | Team Scanner WebService | SCANNER-WEB-09-104 | Configuration binding for Mongo, MinIO, queue, feature flags; startup diagnostics and fail-fast policy. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-201 | Worker host bootstrap with Authority auth, hosted services, and graceful shutdown semantics. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-202 | Lease/heartbeat loop with retry+jitter, poison-job quarantine, structured logging. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-203 | Analyzer dispatch skeleton emitting deterministic stage progress and honoring cancellation tokens. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-204 | Worker metrics (queue latency, stage duration, failure counts) with OpenTelemetry resource wiring. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-205 | Harden heartbeat jitter so lease safety margin stays ≥3× and cover with regression tests + optional live queue smoke run. | -| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | DONE | Policy Guild | POLICY-CORE-09-001 | Policy schema + binder + diagnostics. | -| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | DONE | Policy Guild | POLICY-CORE-09-002 | Policy snapshot store + revision digests. | -| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | DONE | Policy Guild | POLICY-CORE-09-003 | `/policy/preview` API (image digest → projected verdict diff). | -| Sprint 9 | DevOps Foundations | ops/devops/TASKS.md | DONE (2025-10-19) | DevOps Guild | DEVOPS-HELM-09-001 | Helm/Compose environment profiles (dev/staging/airgap) with deterministic digests. | -| Sprint 9 | Docs & Governance | docs/TASKS.md | DONE (2025-10-19) | Docs Guild, DevEx | DOCS-ADR-09-001 | Establish ADR process and template. | -| Sprint 9 | Docs & Governance | docs/TASKS.md | DONE (2025-10-19) | Docs Guild, Platform Events | DOCS-EVENTS-09-002 | Publish event schema catalog (`docs/events/`) for critical envelopes. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Storage/TASKS.md | DONE (2025-10-19) | Team Scanner Storage | SCANNER-STORAGE-09-301 | Mongo catalog schemas/indexes for images, layers, artifacts, jobs, lifecycle rules plus migrations. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Storage/TASKS.md | DONE (2025-10-19) | Team Scanner Storage | SCANNER-STORAGE-09-302 | MinIO layout, immutability policies, client abstraction, and configuration binding. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Storage/TASKS.md | DONE (2025-10-19) | Team Scanner Storage | SCANNER-STORAGE-09-303 | Repositories/services with dual-write feature flag, deterministic digests, TTL enforcement tests. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Queue/TASKS.md | DONE (2025-10-19) | Team Scanner Queue | SCANNER-QUEUE-09-401 | Queue abstraction + Redis Streams adapter with ack/claim APIs and idempotency tokens. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Queue/TASKS.md | DONE (2025-10-19) | Team Scanner Queue | SCANNER-QUEUE-09-402 | Pluggable backend support (Redis, NATS) with configuration binding, health probes, failover docs. | -| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Queue/TASKS.md | DONE (2025-10-19) | Team Scanner Queue | SCANNER-QUEUE-09-403 | Retry + dead-letter strategy with structured logs/metrics for offline deployments. | +Closed sprint tasks archived from SPRINTS.md on 2025-10-19. + +| Sprint | Theme | Tasks File Path | Status | Type of Specialist | Task ID | Task Description | +| --- | --- | --- | --- | --- | --- | --- | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-12) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-001 | SemVer primitive range-style metadata
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Concelier.Models/AGENTS.md. This task lays the groundwork—complete the SemVer helper updates before teammates pick up FEEDMODELS-SCHEMA-01-002/003 and FEEDMODELS-SCHEMA-02-900. Use ./src/FASTER_MODELING_AND_NORMALIZATION.md for the target rule structure. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-11) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-002 | Provenance decision rationale field
Instructions to work:
AdvisoryProvenance now carries `decisionReason` and docs/tests were updated. Connectors and merge tasks should populate the field when applying precedence/freshness/tie-breaker logic; see src/StellaOps.Concelier.Models/PROVENANCE_GUIDELINES.md for usage guidance. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-11) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-003 | Normalized version rules collection
Instructions to work:
`AffectedPackage.NormalizedVersions` and supporting comparer/docs/tests shipped. Connector owners must emit rule arrays per ./src/FASTER_MODELING_AND_NORMALIZATION.md and report progress via FEEDMERGE-COORD-02-900 so merge/storage backfills can proceed. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-12) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-02-900 | Range primitives for SemVer/EVR/NEVRA metadata
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Concelier.Models/AGENTS.md before resuming this stalled effort. Confirm helpers align with the new `NormalizedVersions` representation so connectors finishing in Sprint 2 can emit consistent metadata. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Normalization/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDNORM-NORM-02-001 | SemVer normalized rule emitter
Shared `SemVerRangeRuleBuilder` now outputs primitives + normalized rules per `FASTER_MODELING_AND_NORMALIZATION.md`; CVE/GHSA connectors consuming the API have verified fixtures. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill
AdvisoryStore dual-writes flattened `normalizedVersions` when `concelier.storage.enableSemVerStyle` is set; migration `20251011-semver-style-backfill` updates historical records and docs outline the rollout. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-002 | Provenance decision reason persistence
Storage now persists `provenance.decisionReason` for advisories and merge events; tests cover round-trips. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-003 | Normalized versions indexing
Bootstrapper seeds compound/sparse indexes for flattened normalized rules and `docs/dev/mongo_indices.md` documents query guidance. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-TESTS-02-004 | Restore AdvisoryStore build after normalized versions refactor
Updated constructors/tests keep storage suites passing with the new feature flag defaults. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-ENGINE-01-002 | Plumb Authority client resilience options
WebService wires `authority.resilience.*` into `AddStellaOpsAuthClient` and adds binding coverage via `AuthorityClientResilienceOptionsAreBound`. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-003 | Author ops guidance for resilience tuning
Install/runbooks document connected vs air-gapped resilience profiles and monitoring hooks. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-004 | Document authority bypass logging patterns
Operator guides now call out `route/status/subject/clientId/scopes/bypass/remote` audit fields and SIEM triggers. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-005 | Update Concelier operator guide for enforcement cutoff
Install guide reiterates the 2025-12-31 cutoff and links audit signals to the rollout checklist. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | SEC3.HOST | Rate limiter policy binding
Authority host now applies configuration-driven fixed windows to `/token`, `/authorize`, and `/internal/*`; integration tests assert 429 + `Retry-After` headers; docs/config samples refreshed for Docs guild diagrams. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | SEC3.BUILD | Authority rate-limiter follow-through
`Security.RateLimiting` now fronts token/authorize/internal limiters; Authority + Configuration matrices (`dotnet test src/StellaOps.Authority/StellaOps.Authority.sln`, `dotnet test src/StellaOps.Configuration.Tests/StellaOps.Configuration.Tests.csproj`) passed on 2025-10-11; awaiting #authority-core broadcast. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-14) | Team Authority Platform & Security Guild | AUTHCORE-BUILD-OPENIDDICT / AUTHCORE-STORAGE-DEVICE-TOKENS / AUTHCORE-BOOTSTRAP-INVITES | Address remaining Authority compile blockers (OpenIddict transaction shim, token device document, bootstrap invite cleanup) so `dotnet build src/StellaOps.Authority.sln` returns success. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | PLG6.DOC | Plugin developer guide polish
Section 9 now documents rate limiter metadata, config keys, and lockout interplay; YAML samples updated alongside Authority config templates. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-001 | Fetch pipeline & state tracking
Summary planner now drives monthly/yearly VINCE fetches, persists pending summaries/notes, and hydrates VINCE detail queue with telemetry.
Team instructions: Read ./AGENTS.md and src/StellaOps.Concelier.Connector.CertCc/AGENTS.md. Coordinate daily with Models/Merge leads so new normalizedVersions output and provenance tags stay aligned with ./src/FASTER_MODELING_AND_NORMALIZATION.md. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-002 | VINCE note detail fetcher
Summary planner queues VINCE note detail endpoints, persists raw JSON with SHA/ETag metadata, and records retry/backoff metrics. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-003 | DTO & parser implementation
Added VINCE DTO aggregate, Markdown→text sanitizer, vendor/status/vulnerability parsers, and parser regression fixture. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-004 | Canonical mapping & range primitives
VINCE DTO aggregate flows through `CertCcMapper`, emitting vendor range primitives + normalized version rules that persist via `_advisoryStore`. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-005 | Deterministic fixtures/tests
Snapshot harness refreshed 2025-10-12; `certcc-*.snapshot.json` regenerated and regression suite green without UPDATE flag drift. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-006 | Telemetry & documentation
`CertCcDiagnostics` publishes summary/detail/parse/map metrics (meter `StellaOps.Concelier.Connector.CertCc`), README documents instruments, and log guidance captured for Ops on 2025-10-12. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-007 | Connector test harness remediation
Harness now wires `AddSourceCommon`, resets `FakeTimeProvider`, and passes canned-response regression run dated 2025-10-12. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-008 | Snapshot coverage handoff
Fixtures regenerated with normalized ranges + provenance fields on 2025-10-11; QA handoff notes published and merge backfill unblocked. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-012 | Schema sync & snapshot regen follow-up
Fixtures regenerated with normalizedVersions + provenance decision reasons; handoff notes updated for Merge backfill 2025-10-12. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-009 | Detail/map reintegration plan
Staged reintegration plan published in `src/StellaOps.Concelier.Connector.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md`; coordinates enablement with FEEDCONN-CERTCC-02-004. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-010 | Partial-detail graceful degradation
Detail fetch now tolerates 404/403/410 responses and regression tests cover mixed endpoint availability. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Distro.RedHat/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-REDHAT-02-001 | Fixture validation sweep
Instructions to work:
Fixtures regenerated post-model-helper rollout; provenance ordering and normalizedVersions scaffolding verified via tests. Conflict resolver deltas logged in src/StellaOps.Concelier.Connector.Distro.RedHat/CONFLICT_RESOLVER_NOTES.md for Sprint 3 consumers. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-12) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-001 | Canonical mapping & range primitives
Mapper emits SemVer rules (`scheme=apple:*`); fixtures regenerated with trimmed references + new RSR coverage, update tooling finalized. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-002 | Deterministic fixtures/tests
Sanitized live fixtures + regression snapshots wired into tests; normalized rule coverage asserted. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-003 | Telemetry & documentation
Apple meter metrics wired into Concelier WebService OpenTelemetry configuration; README and fixtures document normalizedVersions coverage. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-12) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-004 | Live HTML regression sweep
Sanitised HT125326/HT125328/HT106355/HT214108/HT215500 fixtures recorded and regression tests green on 2025-10-12. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-005 | Fixture regeneration tooling
`UPDATE_APPLE_FIXTURES=1` flow fetches & rewrites fixtures; README documents usage.
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Concelier.Connector.Vndr.Apple/AGENTS.md. Resume stalled tasks, ensuring normalizedVersions output and fixtures align with ./src/FASTER_MODELING_AND_NORMALIZATION.md before handing data to the conflict sprint. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-GHSA-02-001 | GHSA normalized versions & provenance
Team instructions: Read ./AGENTS.md and each module's AGENTS file. Adopt the `NormalizedVersions` array emitted by the models sprint, wiring provenance `decisionReason` where merge overrides occur. Follow ./src/FASTER_MODELING_AND_NORMALIZATION.md; report via src/StellaOps.Concelier.Merge/TASKS.md (FEEDMERGE-COORD-02-900). Progress 2025-10-11: GHSA/OSV emit normalized arrays with refreshed fixtures; CVE mapper now surfaces SemVer normalized ranges; NVD/KEV adoption pending; outstanding follow-ups include FEEDSTORAGE-DATA-02-001, FEEDMERGE-ENGINE-02-002, and rolling `tools/FixtureUpdater` updates across connectors. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-OSV-02-003 | OSV normalized versions & freshness | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-NVD-02-002 | NVD normalized versions & timestamps | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Cve/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-CVE-02-003 | CVE normalized versions uplift | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Kev/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-KEV-02-003 | KEV normalized versions propagation | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-OSV-04-003 | OSV parity fixture refresh | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-10) | Team WebService & Authority | FEEDWEB-DOCS-01-001 | Document authority toggle & scope requirements
Quickstart carries toggle/scope guidance pending docs guild review (no change this sprint). | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-003 | Author ops guidance for resilience tuning
Operator docs now outline connected vs air-gapped resilience profiles and monitoring cues. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-004 | Document authority bypass logging patterns
Audit logging guidance highlights `route/status/subject/clientId/scopes/bypass/remote` fields and SIEM alerts. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-005 | Update Concelier operator guide for enforcement cutoff
Install guide reiterates the 2025-12-31 cutoff and ties audit signals to rollout checks. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-OPS-01-006 | Rename plugin drop directory to namespaced path
Build outputs, tests, and docs now target `StellaOps.Concelier.PluginBinaries`/`StellaOps.Authority.PluginBinaries`. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-OPS-01-007 | Authority resilience adoption
Deployment docs and CLI notes explain the LIB5 resilience knobs for rollout.
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Concelier.WebService/AGENTS.md. These items were mid-flight; resume implementation ensuring docs/operators receive timely updates. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHCORE-ENGINE-01-001 | CORE8.RL — Rate limiter plumbing validated; integration tests green and docs handoff recorded for middleware ordering + Retry-After headers (see `docs/dev/authority-rate-limit-tuning-outline.md` for continuing guidance). | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHCRYPTO-ENGINE-01-001 | SEC3.A — Shared metadata resolver confirmed via host test run; SEC3.B now unblocked for tuning guidance (outline captured in `docs/dev/authority-rate-limit-tuning-outline.md`). | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-13) | Team Authority Platform & Security Guild | AUTHSEC-DOCS-01-002 | SEC3.B — Published `docs/security/rate-limits.md` with tuning matrix, alert thresholds, and lockout interplay guidance; Docs guild can lift copy into plugin guide. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-14) | Team Authority Platform & Security Guild | AUTHSEC-CRYPTO-02-001 | SEC5.B1 — Introduce libsodium signing provider and parity tests to unblock CLI verification enhancements. | +| Sprint 1 | Bootstrap & Replay Hardening | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-14) | Security Guild | AUTHSEC-CRYPTO-02-004 | SEC5.D/E — Finish bootstrap invite lifecycle (API/store/cleanup) and token device heuristics; build currently red due to pending handler integration. | +| Sprint 1 | Developer Tooling | src/StellaOps.Cli/TASKS.md | DONE (2025-10-15) | DevEx/CLI | AUTHCLI-DIAG-01-001 | Surface password policy diagnostics in CLI startup/output so operators see weakened overrides immediately.
CLI now loads Authority plug-ins at startup, logs weakened password policies (length/complexity), and regression coverage lives in `StellaOps.Cli.Tests/Services/AuthorityDiagnosticsReporterTests`. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHPLUG-DOCS-01-001 | PLG6.DOC — Developer guide copy + diagrams merged 2025-10-11; limiter guidance incorporated and handed to Docs guild for asset export. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Normalization/TASKS.md | DONE (2025-10-12) | Team Normalization & Storage Backbone | FEEDNORM-NORM-02-001 | SemVer normalized rule emitter
`SemVerRangeRuleBuilder` shipped 2025-10-12 with comparator/`||` support and fixtures aligning to `FASTER_MODELING_AND_NORMALIZATION.md`. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-002 | Provenance decision reason persistence | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-003 | Normalized versions indexing
Indexes seeded + docs updated 2025-10-11 to cover flattened normalized rules for connector adoption. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDMERGE-ENGINE-02-002 | Normalized versions union & dedupe
Affected package resolver unions/dedupes normalized rules, stamps merge provenance with `decisionReason`, and tests cover the rollout. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-001 | GHSA normalized versions & provenance | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-004 | GHSA credits & ecosystem severity mapping | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-005 | GitHub quota monitoring & retries | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-006 | Production credential & scheduler rollout | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-007 | Credit parity regression fixtures | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-002 | NVD normalized versions & timestamps | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-004 | NVD CVSS & CWE precedence payloads | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-005 | NVD merge/export parity regression | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-003 | OSV normalized versions & freshness | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-004 | OSV references & credits alignment | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-005 | Fixture updater workflow
Resolved 2025-10-12: OSV mapper now derives canonical PURLs for Go + scoped npm packages when raw payloads omit `purl`; conflict fixtures unchanged for invalid npm names. Verified via `dotnet test src/StellaOps.Concelier.Connector.Osv.Tests`, `src/StellaOps.Concelier.Connector.Ghsa.Tests`, `src/StellaOps.Concelier.Connector.Nvd.Tests`, and backbone normalization/storage suites. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Acsc/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-ACSC-02-001 … 02-008 | Fetch→parse→map pipeline, fixtures, diagnostics, and README finished 2025-10-12; downstream export parity captured via FEEDEXPORT-JSON-04-001 / FEEDEXPORT-TRIVY-04-001 (completed). | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Cccs/TASKS.md | DONE (2025-10-16) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CCCS-02-001 … 02-008 | Observability meter, historical harvest plan, and DOM sanitizer refinements wrapped; ops notes live under `docs/ops/concelier-cccs-operations.md` with fixtures validating EN/FR list handling. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.CertBund/TASKS.md | DONE (2025-10-15) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CERTBUND-02-001 … 02-008 | Telemetry/docs (02-006) and history/locale sweep (02-007) completed alongside pipeline; runbook `docs/ops/concelier-certbund-operations.md` captures locale guidance and offline packaging. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Kisa/TASKS.md | DONE (2025-10-14) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-KISA-02-001 … 02-007 | Connector, tests, and telemetry/docs (02-006) finalized; localisation notes in `docs/dev/kisa_connector_notes.md` complete rollout. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ru.Bdu/TASKS.md | DONE (2025-10-14) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-RUBDU-02-001 … 02-008 | Fetch/parser/mapper refinements, regression fixtures, telemetry/docs, access options, and trusted root packaging all landed; README documents offline access strategy. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ru.Nkcki/TASKS.md | DONE (2025-10-13) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-NKCKI-02-001 … 02-008 | Listing fetch, parser, mapper, fixtures, telemetry/docs, and archive plan finished; Mongo2Go/libcrypto dependency resolved via bundled OpenSSL noted in ops guide. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ics.Cisa/TASKS.md | DONE (2025-10-16) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-ICSCISA-02-001 … 02-011 | Feed parser attachment fixes, SemVer exact values, regression suites, telemetry/docs updates, and handover complete; ops runbook now details attachment verification + proxy usage. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Vndr.Cisco/TASKS.md | DONE (2025-10-14) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CISCO-02-001 … 02-007 | OAuth fetch pipeline, DTO/mapping, tests, and telemetry/docs shipped; monitoring/export integration follow-ups recorded in Ops docs and exporter backlog (completed). | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Vndr.Msrc/TASKS.md | DONE (2025-10-15) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-MSRC-02-001 … 02-008 | Azure AD onboarding (02-008) unblocked fetch/parse/map pipeline; fixtures, telemetry/docs, and Offline Kit guidance published in `docs/ops/concelier-msrc-operations.md`. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Cve/TASKS.md | DONE (2025-10-15) | Team Connector Support & Monitoring | FEEDCONN-CVE-02-001 … 02-002 | CVE data-source selection, fetch pipeline, and docs landed 2025-10-10. 2025-10-15: smoke verified using the seeded mirror fallback; connector now logs a warning and pulls from `seed-data/cve/` until live CVE Services credentials arrive. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Kev/TASKS.md | DONE (2025-10-12) | Team Connector Support & Monitoring | FEEDCONN-KEV-02-001 … 02-002 | KEV catalog ingestion, fixtures, telemetry, and schema validation completed 2025-10-12; ops dashboard published. | +| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-01-001 | Canonical schema docs refresh
Updated canonical schema + provenance guides with SemVer style, normalized version rules, decision reason change log, and migration notes. | +| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-02-001 | Concelier-SemVer Playbook
Published merge playbook covering mapper patterns, dedupe flow, indexes, and rollout checklist. | +| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-02-002 | Normalized versions query guide
Delivered Mongo index/query addendum with `$unwind` recipes, dedupe checks, and operational checklist.
Instructions to work:
DONE Read ./AGENTS.md and docs/AGENTS.md. Document every schema/index/query change produced in Sprint 1-2 leveraging ./src/FASTER_MODELING_AND_NORMALIZATION.md. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-03-001 | Canonical merger implementation
`CanonicalMerger` ships with freshness/tie-breaker logic, provenance, and unit coverage feeding Merge. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-03-002 | Field precedence and tie-breaker map
Field precedence tables and tie-breaker metrics wired into the canonical merge flow; docs/tests updated.
Instructions to work:
Read ./AGENTS.md and core AGENTS. Implement the conflict resolver exactly as specified in ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md, coordinating with Merge and Storage teammates. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-DATA-03-001 | Merge event provenance audit prep
Merge events now persist `fieldDecisions` and analytics-ready provenance snapshots. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill
Dual-write/backfill flag delivered; migration + options validated in tests. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-TESTS-02-004 | Restore AdvisoryStore build after normalized versions refactor
Storage tests adjusted for normalized versions/decision reasons.
Instructions to work:
Read ./AGENTS.md and storage AGENTS. Extend merge events with decision reasons and analytics views to support the conflict rules, and deliver the dual-write/backfill for `NormalizedVersions` + `decisionReason` so connectors can roll out safely. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-001 | GHSA/NVD/OSV conflict rules
Merge pipeline consumes `CanonicalMerger` output prior to precedence merge. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-002 | Override metrics instrumentation
Merge events capture per-field decisions; counters/logs align with conflict rules. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-003 | Reference & credit union pipeline
Canonical merge preserves unions with updated tests. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-QA-04-001 | End-to-end conflict regression suite
Added regression tests (`AdvisoryMergeServiceTests`) covering canonical + precedence flow.
Instructions to work:
Read ./AGENTS.md and merge AGENTS. Integrate the canonical merger, instrument metrics, and deliver comprehensive regression tests following ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-GHSA-04-002 | GHSA conflict regression fixtures | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-NVD-04-002 | NVD conflict regression fixtures | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-OSV-04-002 | OSV conflict regression fixtures
Instructions to work:
Read ./AGENTS.md and module AGENTS. Produce fixture triples supporting the precedence/tie-breaker paths defined in ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md and hand them to Merge QA. | +| Sprint 3 | Conflict Resolution Integration & Communications | docs/TASKS.md | DONE (2025-10-11) | Team Documentation Guild – Conflict Guidance | FEEDDOCS-DOCS-05-001 | Concelier Conflict Rules
Runbook published at `docs/ops/concelier-conflict-resolution.md`; metrics/log guidance aligned with Sprint 3 merge counters. | +| Sprint 3 | Conflict Resolution Integration & Communications | docs/TASKS.md | DONE (2025-10-16) | Team Documentation Guild – Conflict Guidance | FEEDDOCS-DOCS-05-002 | Conflict runbook ops rollout
Ops review completed, alert thresholds applied, and change log appended in `docs/ops/concelier-conflict-resolution.md`; task closed after connector signals verified. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-15) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-04-001 | Advisory schema parity (description/CWE/canonical metric)
Extend `Advisory` and related records with description text, CWE collection, and canonical metric pointer; refresh validation + serializer determinism tests. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-15) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-04-003 | Canonical merger parity for new fields
Teach `CanonicalMerger` to populate description, CWEResults, and canonical metric pointer with provenance + regression coverage. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-15) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-04-004 | Reference normalization & freshness instrumentation cleanup
Implement URL normalization for reference dedupe, align freshness-sensitive instrumentation, and add analytics tests. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-15) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-004 | Merge pipeline parity for new advisory fields
Ensure merge service + merge events surface description/CWE/canonical metric decisions with updated metrics/tests. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-15) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-005 | Connector coordination for new advisory fields
GHSA/NVD/OSV connectors now ship description, CWE, and canonical metric data with refreshed fixtures; merge coordination log updated and exporters notified. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Exporter.Json/TASKS.md | DONE (2025-10-15) | Team Exporters – JSON | FEEDEXPORT-JSON-04-001 | Surface new advisory fields in JSON exporter
Update schemas/offline bundle + fixtures once model/core parity lands.
2025-10-15: `dotnet test src/StellaOps.Concelier.Exporter.Json.Tests` validated canonical metric/CWE emission. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Exporter.TrivyDb/TASKS.md | DONE (2025-10-15) | Team Exporters – Trivy DB | FEEDEXPORT-TRIVY-04-001 | Propagate new advisory fields into Trivy DB package
Extend Bolt builder, metadata, and regression tests for the expanded schema.
2025-10-15: `dotnet test src/StellaOps.Concelier.Exporter.TrivyDb.Tests` confirmed canonical metric/CWE propagation. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-16) | Team Connector Regression Fixtures | FEEDCONN-GHSA-04-004 | Harden CVSS fallback so canonical metric ids persist when GitHub omits vectors; extend fixtures and document severity precedence hand-off to Merge. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-16) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-04-005 | Map OSV advisories lacking CVSS vectors to canonical metric ids/notes and document CWE provenance quirks; schedule parity fixture updates. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-15) | Team Excititor Core & Policy | EXCITITOR-CORE-01-001 | Stand up canonical VEX claim/consensus records with deterministic serializers so Storage/Exports share a stable contract. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-15) | Team Excititor Core & Policy | EXCITITOR-CORE-01-002 | Implement trust-weighted consensus resolver with baseline policy weights, justification gates, telemetry output, and majority/tie handling. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-15) | Team Excititor Core & Policy | EXCITITOR-CORE-01-003 | Publish shared connector/exporter/attestation abstractions and deterministic query signature utilities for cache/attestation workflows. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-15) | Team Excititor Policy | EXCITITOR-POLICY-01-001 | Established policy options & snapshot provider covering baseline weights/overrides. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-15) | Team Excititor Policy | EXCITITOR-POLICY-01-002 | Policy evaluator now feeds consensus resolver with immutable snapshots. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-16) | Team Excititor Policy | EXCITITOR-POLICY-01-003 | Author policy diagnostics, CLI/WebService surfacing, and documentation updates. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-16) | Team Excititor Policy | EXCITITOR-POLICY-01-004 | Implement YAML/JSON schema validation and deterministic diagnostics for operator bundles. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-16) | Team Excititor Policy | EXCITITOR-POLICY-01-005 | Add policy change tracking, snapshot digests, and telemetry/logging hooks. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-15) | Team Excititor Storage | EXCITITOR-STORAGE-01-001 | Mongo mapping registry plus raw/export entities and DI extensions in place. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-16) | Team Excititor Storage | EXCITITOR-STORAGE-01-004 | Build provider/consensus/cache class maps and related collections. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Export/TASKS.md | DONE (2025-10-15) | Team Excititor Export | EXCITITOR-EXPORT-01-001 | Export engine delivers cache lookup, manifest creation, and policy integration. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Export/TASKS.md | DONE (2025-10-17) | Team Excititor Export | EXCITITOR-EXPORT-01-004 | Connect export engine to attestation client and persist Rekor metadata. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Attestation/TASKS.md | DONE (2025-10-16) | Team Excititor Attestation | EXCITITOR-ATTEST-01-001 | Implement in-toto predicate + DSSE builder providing envelopes for export attestation. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Connectors.Abstractions/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors | EXCITITOR-CONN-ABS-01-001 | Deliver shared connector context/base classes so provider plug-ins can be activated via WebService/Worker. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.WebService/TASKS.md | DONE (2025-10-17) | Team Excititor WebService | EXCITITOR-WEB-01-001 | Scaffold minimal API host, DI, and `/excititor/status` endpoint integrating policy, storage, export, and attestation services. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Worker/TASKS.md | DONE (2025-10-17) | Team Excititor Worker | EXCITITOR-WORKER-01-001 | Create Worker host with provider scheduling and logging to drive recurring pulls/reconciliation. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Formats | EXCITITOR-FMT-CSAF-01-001 | Implement CSAF normalizer foundation translating provider documents into `VexClaim` entries. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.CycloneDX/TASKS.md | DONE (2025-10-17) | Team Excititor Formats | EXCITITOR-FMT-CYCLONE-01-001 | Implement CycloneDX VEX normalizer capturing `analysis` state and component references. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.OpenVEX/TASKS.md | DONE (2025-10-17) | Team Excititor Formats | EXCITITOR-FMT-OPENVEX-01-001 | Implement OpenVEX normalizer to ingest attestations into canonical claims with provenance. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-001 | Ship Red Hat CSAF provider metadata discovery enabling incremental pulls. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-002 | Fetch CSAF windows with ETag handling, resume tokens, quarantine on schema errors, and persist raw docs. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-003 | Populate provider trust overrides (cosign issuer, identity regex) and provenance hints for policy evaluation/logging. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-004 | Persist resume cursors (last updated timestamp/document hashes) in storage and reload during fetch to avoid duplicates. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-005 | Register connector in Worker/WebService DI, add scheduled jobs, and document CLI triggers for Red Hat CSAF pulls. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-006 | Add CSAF normalization parity fixtures ensuring RHSA-specific metadata is preserved. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Cisco.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Cisco | EXCITITOR-CONN-CISCO-01-001 | Implement Cisco CSAF endpoint discovery/auth to unlock paginated pulls. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Cisco.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Cisco | EXCITITOR-CONN-CISCO-01-002 | Implement Cisco CSAF paginated fetch loop with dedupe and raw persistence support. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – SUSE | EXCITITOR-CONN-SUSE-01-001 | Build Rancher VEX Hub discovery/subscription path with offline snapshot support. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.MSRC.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – MSRC | EXCITITOR-CONN-MS-01-001 | Deliver AAD onboarding/token cache for MSRC CSAF ingestion. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Oracle.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Oracle | EXCITITOR-CONN-ORACLE-01-001 | Implement Oracle CSAF catalogue discovery with CPU calendar awareness. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Ubuntu | EXCITITOR-CONN-UBUNTU-01-001 | Implement Ubuntu CSAF discovery and channel selection for USN ingestion. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/TASKS.md | DONE (2025-10-18) | Team Excititor Connectors – OCI | EXCITITOR-CONN-OCI-01-001 | Wire OCI discovery/auth to fetch OpenVEX attestations for configured images. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/TASKS.md | DONE (2025-10-18) | Team Excititor Connectors – OCI | EXCITITOR-CONN-OCI-01-002 | Attestation fetch & verify loop – download DSSE attestations, trigger verification, handle retries/backoff, persist raw statements. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/TASKS.md | DONE (2025-10-18) | Team Excititor Connectors – OCI | EXCITITOR-CONN-OCI-01-003 | Provenance metadata & policy hooks – emit image, subject digest, issuer, and trust metadata for policy weighting/logging. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Cli/TASKS.md | DONE (2025-10-18) | DevEx/CLI | EXCITITOR-CLI-01-001 | Add `excititor` CLI verbs bridging to WebService with consistent auth and offline UX. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-19) | Team Excititor Core & Policy | EXCITITOR-CORE-02-001 | Context signal schema prep – extend consensus models with severity/KEV/EPSS fields and update canonical serializers. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-19) | Team Excititor Policy | EXCITITOR-POLICY-02-001 | Scoring coefficients & weight ceilings – add α/β options, weight boosts, and validation guidance. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Attestation/TASKS.md | DONE (2025-10-16) | Team Excititor Attestation | EXCITITOR-ATTEST-01-002 | Rekor v2 client integration – ship transparency log client with retries and offline queue. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Core/TASKS.md | DONE (2025-10-18) | Team Scanner Core | SCANNER-CORE-09-501 | Define shared DTOs (ScanJob, ProgressEvent), error taxonomy, and deterministic ID/timestamp helpers aligning with `ARCHITECTURE_SCANNER.md` §3–§4. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Core/TASKS.md | DONE (2025-10-18) | Team Scanner Core | SCANNER-CORE-09-502 | Observability helpers (correlation IDs, logging scopes, metric namespacing, deterministic hashes) consumed by WebService/Worker. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Core/TASKS.md | DONE (2025-10-18) | Team Scanner Core | SCANNER-CORE-09-503 | Security utilities: Authority client factory, OpTok caching, DPoP verifier, restart-time plug-in guardrails for scanner components. | +| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-001 | Buildx driver scaffold + handshake with Scanner.Emit (local CAS). | +| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-002 | OCI annotations + provenance hand-off to Attestor. | +| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-003 | CI demo: minimal SBOM push & backend report wiring. | +| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-004 | Stabilize descriptor nonce derivation so repeated builds emit deterministic placeholders. | +| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-005 | Integrate determinism guard into GitHub/Gitea workflows and archive proof artifacts. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-18) | Team Scanner WebService | SCANNER-WEB-09-101 | Minimal API host with Authority enforcement, health/ready endpoints, and restart-time plug-in loader per architecture §1, §4. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-18) | Team Scanner WebService | SCANNER-WEB-09-102 | `/api/v1/scans` submission/status endpoints with deterministic IDs, validation, and cancellation support. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-19) | Team Scanner WebService | SCANNER-WEB-09-104 | Configuration binding for Mongo, MinIO, queue, feature flags; startup diagnostics and fail-fast policy. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-201 | Worker host bootstrap with Authority auth, hosted services, and graceful shutdown semantics. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-202 | Lease/heartbeat loop with retry+jitter, poison-job quarantine, structured logging. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-203 | Analyzer dispatch skeleton emitting deterministic stage progress and honoring cancellation tokens. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-204 | Worker metrics (queue latency, stage duration, failure counts) with OpenTelemetry resource wiring. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-205 | Harden heartbeat jitter so lease safety margin stays ≥3× and cover with regression tests + optional live queue smoke run. | +| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | DONE | Policy Guild | POLICY-CORE-09-001 | Policy schema + binder + diagnostics. | +| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | DONE | Policy Guild | POLICY-CORE-09-002 | Policy snapshot store + revision digests. | +| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | DONE | Policy Guild | POLICY-CORE-09-003 | `/policy/preview` API (image digest → projected verdict diff). | +| Sprint 9 | DevOps Foundations | ops/devops/TASKS.md | DONE (2025-10-19) | DevOps Guild | DEVOPS-HELM-09-001 | Helm/Compose environment profiles (dev/staging/airgap) with deterministic digests. | +| Sprint 9 | Docs & Governance | docs/TASKS.md | DONE (2025-10-19) | Docs Guild, DevEx | DOCS-ADR-09-001 | Establish ADR process and template. | +| Sprint 9 | Docs & Governance | docs/TASKS.md | DONE (2025-10-19) | Docs Guild, Platform Events | DOCS-EVENTS-09-002 | Publish event schema catalog (`docs/events/`) for critical envelopes. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Storage/TASKS.md | DONE (2025-10-19) | Team Scanner Storage | SCANNER-STORAGE-09-301 | Mongo catalog schemas/indexes for images, layers, artifacts, jobs, lifecycle rules plus migrations. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Storage/TASKS.md | DONE (2025-10-19) | Team Scanner Storage | SCANNER-STORAGE-09-302 | MinIO layout, immutability policies, client abstraction, and configuration binding. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Storage/TASKS.md | DONE (2025-10-19) | Team Scanner Storage | SCANNER-STORAGE-09-303 | Repositories/services with dual-write feature flag, deterministic digests, TTL enforcement tests. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Queue/TASKS.md | DONE (2025-10-19) | Team Scanner Queue | SCANNER-QUEUE-09-401 | Queue abstraction + Redis Streams adapter with ack/claim APIs and idempotency tokens. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Queue/TASKS.md | DONE (2025-10-19) | Team Scanner Queue | SCANNER-QUEUE-09-402 | Pluggable backend support (Redis, NATS) with configuration binding, health probes, failover docs. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Queue/TASKS.md | DONE (2025-10-19) | Team Scanner Queue | SCANNER-QUEUE-09-403 | Retry + dead-letter strategy with structured logs/metrics for offline deployments. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-GHSA-02-001 | GHSA normalized versions & provenance
Team instructions: Read ./AGENTS.md and each module's AGENTS file. Adopt the `NormalizedVersions` array emitted by the models sprint, wiring provenance `decisionReason` where merge overrides occur. Follow ./src/FASTER_MODELING_AND_NORMALIZATION.md; report via src/StellaOps.Concelier.Merge/TASKS.md (FEEDMERGE-COORD-02-900). Progress 2025-10-11: GHSA/OSV emit normalized arrays with refreshed fixtures; CVE mapper now surfaces SemVer normalized ranges; NVD/KEV adoption pending; outstanding follow-ups include FEEDSTORAGE-DATA-02-001, FEEDMERGE-ENGINE-02-002, and rolling `tools/FixtureUpdater` updates across connectors.
Progress 2025-10-20: Coordination matrix + rollout dashboard refreshed; upcoming deadlines tracked (Cccs/Cisco 2025-10-21, CertBund 2025-10-22, ICS-CISA 2025-10-23, KISA 2025-10-24) with escalation path documented in FEEDMERGE-COORD-02-900.| +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-19) | Team WebService & Authority | FEEDWEB-OPS-01-006 | Rename plugin drop directory to namespaced path
Build outputs now point at `StellaOps.Concelier.PluginBinaries`/`StellaOps.Authority.PluginBinaries`; defaults/docs/tests updated to reflect the new layout. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-19) | Team Excititor Storage | EXCITITOR-STORAGE-02-001 | Statement events & scoring signals – immutable VEX statements store, consensus signal fields, and migration `20251019-consensus-signals-statements` with tests (`dotnet test src/StellaOps.Excititor.Core.Tests/StellaOps.Excititor.Core.Tests.csproj`, `dotnet test src/StellaOps.Excititor.Storage.Mongo.Tests/StellaOps.Excititor.Storage.Mongo.Tests.csproj`). | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-19) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-07-001 | Advisory event log & asOf queries – surface immutable statements and replay capability. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-19) | Concelier WebService Guild | FEEDWEB-EVENTS-07-001 | Advisory event replay API – expose `/concelier/advisories/{key}/replay` with `asOf` filter, hex hashes, and conflict data. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-20) | BE-Merge | FEEDMERGE-ENGINE-07-001 | Conflict sets & explainers – persist conflict materialization and replay hashes for merge decisions. | +| Sprint 8 | Mongo strengthening | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-19) | Team Normalization & Storage Backbone | FEEDSTORAGE-MONGO-08-001 | Causal-consistent Concelier storage sessions
Scoped session facilitator registered, repositories accept optional session handles, and replica-set failover tests verify read-your-write + monotonic reads. | +| Sprint 8 | Mongo strengthening | src/StellaOps.Authority/TASKS.md | DONE (2025-10-19) | Authority Core & Storage Guild | AUTHSTORAGE-MONGO-08-001 | Harden Authority Mongo usage
Scoped Mongo sessions with majority read/write concerns wired through stores and GraphQL/HTTP pipelines; replica-set election regression validated. | +| Sprint 8 | Mongo strengthening | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-19) | Team Excititor Storage | EXCITITOR-STORAGE-MONGO-08-001 | Causal consistency for Excititor repositories
Session-scoped repositories shipped with new Mongo records, orchestrators/workers now share scoped sessions, and replica-set failover coverage added via `dotnet test src/StellaOps.Excititor.Storage.Mongo.Tests/StellaOps.Excititor.Storage.Mongo.Tests.csproj`. | +| Sprint 8 | Platform Maintenance | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-19) | Team Excititor Storage | EXCITITOR-STORAGE-03-001 | Statement backfill tooling – shipped admin backfill endpoint, CLI hook (`stellaops excititor backfill-statements`), integration tests, and operator runbook (`docs/dev/EXCITITOR_STATEMENT_BACKFILL.md`). | +| Sprint 8 | Mirror Distribution | src/StellaOps.Concelier.Exporter.Json/TASKS.md | DONE (2025-10-19) | Concelier Export Guild | CONCELIER-EXPORT-08-201 | Mirror bundle + domain manifest – produce signed JSON aggregates for `*.stella-ops.org` mirrors. | +| Sprint 8 | Mirror Distribution | src/StellaOps.Concelier.Exporter.TrivyDb/TASKS.md | DONE (2025-10-19) | Concelier Export Guild | CONCELIER-EXPORT-08-202 | Mirror-ready Trivy DB bundles – mirror options emit per-domain manifests/metadata/db archives with deterministic digests for downstream sync. | +| Sprint 8 | Mirror Distribution | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-20) | Concelier WebService Guild | CONCELIER-WEB-08-201 | Mirror distribution endpoints – expose domain-scoped index/download APIs with auth/quota. | +| Sprint 8 | Mirror Distribution | ops/devops/TASKS.md | DONE (2025-10-19) | DevOps Guild | DEVOPS-MIRROR-08-001 | Managed mirror deployments for `*.stella-ops.org` – Helm/Compose overlays, CDN, runbooks. | +| Sprint 8 | Plugin Infrastructure | src/StellaOps.Plugin/TASKS.md | DONE (2025-10-20) | Plugin Platform Guild, Authority Core | PLUGIN-DI-08-003 | Refactor Authority identity-provider registry to resolve scoped plugin services on-demand.
Introduce factory pattern aligned with scoped lifetimes decided in coordination workshop. | +| Sprint 8 | Plugin Infrastructure | src/StellaOps.Plugin/TASKS.md | DONE (2025-10-20) | Plugin Platform Guild, Authority Core | PLUGIN-DI-08-004 | Update Authority plugin loader to activate registrars with DI support and scoped service awareness.
Add two-phase initialization allowing scoped dependencies post-container build. | +| Sprint 8 | Plugin Infrastructure | src/StellaOps.Plugin/TASKS.md | DONE (2025-10-20) | Plugin Platform Guild, Authority Core | PLUGIN-DI-08-005 | Provide scoped-safe bootstrap execution for Authority plugins.
Implement scope-per-run pattern for hosted bootstrap tasks and document migration guidance. | +| Sprint 10 | DevOps Security | ops/devops/TASKS.md | DONE (2025-10-20) | DevOps Guild | DEVOPS-SEC-10-301 | Address NU1902/NU1903 advisories for `MongoDB.Driver` 2.12.0 and `SharpCompress` 0.23.0; Wave 0A prerequisites confirmed complete before remediation work. | +| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Authority/TASKS.md | DONE (2025-10-20) | Authority Core & Security Guild | AUTH-DPOP-11-001 | Implement DPoP proof validation + nonce handling for high-value audiences per architecture. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.WebService/TASKS.md | DONE (2025-10-19) | Notify WebService Guild | NOTIFY-WEB-15-103 | Delivery history & test-send endpoints. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Slack/TASKS.md | DONE (2025-10-20) | Notify Connectors Guild | NOTIFY-CONN-SLACK-15-502 | Slack health/test-send support. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Teams/TASKS.md | DONE (2025-10-20) | Notify Connectors Guild | NOTIFY-CONN-TEAMS-15-602 | Teams health/test-send support. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Teams/TASKS.md | DONE (2025-10-20) | Notify Connectors Guild | NOTIFY-CONN-TEAMS-15-604 | Teams health endpoint metadata alignment. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Slack/TASKS.md | DONE (2025-10-20) | Notify Connectors Guild | NOTIFY-CONN-SLACK-15-503 | Package Slack connector as restart-time plug-in (manifest + host registration). | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Teams/TASKS.md | DONE (2025-10-20) | Notify Connectors Guild | NOTIFY-CONN-TEAMS-15-603 | Package Teams connector as restart-time plug-in (manifest + host registration). | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Email/TASKS.md | DONE (2025-10-20) | Notify Connectors Guild | NOTIFY-CONN-EMAIL-15-703 | Package Email connector as restart-time plug-in (manifest + host registration). | +| Sprint 15 | Notify Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-20) | Scanner WebService Guild | SCANNER-EVENTS-15-201 | Emit `scanner.report.ready` + `scanner.scan.completed` events. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Webhook/TASKS.md | DONE (2025-10-20) | Notify Connectors Guild | NOTIFY-CONN-WEBHOOK-15-803 | Package Webhook connector as restart-time plug-in (manifest + host registration). | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Models/TASKS.md | DONE (2025-10-20) | Scheduler Models Guild | SCHED-MODELS-16-103 | Versioning/migration helpers for schedules/runs. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Queue/TASKS.md | DONE (2025-10-20) | Scheduler Queue Guild | SCHED-QUEUE-16-401 | Queue abstraction + Redis Streams adapter. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Queue/TASKS.md | DONE (2025-10-20) | Scheduler Queue Guild | SCHED-QUEUE-16-402 | NATS JetStream adapter with health probes. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.ImpactIndex/TASKS.md | DONE (2025-10-20) | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-300 | **STUB** ImpactIndex ingest/query using fixtures (to be removed by SP16 completion). | diff --git a/SPRINTS_VEXER.md b/SPRINTS_VEXER.md deleted file mode 100644 index cf099e9f..00000000 --- a/SPRINTS_VEXER.md +++ /dev/null @@ -1,2 +0,0 @@ -| Sprint | Theme | Tasks File Path | Status | Type of Specialist | Task ID | Task Description | -| --- | --- | --- | --- | --- | --- | --- | diff --git a/bench/Scanner.Analyzers/README.md b/bench/Scanner.Analyzers/README.md index e16c899a..b1fa030f 100644 --- a/bench/Scanner.Analyzers/README.md +++ b/bench/Scanner.Analyzers/README.md @@ -2,35 +2,41 @@ The bench harness exercises the language analyzers against representative filesystem layouts so that regressions are caught before they ship. -## Layout -- `run-bench.js` – Node.js script that traverses the sample `node_modules/` and `site-packages/` trees, replicating the package discovery work performed by the upcoming analyzers. -- `config.json` – Declarative list of scenarios the harness executes. Each scenario points at a directory in `samples/`. -- `baseline.csv` – Reference numbers captured on the 4 vCPU warm rig described in `docs/12_PERFORMANCE_WORKBOOK.md`. CI publishes fresh CSVs so perf trends stay visible. - -## Running locally - -```bash -cd bench/Scanner.Analyzers -node run-bench.js --out baseline.csv --samples ../.. -``` - -The harness prints a table to stdout and writes the CSV (if `--out` is specified) with the following headers: - -``` -scenario,iterations,sample_count,mean_ms,p95_ms,max_ms -``` - -Use `--iterations` to override the default (5 passes per scenario) and `--threshold-ms` to customize the failure budget. Budgets default to 5 000 ms, aligned with the SBOM compose objective. - -## Adding scenarios -1. Drop the fixture tree under `samples//...`. -2. Append a new scenario entry to `config.json` describing: - - `id` – snake_case scenario name (also used in CSV). - - `label` – human-friendly description shown in logs. - - `root` – path to the directory that will be scanned. - - `matcher` – glob describing files that will be parsed (POSIX `**` patterns). - - `parser` – `node` or `python` to choose the metadata reader. -3. Re-run `node run-bench.js --out baseline.csv`. -4. Commit both the fixture and updated baseline. - -The harness is intentionally dependency-free to remain runnable inside minimal CI runners. +## Layout +- `StellaOps.Bench.ScannerAnalyzers/` – .NET 10 console harness that executes the real language analyzers (and fallback metadata walks for ecosystems that are still underway). +- `config.json` – Declarative list of scenarios the harness executes. Each scenario points at a directory in `samples/`. +- `baseline.csv` – Reference numbers captured on the 4 vCPU warm rig described in `docs/12_PERFORMANCE_WORKBOOK.md`. CI publishes fresh CSVs so perf trends stay visible. + +## Current scenarios +- `node_monorepo_walk` → runs the Node analyzer across `samples/runtime/npm-monorepo`. +- `java_demo_archive` → runs the Java analyzer against `samples/runtime/java-demo/libs/demo.jar`. +- `python_site_packages_walk` → temporary metadata walk over `samples/runtime/python-venv` until the Python analyzer lands. + +## Running locally + +```bash +dotnet run \ + --project bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/StellaOps.Bench.ScannerAnalyzers.csproj \ + -- \ + --repo-root . \ + --out bench/Scanner.Analyzers/baseline.csv +``` + +The harness prints a table to stdout and writes the CSV (if `--out` is specified) with the following headers: + +``` +scenario,iterations,sample_count,mean_ms,p95_ms,max_ms +``` + +Use `--iterations` to override the default (5 passes per scenario) and `--threshold-ms` to customize the failure budget. Budgets default to 5 000 ms (or per-scenario overrides in `config.json`), aligned with the SBOM compose objective. + +## Adding scenarios +1. Drop the fixture tree under `samples//...`. +2. Append a new scenario entry to `config.json` describing: + - `id` – snake_case scenario name (also used in CSV). + - `label` – human-friendly description shown in logs. + - `root` – path to the directory that will be scanned. + - For analyzer-backed scenarios, set `analyzers` to the list of language analyzer ids (for example, `["node"]`). + - For temporary metadata walks (used until the analyzer ships), provide `parser` (`node` or `python`) and the `matcher` glob describing files to parse. +3. Re-run the harness (`dotnet run … --out baseline.csv`). +4. Commit both the fixture and updated baseline. diff --git a/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/BenchmarkConfig.cs b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/BenchmarkConfig.cs new file mode 100644 index 00000000..48184508 --- /dev/null +++ b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/BenchmarkConfig.cs @@ -0,0 +1,104 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace StellaOps.Bench.ScannerAnalyzers; + +internal sealed record BenchmarkConfig +{ + [JsonPropertyName("iterations")] + public int? Iterations { get; init; } + + [JsonPropertyName("thresholdMs")] + public double? ThresholdMs { get; init; } + + [JsonPropertyName("scenarios")] + public List Scenarios { get; init; } = new(); + + public static async Task LoadAsync(string path) + { + if (string.IsNullOrWhiteSpace(path)) + { + throw new ArgumentException("Config path is required.", nameof(path)); + } + + await using var stream = File.OpenRead(path); + var config = await JsonSerializer.DeserializeAsync(stream, SerializerOptions).ConfigureAwait(false); + if (config is null) + { + throw new InvalidOperationException($"Failed to parse benchmark config '{path}'."); + } + + if (config.Scenarios.Count == 0) + { + throw new InvalidOperationException("config.scenarios must declare at least one scenario."); + } + + foreach (var scenario in config.Scenarios) + { + scenario.Validate(); + } + + return config; + } + + private static JsonSerializerOptions SerializerOptions => new() + { + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + AllowTrailingCommas = true, + }; +} + +internal sealed record BenchmarkScenarioConfig +{ + [JsonPropertyName("id")] + public string? Id { get; init; } + + [JsonPropertyName("label")] + public string? Label { get; init; } + + [JsonPropertyName("root")] + public string? Root { get; init; } + + [JsonPropertyName("analyzers")] + public List? Analyzers { get; init; } + + [JsonPropertyName("matcher")] + public string? Matcher { get; init; } + + [JsonPropertyName("parser")] + public string? Parser { get; init; } + + [JsonPropertyName("thresholdMs")] + public double? ThresholdMs { get; init; } + + public bool HasAnalyzers => Analyzers is { Count: > 0 }; + + public void Validate() + { + if (string.IsNullOrWhiteSpace(Id)) + { + throw new InvalidOperationException("scenario.id is required."); + } + + if (string.IsNullOrWhiteSpace(Root)) + { + throw new InvalidOperationException($"Scenario '{Id}' must specify a root path."); + } + + if (HasAnalyzers) + { + return; + } + + if (string.IsNullOrWhiteSpace(Parser)) + { + throw new InvalidOperationException($"Scenario '{Id}' must specify parser or analyzers."); + } + + if (string.IsNullOrWhiteSpace(Matcher)) + { + throw new InvalidOperationException($"Scenario '{Id}' must specify matcher when parser is used."); + } + } +} diff --git a/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Program.cs b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Program.cs new file mode 100644 index 00000000..353c99f2 --- /dev/null +++ b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Program.cs @@ -0,0 +1,302 @@ +using System.Globalization; +using StellaOps.Bench.ScannerAnalyzers.Scenarios; + +namespace StellaOps.Bench.ScannerAnalyzers; + +internal static class Program +{ + public static async Task Main(string[] args) + { + try + { + var options = ProgramOptions.Parse(args); + var config = await BenchmarkConfig.LoadAsync(options.ConfigPath).ConfigureAwait(false); + + var iterations = options.Iterations ?? config.Iterations ?? 5; + var thresholdMs = options.ThresholdMs ?? config.ThresholdMs ?? 5000; + var repoRoot = ResolveRepoRoot(options.RepoRoot, options.ConfigPath); + + var results = new List(); + var failures = new List(); + + foreach (var scenario in config.Scenarios) + { + var runner = ScenarioRunnerFactory.Create(scenario); + var scenarioRoot = ResolveScenarioRoot(repoRoot, scenario.Root!); + + var execution = await runner.ExecuteAsync(scenarioRoot, iterations, CancellationToken.None).ConfigureAwait(false); + var stats = ScenarioStatistics.FromDurations(execution.Durations); + var scenarioThreshold = scenario.ThresholdMs ?? thresholdMs; + + results.Add(new ScenarioResult( + scenario.Id!, + scenario.Label ?? scenario.Id!, + execution.SampleCount, + stats.MeanMs, + stats.P95Ms, + stats.MaxMs, + iterations)); + + if (stats.MaxMs > scenarioThreshold) + { + failures.Add($"{scenario.Id} exceeded threshold: {stats.MaxMs:F2} ms > {scenarioThreshold:F2} ms"); + } + } + + TablePrinter.Print(results); + + if (!string.IsNullOrWhiteSpace(options.OutPath)) + { + CsvWriter.Write(options.OutPath!, results); + } + + if (failures.Count > 0) + { + Console.Error.WriteLine(); + Console.Error.WriteLine("Performance threshold exceeded:"); + foreach (var failure in failures) + { + Console.Error.WriteLine($" - {failure}"); + } + + return 1; + } + + return 0; + } + catch (Exception ex) + { + Console.Error.WriteLine(ex.Message); + return 1; + } + } + + private static string ResolveRepoRoot(string? overridePath, string configPath) + { + if (!string.IsNullOrWhiteSpace(overridePath)) + { + return Path.GetFullPath(overridePath); + } + + var configDirectory = Path.GetDirectoryName(configPath); + if (string.IsNullOrWhiteSpace(configDirectory)) + { + return Directory.GetCurrentDirectory(); + } + + return Path.GetFullPath(Path.Combine(configDirectory, "..", "..")); + } + + private static string ResolveScenarioRoot(string repoRoot, string relativeRoot) + { + if (string.IsNullOrWhiteSpace(relativeRoot)) + { + throw new InvalidOperationException("Scenario root is required."); + } + + var combined = Path.GetFullPath(Path.Combine(repoRoot, relativeRoot)); + if (!PathUtilities.IsWithinRoot(repoRoot, combined)) + { + throw new InvalidOperationException($"Scenario root '{relativeRoot}' escapes repository root '{repoRoot}'."); + } + + if (!Directory.Exists(combined)) + { + throw new DirectoryNotFoundException($"Scenario root '{combined}' does not exist."); + } + + return combined; + } + + private sealed record ProgramOptions(string ConfigPath, int? Iterations, double? ThresholdMs, string? OutPath, string? RepoRoot) + { + public static ProgramOptions Parse(string[] args) + { + var configPath = DefaultConfigPath(); + int? iterations = null; + double? thresholdMs = null; + string? outPath = null; + string? repoRoot = null; + + for (var index = 0; index < args.Length; index++) + { + var current = args[index]; + switch (current) + { + case "--config": + EnsureNext(args, index); + configPath = Path.GetFullPath(args[++index]); + break; + case "--iterations": + EnsureNext(args, index); + iterations = int.Parse(args[++index], CultureInfo.InvariantCulture); + break; + case "--threshold-ms": + EnsureNext(args, index); + thresholdMs = double.Parse(args[++index], CultureInfo.InvariantCulture); + break; + case "--out": + EnsureNext(args, index); + outPath = args[++index]; + break; + case "--repo-root": + case "--samples": + EnsureNext(args, index); + repoRoot = args[++index]; + break; + default: + throw new ArgumentException($"Unknown argument: {current}", nameof(args)); + } + } + + return new ProgramOptions(configPath, iterations, thresholdMs, outPath, repoRoot); + } + + private static string DefaultConfigPath() + { + var binaryDir = AppContext.BaseDirectory; + var projectRoot = Path.GetFullPath(Path.Combine(binaryDir, "..", "..", "..")); + var configDirectory = Path.GetFullPath(Path.Combine(projectRoot, "..")); + return Path.Combine(configDirectory, "config.json"); + } + + private static void EnsureNext(string[] args, int index) + { + if (index + 1 >= args.Length) + { + throw new ArgumentException("Missing value for argument.", nameof(args)); + } + } + } + + private sealed record ScenarioResult( + string Id, + string Label, + int SampleCount, + double MeanMs, + double P95Ms, + double MaxMs, + int Iterations); + + private sealed record ScenarioStatistics(double MeanMs, double P95Ms, double MaxMs) + { + public static ScenarioStatistics FromDurations(IReadOnlyList durations) + { + if (durations.Count == 0) + { + return new ScenarioStatistics(0, 0, 0); + } + + var sorted = durations.ToArray(); + Array.Sort(sorted); + + var total = 0d; + foreach (var value in durations) + { + total += value; + } + + var mean = total / durations.Count; + var p95 = Percentile(sorted, 95); + var max = sorted[^1]; + + return new ScenarioStatistics(mean, p95, max); + } + + private static double Percentile(IReadOnlyList sorted, double percentile) + { + if (sorted.Count == 0) + { + return 0; + } + + var rank = (percentile / 100d) * (sorted.Count - 1); + var lower = (int)Math.Floor(rank); + var upper = (int)Math.Ceiling(rank); + var weight = rank - lower; + + if (upper >= sorted.Count) + { + return sorted[lower]; + } + + return sorted[lower] + weight * (sorted[upper] - sorted[lower]); + } + } + + private static class TablePrinter + { + public static void Print(IEnumerable results) + { + Console.WriteLine("Scenario | Count | Mean(ms) | P95(ms) | Max(ms)"); + Console.WriteLine("---------------------------- | ----- | --------- | --------- | ----------"); + foreach (var row in results) + { + Console.WriteLine(FormatRow(row)); + } + } + + private static string FormatRow(ScenarioResult row) + { + var idColumn = row.Id.Length <= 28 + ? row.Id.PadRight(28) + : row.Id[..28]; + + return string.Join(" | ", new[] + { + idColumn, + row.SampleCount.ToString(CultureInfo.InvariantCulture).PadLeft(5), + row.MeanMs.ToString("F2", CultureInfo.InvariantCulture).PadLeft(9), + row.P95Ms.ToString("F2", CultureInfo.InvariantCulture).PadLeft(9), + row.MaxMs.ToString("F2", CultureInfo.InvariantCulture).PadLeft(10), + }); + } + } + + private static class CsvWriter + { + public static void Write(string path, IEnumerable results) + { + var resolvedPath = Path.GetFullPath(path); + var directory = Path.GetDirectoryName(resolvedPath); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + using var stream = new FileStream(resolvedPath, FileMode.Create, FileAccess.Write, FileShare.None); + using var writer = new StreamWriter(stream); + writer.WriteLine("scenario,iterations,sample_count,mean_ms,p95_ms,max_ms"); + + foreach (var row in results) + { + writer.Write(row.Id); + writer.Write(','); + writer.Write(row.Iterations.ToString(CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(row.SampleCount.ToString(CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(row.MeanMs.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(row.P95Ms.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(row.MaxMs.ToString("F4", CultureInfo.InvariantCulture)); + writer.WriteLine(); + } + } + } + + internal static class PathUtilities + { + public static bool IsWithinRoot(string root, string candidate) + { + var relative = Path.GetRelativePath(root, candidate); + if (string.IsNullOrEmpty(relative) || relative == ".") + { + return true; + } + + return !relative.StartsWith("..", StringComparison.Ordinal) && !Path.IsPathRooted(relative); + } + } +} diff --git a/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/ScenarioRunners.cs b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/ScenarioRunners.cs new file mode 100644 index 00000000..854c4668 --- /dev/null +++ b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/ScenarioRunners.cs @@ -0,0 +1,279 @@ +using System.Diagnostics; +using System.Text; +using System.Linq; +using System.Text.Json; +using System.Text.RegularExpressions; +using StellaOps.Scanner.Analyzers.Lang; +using StellaOps.Scanner.Analyzers.Lang.Java; +using StellaOps.Scanner.Analyzers.Lang.Node; + +namespace StellaOps.Bench.ScannerAnalyzers.Scenarios; + +internal interface IScenarioRunner +{ + Task ExecuteAsync(string rootPath, int iterations, CancellationToken cancellationToken); +} + +internal sealed record ScenarioExecutionResult(double[] Durations, int SampleCount); + +internal static class ScenarioRunnerFactory +{ + public static IScenarioRunner Create(BenchmarkScenarioConfig scenario) + { + if (scenario.HasAnalyzers) + { + return new LanguageAnalyzerScenarioRunner(scenario.Analyzers!); + } + + if (string.IsNullOrWhiteSpace(scenario.Parser) || string.IsNullOrWhiteSpace(scenario.Matcher)) + { + throw new InvalidOperationException($"Scenario '{scenario.Id}' missing parser or matcher configuration."); + } + + return new MetadataWalkScenarioRunner(scenario.Parser, scenario.Matcher); + } +} + +internal sealed class LanguageAnalyzerScenarioRunner : IScenarioRunner +{ + private readonly IReadOnlyList> _analyzerFactories; + + public LanguageAnalyzerScenarioRunner(IEnumerable analyzerIds) + { + if (analyzerIds is null) + { + throw new ArgumentNullException(nameof(analyzerIds)); + } + + _analyzerFactories = analyzerIds + .Where(static id => !string.IsNullOrWhiteSpace(id)) + .Select(CreateFactory) + .ToArray(); + + if (_analyzerFactories.Count == 0) + { + throw new InvalidOperationException("At least one analyzer id must be provided."); + } + } + + public async Task ExecuteAsync(string rootPath, int iterations, CancellationToken cancellationToken) + { + if (iterations <= 0) + { + throw new ArgumentOutOfRangeException(nameof(iterations), iterations, "Iterations must be positive."); + } + + var analyzers = _analyzerFactories.Select(factory => factory()).ToArray(); + var engine = new LanguageAnalyzerEngine(analyzers); + var durations = new double[iterations]; + var componentCount = -1; + + for (var i = 0; i < iterations; i++) + { + cancellationToken.ThrowIfCancellationRequested(); + + var context = new LanguageAnalyzerContext(rootPath, TimeProvider.System); + var stopwatch = Stopwatch.StartNew(); + var result = await engine.AnalyzeAsync(context, cancellationToken).ConfigureAwait(false); + stopwatch.Stop(); + + durations[i] = stopwatch.Elapsed.TotalMilliseconds; + + var currentCount = result.Components.Count; + if (componentCount < 0) + { + componentCount = currentCount; + } + else if (componentCount != currentCount) + { + throw new InvalidOperationException($"Analyzer output count changed between iterations ({componentCount} vs {currentCount})."); + } + } + + if (componentCount < 0) + { + componentCount = 0; + } + + return new ScenarioExecutionResult(durations, componentCount); + } + + private static Func CreateFactory(string analyzerId) + { + var id = analyzerId.Trim().ToLowerInvariant(); + return id switch + { + "java" => static () => new JavaLanguageAnalyzer(), + "node" => static () => new NodeLanguageAnalyzer(), + _ => throw new InvalidOperationException($"Unsupported analyzer '{analyzerId}'."), + }; + } +} + +internal sealed class MetadataWalkScenarioRunner : IScenarioRunner +{ + private readonly Regex _matcher; + private readonly string _parserKind; + + public MetadataWalkScenarioRunner(string parserKind, string globPattern) + { + _parserKind = parserKind?.Trim().ToLowerInvariant() ?? throw new ArgumentNullException(nameof(parserKind)); + _matcher = GlobToRegex(globPattern ?? throw new ArgumentNullException(nameof(globPattern))); + } + + public async Task ExecuteAsync(string rootPath, int iterations, CancellationToken cancellationToken) + { + if (iterations <= 0) + { + throw new ArgumentOutOfRangeException(nameof(iterations), iterations, "Iterations must be positive."); + } + + var durations = new double[iterations]; + var sampleCount = -1; + + for (var i = 0; i < iterations; i++) + { + cancellationToken.ThrowIfCancellationRequested(); + + var stopwatch = Stopwatch.StartNew(); + var files = EnumerateMatchingFiles(rootPath); + if (files.Count == 0) + { + throw new InvalidOperationException($"Parser '{_parserKind}' matched zero files under '{rootPath}'."); + } + + foreach (var file in files) + { + cancellationToken.ThrowIfCancellationRequested(); + await ParseAsync(file).ConfigureAwait(false); + } + + stopwatch.Stop(); + durations[i] = stopwatch.Elapsed.TotalMilliseconds; + + if (sampleCount < 0) + { + sampleCount = files.Count; + } + else if (sampleCount != files.Count) + { + throw new InvalidOperationException($"File count changed between iterations ({sampleCount} vs {files.Count})."); + } + } + + if (sampleCount < 0) + { + sampleCount = 0; + } + + return new ScenarioExecutionResult(durations, sampleCount); + } + + private async ValueTask ParseAsync(string filePath) + { + switch (_parserKind) + { + case "node": + { + using var stream = File.OpenRead(filePath); + using var document = await JsonDocument.ParseAsync(stream).ConfigureAwait(false); + + if (!document.RootElement.TryGetProperty("name", out var name) || name.ValueKind != JsonValueKind.String) + { + throw new InvalidOperationException($"package.json '{filePath}' missing name."); + } + + if (!document.RootElement.TryGetProperty("version", out var version) || version.ValueKind != JsonValueKind.String) + { + throw new InvalidOperationException($"package.json '{filePath}' missing version."); + } + } + break; + case "python": + { + var (name, version) = await ParsePythonMetadataAsync(filePath).ConfigureAwait(false); + if (string.IsNullOrEmpty(name) || string.IsNullOrEmpty(version)) + { + throw new InvalidOperationException($"METADATA '{filePath}' missing Name/Version."); + } + } + break; + default: + throw new InvalidOperationException($"Unknown parser '{_parserKind}'."); + } + } + + private static async Task<(string? Name, string? Version)> ParsePythonMetadataAsync(string filePath) + { + using var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete); + using var reader = new StreamReader(stream); + + string? name = null; + string? version = null; + + while (await reader.ReadLineAsync().ConfigureAwait(false) is { } line) + { + if (line.StartsWith("Name:", StringComparison.OrdinalIgnoreCase)) + { + name ??= line[5..].Trim(); + } + else if (line.StartsWith("Version:", StringComparison.OrdinalIgnoreCase)) + { + version ??= line[8..].Trim(); + } + + if (!string.IsNullOrEmpty(name) && !string.IsNullOrEmpty(version)) + { + break; + } + } + + return (name, version); + } + + private IReadOnlyList EnumerateMatchingFiles(string rootPath) + { + var files = new List(); + var stack = new Stack(); + stack.Push(rootPath); + + while (stack.Count > 0) + { + var current = stack.Pop(); + foreach (var directory in Directory.EnumerateDirectories(current)) + { + stack.Push(directory); + } + + foreach (var file in Directory.EnumerateFiles(current)) + { + var relative = Path.GetRelativePath(rootPath, file).Replace('\\', '/'); + if (_matcher.IsMatch(relative)) + { + files.Add(file); + } + } + } + + return files; + } + + private static Regex GlobToRegex(string pattern) + { + if (string.IsNullOrWhiteSpace(pattern)) + { + throw new ArgumentException("Glob pattern is required.", nameof(pattern)); + } + + var normalized = pattern.Replace("\\", "/"); + normalized = normalized.Replace("**", "\u0001"); + normalized = normalized.Replace("*", "\u0002"); + + var escaped = Regex.Escape(normalized); + escaped = escaped.Replace("\u0001/", "(?:.*/)?", StringComparison.Ordinal); + escaped = escaped.Replace("\u0001", ".*", StringComparison.Ordinal); + escaped = escaped.Replace("\u0002", "[^/]*", StringComparison.Ordinal); + + return new Regex("^" + escaped + "$", RegexOptions.Compiled | RegexOptions.CultureInvariant); + } +} diff --git a/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/StellaOps.Bench.ScannerAnalyzers.csproj b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/StellaOps.Bench.ScannerAnalyzers.csproj new file mode 100644 index 00000000..e7467b36 --- /dev/null +++ b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/StellaOps.Bench.ScannerAnalyzers.csproj @@ -0,0 +1,16 @@ + + + Exe + net10.0 + enable + enable + preview + true + + + + + + + + diff --git a/bench/Scanner.Analyzers/baseline.csv b/bench/Scanner.Analyzers/baseline.csv index eb0770a6..10fbdcbd 100644 --- a/bench/Scanner.Analyzers/baseline.csv +++ b/bench/Scanner.Analyzers/baseline.csv @@ -1,3 +1,4 @@ scenario,iterations,sample_count,mean_ms,p95_ms,max_ms -node_monorepo_walk,5,4,233.9428,319.8564,344.4611 -python_site_packages_walk,5,3,72.9166,74.8970,74.9884 +node_monorepo_walk,5,4,4.2314,15.3277,18.9984 +java_demo_archive,5,1,4.5572,17.3489,21.5472 +python_site_packages_walk,5,3,2.0049,6.4230,7.8832 diff --git a/bench/Scanner.Analyzers/config.json b/bench/Scanner.Analyzers/config.json index ec86eecf..0c9383bd 100644 --- a/bench/Scanner.Analyzers/config.json +++ b/bench/Scanner.Analyzers/config.json @@ -2,17 +2,26 @@ "thresholdMs": 5000, "iterations": 5, "scenarios": [ - { - "id": "node_monorepo_walk", - "label": "Node.js monorepo package.json harvest", - "root": "samples/runtime/npm-monorepo/node_modules", - "matcher": "**/package.json", - "parser": "node" - }, - { - "id": "python_site_packages_walk", - "label": "Python site-packages dist-info crawl", - "root": "samples/runtime/python-venv/lib/python3.11/site-packages", + { + "id": "node_monorepo_walk", + "label": "Node.js analyzer on monorepo fixture", + "root": "samples/runtime/npm-monorepo", + "analyzers": [ + "node" + ] + }, + { + "id": "java_demo_archive", + "label": "Java analyzer on demo jar", + "root": "samples/runtime/java-demo", + "analyzers": [ + "java" + ] + }, + { + "id": "python_site_packages_walk", + "label": "Python site-packages dist-info crawl", + "root": "samples/runtime/python-venv/lib/python3.11/site-packages", "matcher": "**/*.dist-info/METADATA", "parser": "python" } diff --git a/bench/Scanner.Analyzers/run-bench.js b/bench/Scanner.Analyzers/run-bench.js deleted file mode 100644 index 7ed00823..00000000 --- a/bench/Scanner.Analyzers/run-bench.js +++ /dev/null @@ -1,249 +0,0 @@ -#!/usr/bin/env node -'use strict'; - -const fs = require('fs'); -const path = require('path'); -const { performance } = require('perf_hooks'); - -function globToRegExp(pattern) { - let working = pattern - .replace(/\*\*/g, ':::DOUBLE_WILDCARD:::') - .replace(/\*/g, ':::SINGLE_WILDCARD:::'); - working = working.replace(/([.+^${}()|[\]\\])/g, '\\$1'); - working = working - .replace(/:::DOUBLE_WILDCARD:::\//g, '(?:.*/)?') - .replace(/:::DOUBLE_WILDCARD:::/g, '.*') - .replace(/:::SINGLE_WILDCARD:::/g, '[^/]*'); - return new RegExp(`^${working}$`); -} - -function walkFiles(root, matcher) { - const out = []; - const stack = [root]; - while (stack.length) { - const current = stack.pop(); - const stat = fs.statSync(current, { throwIfNoEntry: true }); - if (stat.isDirectory()) { - const entries = fs.readdirSync(current); - for (const entry of entries) { - stack.push(path.join(current, entry)); - } - } else if (stat.isFile()) { - const relativePath = path.relative(root, current).replace(/\\/g, '/'); - if (matcher.test(relativePath)) { - out.push(current); - } - } - } - return out; -} - -function parseArgs(argv) { - const args = { - config: path.join(__dirname, 'config.json'), - iterations: undefined, - thresholdMs: undefined, - out: undefined, - repoRoot: path.join(__dirname, '..', '..'), - }; - - for (let i = 2; i < argv.length; i++) { - const current = argv[i]; - switch (current) { - case '--config': - args.config = argv[++i]; - break; - case '--iterations': - args.iterations = Number(argv[++i]); - break; - case '--threshold-ms': - args.thresholdMs = Number(argv[++i]); - break; - case '--out': - args.out = argv[++i]; - break; - case '--repo-root': - case '--samples': - args.repoRoot = argv[++i]; - break; - default: - throw new Error(`Unknown argument: ${current}`); - } - } - - return args; -} - -function loadConfig(configPath) { - const json = fs.readFileSync(configPath, 'utf8'); - const cfg = JSON.parse(json); - if (!Array.isArray(cfg.scenarios) || cfg.scenarios.length === 0) { - throw new Error('config.scenarios must be a non-empty array'); - } - return cfg; -} - -function ensureWithinRepo(repoRoot, target) { - const relative = path.relative(repoRoot, target); - if (relative === '' || relative === '.') { - return true; - } - return !relative.startsWith('..') && !path.isAbsolute(relative); -} - -function parseNodePackage(contents) { - const parsed = JSON.parse(contents); - if (!parsed.name || !parsed.version) { - throw new Error('package.json missing name/version'); - } - return { name: parsed.name, version: parsed.version }; -} - -function parsePythonMetadata(contents) { - let name; - let version; - for (const line of contents.split(/\r?\n/)) { - if (!name && line.startsWith('Name:')) { - name = line.slice(5).trim(); - } else if (!version && line.startsWith('Version:')) { - version = line.slice(8).trim(); - } - if (name && version) { - break; - } - } - if (!name || !version) { - throw new Error('METADATA missing Name/Version headers'); - } - return { name, version }; -} - -function formatRow(row) { - const cols = [ - row.id.padEnd(28), - row.sampleCount.toString().padStart(5), - row.meanMs.toFixed(2).padStart(9), - row.p95Ms.toFixed(2).padStart(9), - row.maxMs.toFixed(2).padStart(9), - ]; - return cols.join(' | '); -} - -function percentile(sortedDurations, percentile) { - if (sortedDurations.length === 0) { - return 0; - } - const rank = (percentile / 100) * (sortedDurations.length - 1); - const lower = Math.floor(rank); - const upper = Math.ceil(rank); - const weight = rank - lower; - if (upper >= sortedDurations.length) { - return sortedDurations[lower]; - } - return sortedDurations[lower] + weight * (sortedDurations[upper] - sortedDurations[lower]); -} - -function main() { - const args = parseArgs(process.argv); - const cfg = loadConfig(args.config); - const iterations = args.iterations ?? cfg.iterations ?? 5; - const thresholdMs = args.thresholdMs ?? cfg.thresholdMs ?? 5000; - - const results = []; - const failures = []; - - for (const scenario of cfg.scenarios) { - const scenarioRoot = path.resolve(args.repoRoot, scenario.root); - if (!ensureWithinRepo(args.repoRoot, scenarioRoot)) { - throw new Error(`Scenario root ${scenario.root} escapes repo root ${args.repoRoot}`); - } - if (!fs.existsSync(scenarioRoot)) { - throw new Error(`Scenario root ${scenarioRoot} does not exist`); - } - - const matcher = globToRegExp(scenario.matcher.replace(/\\/g, '/')); - const durations = []; - let sampleCount = 0; - - for (let attempt = 0; attempt < iterations; attempt++) { - const start = performance.now(); - const files = walkFiles(scenarioRoot, matcher); - if (files.length === 0) { - throw new Error(`Scenario ${scenario.id} matched no files`); - } - - for (const filePath of files) { - const contents = fs.readFileSync(filePath, 'utf8'); - if (scenario.parser === 'node') { - parseNodePackage(contents); - } else if (scenario.parser === 'python') { - parsePythonMetadata(contents); - } else { - throw new Error(`Unknown parser ${scenario.parser} for scenario ${scenario.id}`); - } - } - const end = performance.now(); - durations.push(end - start); - sampleCount = files.length; - } - - durations.sort((a, b) => a - b); - const mean = durations.reduce((acc, value) => acc + value, 0) / durations.length; - const p95 = percentile(durations, 95); - const max = durations[durations.length - 1]; - - if (max > thresholdMs) { - failures.push(`${scenario.id} exceeded threshold: ${(max).toFixed(2)} ms > ${thresholdMs} ms`); - } - - results.push({ - id: scenario.id, - label: scenario.label, - sampleCount, - meanMs: mean, - p95Ms: p95, - maxMs: max, - iterations, - }); - } - - console.log('Scenario | Count | Mean(ms) | P95(ms) | Max(ms)'); - console.log('---------------------------- | ----- | --------- | --------- | ----------'); - for (const row of results) { - console.log(formatRow(row)); - } - - if (args.out) { - const header = 'scenario,iterations,sample_count,mean_ms,p95_ms,max_ms\n'; - const csvRows = results - .map((row) => - [ - row.id, - row.iterations, - row.sampleCount, - row.meanMs.toFixed(4), - row.p95Ms.toFixed(4), - row.maxMs.toFixed(4), - ].join(',') - ) - .join('\n'); - fs.writeFileSync(args.out, header + csvRows + '\n', 'utf8'); - } - - if (failures.length > 0) { - console.error('\nPerformance threshold exceeded:'); - for (const failure of failures) { - console.error(` - ${failure}`); - } - process.exitCode = 1; - } -} - -if (require.main === module) { - try { - main(); - } catch (err) { - console.error(err instanceof Error ? err.message : err); - process.exit(1); - } -} diff --git a/bench/TASKS.md b/bench/TASKS.md index de8ef5b7..f77bbfc3 100644 --- a/bench/TASKS.md +++ b/bench/TASKS.md @@ -3,6 +3,6 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| | BENCH-SCANNER-10-001 | DONE | Bench Guild, Scanner Team | SCANNER-ANALYZERS-LANG-10-303 | Analyzer microbench harness (node_modules, site-packages) + baseline CSV. | Harness committed under `bench/Scanner.Analyzers`; baseline CSV recorded; CI job publishes results. | -| BENCH-SCANNER-10-002 | TODO | Bench Guild, Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-301..309 | Wire real language analyzers into bench harness & refresh baselines post-implementation. | Harness executes analyzer assemblies end-to-end; updated baseline committed; CI trend doc linked. | +| BENCH-SCANNER-10-002 | DONE (2025-10-21) | Bench Guild, Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-301..309 | Wire real language analyzers into bench harness & refresh baselines post-implementation. | Harness executes analyzer assemblies end-to-end; updated baseline committed; CI trend doc linked. | | BENCH-IMPACT-16-001 | TODO | Bench Guild, Scheduler Team | SCHED-IMPACT-16-301 | ImpactIndex throughput bench (resolve 10k productKeys) + RAM profile. | Benchmark script ready; baseline metrics recorded; alert thresholds defined. | | BENCH-NOTIFY-15-001 | TODO | Bench Guild, Notify Team | NOTIFY-ENGINE-15-301 | Notify dispatch throughput bench (vary rule density) with results CSV. | Bench executed; results stored; regression alert configured. | diff --git a/deploy/README.md b/deploy/README.md index 0353366c..0125c880 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -17,4 +17,21 @@ This directory contains deterministic deployment bundles for the core Stella Ops 3. Run `deploy/tools/validate-profiles.sh` (requires Docker CLI and Helm) to ensure the bundles lint and template cleanly. 4. Commit the change alongside any documentation updates (e.g. install guide cross-links). -Maintaining the digest linkage keeps offline/air-gapped installs reproducible and avoids tag drift between environments. +Maintaining the digest linkage keeps offline/air-gapped installs reproducible and avoids tag drift between environments. + +## CI smoke checks + +The `.gitea/workflows/build-test-deploy.yml` pipeline includes a `notify-smoke` stage that validates scanner event propagation after staging deployments. Configure the following repository secrets (or environment-level secrets) so the job can connect to Redis and the Notify API: + +- `NOTIFY_SMOKE_REDIS_DSN` – Redis connection string (`redis://user:pass@host:port/db`). +- `NOTIFY_SMOKE_NOTIFY_BASEURL` – Base URL for the staging Notify WebService (e.g. `https://notify.stage.stella-ops.internal`). +- `NOTIFY_SMOKE_NOTIFY_TOKEN` – OAuth bearer token (service account) with permission to read deliveries. +- `NOTIFY_SMOKE_NOTIFY_TENANT` – Tenant identifier used for the smoke validation requests. +- *(Optional)* `NOTIFY_SMOKE_NOTIFY_TENANT_HEADER` – Override for the tenant header name (defaults to `X-StellaOps-Tenant`). + +Define the following repository variables (or secrets) to drive the assertions performed by the smoke check: + +- `NOTIFY_SMOKE_EXPECT_KINDS` – Comma-separated event kinds the checker must observe (for example `scanner.report.ready,scanner.scan.completed`). +- `NOTIFY_SMOKE_LOOKBACK_MINUTES` – Time window (in minutes) used when scanning the Redis stream for recent events (for example `30`). + +All of the above values are required—the workflow fails fast with a descriptive error if any are missing or empty. Provide the variables at the organisation or repository scope before enabling the smoke stage. diff --git a/deploy/compose/README.md b/deploy/compose/README.md index f8c2ccc3..73a3ef46 100644 --- a/deploy/compose/README.md +++ b/deploy/compose/README.md @@ -20,12 +20,25 @@ docker compose --env-file dev.env -f docker-compose.dev.yaml config docker compose --env-file dev.env -f docker-compose.dev.yaml up -d ``` -The stage and airgap variants behave the same way—swap the file names accordingly. All profiles expose 443/8443 for the UI and REST APIs, and they share a `stellaops` Docker network scoped to the compose project. - -### Updating to a new release - -1. Import the new manifest into `deploy/releases/` (see `deploy/README.md`). -2. Update image digests in the relevant Compose file(s). -3. Re-run `docker compose config` to confirm the bundle is deterministic. +The stage and airgap variants behave the same way—swap the file names accordingly. All profiles expose 443/8443 for the UI and REST APIs, and they share a `stellaops` Docker network scoped to the compose project. + +### Scanner event stream settings + +Scanner WebService can emit signed `scanner.report.*` events to Redis Streams when `SCANNER__EVENTS__ENABLED=true`. Each profile ships environment placeholders you can override in the `.env` file: + +- `SCANNER_EVENTS_ENABLED` – toggle emission on/off (defaults to `false`). +- `SCANNER_EVENTS_DRIVER` – currently only `redis` is supported. +- `SCANNER_EVENTS_DSN` – Redis endpoint; leave blank to reuse the queue DSN when it uses `redis://`. +- `SCANNER_EVENTS_STREAM` – stream name (`stella.events` by default). +- `SCANNER_EVENTS_PUBLISH_TIMEOUT_SECONDS` – per-publish timeout window (defaults to `5`). +- `SCANNER_EVENTS_MAX_STREAM_LENGTH` – max stream length before Redis trims entries (defaults to `10000`). + +Helm values mirror the same knobs under each service’s `env` map (see `deploy/helm/stellaops/values-*.yaml`). + +### Updating to a new release + +1. Import the new manifest into `deploy/releases/` (see `deploy/README.md`). +2. Update image digests in the relevant Compose file(s). +3. Re-run `docker compose config` to confirm the bundle is deterministic. Keep digests synchronized between Compose, Helm, and the release manifest to preserve reproducibility guarantees. `deploy/tools/validate-profiles.sh` performs a quick audit. diff --git a/deploy/compose/docker-compose.airgap.yaml b/deploy/compose/docker-compose.airgap.yaml index fea80dba..8930ba98 100644 --- a/deploy/compose/docker-compose.airgap.yaml +++ b/deploy/compose/docker-compose.airgap.yaml @@ -136,10 +136,16 @@ services: - nats environment: SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" - SCANNER__STORAGE__S3__ENDPOINT: "http://minio:9000" - SCANNER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}" - SCANNER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}" - SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}" + SCANNER__STORAGE__S3__ENDPOINT: "http://minio:9000" + SCANNER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}" + SCANNER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}" + SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}" + SCANNER__EVENTS__ENABLED: "${SCANNER_EVENTS_ENABLED:-false}" + SCANNER__EVENTS__DRIVER: "${SCANNER_EVENTS_DRIVER:-redis}" + SCANNER__EVENTS__DSN: "${SCANNER_EVENTS_DSN:-}" + SCANNER__EVENTS__STREAM: "${SCANNER_EVENTS_STREAM:-stella.events}" + SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "${SCANNER_EVENTS_PUBLISH_TIMEOUT_SECONDS:-5}" + SCANNER__EVENTS__MAXSTREAMLENGTH: "${SCANNER_EVENTS_MAX_STREAM_LENGTH:-10000}" ports: - "${SCANNER_WEB_PORT:-8444}:8444" networks: diff --git a/deploy/compose/docker-compose.dev.yaml b/deploy/compose/docker-compose.dev.yaml index c9f5d572..6f582886 100644 --- a/deploy/compose/docker-compose.dev.yaml +++ b/deploy/compose/docker-compose.dev.yaml @@ -134,10 +134,16 @@ services: - nats environment: SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" - SCANNER__STORAGE__S3__ENDPOINT: "http://minio:9000" - SCANNER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}" - SCANNER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}" - SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}" + SCANNER__STORAGE__S3__ENDPOINT: "http://minio:9000" + SCANNER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}" + SCANNER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}" + SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}" + SCANNER__EVENTS__ENABLED: "${SCANNER_EVENTS_ENABLED:-false}" + SCANNER__EVENTS__DRIVER: "${SCANNER_EVENTS_DRIVER:-redis}" + SCANNER__EVENTS__DSN: "${SCANNER_EVENTS_DSN:-}" + SCANNER__EVENTS__STREAM: "${SCANNER_EVENTS_STREAM:-stella.events}" + SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "${SCANNER_EVENTS_PUBLISH_TIMEOUT_SECONDS:-5}" + SCANNER__EVENTS__MAXSTREAMLENGTH: "${SCANNER_EVENTS_MAX_STREAM_LENGTH:-10000}" ports: - "${SCANNER_WEB_PORT:-8444}:8444" networks: diff --git a/deploy/compose/docker-compose.stage.yaml b/deploy/compose/docker-compose.stage.yaml index 064d7c8c..a5f68028 100644 --- a/deploy/compose/docker-compose.stage.yaml +++ b/deploy/compose/docker-compose.stage.yaml @@ -134,10 +134,16 @@ services: - nats environment: SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" - SCANNER__STORAGE__S3__ENDPOINT: "http://minio:9000" - SCANNER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}" - SCANNER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}" - SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}" + SCANNER__STORAGE__S3__ENDPOINT: "http://minio:9000" + SCANNER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}" + SCANNER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}" + SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}" + SCANNER__EVENTS__ENABLED: "${SCANNER_EVENTS_ENABLED:-false}" + SCANNER__EVENTS__DRIVER: "${SCANNER_EVENTS_DRIVER:-redis}" + SCANNER__EVENTS__DSN: "${SCANNER_EVENTS_DSN:-}" + SCANNER__EVENTS__STREAM: "${SCANNER_EVENTS_STREAM:-stella.events}" + SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "${SCANNER_EVENTS_PUBLISH_TIMEOUT_SECONDS:-5}" + SCANNER__EVENTS__MAXSTREAMLENGTH: "${SCANNER_EVENTS_MAX_STREAM_LENGTH:-10000}" ports: - "${SCANNER_WEB_PORT:-8444}:8444" networks: diff --git a/deploy/compose/env/airgap.env.example b/deploy/compose/env/airgap.env.example index e8518edf..11190d3f 100644 --- a/deploy/compose/env/airgap.env.example +++ b/deploy/compose/env/airgap.env.example @@ -10,8 +10,15 @@ SIGNER_POE_INTROSPECT_URL=file:///offline/poe/introspect.json SIGNER_PORT=8441 ATTESTOR_PORT=8442 CONCELIER_PORT=8445 -SCANNER_WEB_PORT=8444 -UI_PORT=9443 -NATS_CLIENT_PORT=24222 -SCANNER_QUEUE_BROKER=nats://nats:4222 -AUTHORITY_OFFLINE_CACHE_TOLERANCE=00:45:00 +SCANNER_WEB_PORT=8444 +UI_PORT=9443 +NATS_CLIENT_PORT=24222 +SCANNER_QUEUE_BROKER=nats://nats:4222 +AUTHORITY_OFFLINE_CACHE_TOLERANCE=00:45:00 +SCANNER_EVENTS_ENABLED=false +SCANNER_EVENTS_DRIVER=redis +# Leave SCANNER_EVENTS_DSN empty to inherit the Redis queue DSN when SCANNER_QUEUE_BROKER uses redis://. +SCANNER_EVENTS_DSN= +SCANNER_EVENTS_STREAM=stella.events +SCANNER_EVENTS_PUBLISH_TIMEOUT_SECONDS=5 +SCANNER_EVENTS_MAX_STREAM_LENGTH=10000 diff --git a/deploy/compose/env/dev.env.example b/deploy/compose/env/dev.env.example index 3cc077b1..af218de8 100644 --- a/deploy/compose/env/dev.env.example +++ b/deploy/compose/env/dev.env.example @@ -10,7 +10,14 @@ SIGNER_POE_INTROSPECT_URL=https://licensing.svc.local/introspect SIGNER_PORT=8441 ATTESTOR_PORT=8442 CONCELIER_PORT=8445 -SCANNER_WEB_PORT=8444 -UI_PORT=8443 -NATS_CLIENT_PORT=4222 -SCANNER_QUEUE_BROKER=nats://nats:4222 +SCANNER_WEB_PORT=8444 +UI_PORT=8443 +NATS_CLIENT_PORT=4222 +SCANNER_QUEUE_BROKER=nats://nats:4222 +SCANNER_EVENTS_ENABLED=false +SCANNER_EVENTS_DRIVER=redis +# Leave SCANNER_EVENTS_DSN empty to inherit the Redis queue DSN when SCANNER_QUEUE_BROKER uses redis://. +SCANNER_EVENTS_DSN= +SCANNER_EVENTS_STREAM=stella.events +SCANNER_EVENTS_PUBLISH_TIMEOUT_SECONDS=5 +SCANNER_EVENTS_MAX_STREAM_LENGTH=10000 diff --git a/deploy/compose/env/stage.env.example b/deploy/compose/env/stage.env.example index 4ac1cb38..477e62d1 100644 --- a/deploy/compose/env/stage.env.example +++ b/deploy/compose/env/stage.env.example @@ -10,7 +10,14 @@ SIGNER_POE_INTROSPECT_URL=https://licensing.stage.stella-ops.internal/introspect SIGNER_PORT=8441 ATTESTOR_PORT=8442 CONCELIER_PORT=8445 -SCANNER_WEB_PORT=8444 -UI_PORT=8443 -NATS_CLIENT_PORT=4222 -SCANNER_QUEUE_BROKER=nats://nats:4222 +SCANNER_WEB_PORT=8444 +UI_PORT=8443 +NATS_CLIENT_PORT=4222 +SCANNER_QUEUE_BROKER=nats://nats:4222 +SCANNER_EVENTS_ENABLED=false +SCANNER_EVENTS_DRIVER=redis +# Leave SCANNER_EVENTS_DSN empty to inherit the Redis queue DSN when SCANNER_QUEUE_BROKER uses redis://. +SCANNER_EVENTS_DSN= +SCANNER_EVENTS_STREAM=stella.events +SCANNER_EVENTS_PUBLISH_TIMEOUT_SECONDS=5 +SCANNER_EVENTS_MAX_STREAM_LENGTH=10000 diff --git a/deploy/helm/stellaops/values-airgap.yaml b/deploy/helm/stellaops/values-airgap.yaml index 8e9bf1f2..158c48c5 100644 --- a/deploy/helm/stellaops/values-airgap.yaml +++ b/deploy/helm/stellaops/values-airgap.yaml @@ -97,20 +97,32 @@ services: image: registry.stella-ops.org/stellaops/scanner-web@sha256:3df8ca21878126758203c1a0444e39fd97f77ddacf04a69685cda9f1e5e94718 service: port: 8444 - env: - SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017" - SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" - SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops-airgap" - SCANNER__STORAGE__S3__SECRETACCESSKEY: "airgap-minio-secret" - SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" + env: + SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017" + SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" + SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops-airgap" + SCANNER__STORAGE__S3__SECRETACCESSKEY: "airgap-minio-secret" + SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" + SCANNER__EVENTS__ENABLED: "false" + SCANNER__EVENTS__DRIVER: "redis" + SCANNER__EVENTS__DSN: "" + SCANNER__EVENTS__STREAM: "stella.events" + SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "5" + SCANNER__EVENTS__MAXSTREAMLENGTH: "10000" scanner-worker: image: registry.stella-ops.org/stellaops/scanner-worker@sha256:eea5d6cfe7835950c5ec7a735a651f2f0d727d3e470cf9027a4a402ea89c4fb5 - env: - SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017" - SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" - SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops-airgap" - SCANNER__STORAGE__S3__SECRETACCESSKEY: "airgap-minio-secret" - SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" + env: + SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017" + SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" + SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops-airgap" + SCANNER__STORAGE__S3__SECRETACCESSKEY: "airgap-minio-secret" + SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" + SCANNER__EVENTS__ENABLED: "false" + SCANNER__EVENTS__DRIVER: "redis" + SCANNER__EVENTS__DSN: "" + SCANNER__EVENTS__STREAM: "stella.events" + SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "5" + SCANNER__EVENTS__MAXSTREAMLENGTH: "10000" notify-web: image: registry.stella-ops.org/stellaops/notify-web:2025.09.2 service: diff --git a/deploy/helm/stellaops/values-dev.yaml b/deploy/helm/stellaops/values-dev.yaml index 9cacb1ef..2f13c090 100644 --- a/deploy/helm/stellaops/values-dev.yaml +++ b/deploy/helm/stellaops/values-dev.yaml @@ -96,20 +96,32 @@ services: image: registry.stella-ops.org/stellaops/scanner-web@sha256:e0dfdb087e330585a5953029fb4757f5abdf7610820a085bd61b457dbead9a11 service: port: 8444 - env: - SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017" - SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" - SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops" - SCANNER__STORAGE__S3__SECRETACCESSKEY: "dev-minio-secret" - SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" + env: + SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017" + SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" + SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops" + SCANNER__STORAGE__S3__SECRETACCESSKEY: "dev-minio-secret" + SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" + SCANNER__EVENTS__ENABLED: "false" + SCANNER__EVENTS__DRIVER: "redis" + SCANNER__EVENTS__DSN: "" + SCANNER__EVENTS__STREAM: "stella.events" + SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "5" + SCANNER__EVENTS__MAXSTREAMLENGTH: "10000" scanner-worker: image: registry.stella-ops.org/stellaops/scanner-worker@sha256:92dda42f6f64b2d9522104a5c9ffb61d37b34dd193132b68457a259748008f37 - env: - SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017" - SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" - SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops" - SCANNER__STORAGE__S3__SECRETACCESSKEY: "dev-minio-secret" - SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" + env: + SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017" + SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" + SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops" + SCANNER__STORAGE__S3__SECRETACCESSKEY: "dev-minio-secret" + SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" + SCANNER__EVENTS__ENABLED: "false" + SCANNER__EVENTS__DRIVER: "redis" + SCANNER__EVENTS__DSN: "" + SCANNER__EVENTS__STREAM: "stella.events" + SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "5" + SCANNER__EVENTS__MAXSTREAMLENGTH: "10000" notify-web: image: registry.stella-ops.org/stellaops/notify-web:2025.10.0-edge service: diff --git a/deploy/helm/stellaops/values-stage.yaml b/deploy/helm/stellaops/values-stage.yaml index 29b906fb..8777e0be 100644 --- a/deploy/helm/stellaops/values-stage.yaml +++ b/deploy/helm/stellaops/values-stage.yaml @@ -96,21 +96,33 @@ services: image: registry.stella-ops.org/stellaops/scanner-web@sha256:14b23448c3f9586a9156370b3e8c1991b61907efa666ca37dd3aaed1e79fe3b7 service: port: 8444 - env: - SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017" - SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" - SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops-stage" - SCANNER__STORAGE__S3__SECRETACCESSKEY: "stage-minio-secret" - SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" + env: + SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017" + SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" + SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops-stage" + SCANNER__STORAGE__S3__SECRETACCESSKEY: "stage-minio-secret" + SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" + SCANNER__EVENTS__ENABLED: "false" + SCANNER__EVENTS__DRIVER: "redis" + SCANNER__EVENTS__DSN: "" + SCANNER__EVENTS__STREAM: "stella.events" + SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "5" + SCANNER__EVENTS__MAXSTREAMLENGTH: "10000" scanner-worker: image: registry.stella-ops.org/stellaops/scanner-worker@sha256:32e25e76386eb9ea8bee0a1ad546775db9a2df989fab61ac877e351881960dab replicas: 2 - env: - SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017" - SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" - SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops-stage" - SCANNER__STORAGE__S3__SECRETACCESSKEY: "stage-minio-secret" - SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" + env: + SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017" + SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" + SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops-stage" + SCANNER__STORAGE__S3__SECRETACCESSKEY: "stage-minio-secret" + SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" + SCANNER__EVENTS__ENABLED: "false" + SCANNER__EVENTS__DRIVER: "redis" + SCANNER__EVENTS__DSN: "" + SCANNER__EVENTS__STREAM: "stella.events" + SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "5" + SCANNER__EVENTS__MAXSTREAMLENGTH: "10000" notify-web: image: registry.stella-ops.org/stellaops/notify-web:2025.09.2 service: diff --git a/docs/09_API_CLI_REFERENCE.md b/docs/09_API_CLI_REFERENCE.md index 94daac59..c08069f7 100755 --- a/docs/09_API_CLI_REFERENCE.md +++ b/docs/09_API_CLI_REFERENCE.md @@ -489,6 +489,71 @@ Returns `202 Accepted` and `Location: /attest/{id}` for async verify. --- +### 2.8 Runtime – Ingest Observer Events *(SCANNER-RUNTIME-12-301)* + +``` +POST /api/v1/runtime/events +Authorization: Bearer +Content-Type: application/json +``` + +| Requirement | Details | +|-------------|---------| +| Auth scope | `scanner.runtime.ingest` | +| Batch size | ≤ **256** envelopes (`scanner.runtime.maxBatchSize`, configurable) | +| Payload cap | ≤ **1 MiB** serialized JSON (`scanner.runtime.maxPayloadBytes`) | +| Rate limits | Per-tenant and per-node token buckets (default 200 events/s tenant, 50 events/s node, burst 200) – excess returns **429** with `Retry-After`. | +| TTL | Runtime events retained **45 days** by default (`scanner.runtime.eventTtlDays`). | + +**Request body** + +```json +{ + "batchId": "node-a-2025-10-20T15:03:12Z", + "events": [ + { + "schemaVersion": "zastava.runtime.event@v1", + "event": { + "eventId": "evt-2f9c02b8", + "when": "2025-10-20T15:03:08Z", + "kind": "ContainerStart", + "tenant": "tenant-alpha", + "node": "cluster-a/node-01", + "runtime": { "engine": "containerd", "version": "1.7.19" }, + "workload": { + "platform": "kubernetes", + "namespace": "payments", + "pod": "api-7c9fbbd8b7-ktd84", + "container": "api", + "containerId": "containerd://bead5...", + "imageRef": "ghcr.io/acme/api@sha256:deadbeef" + }, + "process": { "pid": 12345, "entrypoint": ["/start.sh", "--serve"] }, + "loadedLibs": [ + { "path": "/lib/x86_64-linux-gnu/libssl.so.3", "inode": 123456, "sha256": "abc123..." } + ], + "posture": { "imageSigned": true, "sbomReferrer": "present" }, + "delta": { "baselineImageDigest": "sha256:deadbeef" }, + "evidence": [ { "signal": "proc.maps", "value": "libssl.so.3@0x7f..." } ], + "annotations": { "observerVersion": "1.0.0" } + } + } + ] +} +``` + +**Responses** + +| Code | Body | Notes | +|------|------|-------| +| `202 Accepted` | `{ "accepted": 128, "duplicates": 2 }` | Batch persisted; duplicates are ignored via unique `eventId`. | +| `400 Bad Request` | Problem+JSON | Validation failures – empty batch, duplicate IDs, unsupported schema version, payload too large. | +| `429 Too Many Requests` | Problem+JSON | Per-tenant/node rate limit exceeded; `Retry-After` header emitted in seconds. | + +Persisted documents capture the canonical envelope (`payload` field), tenant/node metadata, and set an automatic TTL on `expiresAt`. Observers should retry rejected batches with exponential backoff honouring the provided `Retry-After` hint. + +--- + ## 3 StellaOps CLI (`stellaops-cli`) The new CLI is built on **System.CommandLine 2.0.0‑beta5** and mirrors the Concelier backend REST API. @@ -521,6 +586,9 @@ See `docs/dev/32_AUTH_CLIENT_GUIDE.md` for recommended profiles (online vs. air- | `stellaops-cli auth ` | Manage cached tokens for StellaOps Authority | `auth login --force` (ignore cache)
`auth status`
`auth whoami` | Uses `StellaOps.Auth.Client`; honours `StellaOps:Authority:*` configuration, stores tokens under `~/.stellaops/tokens` by default, and `whoami` prints subject/scope/expiry | | `stellaops-cli auth revoke export` | Export the Authority revocation bundle | `--output ` (defaults to CWD) | Writes `revocation-bundle.json`, `.json.jws`, and `.json.sha256`; verifies the digest locally and includes key metadata in the log summary. | | `stellaops-cli auth revoke verify` | Validate a revocation bundle offline | `--bundle ` `--signature ` `--key `
`--verbose` | Verifies detached JWS signatures, reports the computed SHA-256, and can fall back to cached JWKS when `--key` is omitted. | +| `stellaops-cli offline kit pull` | Download the latest offline kit bundle and manifest | `--bundle-id ` (optional)
`--destination `
`--overwrite`
`--no-resume` | Streams the bundle + manifest from the configured mirror/backend, resumes interrupted downloads, verifies SHA-256, and writes signatures plus a `.metadata.json` manifest alongside the artefacts. | +| `stellaops-cli offline kit import` | Upload an offline kit bundle to the backend | `` (argument)
`--manifest `
`--bundle-signature `
`--manifest-signature ` | Validates digests when metadata is present, then posts multipart payloads to `POST /api/offline-kit/import`; logs the submitted import ID/status for air-gapped rollout tracking. | +| `stellaops-cli offline kit status` | Display imported offline kit details | `--json` | Shows bundle id/kind, captured/imported timestamps, digests, and component versions; `--json` emits machine-readable output for scripting. | | `stellaops-cli config show` | Display resolved configuration | — | Masks secret values; helpful for air‑gapped installs | | `stellaops-cli runtime policy test` | Ask Scanner.WebService for runtime verdicts (Webhook parity) | `--image/-i ` (repeatable, comma/space lists supported)
`--file/-f `
`--namespace/--ns `
`--label/-l key=value` (repeatable)
`--json` | Posts to `POST /api/v1/scanner/policy/runtime`, deduplicates image digests, and prints TTL/policy revision plus per-image columns for signed state, SBOM referrers, quieted-by metadata, confidence, and Rekor attestation (uuid + verified flag). Accepts newline/whitespace-delimited stdin when piped; `--json` emits the raw response without additional logging. | @@ -602,6 +670,8 @@ Authority-backed auth workflow: Tokens live in `~/.stellaops/tokens` unless `StellaOps:Authority:TokenCacheDirectory` overrides it. Cached tokens are reused offline until they expire; the CLI surfaces clear errors if refresh fails. +For offline workflows, configure `StellaOps:Offline:KitsDirectory` (or `STELLAOPS_OFFLINE_KITS_DIR`) to control where bundles, manifests, and metadata are stored, and `StellaOps:Offline:KitMirror` (or `STELLAOPS_OFFLINE_MIRROR_URL`) to override the download base URL when pulling from a mirror. + **Configuration file template** ```jsonc @@ -614,6 +684,10 @@ Tokens live in `~/.stellaops/tokens` unless `StellaOps:Authority:TokenCacheDirec "DefaultRunner": "docker", "ScannerSignaturePublicKeyPath": "", "ScannerDownloadAttempts": 3, + "Offline": { + "KitsDirectory": "offline-kits", + "KitMirror": "https://get.stella-ops.org/ouk/" + }, "Authority": { "Url": "https://authority.example.org", "ClientId": "concelier-cli", diff --git a/docs/11_AUTHORITY.md b/docs/11_AUTHORITY.md index 58e8ffb4..3bc03df2 100644 --- a/docs/11_AUTHORITY.md +++ b/docs/11_AUTHORITY.md @@ -151,15 +151,35 @@ All administrative calls emit `AuthEventRecord` entries enriched with correlatio Authority now understands two flavours of sender-constrained OAuth clients: -- **DPoP proof-of-possession** – clients sign a `DPoP` header for `/token` requests. Authority validates the JWK thumbprint, HTTP method/URI, and replay window, then stamps the resulting access token with `cnf.jkt` so downstream services can verify the same key is reused. - - Configure under `security.senderConstraints.dpop`. `allowedAlgorithms`, `proofLifetime`, and `replayWindow` are enforced at validation time. - - `security.senderConstraints.dpop.nonce.enabled` enables nonce challenges for high-value audiences (`requiredAudiences`, normalised to case-insensitive strings). When a nonce is required but missing or expired, `/token` replies with `WWW-Authenticate: DPoP error="use_dpop_nonce"` (and, when available, a fresh `DPoP-Nonce` header). Clients must retry with the issued nonce embedded in the proof. - - `security.senderConstraints.dpop.nonce.store` selects `memory` (default) or `redis`. When `redis` is configured, set `security.senderConstraints.dpop.nonce.redisConnectionString` so replicas share nonce issuance and high-value clients avoid replay gaps during failover. - - Declare client `audiences` in bootstrap manifests or plug-in provisioning metadata; Authority now defaults the token `aud` claim and `resource` indicator from this list, which is also used to trigger nonce enforcement for audiences such as `signer` and `attestor`. -- **Mutual TLS clients** – client registrations may declare an mTLS binding (`senderConstraint: mtls`). When enabled via `security.senderConstraints.mtls`, Authority validates the presented client certificate against stored bindings (`certificateBindings[]`), optional chain verification, and timing windows. Successful requests embed `cnf.x5t#S256` into the access token so resource servers can enforce the certificate thumbprint. - - Certificate bindings record the certificate thumbprint, optional SANs, subject/issuer metadata, and activation windows. Operators can enforce subject regexes, SAN type allow-lists (`dns`, `uri`, `ip`), trusted certificate authorities, and rotation grace via `security.senderConstraints.mtls.*`. - -Both modes persist additional metadata in `authority_tokens`: `senderConstraint` records the enforced policy, while `senderKeyThumbprint` stores the DPoP JWK thumbprint or mTLS certificate hash captured at issuance. Downstream services can rely on these fields (and the corresponding `cnf` claim) when auditing offline copies of the token store. +- **DPoP proof-of-possession** – clients sign a `DPoP` header for `/token` requests. Authority validates the JWK thumbprint, HTTP method/URI, and replay window, then stamps the resulting access token with `cnf.jkt` so downstream services can verify the same key is reused. + - Configure under `security.senderConstraints.dpop`. `allowedAlgorithms`, `proofLifetime`, and `replayWindow` are enforced at validation time. + - `security.senderConstraints.dpop.nonce.enabled` enables nonce challenges for high-value audiences (`requiredAudiences`, normalised to case-insensitive strings). When a nonce is required but missing or expired, `/token` replies with `WWW-Authenticate: DPoP error="use_dpop_nonce"` (and, when available, a fresh `DPoP-Nonce` header). Clients must retry with the issued nonce embedded in the proof. + - `security.senderConstraints.dpop.nonce.store` selects `memory` (default) or `redis`. When `redis` is configured, set `security.senderConstraints.dpop.nonce.redisConnectionString` so replicas share nonce issuance and high-value clients avoid replay gaps during failover. + - Example (enabling Redis-backed nonces; adjust audiences per deployment): + ```yaml + security: + senderConstraints: + dpop: + enabled: true + proofLifetime: "00:02:00" + replayWindow: "00:05:00" + allowedAlgorithms: [ "ES256", "ES384" ] + nonce: + enabled: true + ttl: "00:10:00" + maxIssuancePerMinute: 120 + store: "redis" + redisConnectionString: "redis://authority-redis:6379?ssl=false" + requiredAudiences: + - "signer" + - "attestor" + ``` + Operators can override any field via environment variables (e.g. `STELLAOPS_AUTHORITY__SECURITY__SENDERCONSTRAINTS__DPOP__NONCE__STORE=redis`). + - Declare client `audiences` in bootstrap manifests or plug-in provisioning metadata; Authority now defaults the token `aud` claim and `resource` indicator from this list, which is also used to trigger nonce enforcement for audiences such as `signer` and `attestor`. +- **Mutual TLS clients** – client registrations may declare an mTLS binding (`senderConstraint: mtls`). When enabled via `security.senderConstraints.mtls`, Authority validates the presented client certificate against stored bindings (`certificateBindings[]`), optional chain verification, and timing windows. Successful requests embed `cnf.x5t#S256` into the access token so resource servers can enforce the certificate thumbprint. + - Certificate bindings record the certificate thumbprint, optional SANs, subject/issuer metadata, and activation windows. Operators can enforce subject regexes, SAN type allow-lists (`dns`, `uri`, `ip`), trusted certificate authorities, and rotation grace via `security.senderConstraints.mtls.*`. + +Both modes persist additional metadata in `authority_tokens`: `senderConstraint` records the enforced policy, while `senderKeyThumbprint` stores the DPoP JWK thumbprint or mTLS certificate hash captured at issuance. Downstream services can rely on these fields (and the corresponding `cnf` claim) when auditing offline copies of the token store. ## 8. Offline & Sovereign Operation - **No outbound dependencies:** Authority only contacts MongoDB and local plugins. Discovery and JWKS are cached by clients with offline tolerances (`AllowOfflineCacheFallback`, `OfflineCacheTolerance`). Operators should mirror these responses for air-gapped use. diff --git a/docs/12_PERFORMANCE_WORKBOOK.md b/docs/12_PERFORMANCE_WORKBOOK.md index 192fecc3..9fdb9355 100755 --- a/docs/12_PERFORMANCE_WORKBOOK.md +++ b/docs/12_PERFORMANCE_WORKBOOK.md @@ -55,10 +55,11 @@ ## 3 Test Harness -* **Runner** – `perf/run.sh`, accepts `--phase` and `--samples`. -* **Metrics** – Prometheus + `jq` extracts; aggregated via `scripts/aggregate.ts`. -* **CI** – GitLab CI job *benchmark* publishes JSON to `bench‑artifacts/`. -* **Visualisation** – Grafana dashboard *Stella‑Perf* (provisioned JSON). +* **Runner** – `perf/run.sh`, accepts `--phase` and `--samples`. +* **Language analyzers microbench** – `dotnet run --project bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/StellaOps.Bench.ScannerAnalyzers.csproj -- --repo-root . --out bench/Scanner.Analyzers/baseline.csv` produces deterministic CSVs for analyzer scenarios (Node today, others as they land). +* **Metrics** – Prometheus + `jq` extracts; aggregated via `scripts/aggregate.ts`. +* **CI** – GitLab CI job *benchmark* publishes JSON to `bench‑artifacts/`. +* **Visualisation** – Grafana dashboard *Stella‑Perf* (provisioned JSON). > **Note** – harness mounts `/var/cache/trivy` tmpfs to avoid disk noise. diff --git a/docs/24_OFFLINE_KIT.md b/docs/24_OFFLINE_KIT.md index f4f47355..49e7fea0 100755 --- a/docs/24_OFFLINE_KIT.md +++ b/docs/24_OFFLINE_KIT.md @@ -37,7 +37,9 @@ cosign verify-blob \ --key https://stella-ops.org/keys/cosign.pub \ --signature stella-ops-offline-kit-.tgz.sig \ stella-ops-offline-kit-.tgz -```` +```` + +**CLI shortcut.** `stellaops-cli offline kit pull --destination ./offline-kit` downloads the bundle, manifest, and detached signatures in one step, resumes partial transfers, and writes a `.metadata.json` summary for later import. Verification prints **OK** and the SHA‑256 digest; cross‑check against the [changelog](https://git.stella-ops.org/stella-ops/offline-kit/-/releases). @@ -60,11 +62,22 @@ The manifest enumerates every artefact (`name`, `sha256`, `size`, `capturedAt`) ## 2 · Import on the air‑gapped host ```bash -docker compose --env-file .env \ - -f docker-compose.stella-ops.yml \ - exec stella-ops \ - stella admin import-offline-usage-kit stella-ops-offline-kit-.tgz -``` +docker compose --env-file .env \ + -f docker-compose.stella-ops.yml \ + exec stella-ops \ + stella admin import-offline-usage-kit stella-ops-offline-kit-.tgz +``` + +Alternatively, run + +```bash +stellaops-cli offline kit import stella-ops-offline-kit-.tgz \ + --manifest offline-manifest-.json \ + --bundle-signature stella-ops-offline-kit-.tgz.sig \ + --manifest-signature offline-manifest-.json.jws +``` + +The CLI validates recorded digests (when `.metadata.json` is present) before streaming the multipart payload to `/api/offline-kit/import`. * The CLI validates the Cosign signature **before** activation. * Old feeds are kept until the new bundle is fully verified. diff --git a/docs/ARCHITECTURE_AUTHORITY.md b/docs/ARCHITECTURE_AUTHORITY.md index f4f92f4d..bfa5f8d0 100644 --- a/docs/ARCHITECTURE_AUTHORITY.md +++ b/docs/ARCHITECTURE_AUTHORITY.md @@ -275,24 +275,56 @@ Every Stella Ops service that consumes Authority tokens **must**: ```yaml authority: issuer: "https://authority.internal" - keys: - algs: [ "EdDSA", "ES256" ] - rotationDays: 60 - storage: kms://cluster-kms/authority-signing - tokens: - accessTtlSeconds: 180 - enableRefreshTokens: false - clockSkewSeconds: 60 - dpop: - enable: true - nonce: - enable: true - ttlSeconds: 600 - store: redis - redisConnectionString: "redis://authority-redis:6379?ssl=false" - mtls: - enable: true - caBundleFile: /etc/ssl/mtls/clients-ca.pem + signing: + enabled: true + activeKeyId: "authority-signing-2025" + keyPath: "../certificates/authority-signing-2025.pem" + algorithm: "ES256" + keySource: "file" + security: + rateLimiting: + token: + enabled: true + permitLimit: 30 + window: "00:01:00" + queueLimit: 0 + authorize: + enabled: true + permitLimit: 60 + window: "00:01:00" + queueLimit: 10 + internal: + enabled: false + permitLimit: 5 + window: "00:01:00" + queueLimit: 0 + senderConstraints: + dpop: + enabled: true + allowedAlgorithms: [ "ES256", "ES384" ] + proofLifetime: "00:02:00" + allowedClockSkew: "00:00:30" + replayWindow: "00:05:00" + nonce: + enabled: true + ttl: "00:10:00" + maxIssuancePerMinute: 120 + store: "redis" + redisConnectionString: "redis://authority-redis:6379?ssl=false" + requiredAudiences: + - "signer" + - "attestor" + mtls: + enabled: true + requireChainValidation: true + rotationGrace: "00:15:00" + enforceForAudiences: + - "signer" + allowedSanTypes: + - "dns" + - "uri" + allowedCertificateAuthorities: + - "/etc/ssl/mtls/clients-ca.pem" clients: - clientId: scanner-web grantTypes: [ "client_credentials" ] @@ -407,4 +439,3 @@ Signer validates that `hash(JWK)` in the proof matches `cnf.jkt` in the token. 2. **Add**: mTLS‑bound tokens for Signer/Attestor; device code for CLI; optional introspection. 3. **Hardening**: DPoP nonce support; full audit pipeline; HA tuning. 4. **UX**: Tenant/installation admin UI; role→scope editors; client bootstrap wizards. - diff --git a/docs/ARCHITECTURE_EXCITITOR.md b/docs/ARCHITECTURE_EXCITITOR.md index 34dbab49..be3d1718 100644 --- a/docs/ARCHITECTURE_EXCITITOR.md +++ b/docs/ARCHITECTURE_EXCITITOR.md @@ -299,9 +299,9 @@ POST /consensus/search body: { vulnIds?: string[], productKeys?: string[], policyRevisionId?: string, since?: timestamp, limit?: int, pageToken?: string } → { entries[], nextPageToken? } -POST /resolve - body: { purls: string[], vulnIds: string[], policyRevisionId?: string } - → { results: [ { vulnId, productKey, rollupStatus, sources[] } ] } +POST /excititor/resolve (scope: vex.read) + body: { productKeys?: string[], purls?: string[], vulnerabilityIds: string[], policyRevisionId?: string } + → { policy, resolvedAt, results: [ { vulnerabilityId, productKey, status, sources[], conflicts[], decisions[], signals?, summary?, envelope: { artifact, contentSignature?, attestation?, attestationEnvelope?, attestationSignature? } } ] } ``` ### 7.2 Exports (cacheable snapshots) @@ -388,12 +388,13 @@ excititor: With storage configured, the WebService exposes the following ingress and diagnostic APIs: -* `GET /excititor/status` – returns the active storage configuration and registered artifact stores. -* `GET /excititor/health` – simple liveness probe. -* `POST /excititor/statements` – accepts normalized VEX statements and persists them via `IVexClaimStore`; use this for migrations/backfills. -* `GET /excititor/statements/{vulnId}/{productKey}?since=` – returns the immutable statement log for a vulnerability/product pair. - -Run the ingestion endpoint once after applying migration `20251019-consensus-signals-statements` to repopulate historical statements with the new severity/KEV/EPSS signal fields. +* `GET /excititor/status` – returns the active storage configuration and registered artifact stores. +* `GET /excititor/health` – simple liveness probe. +* `POST /excititor/statements` – accepts normalized VEX statements and persists them via `IVexClaimStore`; use this for migrations/backfills. +* `GET /excititor/statements/{vulnId}/{productKey}?since=` – returns the immutable statement log for a vulnerability/product pair. +* `POST /excititor/resolve` – requires `vex.read` scope; accepts up to 256 `(vulnId, productKey)` pairs via `productKeys` or `purls` and returns deterministic consensus results, decision telemetry, and a signed envelope (`artifact` digest, optional signer signature, optional attestation metadata + DSSE envelope). Returns **409 Conflict** when the requested `policyRevisionId` mismatches the active snapshot. + +Run the ingestion endpoint once after applying migration `20251019-consensus-signals-statements` to repopulate historical statements with the new severity/KEV/EPSS signal fields. * `weights.ceiling` raises the deterministic clamp applied to provider tiers/overrides (range 1.0‒5.0). Values outside the range are clamped with warnings so operators can spot typos. * `scoring.alpha` / `scoring.beta` configure KEV/EPSS boosts for the Phase 1 → Phase 2 scoring pipeline. Defaults (0.25, 0.5) preserve prior behaviour; negative or excessively large values fall back with diagnostics. @@ -410,7 +411,7 @@ Run the ingestion endpoint once after applying migration `20251019-consensus-sig --- -## 11) Performance & scale +## 11) Performance & scale * **Targets:** @@ -423,9 +424,42 @@ Run the ingestion endpoint once after applying migration `20251019-consensus-sig * WebService handles control APIs; **Worker** background services (same image) execute fetch/normalize in parallel with rate‑limits; Mongo writes batched; upserts by natural keys. * Exports stream straight to S3 (MinIO) with rolling buffers. -* **Caching:** - - * `vex.cache` maps query signatures → export; TTL to avoid stampedes; optimistic reuse unless `force`. +* **Caching:** + + * `vex.cache` maps query signatures → export; TTL to avoid stampedes; optimistic reuse unless `force`. + +### 11.1 Worker TTL refresh controls + +Excititor.Worker ships with a background refresh service that re-evaluates stale consensus rows and applies stability dampers before publishing status flips. Operators can tune its behaviour through the following configuration (shown in `appsettings.json` syntax): + +```jsonc +{ + "Excititor": { + "Worker": { + "Refresh": { + "Enabled": true, + "ConsensusTtl": "02:00:00", // refresh consensus older than 2 hours + "ScanInterval": "00:10:00", // sweep cadence + "ScanBatchSize": 250, // max documents examined per sweep + "Damper": { + "Minimum": "1.00:00:00", // lower bound before status flip publishes + "Maximum": "2.00:00:00", // upper bound guardrail + "DefaultDuration": "1.12:00:00", + "Rules": [ + { "MinWeight": 0.90, "Duration": "1.00:00:00" }, + { "MinWeight": 0.75, "Duration": "1.06:00:00" }, + { "MinWeight": 0.50, "Duration": "1.12:00:00" } + ] + } + } + } + } +} +``` + +* `ConsensusTtl` governs when the worker issues a fresh resolve for cached consensus data. +* `Damper` lengths are clamped between `Minimum`/`Maximum`; duration is bypassed when component fingerprints (`VexProduct.ComponentIdentifiers`) change. +* The same keys are available through environment variables (e.g., `Excititor__Worker__Refresh__ConsensusTtl=02:00:00`). --- @@ -457,7 +491,7 @@ Run the ingestion endpoint once after applying migration `20251019-consensus-sig ## 14) Integration points -* **Backend Policy Engine** (in Scanner.WebService): calls `POST /resolve` with batched `(purl, vulnId)` pairs to fetch `rollupStatus + sources`. +* **Backend Policy Engine** (in Scanner.WebService): calls `POST /excititor/resolve` (scope `vex.read`) with batched `(purl, vulnId)` pairs to fetch `rollupStatus + sources`. * **Concelier**: provides alias graph (CVE↔vendor IDs) and may supply VEX‑adjacent metadata (e.g., KEV flag) for policy escalation. * **UI**: VEX explorer screens use `/claims/search` and `/consensus/search`; show conflicts & provenance. * **CLI**: `stellaops vex export --consensus --since 7d --out vex.json` for audits. @@ -474,7 +508,7 @@ Run the ingestion endpoint once after applying migration `20251019-consensus-sig ## 16) Rollout plan (incremental) -1. **MVP**: OpenVEX + CSAF connectors for 3 major providers (e.g., Red Hat/SUSE/Ubuntu), normalization + consensus + `/resolve`. +1. **MVP**: OpenVEX + CSAF connectors for 3 major providers (e.g., Red Hat/SUSE/Ubuntu), normalization + consensus + `/excititor/resolve`. 2. **Signature policies**: PGP for distros; cosign for OCI. 3. **Exports + optional attestation**. 4. **CycloneDX VEX** connectors; platform claim expansion tables; UI explorer. diff --git a/docs/ARCHITECTURE_NOTIFY.md b/docs/ARCHITECTURE_NOTIFY.md index f9006782..b953de2f 100644 --- a/docs/ARCHITECTURE_NOTIFY.md +++ b/docs/ARCHITECTURE_NOTIFY.md @@ -230,7 +230,7 @@ public interface INotifyConnector { **Channel mapping**: * Slack: title + blocks, limited to 50 blocks/3000 chars per section; long lists → link to UI. -* Teams: Adaptive Card schema 1.5; fallback text for older channels. +* Teams: Adaptive Card schema 1.5; fallback text for older channels (surfaced as `teams.fallbackText` metadata alongside webhook hash). * Email: HTML + text; inline table of top N findings, rest behind UI link. * Webhook: JSON with `event`, `ruleId`, `actionId`, `summary`, `links`, and raw `payload` subset. @@ -299,7 +299,7 @@ Internal tooling can hit `/internal/notify//normalize` to upgrade legacy * `POST /channels` | `GET /channels` | `GET /channels/{id}` | `PATCH /channels/{id}` | `DELETE /channels/{id}` * `POST /channels/{id}/test` → send sample message (no rule evaluation); returns `202 Accepted` with rendered preview + metadata (base keys: `channelType`, `target`, `previewProvider`, `traceId` + connector-specific entries); governed by `api.rateLimits:testSend`. - * `GET /channels/{id}/health` → connector self‑check +* `GET /channels/{id}/health` → connector self‑check (returns redacted metadata: secret refs hashed, sensitive config keys masked, fallbacks noted via `teams.fallbackText`/`teams.validation.*`) * **Rules** diff --git a/docs/ARCHITECTURE_SCANNER.md b/docs/ARCHITECTURE_SCANNER.md index fae81829..fd6fc1dd 100644 --- a/docs/ARCHITECTURE_SCANNER.md +++ b/docs/ARCHITECTURE_SCANNER.md @@ -164,7 +164,7 @@ GET /healthz | /readyz | /metrics ### Report events -When `scanner.events.enabled = true`, the WebService serialises the signed report (canonical JSON + DSSE envelope) with `NotifyCanonicalJsonSerializer` and publishes two Redis Stream entries (`scanner.report.ready`, `scanner.scan.completed`) to the configured stream (default `stella.events`). The stream fields carry the whole envelope plus lightweight headers (`kind`, `tenant`, `ts`) so Notify and UI timelines can consume the event bus without recomputing signatures. Publish timeouts and bounded stream length are controlled via `scanner:events:publishTimeoutSeconds` and `scanner:events:maxStreamLength`. If the queue driver is already Redis and no explicit events DSN is provided, the host reuses the queue connection and auto-enables event emission so deployments get live envelopes without extra wiring. +When `scanner.events.enabled = true`, the WebService serialises the signed report (canonical JSON + DSSE envelope) with `NotifyCanonicalJsonSerializer` and publishes two Redis Stream entries (`scanner.report.ready`, `scanner.scan.completed`) to the configured stream (default `stella.events`). The stream fields carry the whole envelope plus lightweight headers (`kind`, `tenant`, `ts`) so Notify and UI timelines can consume the event bus without recomputing signatures. Publish timeouts and bounded stream length are controlled via `scanner:events:publishTimeoutSeconds` and `scanner:events:maxStreamLength`. If the queue driver is already Redis and no explicit events DSN is provided, the host reuses the queue connection and auto-enables event emission so deployments get live envelopes without extra wiring. Compose/Helm bundles expose the same knobs via the `SCANNER__EVENTS__*` environment variables for quick tuning. --- diff --git a/docs/ARCHITECTURE_VEXER.md b/docs/ARCHITECTURE_VEXER.md index 1f6920f9..0ea54de5 100644 --- a/docs/ARCHITECTURE_VEXER.md +++ b/docs/ARCHITECTURE_VEXER.md @@ -286,9 +286,9 @@ POST /consensus/search body: { vulnIds?: string[], productKeys?: string[], policyRevisionId?: string, since?: timestamp, limit?: int, pageToken?: string } → { entries[], nextPageToken? } -POST /resolve - body: { purls: string[], vulnIds: string[], policyRevisionId?: string } - → { results: [ { vulnId, productKey, rollupStatus, sources[] } ] } +POST /excititor/resolve (scope: vex.read) + body: { productKeys?: string[], purls?: string[], vulnerabilityIds: string[], policyRevisionId?: string } + → { policy, resolvedAt, results: [ { vulnerabilityId, productKey, status, sources[], conflicts[], decisions[], signals?, summary?, envelope: { artifact, contentSignature?, attestation?, attestationEnvelope?, attestationSignature? } } ] } ``` ### 7.2 Exports (cacheable snapshots) @@ -426,7 +426,7 @@ vexer: ## 14) Integration points -* **Backend Policy Engine** (in Scanner.WebService): calls `POST /resolve` with batched `(purl, vulnId)` pairs to fetch `rollupStatus + sources`. +* **Backend Policy Engine** (in Scanner.WebService): calls `POST /excititor/resolve` (scope `vex.read`) with batched `(purl, vulnId)` pairs to fetch `rollupStatus + sources`. * **Feedser**: provides alias graph (CVE↔vendor IDs) and may supply VEX‑adjacent metadata (e.g., KEV flag) for policy escalation. * **UI**: VEX explorer screens use `/claims/search` and `/consensus/search`; show conflicts & provenance. * **CLI**: `stellaops vex export --consensus --since 7d --out vex.json` for audits. @@ -443,7 +443,7 @@ vexer: ## 16) Rollout plan (incremental) -1. **MVP**: OpenVEX + CSAF connectors for 3 major providers (e.g., Red Hat/SUSE/Ubuntu), normalization + consensus + `/resolve`. +1. **MVP**: OpenVEX + CSAF connectors for 3 major providers (e.g., Red Hat/SUSE/Ubuntu), normalization + consensus + `/excititor/resolve`. 2. **Signature policies**: PGP for distros; cosign for OCI. 3. **Exports + optional attestation**. 4. **CycloneDX VEX** connectors; platform claim expansion tables; UI explorer. diff --git a/docs/TASKS.md b/docs/TASKS.md index ad2480e8..9b26b9ed 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -15,6 +15,7 @@ | DOCS-EVENTS-09-004 | DONE (2025-10-19) | Docs Guild, Scanner WebService | SCANNER-EVENTS-15-201 | Refresh scanner event docs to mirror DSSE-backed report fields, document `scanner.scan.completed`, and capture canonical sample validation. | Schemas updated for new payload shape; README references DSSE reuse and validation test; samples align with emitted events. | | PLATFORM-EVENTS-09-401 | DONE (2025-10-19) | Platform Events Guild | DOCS-EVENTS-09-003 | Embed canonical event samples into contract/integration tests and ensure CI validates payloads against published schemas. | Notify/Scheduler contract suites exercise samples; CI job validates samples with `ajv-cli`; Platform Events changelog notes coverage. | | RUNTIME-GUILD-09-402 | DONE (2025-10-19) | Runtime Guild | SCANNER-POLICY-09-107 | Confirm Scanner WebService surfaces `quietedFindingCount` and progress hints to runtime consumers; document readiness checklist. | Runtime verification run captures enriched payload; checklist/doc updates merged; stakeholders acknowledge availability. | +| DOCS-CONCELIER-07-201 | TODO | Docs Guild, Concelier WebService | FEEDWEB-DOCS-01-001 | Final editorial review and publish pass for Concelier authority toggle documentation (Quickstart + operator guide). | Review feedback resolved, publish PR merged, release notes updated with documentation pointer. | | DOCS-RUNTIME-17-004 | TODO | Docs Guild, Runtime Guild | SCANNER-EMIT-17-701, ZASTAVA-OBS-17-005, DEVOPS-REL-17-002 | Document build-id workflows: SBOM exposure, runtime event payloads, debug-store layout, and operator guidance for symbol retrieval. | Architecture + operator docs updated with build-id sections, examples show `readelf` output + debuginfod usage, references linked from Offline Kit/Release guides. | > Update statuses (TODO/DOING/REVIEW/DONE/BLOCKED) as progress changes. Keep guides in sync with configuration samples under `etc/`. diff --git a/docs/dev/authority-dpop-mtls-plan.md b/docs/dev/authority-dpop-mtls-plan.md index f89c7386..f454c25f 100644 --- a/docs/dev/authority-dpop-mtls-plan.md +++ b/docs/dev/authority-dpop-mtls-plan.md @@ -11,7 +11,9 @@ - Operator-facing configuration, auditing, and observability. - Out of scope: PoE enforcement (Signer) and CLI/UI client UX; those teams consume the new capabilities. -> **Status update (2025-10-19):** `ValidateDpopProofHandler`, `AuthorityClientCertificateValidator`, and the supporting storage/audit plumbing now live in `src/StellaOps.Authority`. DPoP proofs populate `cnf.jkt`, mTLS bindings enforce certificate thumbprints via `cnf.x5t#S256`, and token documents persist the sender constraint metadata. In-memory nonce issuance is wired (Redis implementation to follow). Documentation and configuration references were updated (`docs/11_AUTHORITY.md`). Targeted unit/integration tests were added; running the broader test suite is currently blocked by pre-existing `StellaOps.Concelier.Storage.Mongo` build errors. +> **Status update (2025-10-19):** `ValidateDpopProofHandler`, `AuthorityClientCertificateValidator`, and the supporting storage/audit plumbing now live in `src/StellaOps.Authority`. DPoP proofs populate `cnf.jkt`, mTLS bindings enforce certificate thumbprints via `cnf.x5t#S256`, and token documents persist the sender constraint metadata. In-memory nonce issuance is wired (Redis implementation to follow). Documentation and configuration references were updated (`docs/11_AUTHORITY.md`). Targeted unit/integration tests were added; running the broader test suite is currently blocked by pre-existing `StellaOps.Concelier.Storage.Mongo` build errors. +> +> **Status update (2025-10-20):** Redis-backed nonce configuration is exposed through `security.senderConstraints.dpop.nonce` with sample YAML (`etc/authority.yaml.sample`) and architecture docs refreshed. Operator guide now includes concrete Redis/required audiences snippet; nonce challenge regression remains covered by `ValidateDpopProof_IssuesNonceChallenge_WhenNonceMissing`. ## Design Summary - Extract the existing Scanner `DpopProofValidator` stack into a shared `StellaOps.Auth.Security` library used by Authority and resource servers. diff --git a/docs/dev/authority-plugin-di-coordination.md b/docs/dev/authority-plugin-di-coordination.md index c79470ba..50e1f499 100644 --- a/docs/dev/authority-plugin-di-coordination.md +++ b/docs/dev/authority-plugin-di-coordination.md @@ -1,7 +1,7 @@ -# Authority Plug-in Scoped Service Coordination - -> Created: 2025-10-19 — Plugin Platform Guild & Authority Core -> Status: Scheduled (session confirmed for 2025-10-20 15:00–16:00 UTC) +# Authority Plug-in Scoped Service Coordination + +> Created: 2025-10-19 — Plugin Platform Guild & Authority Core +> Status: Completed (workshop held 2025-10-20 15:00–16:05 UTC) This document tracks preparation, agenda, and outcomes for the scoped-service workshop required before implementing PLUGIN-DI-08-002. @@ -27,9 +27,28 @@ This document tracks preparation, agenda, and outcomes for the scoped-service wo - Audit background jobs that assume singleton lifetimes. - Identify plug-in health checks/telemetry surfaces impacted by scoped lifetimes. -### Pre-work References - -- _Add links, file paths, or notes here prior to the session._ +### Pre-work References + +| Focus | Path | Notes | +|-------|------|-------| +| Host DI wiring | `src/StellaOps.Authority/StellaOps.Authority/Program.cs:159` | Startup registers `IAuthorityIdentityProviderRegistry` as a singleton and invokes `AuthorityPluginLoader.RegisterPlugins(...)` before the container is built. Any scoped plugin services will currently be captured in the singleton registry context. | +| Registrar discovery | `src/StellaOps.Authority/StellaOps.Authority/Plugins/AuthorityPluginLoader.cs:46` | Loader instantiates `IAuthorityPluginRegistrar` implementations via `Activator.CreateInstance`, so registrars cannot depend on host services yet. Need agreement on whether to move discovery post-build or introduce `ActivatorUtilities`. | +| Registry aggregation | `src/StellaOps.Authority/StellaOps.Authority/AuthorityIdentityProviderRegistry.cs:16` | Registry caches `IIdentityProviderPlugin` instances at construction time. With scoped lifetimes we must revisit how providers are resolved (factory vs accessor). | +| Standard registrar services | `src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StandardPluginRegistrar.cs:21` | All plugin services are registered as singletons today (`StandardUserCredentialStore`, `StandardClientProvisioningStore`, hosted bootstrapper). This registrar is our baseline for migrating to scoped bindings. | +| Hosted bootstrapper | `src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Bootstrap/StandardPluginBootstrapper.cs:17` | Background job directly consumes `StandardUserCredentialStore`. If the store becomes scoped we will need an `IServiceScopeFactory` bridge. | +| Password grant handler | `src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/PasswordGrantHandlers.cs:26` | Password flow resolves `IIdentityProviderPlugin` during scoped requests. Scope semantics must ensure credential stores stay cancellation-aware. | +| Client credential handler | `src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/ClientCredentialsHandlers.cs:21` | Handler fetches provider + `ClientProvisioning` store; confirms need for consistent scoping in both user and client flows. | + +## Preliminary Findings — 2025-10-20 + +- `IAuthorityIdentityProviderRegistry` must stop materialising provider singletons when scoped lifetimes land. Options to evaluate: make the registry itself scoped, convert it to a factory over `IServiceProvider`, or cache lightweight descriptors and resolve implementations on-demand. +- `AuthorityPluginLoader` instantiates registrars without DI support. To let registrars request scoped helpers (e.g. `IServiceScopeFactory`) we may need a two-phase registration: discover types at build time, defer execution until the container is available. +- Hosted bootstrap tasks (e.g. `StandardPluginBootstrapper`) will break if their dependencies become scoped. Workshop should align on using scoped pipelines inside `StartAsync` or shifting bootstrap work to queued jobs. +- Standard plugin stores assume singleton access to Mongo collections and password hashing utilities. If we embrace scoped stores, document thread-safety expectations and reuse of Mongo clients across scopes. +- OpenIddict handlers already run as scoped services; once providers move to scoped lifetimes we must ensure the new resolution path stays cancellation-aware and avoids redundant service resolution per request. +- 2025-10-20 (PLUGIN-DI-08-003): Registry implementation updated to expose metadata + scoped handles; OpenIddict flows, bootstrap endpoints, and `/health` now resolve providers via scoped leases with accompanying test coverage. +- 2025-10-20 (PLUGIN-DI-08-004): Authority plugin loader now instantiates registrars via scoped DI activations and honours `[ServiceBinding]` metadata in plugin assemblies. +- 2025-10-20 (PLUGIN-DI-08-005): `StandardPluginBootstrapper` shifted to scope-per-run execution using `IServiceScopeFactory`, enabling future scoped stores without singleton leaks. ## Draft Agenda @@ -39,14 +58,20 @@ This document tracks preparation, agenda, and outcomes for the scoped-service wo 4. Action items & owners (10 min) — capture code/docs/test tasks with due dates. 5. Risks & follow-ups (5 min) — dependencies, rollout sequencing. -## Notes - -- _Pending coordination session; populate afterwards._ +## Notes + +- Session opened with recap of scoped-service goals and PLUGIN-DI-08-001 changes, confirming Authority readiness to adopt `[ServiceBinding]` metadata. +- Agreed to treat `IAuthorityIdentityProviderRegistry` as a scoped-factory facade rather than a singleton cache; registry will track descriptors and resolve implementations on-demand per request/worker scope. +- Standard plug-in bootstrap will create scopes via `IServiceScopeFactory` and pass cancellation tokens through to avoid lingering singleton references. +- Authority Plugin Loader will enumerate plug-in assemblies at startup but defer registrar activation until a scoped service provider is available, aligning with PLUGIN-DI-08-004 implementation. +- Follow-up engineering tasks assigned to land PLUGIN-DI-08-002 code path adjustments and Authority host updates before 2025-10-24. ## Action Item Log | Item | Owner | Due | Status | Notes | |------|-------|-----|--------|-------| -| Confirm meeting time | Alicia Rivera | 2025-10-19 15:30 UTC | DONE | Calendar invite sent; all required attendees accepted | -| Compile Authority plug-in DI entry points | Jasmin Patel | 2025-10-20 | IN PROGRESS | Gather current Authority plug-in registrars, background jobs, and helper factories that assume singleton lifetimes; add the list with file paths to **Pre-work References** in this document before 2025-10-20 12:00 UTC. | -| Outline scoped-session pattern for background jobs | Leah Chen | Post-session | BLOCKED | Requires meeting outcomes | +| Confirm meeting time | Alicia Rivera | 2025-10-19 15:30 UTC | DONE | Calendar invite sent; all required attendees accepted | +| Compile Authority plug-in DI entry points | Jasmin Patel | 2025-10-20 | DONE (2025-10-20) | Scoped-service touchpoints summarised in **Pre-work References** and **Preliminary Findings** ahead of the workshop. | +| Outline scoped-session pattern for background jobs | Leah Chen | 2025-10-21 | DONE (2025-10-20) | Pattern agreed: bootstrap services must open transient scopes per execution via `IServiceScopeFactory`; document update to follow in PLUGIN-DI-08-002 patch. | +| Update PLUGIN-DI-08-002 implementation plan | Alicia Rivera | 2025-10-21 | DONE (2025-10-20) | Task board + SPRINTS updated with scoped-integration delivery notes and test references. | +| Sync Authority host backlog | Mohan Singh | 2025-10-21 | DONE (2025-10-20) | Authority/Plugin TASKS.md and SPRINTS entries reflect scoped-service completion. | diff --git a/docs/dev/normalized_versions_rollout.md b/docs/dev/normalized_versions_rollout.md index 2a5b6091..b163d351 100644 --- a/docs/dev/normalized_versions_rollout.md +++ b/docs/dev/normalized_versions_rollout.md @@ -1,6 +1,6 @@ # Normalized Versions Rollout Dashboard (Sprint 2 – Concelier) -_Status date: 2025-10-12 17:05 UTC_ +_Status date: 2025-10-20 19:10 UTC_ This dashboard tracks connector readiness for emitting `AffectedPackage.NormalizedVersions` arrays and highlights upcoming coordination checkpoints. Use it alongside: @@ -10,30 +10,32 @@ This dashboard tracks connector readiness for emitting `AffectedPackage.Normaliz ## Key milestones -- **2025-10-12** – Normalization finalized `SemVerRangeRuleBuilder` API contract (multi-segment comparators + notes), connector review opens. -- **2025-10-17** – Connector owners to post fixture PRs showing `NormalizedVersions` arrays (even if feature-flagged). -- **2025-10-18** – Merge cross-connector review to validate consistent field usage before enabling union logic. +- **2025-10-21** – Cccs and Cisco connectors finalize normalized rule emission and share merge-counter screenshots. +- **2025-10-22** – CertBund localisation translator reviewed; blockers escalated if localisation guidance slips. +- **2025-10-23** – ICS-CISA confirms SemVer reuse vs new firmware scheme and files Models ticket if needed. +- **2025-10-24** – KISA firmware scheme proposal due; Merge provides same-day review. +- **2025-10-25** – Merge runs cross-connector validation before enabling normalized-rule union logic by default. ## Connector readiness matrix | Connector | Owner team | Normalized versions status | Last update | Next action / link | |-----------|------------|---------------------------|-------------|--------------------| -| Acsc | BE-Conn-ACSC | ❌ Not started – mapper pending | 2025-10-11 | Design DTOs + mapper with normalized rule array; see `src/StellaOps.Concelier.Connector.Acsc/TASKS.md`. | -| Cccs | BE-Conn-CCCS | ⚠️ Scheduled – helper ready, implementation due 2025-10-21 | 2025-10-19 | Apply Merge-provided trailing-version helper to emit `NormalizedVersions`; update mapper/tests per `src/StellaOps.Concelier.Connector.Cccs/TASKS.md`. | -| CertBund | BE-Conn-CERTBUND | ⚠️ Follow-up – translate `versions` strings to normalized rules | 2025-10-19 | Build `bis`/`alle` translator + fixtures before 2025-10-22 per `src/StellaOps.Concelier.Connector.CertBund/TASKS.md`. | -| CertCc | BE-Conn-CERTCC | ⚠️ In progress – fetch pipeline DOING | 2025-10-11 | Implement VINCE mapper with SemVer/NEVRA rules; unblock snapshot regeneration; `src/StellaOps.Concelier.Connector.CertCc/TASKS.md`. | -| Kev | BE-Conn-KEV | ✅ Normalized catalog/due-date rules verified | 2025-10-12 | Fixtures reconfirmed via `dotnet test src/StellaOps.Concelier.Connector.Kev.Tests`; `src/StellaOps.Concelier.Connector.Kev/TASKS.md`. | -| Cve | BE-Conn-CVE | ✅ Normalized SemVer rules verified | 2025-10-12 | Snapshot parity green (`dotnet test src/StellaOps.Concelier.Connector.Cve.Tests`); `src/StellaOps.Concelier.Connector.Cve/TASKS.md`. | -| Ghsa | BE-Conn-GHSA | ⚠️ DOING – normalized rollout task active | 2025-10-11 18:45 UTC | Wire `SemVerRangeRuleBuilder` + refresh fixtures; `src/StellaOps.Concelier.Connector.Ghsa/TASKS.md`. | -| Osv | BE-Conn-OSV | ✅ SemVer mapper & parity fixtures verified | 2025-10-12 | GHSA parity regression passing (`dotnet test src/StellaOps.Concelier.Connector.Osv.Tests`); `src/StellaOps.Concelier.Connector.Osv/TASKS.md`. | -| Ics.Cisa | BE-Conn-ICS-CISA | ⚠️ Decision pending – normalize SemVer exacts or escalate scheme | 2025-10-19 | Promote `SemVerPrimitive` outputs into `NormalizedVersions` or file Models ticket by 2025-10-23 (`src/StellaOps.Concelier.Connector.Ics.Cisa/TASKS.md`). | -| Kisa | BE-Conn-KISA | ⚠️ Proposal required – firmware scheme due 2025-10-24 | 2025-10-19 | Draft `kisa.build` (or equivalent) scheme with Models, then emit normalized rules; track in `src/StellaOps.Concelier.Connector.Kisa/TASKS.md`. | -| Ru.Bdu | BE-Conn-BDU | ✅ Raw scheme emitted | 2025-10-14 | Mapper now writes `ru-bdu.raw` normalized rules with provenance + telemetry; `src/StellaOps.Concelier.Connector.Ru.Bdu/TASKS.md`. | -| Ru.Nkcki | BE-Conn-Nkcki | ❌ Not started – mapper TODO | 2025-10-11 | Similar to BDU; ensure Cyrillic provenance preserved; `src/StellaOps.Concelier.Connector.Ru.Nkcki/TASKS.md`. | -| Vndr.Apple | BE-Conn-Apple | ✅ Shipped – emitting normalized arrays | 2025-10-11 | Continue fixture/tooling work; `src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md`. | -| Vndr.Cisco | BE-Conn-Cisco | ⚠️ Scheduled – normalized rule emission due 2025-10-21 | 2025-10-19 | Use Merge helper to persist `NormalizedVersions` alongside SemVer primitives; see `src/StellaOps.Concelier.Connector.Vndr.Cisco/TASKS.md`. | -| Vndr.Msrc | BE-Conn-MSRC | ✅ Map + normalized build rules landed | 2025-10-15 | `MsrcMapper` emits `msrc.build` normalized rules with CVRF references; see `src/StellaOps.Concelier.Connector.Vndr.Msrc/TASKS.md`. | -| Nvd | BE-Conn-NVD | ⚠️ Needs follow-up – mapper complete but normalized array MR pending | 2025-10-11 | Align CVE notes + normalized payload flag; `src/StellaOps.Concelier.Connector.Nvd/TASKS.md`. | +| Acsc | BE-Conn-ACSC | ❌ Not started – normalized helper pending relay stability | 2025-10-20 | Prepare builder integration plan for 2025-10-24 kickoff; update `src/StellaOps.Concelier.Connector.Acsc/TASKS.md` once branch opens. | +| Cccs | BE-Conn-CCCS | ⚠️ DOING – trailing-version helper MR reviewing (due 2025-10-21) | 2025-10-20 | Land helper + fixture refresh, post merge-counter screenshot; `src/StellaOps.Concelier.Connector.Cccs/TASKS.md`. | +| CertBund | BE-Conn-CERTBUND | ⚠️ In progress – localisation translator WIP (due 2025-10-22) | 2025-10-20 | Finish translator + provenance notes, regenerate fixtures; `src/StellaOps.Concelier.Connector.CertBund/TASKS.md`. | +| CertCc | BE-Conn-CERTCC | ✅ Complete – `certcc.vendor` rules emitting | 2025-10-20 | Monitor VINCE payload changes; no action. | +| Kev | BE-Conn-KEV | ✅ Complete – catalog/due-date rules verified | 2025-10-20 | Routine monitoring only. | +| Cve | BE-Conn-CVE | ✅ Complete – SemVer normalized rules live | 2025-10-20 | Keep fixtures in sync as CVE schema evolves. | +| Ghsa | BE-Conn-GHSA | ✅ Complete – rollout merged 2025-10-11 | 2025-10-20 | Maintain parity with OSV ecosystems; no action. | +| Osv | BE-Conn-OSV | ✅ Complete – normalized rules shipping | 2025-10-20 | Watch for new ecosystems; refresh fixtures as needed. | +| Ics.Cisa | BE-Conn-ICS-CISA | ⚠️ Decision pending – exact SemVer promotion due 2025-10-23 | 2025-10-20 | Promote primitives or request new scheme; `src/StellaOps.Concelier.Connector.Ics.Cisa/TASKS.md`. | +| Kisa | BE-Conn-KISA | ⚠️ Proposal drafting – firmware scheme due 2025-10-24 | 2025-10-20 | Finalise `kisa.build` proposal with Models; update mapper/tests; `src/StellaOps.Concelier.Connector.Kisa/TASKS.md`. | +| Ru.Bdu | BE-Conn-BDU | ✅ Complete – `ru-bdu.raw` rules live | 2025-10-20 | Continue monitoring UTF-8 handling; no action. | +| Ru.Nkcki | BE-Conn-Nkcki | ✅ Complete – normalized rules emitted | 2025-10-20 | Maintain transliteration guidance; no action. | +| Vndr.Apple | BE-Conn-Apple | ✅ Complete – normalized arrays emitting | 2025-10-20 | Add beta-channel coverage follow-up; see module README. | +| Vndr.Cisco | BE-Conn-Cisco | ⚠️ DOING – normalized promotion branch open (due 2025-10-21) | 2025-10-20 | Merge helper branch, refresh fixtures, post counters; `src/StellaOps.Concelier.Connector.Vndr.Cisco/TASKS.md`. | +| Vndr.Msrc | BE-Conn-MSRC | ✅ Complete – `msrc.build` rules emitting | 2025-10-20 | Monitor monthly rollups; no action. | +| Nvd | BE-Conn-NVD | ✅ Complete – normalized SemVer output live | 2025-10-20 | Keep provenance aligned with CVE IDs; monitor export parity toggle. | Legend: ✅ complete, ⚠️ in progress/partial, ❌ not started. diff --git a/docs/ops/concelier-mirror-operations.md b/docs/ops/concelier-mirror-operations.md index d93f962c..c06da02d 100644 --- a/docs/ops/concelier-mirror-operations.md +++ b/docs/ops/concelier-mirror-operations.md @@ -33,6 +33,20 @@ Key knobs: Mirror responses carry deterministic cache headers: `/index.json` returns `Cache-Control: public, max-age=60`, while per-domain manifests/bundles include `Cache-Control: public, max-age=300, immutable`. Rate limiting surfaces `Retry-After` when quotas are exceeded. +### 1.2 Mirror connector configuration + +Downstream Concelier instances ingest published bundles using the `StellaOpsMirrorConnector`. Operators running the connector in air‑gapped or limited connectivity environments can tune the following options (environment prefix `CONCELIER__SOURCES__STELLAOPSMIRROR__`): + +- `BASEADDRESS` – absolute mirror root (e.g., `https://mirror-primary.stella-ops.org`). +- `INDEXPATH` – relative path to the mirror index (`/concelier/exports/index.json` by default). +- `DOMAINID` – mirror domain identifier from the index (`primary`, `community`, etc.). +- `HTTPTIMEOUT` – request timeout; raise when mirrors sit behind slow WAN links. +- `SIGNATURE__ENABLED` – require detached JWS verification for `bundle.json`. +- `SIGNATURE__KEYID` / `SIGNATURE__PROVIDER` – expected signing key metadata. +- `SIGNATURE__PUBLICKEYPATH` – PEM fallback used when the mirror key registry is offline. + +The connector keeps a per-export fingerprint (bundle digest + generated-at timestamp) and tracks outstanding document IDs. If a scan is interrupted, the next run resumes parse/map work using the stored fingerprint and pending document lists—no network requests are reissued unless the upstream digest changes. + ## 2. Secret & certificate layout ### Docker Compose (`deploy/compose/docker-compose.mirror.yaml`) diff --git a/docs/updates/2025-10-20-authority-identity-registry.md b/docs/updates/2025-10-20-authority-identity-registry.md new file mode 100644 index 00000000..9282d44c --- /dev/null +++ b/docs/updates/2025-10-20-authority-identity-registry.md @@ -0,0 +1,14 @@ +# 2025-10-20 — Authority Identity Provider Registry & DPoP nonce updates + +## Summary +- Authority host now resolves identity providers through the new metadata/handle pattern introduced in `StellaOps.Authority.Plugins.Abstractions`. Runtime handlers (`ValidateClientCredentialsHandler`, `ValidatePasswordGrantHandler`, `ValidateAccessTokenHandler`, bootstrap endpoints) acquire providers with `IAuthorityIdentityProviderRegistry.AcquireAsync` and rely on metadata (`AuthorityIdentityProviderMetadata`) for capability checks. +- Unit and integration tests build lightweight `ServiceProvider` instances with test plugins, matching production DI behaviour and ensuring the new registry contract is exercised. +- DPoP nonce enforcement now prefers `NormalizedAudiences` when populated and gracefully falls back to the configured `RequiredAudiences`, eliminating the runtime type mismatch that previously surfaced during test runs. + +## Operator impact +- No configuration changes are required; existing YAML and environment-based settings continue to function. +- Documentation examples referencing password/mTLS bootstrap flows remain accurate. The new registry logic simply ensures providers advertised in configuration are resolved deterministically and capability-gated before use. + +## Developer notes +- When adding new identity providers or tests, register plugins via `ServiceCollection` and call `new AuthorityIdentityProviderRegistry(serviceProvider, logger)`. +- For DPoP-required endpoints, populate `security.senderConstraints.dpop.nonce.requiredAudiences` or rely on defaults; both now funnel through the normalized set. diff --git a/docs/updates/2025-10-20-scanner-events.md b/docs/updates/2025-10-20-scanner-events.md new file mode 100644 index 00000000..8166e2e2 --- /dev/null +++ b/docs/updates/2025-10-20-scanner-events.md @@ -0,0 +1,5 @@ +# 2025-10-20 – Scanner Platform Events Hardening + +- Scanner WebService now wires a reusable `IRedisConnectionFactory`, simplifying redis transport testing and reuse for future adapters. +- `/api/v1/reports` integration test (`ReportsEndpointPublishesPlatformEvents`) asserts both report-ready and scan-completed envelopes carry DSSE payloads, scope metadata, and deterministic verdicts. +- Task `SCANNER-EVENTS-15-201` closed after verifying `dotnet test src/StellaOps.Scanner.WebService.Tests/StellaOps.Scanner.WebService.Tests.csproj`. diff --git a/etc/authority.yaml.sample b/etc/authority.yaml.sample index 2daf2a95..27878656 100644 --- a/etc/authority.yaml.sample +++ b/etc/authority.yaml.sample @@ -108,6 +108,52 @@ clients: # CIDR ranges that bypass network-sensitive policies (e.g. on-host cron jobs). # Keep the list tight: localhost is sufficient for most air-gapped installs. -bypassNetworks: - - "127.0.0.1/32" - - "::1/128" +bypassNetworks: + - "127.0.0.1/32" + - "::1/128" + +# Security posture (rate limiting + sender constraints). +security: + rateLimiting: + token: + enabled: true + permitLimit: 30 + window: "00:01:00" + queueLimit: 0 + authorize: + enabled: true + permitLimit: 60 + window: "00:01:00" + queueLimit: 10 + internal: + enabled: false + permitLimit: 5 + window: "00:01:00" + queueLimit: 0 + senderConstraints: + dpop: + enabled: true + allowedAlgorithms: [ "ES256", "ES384" ] + proofLifetime: "00:02:00" + allowedClockSkew: "00:00:30" + replayWindow: "00:05:00" + nonce: + enabled: true + ttl: "00:10:00" + maxIssuancePerMinute: 120 + store: "memory" # Set to "redis" for multi-node Authority deployments. + requiredAudiences: + - "signer" + - "attestor" + # redisConnectionString: "redis://authority-redis:6379?ssl=false" + mtls: + enabled: false + requireChainValidation: true + rotationGrace: "00:15:00" + enforceForAudiences: + - "signer" + allowedSanTypes: + - "dns" + - "uri" + allowedCertificateAuthorities: [ ] + allowedSubjectPatterns: [ ] diff --git a/ops/devops/TASKS.md b/ops/devops/TASKS.md index 924b1e63..1f8ea7a9 100644 --- a/ops/devops/TASKS.md +++ b/ops/devops/TASKS.md @@ -3,12 +3,16 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| | DEVOPS-HELM-09-001 | DONE | DevOps Guild | SCANNER-WEB-09-101 | Create Helm/Compose environment profiles (dev, staging, airgap) with deterministic digests. | Profiles committed under `deploy/`; docs updated; CI smoke deploy passes. | -| DEVOPS-SCANNER-09-204 | TODO | DevOps Guild, Scanner WebService Guild | SCANNER-EVENTS-15-201 | Surface `SCANNER__EVENTS__*` environment variables across docker-compose (dev/stage/airgap) and Helm values, defaulting to share the Redis queue DSN. | Compose/Helm configs ship enabled Redis event publishing with documented overrides; lint jobs updated; docs cross-link to new knobs. | -| DEVOPS-SCANNER-09-205 | TODO | DevOps Guild, Notify Guild | DEVOPS-SCANNER-09-204 | Add Notify smoke stage that tails the Redis stream and asserts `scanner.report.ready`/`scanner.scan.completed` reach Notify WebService in staging. | CI job reads Redis stream during scanner smoke deploy, confirms Notify ingestion via API, alerts on failure. | +| DEVOPS-SCANNER-09-204 | DONE (2025-10-21) | DevOps Guild, Scanner WebService Guild | SCANNER-EVENTS-15-201 | Surface `SCANNER__EVENTS__*` environment variables across docker-compose (dev/stage/airgap) and Helm values, defaulting to share the Redis queue DSN. | Compose/Helm configs ship enabled Redis event publishing with documented overrides; lint jobs updated; docs cross-link to new knobs. | +| DEVOPS-SCANNER-09-205 | DONE (2025-10-21) | DevOps Guild, Notify Guild | DEVOPS-SCANNER-09-204 | Add Notify smoke stage that tails the Redis stream and asserts `scanner.report.ready`/`scanner.scan.completed` reach Notify WebService in staging. | CI job reads Redis stream during scanner smoke deploy, confirms Notify ingestion via API, alerts on failure. | | DEVOPS-PERF-10-001 | DONE | DevOps Guild | BENCH-SCANNER-10-001 | Add perf smoke job (SBOM compose <5 s target) to CI. | CI job runs sample build verifying <5 s; alerts configured. | | DEVOPS-PERF-10-002 | TODO | DevOps Guild | BENCH-SCANNER-10-002 | Publish analyzer bench metrics to Grafana/perf workbook and alarm on ≥20 % regressions. | CI exports JSON for dashboards; Grafana panel wired; Ops on-call doc updated with alert hook. | | DEVOPS-REL-14-001 | TODO | DevOps Guild | SIGNER-API-11-101, ATTESTOR-API-11-201 | Deterministic build/release pipeline with SBOM/provenance, signing, manifest generation. | CI pipeline produces signed images + SBOM/attestations, manifests published with verified hashes, docs updated. | -| DEVOPS-REL-17-002 | TODO | DevOps Guild | DEVOPS-REL-14-001, SCANNER-EMIT-17-701 | Persist stripped-debug artifacts organised by GNU build-id and bundle them into release/offline kits with checksum manifests. | CI job writes `.debug` files under `artifacts/debug/.build-id/`, manifest + checksums published, offline kit includes cache, smoke job proves symbol lookup via build-id. | -| DEVOPS-MIRROR-08-001 | DONE (2025-10-19) | DevOps Guild | DEVOPS-REL-14-001 | Stand up managed mirror profiles for `*.stella-ops.org` (Concelier/Excititor), including Helm/Compose overlays, multi-tenant secrets, CDN caching, and sync documentation. | Infra overlays committed, CI smoke deploy hits mirror endpoints, runbooks published for downstream sync and quota management. | +| DEVOPS-REL-17-002 | TODO | DevOps Guild | DEVOPS-REL-14-001, SCANNER-EMIT-17-701 | Persist stripped-debug artifacts organised by GNU build-id and bundle them into release/offline kits with checksum manifests. | CI job writes `.debug` files under `artifacts/debug/.build-id/`, manifest + checksums published, offline kit includes cache, smoke job proves symbol lookup via build-id. | +| DEVOPS-MIRROR-08-001 | DONE (2025-10-19) | DevOps Guild | DEVOPS-REL-14-001 | Stand up managed mirror profiles for `*.stella-ops.org` (Concelier/Excititor), including Helm/Compose overlays, multi-tenant secrets, CDN caching, and sync documentation. | Infra overlays committed, CI smoke deploy hits mirror endpoints, runbooks published for downstream sync and quota management. | | DEVOPS-SEC-10-301 | DONE (2025-10-20) | DevOps Guild | Wave 0A complete | Address NU1902/NU1903 advisories for `MongoDB.Driver` 2.12.0 and `SharpCompress` 0.23.0 surfaced during scanner cache and worker test runs. | Dependencies bumped to patched releases, audit logs free of NU1902/NU1903 warnings, regression tests green, change log documents upgrade guidance. | +| DEVOPS-LAUNCH-18-100 | TODO | DevOps Guild | - | Finalise production environment footprint (clusters, secrets, network overlays) for full-platform go-live. | IaC/compose overlays committed, secrets placeholders documented, dry-run deploy succeeds in staging. | +| DEVOPS-LAUNCH-18-900 | TODO | DevOps Guild, Module Leads | Wave 0 completion | Collect “full implementation” sign-off from module owners and consolidate launch readiness checklist. | Sign-off record stored under `docs/ops/launch-readiness.md`; outstanding gaps triaged; checklist approved. | +| DEVOPS-LAUNCH-18-001 | TODO | DevOps Guild | DEVOPS-LAUNCH-18-100, DEVOPS-LAUNCH-18-900 | Production launch cutover rehearsal and runbook publication. | `docs/ops/launch-cutover.md` drafted, rehearsal executed with rollback drill, approvals captured. | > Remark (2025-10-20): Repacked `Mongo2Go` local feed to require MongoDB.Driver 3.5.0 + SharpCompress 0.41.0; cache regression tests green and NU1902/NU1903 suppressed. +> Remark (2025-10-21): Compose/Helm profiles now surface `SCANNER__EVENTS__*` toggles with docs pointing at new `.env` placeholders. diff --git a/plugins/notify/email/notify-plugin.json b/plugins/notify/email/notify-plugin.json new file mode 100644 index 00000000..56407f5f --- /dev/null +++ b/plugins/notify/email/notify-plugin.json @@ -0,0 +1,18 @@ +{ + "schemaVersion": "1.0", + "id": "stellaops.notify.connector.email", + "displayName": "StellaOps Email Notify Connector", + "version": "0.1.0-alpha", + "requiresRestart": true, + "entryPoint": { + "type": "dotnet", + "assembly": "StellaOps.Notify.Connectors.Email.dll" + }, + "capabilities": [ + "notify-connector", + "email" + ], + "metadata": { + "org.stellaops.notify.channel.type": "email" + } +} diff --git a/plugins/notify/slack/notify-plugin.json b/plugins/notify/slack/notify-plugin.json new file mode 100644 index 00000000..95fb1dfb --- /dev/null +++ b/plugins/notify/slack/notify-plugin.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": "1.0", + "id": "stellaops.notify.connector.slack", + "displayName": "StellaOps Slack Notify Connector", + "version": "0.1.0-alpha", + "requiresRestart": true, + "entryPoint": { + "type": "dotnet", + "assembly": "StellaOps.Notify.Connectors.Slack.dll" + }, + "capabilities": [ + "notify-connector", + "slack" + ], + "metadata": { + "org.stellaops.notify.channel.type": "slack", + "org.stellaops.notify.connector.requiredScopes": "chat:write,chat:write.public" + } +} diff --git a/plugins/notify/teams/notify-plugin.json b/plugins/notify/teams/notify-plugin.json new file mode 100644 index 00000000..78239596 --- /dev/null +++ b/plugins/notify/teams/notify-plugin.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": "1.0", + "id": "stellaops.notify.connector.teams", + "displayName": "StellaOps Teams Notify Connector", + "version": "0.1.0-alpha", + "requiresRestart": true, + "entryPoint": { + "type": "dotnet", + "assembly": "StellaOps.Notify.Connectors.Teams.dll" + }, + "capabilities": [ + "notify-connector", + "teams" + ], + "metadata": { + "org.stellaops.notify.channel.type": "teams", + "org.stellaops.notify.connector.cardVersion": "1.5" + } +} diff --git a/plugins/notify/webhook/notify-plugin.json b/plugins/notify/webhook/notify-plugin.json new file mode 100644 index 00000000..32b4ead7 --- /dev/null +++ b/plugins/notify/webhook/notify-plugin.json @@ -0,0 +1,18 @@ +{ + "schemaVersion": "1.0", + "id": "stellaops.notify.connector.webhook", + "displayName": "StellaOps Webhook Notify Connector", + "version": "0.1.0-alpha", + "requiresRestart": true, + "entryPoint": { + "type": "dotnet", + "assembly": "StellaOps.Notify.Connectors.Webhook.dll" + }, + "capabilities": [ + "notify-connector", + "webhook" + ], + "metadata": { + "org.stellaops.notify.channel.type": "webhook" + } +} diff --git a/samples/runtime/README.md b/samples/runtime/README.md index 487205d4..0d0dd175 100644 --- a/samples/runtime/README.md +++ b/samples/runtime/README.md @@ -1,6 +1,7 @@ # Runtime Fixtures -Supporting filesystem snippets consumed by analyzer microbenchmarks and integration tests. They are intentionally lightweight yet deterministic so they can be committed to the repository without bloating history. - -- `npm-monorepo/` – trimmed `node_modules/` tree for workspace-style Node.js projects. -- `python-venv/` – selected `site-packages/` entries highlighting `*.dist-info` metadata. +Supporting filesystem snippets consumed by analyzer microbenchmarks and integration tests. They are intentionally lightweight yet deterministic so they can be committed to the repository without bloating history. + +- `npm-monorepo/` – trimmed `node_modules/` tree for workspace-style Node.js projects. +- `java-demo/` – minimal `libs/` directory with a tiny Maven-style JAR for the Java analyzer. +- `python-venv/` – selected `site-packages/` entries highlighting `*.dist-info` metadata. diff --git a/samples/runtime/java-demo/README.md b/samples/runtime/java-demo/README.md new file mode 100644 index 00000000..a9315813 --- /dev/null +++ b/samples/runtime/java-demo/README.md @@ -0,0 +1,5 @@ +# Java Demo Fixture + +Minimal archive tree that exercises the Java language analyzer during microbenchmarks. The `libs/demo.jar` +artefact ships `META-INF/MANIFEST.MF` and `META-INF/maven/com.example/demo/pom.properties` entries so the +analyzer can extract Maven coordinates and manifest metadata without pulling in large third-party jars. diff --git a/samples/runtime/java-demo/libs/demo.jar b/samples/runtime/java-demo/libs/demo.jar new file mode 100644 index 0000000000000000000000000000000000000000..a0ac7d973138e9a80aa35c1703f4a6317d03ba38 GIT binary patch literal 481 zcmWIWW@Zs#U|`^2=r9P4u0DS+AqdE;0Ai4+uWN{-uBV@yzOSR7r<-eVh@P+8XWuiY zeY|z`F7kToYMncCeshq)72^k=&gh=-JnMT_+f#?J>t$QYv?oiVGn`UOCNf};F_)S{Bi)MDO~j$DTo1XwTpos+bnTJi0z zSOu1<&cOY>0h zuiZkeTA%(p<*ZVRrQc+)EID-k`={M3!ntrue true true + $(SolutionDir)plugins\notify + $([System.IO.Path]::GetFullPath('$(MSBuildThisFileDirectory)..\plugins\notify\')) + true + false $([System.IO.Path]::GetFullPath('$(MSBuildThisFileDirectory)..\plugins\scanner\buildx\')) true $([System.IO.Path]::GetFullPath('$(MSBuildThisFileDirectory)..\plugins\scanner\analyzers\os\')) diff --git a/src/Directory.Build.targets b/src/Directory.Build.targets index 14b78768..43e13e17 100644 --- a/src/Directory.Build.targets +++ b/src/Directory.Build.targets @@ -31,6 +31,24 @@ + + + $([System.String]::Copy('$(MSBuildProjectName)').Replace('StellaOps.Notify.Connectors.', '').ToLowerInvariant()) + $(NotifyPluginOutputRoot)\$(NotifyPluginDirectoryName) + + + + + + + + + + + + + + $(ScannerBuildxPluginOutputRoot)\$(MSBuildProjectName) diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardPluginRegistrarTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardPluginRegistrarTests.cs index edaad0a9..1b365ddf 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardPluginRegistrarTests.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardPluginRegistrarTests.cs @@ -78,7 +78,7 @@ public class StandardPluginRegistrarTests var registrar = new StandardPluginRegistrar(); registrar.Register(new AuthorityPluginRegistrationContext(services, pluginContext, configuration)); - var provider = services.BuildServiceProvider(); + using var provider = services.BuildServiceProvider(); var hostedServices = provider.GetServices(); foreach (var hosted in hostedServices) { @@ -88,7 +88,8 @@ public class StandardPluginRegistrarTests } } - var plugin = provider.GetRequiredService(); + using var scope = provider.CreateScope(); + var plugin = scope.ServiceProvider.GetRequiredService(); Assert.Equal("standard", plugin.Type); Assert.True(plugin.Capabilities.SupportsPassword); Assert.False(plugin.Capabilities.SupportsMfa); @@ -138,7 +139,8 @@ public class StandardPluginRegistrarTests registrar.Register(new AuthorityPluginRegistrationContext(services, pluginContext, configuration)); using var provider = services.BuildServiceProvider(); - _ = provider.GetRequiredService(); + using var scope = provider.CreateScope(); + _ = scope.ServiceProvider.GetRequiredService(); Assert.Contains(loggerProvider.Entries, entry => entry.Level == LogLevel.Warning && @@ -176,7 +178,8 @@ public class StandardPluginRegistrarTests registrar.Register(new AuthorityPluginRegistrationContext(services, pluginContext, configuration)); using var provider = services.BuildServiceProvider(); - var plugin = provider.GetRequiredService(); + using var scope = provider.CreateScope(); + var plugin = scope.ServiceProvider.GetRequiredService(); Assert.True(plugin.Capabilities.SupportsPassword); } @@ -215,7 +218,8 @@ public class StandardPluginRegistrarTests registrar.Register(new AuthorityPluginRegistrationContext(services, pluginContext, configuration)); using var provider = services.BuildServiceProvider(); - Assert.Throws(() => provider.GetRequiredService()); + using var scope = provider.CreateScope(); + Assert.Throws(() => scope.ServiceProvider.GetRequiredService()); } [Fact] diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Bootstrap/StandardPluginBootstrapper.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Bootstrap/StandardPluginBootstrapper.cs index 01608007..54d8607a 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Bootstrap/StandardPluginBootstrapper.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Bootstrap/StandardPluginBootstrapper.cs @@ -1,5 +1,6 @@ using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -10,24 +11,25 @@ namespace StellaOps.Authority.Plugin.Standard.Bootstrap; internal sealed class StandardPluginBootstrapper : IHostedService { private readonly string pluginName; - private readonly IOptionsMonitor optionsMonitor; - private readonly StandardUserCredentialStore credentialStore; + private readonly IServiceScopeFactory scopeFactory; private readonly ILogger logger; public StandardPluginBootstrapper( string pluginName, - IOptionsMonitor optionsMonitor, - StandardUserCredentialStore credentialStore, + IServiceScopeFactory scopeFactory, ILogger logger) { this.pluginName = pluginName; - this.optionsMonitor = optionsMonitor; - this.credentialStore = credentialStore; + this.scopeFactory = scopeFactory; this.logger = logger; } public async Task StartAsync(CancellationToken cancellationToken) { + using var scope = scopeFactory.CreateScope(); + var optionsMonitor = scope.ServiceProvider.GetRequiredService>(); + var credentialStore = scope.ServiceProvider.GetRequiredService(); + var options = optionsMonitor.Get(pluginName); if (options.BootstrapUser is null || !options.BootstrapUser.IsConfigured) { diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StandardPluginRegistrar.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StandardPluginRegistrar.cs index d7857413..eeb9a7c0 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StandardPluginRegistrar.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StandardPluginRegistrar.cs @@ -43,7 +43,7 @@ internal sealed class StandardPluginRegistrar : IAuthorityPluginRegistrar }) .ValidateOnStart(); - context.Services.AddSingleton(sp => + context.Services.AddScoped(sp => { var database = sp.GetRequiredService(); var optionsMonitor = sp.GetRequiredService>(); @@ -79,7 +79,7 @@ internal sealed class StandardPluginRegistrar : IAuthorityPluginRegistrar loggerFactory.CreateLogger()); }); - context.Services.AddSingleton(sp => + context.Services.AddScoped(sp => { var clientStore = sp.GetRequiredService(); var revocationStore = sp.GetRequiredService(); @@ -87,7 +87,7 @@ internal sealed class StandardPluginRegistrar : IAuthorityPluginRegistrar return new StandardClientProvisioningStore(pluginName, clientStore, revocationStore, timeProvider); }); - context.Services.AddSingleton(sp => + context.Services.AddScoped(sp => { var store = sp.GetRequiredService(); var clientProvisioningStore = sp.GetRequiredService(); @@ -100,14 +100,13 @@ internal sealed class StandardPluginRegistrar : IAuthorityPluginRegistrar loggerFactory.CreateLogger()); }); - context.Services.AddSingleton(sp => + context.Services.AddScoped(sp => sp.GetRequiredService()); context.Services.AddSingleton(sp => new StandardPluginBootstrapper( pluginName, - sp.GetRequiredService>(), - sp.GetRequiredService(), + sp.GetRequiredService(), sp.GetRequiredService>())); } } diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md index a4caeba9..4b7fb53b 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md @@ -5,10 +5,10 @@ | PLG6.DOC | DONE (2025-10-11) | BE-Auth Plugin, Docs Guild | PLG1–PLG5 | Final polish + diagrams for plugin developer guide (AUTHPLUG-DOCS-01-001). | Docs team delivers copy-edit + exported diagrams; PR merged. | | SEC1.PLG | DONE (2025-10-11) | Security Guild, BE-Auth Plugin | SEC1.A (StellaOps.Cryptography) | Swap Standard plugin hashing to Argon2id via `StellaOps.Cryptography` abstractions; keep PBKDF2 verification for legacy. | ✅ `StandardUserCredentialStore` uses `ICryptoProvider` to hash/check; ✅ Transparent rehash on success; ✅ Unit tests cover tamper + legacy rehash. | | SEC1.OPT | DONE (2025-10-11) | Security Guild | SEC1.PLG | Expose password hashing knobs in `StandardPluginOptions` (`memoryKiB`, `iterations`, `parallelism`, `algorithm`) with validation. | ✅ Options bound from YAML; ✅ Invalid configs throw; ✅ Docs include tuning guidance. | -| SEC2.PLG | DOING (2025-10-14) | Security Guild, Storage Guild | SEC2.A (audit contract) | Emit audit events from password verification outcomes and persist via `IAuthorityLoginAttemptStore`.
⏳ Awaiting AUTH-DPOP-11-001 / AUTH-MTLS-11-002 / PLUGIN-DI-08-001 completion to unlock Wave 0B verification paths. | ✅ Serilog events enriched with subject/client/IP/outcome; ✅ Mongo records written per attempt; ✅ Tests assert success/lockout/failure cases. | -| SEC3.PLG | DOING (2025-10-14) | Security Guild, BE-Auth Plugin | CORE8, SEC3.A (rate limiter) | Ensure lockout responses and rate-limit metadata flow through plugin logs/events (include retry-after).
⏳ Pending AUTH-DPOP-11-001 / AUTH-MTLS-11-002 / PLUGIN-DI-08-001 so limiter telemetry contract matches final authority surface. | ✅ Audit record includes retry-after; ✅ Tests confirm lockout + limiter interplay. | +| SEC2.PLG | BLOCKED (2025-10-21) | Security Guild, Storage Guild | SEC2.A (audit contract) | Emit audit events from password verification outcomes and persist via `IAuthorityLoginAttemptStore`.
⛔ Waiting on AUTH-DPOP-11-001 / AUTH-MTLS-11-002 / PLUGIN-DI-08-001 to stabilise Authority auth surfaces before final verification + publish. | ✅ Serilog events enriched with subject/client/IP/outcome; ✅ Mongo records written per attempt; ✅ Tests assert success/lockout/failure cases. | +| SEC3.PLG | BLOCKED (2025-10-21) | Security Guild, BE-Auth Plugin | CORE8, SEC3.A (rate limiter) | Ensure lockout responses and rate-limit metadata flow through plugin logs/events (include retry-after).
⛔ Pending AUTH-DPOP-11-001 / AUTH-MTLS-11-002 / PLUGIN-DI-08-001 so limiter telemetry contract matches final authority surface. | ✅ Audit record includes retry-after; ✅ Tests confirm lockout + limiter interplay. | | SEC4.PLG | DONE (2025-10-12) | Security Guild | SEC4.A (revocation schema) | Provide plugin hooks so revoked users/clients write reasons for revocation bundle export. | ✅ Revocation exporter consumes plugin data; ✅ Tests cover revoked user/client output. | -| SEC5.PLG | DOING (2025-10-14) | Security Guild | SEC5.A (threat model) | Address plugin-specific mitigations (bootstrap user handling, password policy docs) in threat model backlog.
⏳ Final documentation depends on AUTH-DPOP-11-001 / AUTH-MTLS-11-002 / PLUGIN-DI-08-001 outcomes. | ✅ Threat model lists plugin attack surfaces; ✅ Mitigation items filed. | +| SEC5.PLG | BLOCKED (2025-10-21) | Security Guild | SEC5.A (threat model) | Address plugin-specific mitigations (bootstrap user handling, password policy docs) in threat model backlog.
⛔ Final documentation depends on AUTH-DPOP-11-001 / AUTH-MTLS-11-002 / PLUGIN-DI-08-001 outcomes. | ✅ Threat model lists plugin attack surfaces; ✅ Mitigation items filed. | | PLG4-6.CAPABILITIES | BLOCKED (2025-10-12) | BE-Auth Plugin, Docs Guild | PLG1–PLG3 | Finalise capability metadata exposure, config validation, and developer guide updates; remaining action is Docs polish/diagram export. | ✅ Capability metadata + validation merged; ✅ Plugin guide updated with final copy & diagrams; ✅ Release notes mention new toggles.
⛔ Blocked awaiting Authority rate-limiter stream (CORE8/SEC3) to resume so doc updates reflect final limiter behaviour. | | PLG7.RFC | REVIEW | BE-Auth Plugin, Security Guild | PLG4 | Socialize LDAP plugin RFC (`docs/rfcs/authority-plugin-ldap.md`) and capture guild feedback. | ✅ Guild review sign-off recorded; ✅ Follow-up issues filed in module boards. | | PLG6.DIAGRAM | TODO | Docs Guild | PLG6.DOC | Export final sequence/component diagrams for the developer guide and add offline-friendly assets under `docs/assets/authority`. | ✅ Mermaid sources committed; ✅ Rendered SVG/PNG linked from Section 2 + Section 9; ✅ Docs build preview shared with Plugin + Docs guilds. | diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthorityPluginContracts.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthorityPluginContracts.cs index 6eb184ee..6ad7a6ef 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthorityPluginContracts.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthorityPluginContracts.cs @@ -1,7 +1,10 @@ using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; namespace StellaOps.Authority.Plugins.Abstractions; @@ -95,24 +98,24 @@ public interface IAuthorityPluginRegistry public interface IAuthorityIdentityProviderRegistry { /// - /// Gets all registered identity provider plugins keyed by logical name. + /// Gets metadata for all registered identity provider plugins. /// - IReadOnlyCollection Providers { get; } + IReadOnlyCollection Providers { get; } /// - /// Gets identity providers that advertise password support. + /// Gets metadata for identity providers that advertise password support. /// - IReadOnlyCollection PasswordProviders { get; } + IReadOnlyCollection PasswordProviders { get; } /// - /// Gets identity providers that advertise multi-factor authentication support. + /// Gets metadata for identity providers that advertise multi-factor authentication support. /// - IReadOnlyCollection MfaProviders { get; } + IReadOnlyCollection MfaProviders { get; } /// - /// Gets identity providers that advertise client provisioning support. + /// Gets metadata for identity providers that advertise client provisioning support. /// - IReadOnlyCollection ClientProvisioningProviders { get; } + IReadOnlyCollection ClientProvisioningProviders { get; } /// /// Aggregate capability flags across all registered providers. @@ -120,20 +123,89 @@ public interface IAuthorityIdentityProviderRegistry AuthorityIdentityProviderCapabilities AggregateCapabilities { get; } /// - /// Attempts to resolve an identity provider by name. + /// Attempts to resolve identity provider metadata by name. /// - bool TryGet(string name, [NotNullWhen(true)] out IIdentityProviderPlugin? provider); + bool TryGet(string name, [NotNullWhen(true)] out AuthorityIdentityProviderMetadata? metadata); /// - /// Resolves an identity provider by name or throws when not found. + /// Resolves identity provider metadata by name or throws when not found. /// - IIdentityProviderPlugin GetRequired(string name) + AuthorityIdentityProviderMetadata GetRequired(string name) { - if (TryGet(name, out var provider)) + if (TryGet(name, out var metadata)) { - return provider; + return metadata; } throw new KeyNotFoundException($"Identity provider plugin '{name}' is not registered."); } + + /// + /// Acquires a scoped handle to the specified identity provider. + /// + /// Logical provider name. + /// Cancellation token. + /// Handle managing the provider instance lifetime. + ValueTask AcquireAsync(string name, CancellationToken cancellationToken); +} + +/// +/// Immutable metadata describing a registered identity provider. +/// +/// Logical provider name from the manifest. +/// Provider type identifier. +/// Capability flags advertised by the provider. +public sealed record AuthorityIdentityProviderMetadata( + string Name, + string Type, + AuthorityIdentityProviderCapabilities Capabilities); + +/// +/// Represents a scoped identity provider instance and manages its disposal. +/// +public sealed class AuthorityIdentityProviderHandle : IAsyncDisposable, IDisposable +{ + private readonly AsyncServiceScope scope; + private bool disposed; + + public AuthorityIdentityProviderHandle(AsyncServiceScope scope, AuthorityIdentityProviderMetadata metadata, IIdentityProviderPlugin provider) + { + this.scope = scope; + Metadata = metadata ?? throw new ArgumentNullException(nameof(metadata)); + Provider = provider ?? throw new ArgumentNullException(nameof(provider)); + } + + /// + /// Gets the metadata associated with the provider instance. + /// + public AuthorityIdentityProviderMetadata Metadata { get; } + + /// + /// Gets the active provider instance. + /// + public IIdentityProviderPlugin Provider { get; } + + /// + public void Dispose() + { + if (disposed) + { + return; + } + + disposed = true; + scope.Dispose(); + } + + /// + public async ValueTask DisposeAsync() + { + if (disposed) + { + return; + } + + disposed = true; + await scope.DisposeAsync().ConfigureAwait(false); + } } diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/Identity/AuthorityIdentityProviderRegistryTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Tests/Identity/AuthorityIdentityProviderRegistryTests.cs index a1f2a874..3f0a4034 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Tests/Identity/AuthorityIdentityProviderRegistryTests.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/Identity/AuthorityIdentityProviderRegistryTests.cs @@ -1,15 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; using StellaOps.Authority.Plugins.Abstractions; using Xunit; -using System.Linq; namespace StellaOps.Authority.Tests.Identity; public class AuthorityIdentityProviderRegistryTests { [Fact] - public void RegistryIndexesProvidersAndAggregatesCapabilities() + public async Task RegistryIndexesProvidersAndAggregatesCapabilities() { var providers = new[] { @@ -17,21 +22,25 @@ public class AuthorityIdentityProviderRegistryTests CreateProvider("sso", type: "saml", supportsPassword: false, supportsMfa: true, supportsClientProvisioning: true) }; - var registry = new AuthorityIdentityProviderRegistry(providers, NullLogger.Instance); + using var serviceProvider = BuildServiceProvider(providers); + var registry = new AuthorityIdentityProviderRegistry(serviceProvider, NullLogger.Instance); Assert.Equal(2, registry.Providers.Count); Assert.True(registry.TryGet("standard", out var standard)); - Assert.Same(providers[0], standard); + Assert.Equal("standard", standard!.Name); Assert.Single(registry.PasswordProviders); Assert.Single(registry.MfaProviders); Assert.Single(registry.ClientProvisioningProviders); Assert.True(registry.AggregateCapabilities.SupportsPassword); Assert.True(registry.AggregateCapabilities.SupportsMfa); Assert.True(registry.AggregateCapabilities.SupportsClientProvisioning); + + await using var handle = await registry.AcquireAsync("standard", default); + Assert.Same(providers[0], handle.Provider); } [Fact] - public void RegistryIgnoresDuplicateNames() + public async Task RegistryIgnoresDuplicateNames() { var duplicate = CreateProvider("standard", "ldap", supportsPassword: true, supportsMfa: false, supportsClientProvisioning: false); var providers = new[] @@ -40,12 +49,56 @@ public class AuthorityIdentityProviderRegistryTests duplicate }; - var registry = new AuthorityIdentityProviderRegistry(providers, NullLogger.Instance); + using var serviceProvider = BuildServiceProvider(providers); + var registry = new AuthorityIdentityProviderRegistry(serviceProvider, NullLogger.Instance); Assert.Single(registry.Providers); - Assert.Same(providers[0], registry.Providers.First()); + Assert.Equal("standard", registry.Providers.First().Name); Assert.True(registry.TryGet("standard", out var provider)); - Assert.Same(providers[0], provider); + await using var handle = await registry.AcquireAsync("standard", default); + Assert.Same(providers[0], handle.Provider); + Assert.Equal("standard", provider!.Name); + } + + [Fact] + public async Task AcquireAsync_ReturnsScopedProviderInstances() + { + var configuration = new ConfigurationBuilder().Build(); + var manifest = new AuthorityPluginManifest( + "scoped", + "scoped", + true, + AssemblyName: null, + AssemblyPath: null, + Capabilities: new[] { AuthorityPluginCapabilities.Password }, + Metadata: new Dictionary(StringComparer.OrdinalIgnoreCase), + ConfigPath: string.Empty); + + var context = new AuthorityPluginContext(manifest, configuration); + + var services = new ServiceCollection(); + services.AddScoped(_ => new ScopedIdentityProviderPlugin(context)); + + using var serviceProvider = services.BuildServiceProvider(); + var registry = new AuthorityIdentityProviderRegistry(serviceProvider, NullLogger.Instance); + + await using var first = await registry.AcquireAsync("scoped", default); + await using var second = await registry.AcquireAsync("scoped", default); + + var firstPlugin = Assert.IsType(first.Provider); + var secondPlugin = Assert.IsType(second.Provider); + Assert.NotEqual(firstPlugin.InstanceId, secondPlugin.InstanceId); + } + + private static ServiceProvider BuildServiceProvider(IEnumerable providers) + { + var services = new ServiceCollection(); + foreach (var provider in providers) + { + services.AddSingleton(provider); + } + + return services.BuildServiceProvider(); } private static IIdentityProviderPlugin CreateProvider( @@ -122,4 +175,36 @@ public class AuthorityIdentityProviderRegistryTests public ValueTask CheckHealthAsync(CancellationToken cancellationToken) => ValueTask.FromResult(AuthorityPluginHealthResult.Healthy()); } + + private sealed class ScopedIdentityProviderPlugin : IIdentityProviderPlugin + { + public ScopedIdentityProviderPlugin(AuthorityPluginContext context) + { + Context = context; + InstanceId = Guid.NewGuid(); + Capabilities = new AuthorityIdentityProviderCapabilities( + SupportsPassword: true, + SupportsMfa: false, + SupportsClientProvisioning: false); + } + + public Guid InstanceId { get; } + + public string Name => Context.Manifest.Name; + + public string Type => Context.Manifest.Type; + + public AuthorityPluginContext Context { get; } + + public IUserCredentialStore Credentials => throw new NotImplementedException(); + + public IClaimsEnricher ClaimsEnricher => throw new NotImplementedException(); + + public IClientProvisioningStore? ClientProvisioning => null; + + public AuthorityIdentityProviderCapabilities Capabilities { get; } + + public ValueTask CheckHealthAsync(CancellationToken cancellationToken) + => ValueTask.FromResult(AuthorityPluginHealthResult.Healthy()); + } } diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/Identity/AuthorityIdentityProviderSelectorTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Tests/Identity/AuthorityIdentityProviderSelectorTests.cs index 83decc23..2c15d020 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Tests/Identity/AuthorityIdentityProviderSelectorTests.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/Identity/AuthorityIdentityProviderSelectorTests.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; using OpenIddict.Abstractions; using StellaOps.Authority.OpenIddict; using StellaOps.Authority.Plugins.Abstractions; @@ -67,8 +68,14 @@ public class AuthorityIdentityProviderSelectorTests private static AuthorityIdentityProviderRegistry CreateRegistry(IEnumerable passwordProviders) { - var providers = passwordProviders.ToList(); - return new AuthorityIdentityProviderRegistry(providers, Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + var services = new ServiceCollection(); + foreach (var provider in passwordProviders) + { + services.AddSingleton(provider); + } + + var serviceProvider = services.BuildServiceProvider(); + return new AuthorityIdentityProviderRegistry(serviceProvider, Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); } private static IIdentityProviderPlugin CreateProvider(string name, bool supportsPassword) diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/ClientCredentialsAndTokenHandlersTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/ClientCredentialsAndTokenHandlersTests.cs index 03dd02f9..d36e0f69 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/ClientCredentialsAndTokenHandlersTests.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/ClientCredentialsAndTokenHandlersTests.cs @@ -10,6 +10,7 @@ using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Http.Extensions; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; using Microsoft.Extensions.Primitives; using Microsoft.IdentityModel.Tokens; @@ -350,6 +351,7 @@ public class ClientCredentialsHandlersTests }; options.Security.SenderConstraints.Mtls.Enabled = true; options.Security.SenderConstraints.Mtls.RequireChainValidation = false; + options.Security.SenderConstraints.Mtls.AllowedSanTypes.Clear(); options.Signing.ActiveKeyId = "test-key"; options.Signing.KeyPath = "/tmp/test-key.pem"; options.Storage.ConnectionString = "mongodb://localhost/test"; @@ -394,7 +396,7 @@ public class ClientCredentialsHandlersTests await handler.HandleAsync(context); - Assert.False(context.IsRejected); + Assert.False(context.IsRejected, context.ErrorDescription ?? context.Error); Assert.Equal(AuthoritySenderConstraintKinds.Mtls, context.Transaction.Properties[AuthorityOpenIddictConstants.SenderConstraintProperty]); var expectedBase64 = Base64UrlEncoder.Encode(certificate.GetCertHash(HashAlgorithmName.SHA256)); @@ -581,7 +583,7 @@ public class TokenValidationHandlersTests descriptor: CreateDescriptor(clientDocument), user: userDescriptor); - var registry = new AuthorityIdentityProviderRegistry(new[] { plugin }, NullLogger.Instance); + var registry = CreateRegistryFromPlugins(plugin); var metadataAccessorSuccess = new TestRateLimiterMetadataAccessor(); var auditSinkSuccess = new TestAuthEventSink(); @@ -1073,7 +1075,7 @@ internal static class TestHelpers descriptor: clientDescriptor, user: null); - return new AuthorityIdentityProviderRegistry(new[] { plugin }, NullLogger.Instance); + return CreateRegistryFromPlugins(plugin); } public static TestIdentityProviderPlugin CreatePlugin( @@ -1109,6 +1111,19 @@ internal static class TestHelpers SupportsClientProvisioning: supportsClientProvisioning)); } + public static AuthorityIdentityProviderRegistry CreateRegistryFromPlugins(params IIdentityProviderPlugin[] plugins) + { + var services = new ServiceCollection(); + services.AddLogging(); + foreach (var plugin in plugins) + { + services.AddSingleton(plugin); + } + + var provider = services.BuildServiceProvider(); + return new AuthorityIdentityProviderRegistry(provider, NullLogger.Instance); + } + public static OpenIddictServerTransaction CreateTokenTransaction(string clientId, string? secret, string? scope) { var request = new OpenIddictRequest diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/PasswordGrantHandlersTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/PasswordGrantHandlersTests.cs index b0f502c2..28d3e8ef 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/PasswordGrantHandlersTests.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/PasswordGrantHandlersTests.cs @@ -7,6 +7,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.DependencyInjection; using OpenIddict.Abstractions; using OpenIddict.Server; using OpenIddict.Server.AspNetCore; @@ -97,7 +98,13 @@ public class PasswordGrantHandlersTests private static AuthorityIdentityProviderRegistry CreateRegistry(IUserCredentialStore store) { var plugin = new StubIdentityProviderPlugin("stub", store); - return new AuthorityIdentityProviderRegistry(new[] { plugin }, NullLogger.Instance); + + var services = new ServiceCollection(); + services.AddLogging(); + services.AddSingleton(plugin); + var provider = services.BuildServiceProvider(); + + return new AuthorityIdentityProviderRegistry(provider, NullLogger.Instance); } private static OpenIddictServerTransaction CreatePasswordTransaction(string username, string password) diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/TokenPersistenceIntegrationTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/TokenPersistenceIntegrationTests.cs index 63722c92..9d07a12a 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/TokenPersistenceIntegrationTests.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/TokenPersistenceIntegrationTests.cs @@ -131,9 +131,7 @@ public sealed class TokenPersistenceIntegrationTests descriptor, userDescriptor); - var registry = new AuthorityIdentityProviderRegistry( - new[] { plugin }, - NullLogger.Instance); + var registry = TestHelpers.CreateRegistryFromPlugins(plugin); const string revokedTokenId = "refresh-token-1"; var refreshToken = new AuthorityTokenDocument diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/Plugins/AuthorityPluginLoaderTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Tests/Plugins/AuthorityPluginLoaderTests.cs index ad9c22bc..4fd137ae 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Tests/Plugins/AuthorityPluginLoaderTests.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/Plugins/AuthorityPluginLoaderTests.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using StellaOps.Authority.Plugins; using StellaOps.Authority.Plugins.Abstractions; @@ -67,6 +68,7 @@ public class AuthorityPluginLoaderTests public void RegisterPlugins_RegistersEnabledPlugin_WhenRegistrarAvailable() { var services = new ServiceCollection(); + services.AddLogging(); var hostConfiguration = new ConfigurationBuilder().Build(); var manifest = new AuthorityPluginManifest( @@ -99,6 +101,46 @@ public class AuthorityPluginLoaderTests Assert.NotNull(provider.GetRequiredService()); } + [Fact] + public void RegisterPlugins_ActivatesRegistrarUsingDependencyInjection() + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddSingleton(TimeProvider.System); + + var hostConfiguration = new ConfigurationBuilder().Build(); + + var manifest = new AuthorityPluginManifest( + "di-test", + DiAuthorityPluginRegistrar.PluginTypeIdentifier, + true, + typeof(DiAuthorityPluginRegistrar).Assembly.GetName().Name, + typeof(DiAuthorityPluginRegistrar).Assembly.Location, + Array.Empty(), + new Dictionary(), + "di-test.yaml"); + + var pluginContext = new AuthorityPluginContext(manifest, hostConfiguration); + var descriptor = new AuthorityPluginLoader.LoadedPluginDescriptor( + typeof(DiAuthorityPluginRegistrar).Assembly, + typeof(DiAuthorityPluginRegistrar).Assembly.Location); + + var summary = AuthorityPluginLoader.RegisterPluginsCore( + services, + hostConfiguration, + new[] { pluginContext }, + new[] { descriptor }, + Array.Empty(), + NullLogger.Instance); + + Assert.Contains("di-test", summary.RegisteredPlugins); + + var provider = services.BuildServiceProvider(); + var dependent = provider.GetRequiredService(); + Assert.True(dependent.LoggerWasResolved); + Assert.True(dependent.TimeProviderResolved); + } + private sealed class TestAuthorityPluginRegistrar : IAuthorityPluginRegistrar { public const string PluginTypeIdentifier = "test-plugin"; @@ -114,4 +156,38 @@ public class AuthorityPluginLoaderTests private sealed class TestMarkerService { } + + private sealed class DiAuthorityPluginRegistrar : IAuthorityPluginRegistrar + { + public const string PluginTypeIdentifier = "test-plugin-di"; + + private readonly ILogger logger; + private readonly TimeProvider timeProvider; + + public DiAuthorityPluginRegistrar(ILogger logger, TimeProvider timeProvider) + { + this.logger = logger; + this.timeProvider = timeProvider; + } + + public string PluginType => PluginTypeIdentifier; + + public void Register(AuthorityPluginRegistrationContext context) + { + context.Services.AddSingleton(new DependentService(logger != null, timeProvider != null)); + } + } + + private sealed class DependentService + { + public DependentService(bool loggerResolved, bool timeProviderResolved) + { + LoggerWasResolved = loggerResolved; + TimeProviderResolved = timeProviderResolved; + } + + public bool LoggerWasResolved { get; } + + public bool TimeProviderResolved { get; } + } } diff --git a/src/StellaOps.Authority/StellaOps.Authority/AuthorityIdentityProviderRegistry.cs b/src/StellaOps.Authority/StellaOps.Authority/AuthorityIdentityProviderRegistry.cs index f1289283..2c9b4a06 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/AuthorityIdentityProviderRegistry.cs +++ b/src/StellaOps.Authority/StellaOps.Authority/AuthorityIdentityProviderRegistry.cs @@ -1,5 +1,9 @@ using System.Collections.ObjectModel; using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using StellaOps.Authority.Plugins.Abstractions; @@ -7,29 +11,34 @@ namespace StellaOps.Authority; internal sealed class AuthorityIdentityProviderRegistry : IAuthorityIdentityProviderRegistry { - private readonly IReadOnlyDictionary providersByName; - private readonly ReadOnlyCollection providers; - private readonly ReadOnlyCollection passwordProviders; - private readonly ReadOnlyCollection mfaProviders; - private readonly ReadOnlyCollection clientProvisioningProviders; + private readonly IServiceProvider serviceProvider; + private readonly IReadOnlyDictionary providersByName; + private readonly ReadOnlyCollection providers; + private readonly ReadOnlyCollection passwordProviders; + private readonly ReadOnlyCollection mfaProviders; + private readonly ReadOnlyCollection clientProvisioningProviders; public AuthorityIdentityProviderRegistry( - IEnumerable providerInstances, + IServiceProvider serviceProvider, ILogger logger) { + this.serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider)); logger = logger ?? throw new ArgumentNullException(nameof(logger)); + using var scope = serviceProvider.CreateScope(); + var providerInstances = scope.ServiceProvider.GetServices(); + var orderedProviders = providerInstances? .Where(static p => p is not null) .OrderBy(static p => p.Name, StringComparer.OrdinalIgnoreCase) .ToList() ?? new List(); - var uniqueProviders = new List(orderedProviders.Count); - var password = new List(); - var mfa = new List(); - var clientProvisioning = new List(); + var uniqueProviders = new List(orderedProviders.Count); + var password = new List(); + var mfa = new List(); + var clientProvisioning = new List(); - var dictionary = new Dictionary(StringComparer.OrdinalIgnoreCase); + var dictionary = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (var provider in orderedProviders) { @@ -41,7 +50,9 @@ internal sealed class AuthorityIdentityProviderRegistry : IAuthorityIdentityProv continue; } - if (!dictionary.TryAdd(provider.Name, provider)) + var metadata = new AuthorityIdentityProviderMetadata(provider.Name, provider.Type, provider.Capabilities); + + if (!dictionary.TryAdd(provider.Name, metadata)) { logger.LogWarning( "Duplicate identity provider name '{PluginName}' detected; ignoring additional registration for type '{PluginType}'.", @@ -50,29 +61,29 @@ internal sealed class AuthorityIdentityProviderRegistry : IAuthorityIdentityProv continue; } - uniqueProviders.Add(provider); + uniqueProviders.Add(metadata); - if (provider.Capabilities.SupportsPassword) + if (metadata.Capabilities.SupportsPassword) { - password.Add(provider); + password.Add(metadata); } - if (provider.Capabilities.SupportsMfa) + if (metadata.Capabilities.SupportsMfa) { - mfa.Add(provider); + mfa.Add(metadata); } - if (provider.Capabilities.SupportsClientProvisioning) + if (metadata.Capabilities.SupportsClientProvisioning) { - clientProvisioning.Add(provider); + clientProvisioning.Add(metadata); } } providersByName = dictionary; - providers = new ReadOnlyCollection(uniqueProviders); - passwordProviders = new ReadOnlyCollection(password); - mfaProviders = new ReadOnlyCollection(mfa); - clientProvisioningProviders = new ReadOnlyCollection(clientProvisioning); + providers = new ReadOnlyCollection(uniqueProviders); + passwordProviders = new ReadOnlyCollection(password); + mfaProviders = new ReadOnlyCollection(mfa); + clientProvisioningProviders = new ReadOnlyCollection(clientProvisioning); AggregateCapabilities = new AuthorityIdentityProviderCapabilities( SupportsPassword: passwordProviders.Count > 0, @@ -80,24 +91,56 @@ internal sealed class AuthorityIdentityProviderRegistry : IAuthorityIdentityProv SupportsClientProvisioning: clientProvisioningProviders.Count > 0); } - public IReadOnlyCollection Providers => providers; + public IReadOnlyCollection Providers => providers; - public IReadOnlyCollection PasswordProviders => passwordProviders; + public IReadOnlyCollection PasswordProviders => passwordProviders; - public IReadOnlyCollection MfaProviders => mfaProviders; + public IReadOnlyCollection MfaProviders => mfaProviders; - public IReadOnlyCollection ClientProvisioningProviders => clientProvisioningProviders; + public IReadOnlyCollection ClientProvisioningProviders => clientProvisioningProviders; public AuthorityIdentityProviderCapabilities AggregateCapabilities { get; } - public bool TryGet(string name, [NotNullWhen(true)] out IIdentityProviderPlugin? provider) + public bool TryGet(string name, [NotNullWhen(true)] out AuthorityIdentityProviderMetadata? metadata) { if (string.IsNullOrWhiteSpace(name)) { - provider = null; + metadata = null; return false; } - return providersByName.TryGetValue(name, out provider); + return providersByName.TryGetValue(name, out metadata); + } + + public async ValueTask AcquireAsync(string name, CancellationToken cancellationToken) + { + if (!providersByName.TryGetValue(name, out var metadata)) + { + throw new KeyNotFoundException($"Identity provider plugin '{name}' is not registered."); + } + + cancellationToken.ThrowIfCancellationRequested(); + + var scope = serviceProvider.CreateAsyncScope(); + try + { + var provider = scope.ServiceProvider + .GetServices() + .FirstOrDefault(p => string.Equals(p.Name, metadata.Name, StringComparison.OrdinalIgnoreCase)); + + if (provider is null) + { + await scope.DisposeAsync().ConfigureAwait(false); + throw new InvalidOperationException($"Identity provider plugin '{metadata.Name}' could not be resolved."); + } + + cancellationToken.ThrowIfCancellationRequested(); + return new AuthorityIdentityProviderHandle(scope, metadata, provider); + } + catch + { + await scope.DisposeAsync().ConfigureAwait(false); + throw; + } } } diff --git a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/AuthorityIdentityProviderSelector.cs b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/AuthorityIdentityProviderSelector.cs index 95091a6f..fb5d40f3 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/AuthorityIdentityProviderSelector.cs +++ b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/AuthorityIdentityProviderSelector.cs @@ -1,3 +1,4 @@ +using System.Linq; using OpenIddict.Abstractions; using StellaOps.Authority.Plugins.Abstractions; @@ -50,11 +51,11 @@ internal static class AuthorityIdentityProviderSelector internal sealed record ProviderSelectionResult( bool Succeeded, - IIdentityProviderPlugin? Provider, + AuthorityIdentityProviderMetadata? Provider, string? Error, string? Description) { - public static ProviderSelectionResult Success(IIdentityProviderPlugin provider) + public static ProviderSelectionResult Success(AuthorityIdentityProviderMetadata provider) => new(true, provider, null, null); public static ProviderSelectionResult Failure(string error, string description) diff --git a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/ClientCredentialsHandlers.cs b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/ClientCredentialsHandlers.cs index 92c5acf5..e5998be6 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/ClientCredentialsHandlers.cs +++ b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/ClientCredentialsHandlers.cs @@ -1,5 +1,5 @@ -using System.Collections.Generic; -using System.Collections.Immutable; +using System.Collections.Generic; +using System.Collections.Immutable; using System.Diagnostics; using System.Globalization; using System.Linq; @@ -159,25 +159,28 @@ internal sealed class ValidateClientCredentialsHandler : IOpenIddictServerHandle context.Transaction.Properties[AuthorityOpenIddictConstants.AuditConfidentialProperty] = string.Equals(document.ClientType, "confidential", StringComparison.OrdinalIgnoreCase); - IIdentityProviderPlugin? provider = null; - if (!string.IsNullOrWhiteSpace(document.Plugin)) - { - if (!registry.TryGet(document.Plugin, out provider)) - { - context.Reject(OpenIddictConstants.Errors.InvalidClient, "Configured identity provider is unavailable."); - logger.LogWarning("Client credentials validation failed for {ClientId}: provider {Provider} unavailable.", context.ClientId, document.Plugin); - return; - } - - context.Transaction.Properties[AuthorityOpenIddictConstants.AuditProviderProperty] = provider.Name; - - if (!provider.Capabilities.SupportsClientProvisioning || provider.ClientProvisioning is null) - { - context.Reject(OpenIddictConstants.Errors.UnauthorizedClient, "Associated identity provider does not support client provisioning."); - logger.LogWarning("Client credentials validation failed for {ClientId}: provider {Provider} lacks client provisioning capabilities.", context.ClientId, provider.Name); - return; - } - } + AuthorityIdentityProviderMetadata? providerMetadata = null; + if (!string.IsNullOrWhiteSpace(document.Plugin)) + { + if (!registry.TryGet(document.Plugin, out providerMetadata)) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Configured identity provider is unavailable."); + logger.LogWarning("Client credentials validation failed for {ClientId}: provider {Provider} unavailable.", context.ClientId, document.Plugin); + return; + } + + await using var providerHandle = await registry.AcquireAsync(providerMetadata.Name, context.CancellationToken).ConfigureAwait(false); + var providerInstance = providerHandle.Provider; + + context.Transaction.Properties[AuthorityOpenIddictConstants.AuditProviderProperty] = providerMetadata.Name; + + if (!providerMetadata.Capabilities.SupportsClientProvisioning || providerInstance.ClientProvisioning is null) + { + context.Reject(OpenIddictConstants.Errors.UnauthorizedClient, "Associated identity provider does not support client provisioning."); + logger.LogWarning("Client credentials validation failed for {ClientId}: provider {Provider} lacks client provisioning capabilities.", context.ClientId, providerMetadata.Name); + return; + } + } var allowedGrantTypes = ClientCredentialHandlerHelpers.Split(document.Properties, AuthorityClientMetadataKeys.AllowedGrantTypes); if (allowedGrantTypes.Count > 0 && @@ -191,28 +194,28 @@ internal sealed class ValidateClientCredentialsHandler : IOpenIddictServerHandle var requiresSecret = string.Equals(document.ClientType, "confidential", StringComparison.OrdinalIgnoreCase); if (requiresSecret) { - if (string.IsNullOrWhiteSpace(document.SecretHash)) - { - context.Reject(OpenIddictConstants.Errors.InvalidClient, "Client secret is not configured."); - logger.LogWarning("Client credentials validation failed for {ClientId}: secret not configured.", document.ClientId); - return; - } - - if (string.IsNullOrWhiteSpace(context.ClientSecret) || - !ClientCredentialHandlerHelpers.VerifySecret(context.ClientSecret, document.SecretHash)) - { - context.Reject(OpenIddictConstants.Errors.InvalidClient, "Invalid client credentials."); - logger.LogWarning("Client credentials validation failed for {ClientId}: secret verification failed.", document.ClientId); - return; - } - } - else if (!string.IsNullOrWhiteSpace(context.ClientSecret) && !string.IsNullOrWhiteSpace(document.SecretHash) && - !ClientCredentialHandlerHelpers.VerifySecret(context.ClientSecret, document.SecretHash)) - { - context.Reject(OpenIddictConstants.Errors.InvalidClient, "Invalid client credentials."); - logger.LogWarning("Client credentials validation failed for {ClientId}: secret verification failed.", document.ClientId); - return; - } + if (string.IsNullOrWhiteSpace(document.SecretHash)) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Client secret is not configured."); + logger.LogWarning("Client credentials validation failed for {ClientId}: secret not configured.", document.ClientId); + return; + } + + if (string.IsNullOrWhiteSpace(context.ClientSecret) || + !ClientCredentialHandlerHelpers.VerifySecret(context.ClientSecret, document.SecretHash)) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Invalid client credentials."); + logger.LogWarning("Client credentials validation failed for {ClientId}: secret verification failed.", document.ClientId); + return; + } + } + else if (!string.IsNullOrWhiteSpace(context.ClientSecret) && !string.IsNullOrWhiteSpace(document.SecretHash) && + !ClientCredentialHandlerHelpers.VerifySecret(context.ClientSecret, document.SecretHash)) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Invalid client credentials."); + logger.LogWarning("Client credentials validation failed for {ClientId}: secret verification failed.", document.ClientId); + return; + } var allowedScopes = ClientCredentialHandlerHelpers.Split(document.Properties, AuthorityClientMetadataKeys.AllowedScopes); var resolvedScopes = ClientCredentialHandlerHelpers.ResolveGrantedScopes( @@ -230,11 +233,11 @@ internal sealed class ValidateClientCredentialsHandler : IOpenIddictServerHandle context.Transaction.Properties[AuthorityOpenIddictConstants.AuditGrantedScopesProperty] = resolvedScopes.Scopes; context.Transaction.Properties[AuthorityOpenIddictConstants.ClientTransactionProperty] = document; - if (provider is not null) - { - context.Transaction.Properties[AuthorityOpenIddictConstants.ClientProviderTransactionProperty] = provider.Name; - activity?.SetTag("authority.identity_provider", provider.Name); - } + if (providerMetadata is not null) + { + context.Transaction.Properties[AuthorityOpenIddictConstants.ClientProviderTransactionProperty] = providerMetadata.Name; + activity?.SetTag("authority.identity_provider", providerMetadata.Name); + } context.Transaction.Properties[AuthorityOpenIddictConstants.ClientGrantedScopesProperty] = resolvedScopes.Scopes; logger.LogInformation("Client credentials validated for {ClientId}.", document.ClientId); @@ -373,70 +376,88 @@ internal sealed class HandleClientCredentialsHandler : IOpenIddictServerHandler< _ => new[] { OpenIddictConstants.Destinations.AccessToken } }); - var (provider, descriptor) = await ResolveProviderAsync(context, document).ConfigureAwait(false); - if (context.IsRejected) - { - logger.LogWarning("Client credentials request rejected for {ClientId} during provider resolution.", document.ClientId); - return; - } - - if (provider is null) - { - if (!string.IsNullOrWhiteSpace(document.Plugin)) - { - identity.SetClaim(StellaOpsClaimTypes.IdentityProvider, document.Plugin); - activity?.SetTag("authority.identity_provider", document.Plugin); - } - } - else - { - identity.SetClaim(StellaOpsClaimTypes.IdentityProvider, provider.Name); - activity?.SetTag("authority.identity_provider", provider.Name); - } - - ApplySenderConstraintClaims(context, identity, document); - - var principal = new ClaimsPrincipal(identity); - - var grantedScopes = context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.ClientGrantedScopesProperty, out var scopesValue) && - scopesValue is IReadOnlyList resolvedScopes - ? resolvedScopes - : ClientCredentialHandlerHelpers.Split(document.Properties, AuthorityClientMetadataKeys.AllowedScopes); - - if (grantedScopes.Count > 0) - { - principal.SetScopes(grantedScopes); - } - else - { - principal.SetScopes(Array.Empty()); - } - - if (configuredAudiences.Count > 0) - { - principal.SetAudiences(configuredAudiences); - } - - if (provider is not null && descriptor is not null) - { - var enrichmentContext = new AuthorityClaimsEnrichmentContext(provider.Context, user: null, descriptor); - await provider.ClaimsEnricher.EnrichAsync(identity, enrichmentContext, context.CancellationToken).ConfigureAwait(false); - } - - var session = await sessionAccessor.GetSessionAsync(context.CancellationToken).ConfigureAwait(false); - await PersistTokenAsync(context, document, tokenId, grantedScopes, session, activity).ConfigureAwait(false); - - context.Principal = principal; - context.HandleRequest(); - logger.LogInformation("Issued client credentials access token for {ClientId} with scopes {Scopes}.", document.ClientId, grantedScopes); - } - - private async ValueTask<(IIdentityProviderPlugin? Provider, AuthorityClientDescriptor? Descriptor)> ResolveProviderAsync( - OpenIddictServerEvents.HandleTokenRequestContext context, - AuthorityClientDocument document) - { - string? providerName = null; - if (context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.ClientProviderTransactionProperty, out var providerValue) && + var (providerHandle, descriptor) = await ResolveProviderAsync(context, document).ConfigureAwait(false); + if (context.IsRejected) + { + if (providerHandle is not null) + { + await providerHandle.DisposeAsync().ConfigureAwait(false); + } + + logger.LogWarning("Client credentials request rejected for {ClientId} during provider resolution.", document.ClientId); + return; + } + + AuthorityIdentityProviderHandle? handle = providerHandle; + try + { + var provider = handle?.Provider; + + if (provider is null) + { + if (!string.IsNullOrWhiteSpace(document.Plugin)) + { + identity.SetClaim(StellaOpsClaimTypes.IdentityProvider, document.Plugin); + activity?.SetTag("authority.identity_provider", document.Plugin); + } + } + else + { + identity.SetClaim(StellaOpsClaimTypes.IdentityProvider, provider.Name); + activity?.SetTag("authority.identity_provider", provider.Name); + } + + ApplySenderConstraintClaims(context, identity, document); + + var principal = new ClaimsPrincipal(identity); + + var grantedScopes = context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.ClientGrantedScopesProperty, out var scopesValue) && + scopesValue is IReadOnlyList resolvedScopes + ? resolvedScopes + : ClientCredentialHandlerHelpers.Split(document.Properties, AuthorityClientMetadataKeys.AllowedScopes); + + if (grantedScopes.Count > 0) + { + principal.SetScopes(grantedScopes); + } + else + { + principal.SetScopes(Array.Empty()); + } + + if (configuredAudiences.Count > 0) + { + principal.SetAudiences(configuredAudiences); + } + + if (provider is not null && descriptor is not null) + { + var enrichmentContext = new AuthorityClaimsEnrichmentContext(provider.Context, user: null, descriptor); + await provider.ClaimsEnricher.EnrichAsync(identity, enrichmentContext, context.CancellationToken).ConfigureAwait(false); + } + + var session = await sessionAccessor.GetSessionAsync(context.CancellationToken).ConfigureAwait(false); + await PersistTokenAsync(context, document, tokenId, grantedScopes, session, activity).ConfigureAwait(false); + + context.Principal = principal; + context.HandleRequest(); + logger.LogInformation("Issued client credentials access token for {ClientId} with scopes {Scopes}.", document.ClientId, grantedScopes); + } + finally + { + if (handle is not null) + { + await handle.DisposeAsync().ConfigureAwait(false); + } + } + } + + private async ValueTask<(AuthorityIdentityProviderHandle? Handle, AuthorityClientDescriptor? Descriptor)> ResolveProviderAsync( + OpenIddictServerEvents.HandleTokenRequestContext context, + AuthorityClientDocument document) + { + string? providerName = null; + if (context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.ClientProviderTransactionProperty, out var providerValue) && providerValue is string storedProvider) { providerName = storedProvider; @@ -446,27 +467,46 @@ internal sealed class HandleClientCredentialsHandler : IOpenIddictServerHandler< providerName = document.Plugin; } - if (string.IsNullOrWhiteSpace(providerName)) - { - return (null, null); - } - - if (!registry.TryGet(providerName, out var provider) || provider.ClientProvisioning is null) - { - context.Reject(OpenIddictConstants.Errors.InvalidClient, "Configured identity provider is unavailable."); - return (null, null); - } - - var descriptor = await provider.ClientProvisioning.FindByClientIdAsync(document.ClientId, context.CancellationToken).ConfigureAwait(false); - - if (descriptor is null) - { - context.Reject(OpenIddictConstants.Errors.InvalidClient, "Client registration was not found."); - return (null, null); - } - - return (provider, descriptor); - } + if (string.IsNullOrWhiteSpace(providerName)) + { + return (null, null); + } + + if (!registry.TryGet(providerName, out var metadata)) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Configured identity provider is unavailable."); + return (null, null); + } + + var handle = await registry.AcquireAsync(metadata.Name, context.CancellationToken).ConfigureAwait(false); + try + { + var provider = handle.Provider; + + if (provider.ClientProvisioning is null) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Associated identity provider does not support client provisioning."); + await handle.DisposeAsync().ConfigureAwait(false); + return (null, null); + } + + var descriptor = await provider.ClientProvisioning.FindByClientIdAsync(document.ClientId, context.CancellationToken).ConfigureAwait(false); + + if (descriptor is null) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Client registration was not found."); + await handle.DisposeAsync().ConfigureAwait(false); + return (null, null); + } + + return (handle, descriptor); + } + catch + { + await handle.DisposeAsync().ConfigureAwait(false); + throw; + } + } private async ValueTask PersistTokenAsync( OpenIddictServerEvents.HandleTokenRequestContext context, diff --git a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/DpopHandlers.cs b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/DpopHandlers.cs index c17e9349..52179e0e 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/DpopHandlers.cs +++ b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/DpopHandlers.cs @@ -367,66 +367,79 @@ internal sealed class ValidateDpopProofHandler : IOpenIddictServerHandler configuredAudiences) - { - if (!nonceOptions.Enabled || request is null) - { - return null; - } - - if (request.Resources is not null) - { - foreach (var resource in request.Resources) - { - if (string.IsNullOrWhiteSpace(resource)) - { - continue; - } - - var normalized = resource.Trim(); - if (nonceOptions.RequiredAudiences.Contains(normalized)) - { - return normalized; - } - } - } - - if (request.Audiences is not null) - { - foreach (var audience in request.Audiences) - { - if (string.IsNullOrWhiteSpace(audience)) - { - continue; - } - - var normalized = audience.Trim(); - if (nonceOptions.RequiredAudiences.Contains(normalized)) - { - return normalized; - } - } - } - - if (configuredAudiences is { Count: > 0 }) - { - foreach (var audience in configuredAudiences) - { - if (string.IsNullOrWhiteSpace(audience)) - { - continue; - } - - var normalized = audience.Trim(); - if (nonceOptions.RequiredAudiences.Contains(normalized)) - { - return normalized; - } - } - } - - return null; - } + private static string? ResolveNonceAudience( + OpenIddictRequest request, + AuthorityDpopNonceOptions nonceOptions, + IReadOnlyList configuredAudiences) + { + if (!nonceOptions.Enabled || request is null) + { + return null; + } + + var normalizedAudiences = nonceOptions.NormalizedAudiences; + IReadOnlySet effectiveAudiences; + + if (normalizedAudiences.Count > 0) + { + effectiveAudiences = normalizedAudiences; + } + else if (nonceOptions.RequiredAudiences.Count > 0) + { + effectiveAudiences = nonceOptions.RequiredAudiences.ToHashSet(StringComparer.OrdinalIgnoreCase); + } + else + { + return null; + } + + bool TryMatch(string? candidate, out string normalized) + { + normalized = string.Empty; + if (string.IsNullOrWhiteSpace(candidate)) + { + return false; + } + + normalized = candidate.Trim(); + return effectiveAudiences.Contains(normalized); + } + + if (request.Resources is not null) + { + foreach (var resource in request.Resources) + { + if (TryMatch(resource, out var normalized)) + { + return normalized; + } + } + } + + if (request.Audiences is not null) + { + foreach (var audience in request.Audiences) + { + if (TryMatch(audience, out var normalized)) + { + return normalized; + } + } + } + + if (configuredAudiences is { Count: > 0 }) + { + foreach (var audience in configuredAudiences) + { + if (TryMatch(audience, out var normalized)) + { + return normalized; + } + } + } + + return null; + } private async ValueTask ChallengeNonceAsync( OpenIddictServerEvents.ValidateTokenRequestContext context, diff --git a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/PasswordGrantHandlers.cs b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/PasswordGrantHandlers.cs index 85b5e5f3..9f652a61 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/PasswordGrantHandlers.cs +++ b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/PasswordGrantHandlers.cs @@ -110,6 +110,8 @@ internal sealed class ValidatePasswordGrantHandler : IOpenIddictServerHandler missingOrdered, ILogger? logger) { - var registrarLookup = DiscoverRegistrars(loadedAssemblies, logger); + var registrarCandidates = DiscoverRegistrars(loadedAssemblies); + var pluginTypeLookup = new Dictionary(StringComparer.OrdinalIgnoreCase); + var registrarTypeLookup = new Dictionary(); var registered = new List(); var failures = new List(); @@ -79,7 +82,16 @@ internal static class AuthorityPluginLoader continue; } - if (!registrarLookup.TryGetValue(manifest.Type, out var registrar)) + var activation = TryResolveActivationForManifest( + services, + manifest.Type, + registrarCandidates, + pluginTypeLookup, + registrarTypeLookup, + logger, + out var registrarType); + + if (activation is null || registrarType is null) { var reason = $"No registrar found for plugin type '{manifest.Type}'."; logger?.LogError( @@ -92,7 +104,9 @@ internal static class AuthorityPluginLoader try { - registrar.Register(new AuthorityPluginRegistrationContext(services, pluginContext, configuration)); + PluginServiceRegistration.RegisterAssemblyMetadata(services, registrarType.Assembly, logger); + + activation.Registrar.Register(new AuthorityPluginRegistrationContext(services, pluginContext, configuration)); registered.Add(manifest.Name); logger?.LogInformation( @@ -109,6 +123,10 @@ internal static class AuthorityPluginLoader manifest.Name); failures.Add(new AuthorityPluginRegistrationFailure(manifest.Name, reason)); } + finally + { + activation.Dispose(); + } } if (missingOrdered.Count > 0) @@ -124,11 +142,9 @@ internal static class AuthorityPluginLoader return new AuthorityPluginRegistrationSummary(registered, failures, missingOrdered); } - private static Dictionary DiscoverRegistrars( - IReadOnlyCollection loadedAssemblies, - ILogger? logger) + private static IReadOnlyList DiscoverRegistrars(IReadOnlyCollection loadedAssemblies) { - var lookup = new Dictionary(StringComparer.OrdinalIgnoreCase); + var registrars = new List(); foreach (var descriptor in loadedAssemblies) { @@ -139,43 +155,144 @@ internal static class AuthorityPluginLoader continue; } - try - { - if (Activator.CreateInstance(type) is not IAuthorityPluginRegistrar registrar) - { - continue; - } - - if (string.IsNullOrWhiteSpace(registrar.PluginType)) - { - logger?.LogWarning( - "Authority plugin registrar '{RegistrarType}' returned an empty plugin type and will be ignored.", - type.FullName); - continue; - } - - if (lookup.TryGetValue(registrar.PluginType, out var existing)) - { - logger?.LogWarning( - "Multiple registrars detected for plugin type '{PluginType}'. Replacing '{ExistingType}' with '{RegistrarType}'.", - registrar.PluginType, - existing.GetType().FullName, - type.FullName); - } - - lookup[registrar.PluginType] = registrar; - } - catch (Exception ex) - { - logger?.LogError( - ex, - "Failed to instantiate Authority plugin registrar '{RegistrarType}'.", - type.FullName); - } + registrars.Add(type); } } - return lookup; + return registrars; + } + + private static RegistrarActivation? TryResolveActivationForManifest( + IServiceCollection services, + string pluginType, + IReadOnlyList registrarCandidates, + IDictionary pluginTypeLookup, + IDictionary registrarTypeLookup, + ILogger? logger, + out Type? resolvedType) + { + resolvedType = null; + + if (pluginTypeLookup.TryGetValue(pluginType, out var cachedType)) + { + var cachedActivation = CreateRegistrarActivation(services, cachedType, logger); + if (cachedActivation is null) + { + pluginTypeLookup.Remove(pluginType); + registrarTypeLookup.Remove(cachedType); + return null; + } + + resolvedType = cachedType; + return cachedActivation; + } + + foreach (var candidate in registrarCandidates) + { + if (registrarTypeLookup.TryGetValue(candidate, out var knownType)) + { + if (string.IsNullOrWhiteSpace(knownType)) + { + continue; + } + + if (string.Equals(knownType, pluginType, StringComparison.OrdinalIgnoreCase)) + { + var activation = CreateRegistrarActivation(services, candidate, logger); + if (activation is null) + { + registrarTypeLookup.Remove(candidate); + pluginTypeLookup.Remove(knownType); + return null; + } + + resolvedType = candidate; + return activation; + } + + continue; + } + + var attempt = CreateRegistrarActivation(services, candidate, logger); + if (attempt is null) + { + registrarTypeLookup[candidate] = string.Empty; + continue; + } + + var candidateType = attempt.Registrar.PluginType; + if (string.IsNullOrWhiteSpace(candidateType)) + { + logger?.LogWarning( + "Authority plugin registrar '{RegistrarType}' reported an empty plugin type and will be ignored.", + candidate.FullName); + registrarTypeLookup[candidate] = string.Empty; + attempt.Dispose(); + continue; + } + + registrarTypeLookup[candidate] = candidateType; + pluginTypeLookup[candidateType] = candidate; + + if (string.Equals(candidateType, pluginType, StringComparison.OrdinalIgnoreCase)) + { + resolvedType = candidate; + return attempt; + } + + attempt.Dispose(); + } + + return null; + } + + private static RegistrarActivation? CreateRegistrarActivation(IServiceCollection services, Type registrarType, ILogger? logger) + { + ServiceProvider? provider = null; + IServiceScope? scope = null; + try + { + provider = services.BuildServiceProvider(new ServiceProviderOptions + { + ValidateScopes = true + }); + + scope = provider.CreateScope(); + var registrar = (IAuthorityPluginRegistrar)ActivatorUtilities.GetServiceOrCreateInstance(scope.ServiceProvider, registrarType); + return new RegistrarActivation(provider, scope, registrar); + } + catch (Exception ex) + { + logger?.LogError( + ex, + "Failed to activate Authority plugin registrar '{RegistrarType}'.", + registrarType.FullName); + + scope?.Dispose(); + provider?.Dispose(); + return null; + } + } + + private sealed class RegistrarActivation : IDisposable + { + private readonly ServiceProvider provider; + private readonly IServiceScope scope; + + public RegistrarActivation(ServiceProvider provider, IServiceScope scope, IAuthorityPluginRegistrar registrar) + { + this.provider = provider; + this.scope = scope; + Registrar = registrar; + } + + public IAuthorityPluginRegistrar Registrar { get; } + + public void Dispose() + { + scope.Dispose(); + provider.Dispose(); + } } private static bool IsAssemblyLoaded( diff --git a/src/StellaOps.Authority/StellaOps.Authority/Program.cs b/src/StellaOps.Authority/StellaOps.Authority/Program.cs index 4d032efe..2c3db01e 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/Program.cs +++ b/src/StellaOps.Authority/StellaOps.Authority/Program.cs @@ -416,24 +416,24 @@ if (authorityOptions.Bootstrap.Enabled) return Results.BadRequest(new { error = "invite_provider_mismatch", message = "Invite is limited to a different identity provider." }); } - if (string.IsNullOrWhiteSpace(providerName) || !registry.TryGet(providerName!, out var provider)) + if (string.IsNullOrWhiteSpace(providerName) || !registry.TryGet(providerName!, out var providerMetadata)) { await ReleaseInviteAsync("Specified identity provider was not found."); await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, "Specified identity provider was not found.", null, request.Username, providerName, request.Roles ?? Array.Empty(), inviteToken).ConfigureAwait(false); return Results.BadRequest(new { error = "invalid_provider", message = "Specified identity provider was not found." }); } - if (!provider.Capabilities.SupportsPassword) + if (!providerMetadata.Capabilities.SupportsPassword) { await ReleaseInviteAsync("Selected provider does not support password provisioning."); - await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, "Selected provider does not support password provisioning.", null, request.Username, provider.Name, request.Roles ?? Array.Empty(), inviteToken).ConfigureAwait(false); + await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, "Selected provider does not support password provisioning.", null, request.Username, providerMetadata.Name, request.Roles ?? Array.Empty(), inviteToken).ConfigureAwait(false); return Results.BadRequest(new { error = "unsupported_provider", message = "Selected provider does not support password provisioning." }); } if (string.IsNullOrWhiteSpace(request.Username) || string.IsNullOrEmpty(request.Password)) { await ReleaseInviteAsync("Username and password are required."); - await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, "Username and password are required.", null, request.Username, provider.Name, request.Roles ?? Array.Empty(), inviteToken).ConfigureAwait(false); + await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, "Username and password are required.", null, request.Username, providerMetadata.Name, request.Roles ?? Array.Empty(), inviteToken).ConfigureAwait(false); return Results.BadRequest(new { error = "invalid_request", message = "Username and password are required." }); } @@ -458,6 +458,9 @@ if (authorityOptions.Bootstrap.Enabled) roles, attributes); + await using var providerHandle = await registry.AcquireAsync(providerMetadata.Name, cancellationToken).ConfigureAwait(false); + var provider = providerHandle.Provider; + try { var result = await provider.Credentials.UpsertUserAsync(registration, cancellationToken).ConfigureAwait(false); @@ -465,7 +468,7 @@ if (authorityOptions.Bootstrap.Enabled) if (!result.Succeeded || result.Value is null) { await ReleaseInviteAsync(result.Message ?? "User provisioning failed."); - await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, result.Message ?? "User provisioning failed.", null, request.Username, provider.Name, roles, inviteToken).ConfigureAwait(false); + await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, result.Message ?? "User provisioning failed.", null, request.Username, providerMetadata.Name, roles, inviteToken).ConfigureAwait(false); return Results.BadRequest(new { error = result.ErrorCode ?? "bootstrap_failed", message = result.Message ?? "User provisioning failed." }); } @@ -478,11 +481,11 @@ if (authorityOptions.Bootstrap.Enabled) } } - await WriteBootstrapUserAuditAsync(AuthEventOutcome.Success, null, result.Value.SubjectId, result.Value.Username, provider.Name, roles, inviteToken).ConfigureAwait(false); + await WriteBootstrapUserAuditAsync(AuthEventOutcome.Success, null, result.Value.SubjectId, result.Value.Username, providerMetadata.Name, roles, inviteToken).ConfigureAwait(false); return Results.Ok(new { - provider = provider.Name, + provider = providerMetadata.Name, subjectId = result.Value.SubjectId, username = result.Value.Username }); @@ -701,24 +704,34 @@ if (authorityOptions.Bootstrap.Enabled) return Results.BadRequest(new { error = "invite_provider_mismatch", message = "Invite is limited to a different identity provider." }); } - if (string.IsNullOrWhiteSpace(providerName) || !registry.TryGet(providerName!, out var provider)) + if (string.IsNullOrWhiteSpace(providerName) || !registry.TryGet(providerName!, out var providerMetadata)) { await ReleaseInviteAsync("Specified identity provider was not found."); await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "Specified identity provider was not found.", request.ClientId, null, providerName, request.AllowedScopes ?? Array.Empty(), request?.Confidential, inviteToken).ConfigureAwait(false); return Results.BadRequest(new { error = "invalid_provider", message = "Specified identity provider was not found." }); } - if (!provider.Capabilities.SupportsClientProvisioning || provider.ClientProvisioning is null) + if (!providerMetadata.Capabilities.SupportsClientProvisioning) { await ReleaseInviteAsync("Selected provider does not support client provisioning."); - await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "Selected provider does not support client provisioning.", request.ClientId, null, provider.Name, request.AllowedScopes ?? Array.Empty(), request.Confidential, inviteToken).ConfigureAwait(false); + await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "Selected provider does not support client provisioning.", request.ClientId, null, providerMetadata.Name, request.AllowedScopes ?? Array.Empty(), request.Confidential, inviteToken).ConfigureAwait(false); + return Results.BadRequest(new { error = "unsupported_provider", message = "Selected provider does not support client provisioning." }); + } + + await using var providerHandle = await registry.AcquireAsync(providerMetadata.Name, cancellationToken).ConfigureAwait(false); + var provider = providerHandle.Provider; + + if (provider.ClientProvisioning is null) + { + await ReleaseInviteAsync("Selected provider does not support client provisioning."); + await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "Selected provider does not support client provisioning.", request.ClientId, null, providerMetadata.Name, request.AllowedScopes ?? Array.Empty(), request.Confidential, inviteToken).ConfigureAwait(false); return Results.BadRequest(new { error = "unsupported_provider", message = "Selected provider does not support client provisioning." }); } if (string.IsNullOrWhiteSpace(request.ClientId)) { await ReleaseInviteAsync("ClientId is required."); - await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "ClientId is required.", null, null, provider.Name, request.AllowedScopes ?? Array.Empty(), request.Confidential, inviteToken).ConfigureAwait(false); + await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "ClientId is required.", null, null, providerMetadata.Name, request.AllowedScopes ?? Array.Empty(), request.Confidential, inviteToken).ConfigureAwait(false); return Results.BadRequest(new { error = "invalid_request", message = "ClientId is required." }); } @@ -732,7 +745,7 @@ if (authorityOptions.Bootstrap.Enabled) if (request.Confidential && string.IsNullOrWhiteSpace(request.ClientSecret)) { await ReleaseInviteAsync("Confidential clients require a client secret."); - await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "Confidential clients require a client secret.", request.ClientId, null, provider.Name, request.AllowedScopes ?? Array.Empty(), request.Confidential, inviteToken).ConfigureAwait(false); + await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "Confidential clients require a client secret.", request.ClientId, null, providerMetadata.Name, request.AllowedScopes ?? Array.Empty(), request.Confidential, inviteToken).ConfigureAwait(false); return Results.BadRequest(new { error = "invalid_request", message = "Confidential clients require a client secret." }); } @@ -740,7 +753,7 @@ if (authorityOptions.Bootstrap.Enabled) { var errorMessage = redirectError ?? "Redirect URI validation failed."; await ReleaseInviteAsync(errorMessage); - await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, errorMessage, request.ClientId, null, provider.Name, request.AllowedScopes ?? Array.Empty(), request.Confidential, inviteToken).ConfigureAwait(false); + await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, errorMessage, request.ClientId, null, providerMetadata.Name, request.AllowedScopes ?? Array.Empty(), request.Confidential, inviteToken).ConfigureAwait(false); return Results.BadRequest(new { error = "invalid_request", message = errorMessage }); } @@ -748,7 +761,7 @@ if (authorityOptions.Bootstrap.Enabled) { var errorMessage = postLogoutError ?? "Post-logout redirect URI validation failed."; await ReleaseInviteAsync(errorMessage); - await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, errorMessage, request.ClientId, null, provider.Name, request.AllowedScopes ?? Array.Empty(), request.Confidential, inviteToken).ConfigureAwait(false); + await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, errorMessage, request.ClientId, null, providerMetadata.Name, request.AllowedScopes ?? Array.Empty(), request.Confidential, inviteToken).ConfigureAwait(false); return Results.BadRequest(new { error = "invalid_request", message = errorMessage }); } @@ -765,7 +778,7 @@ if (authorityOptions.Bootstrap.Enabled) if (binding is null || string.IsNullOrWhiteSpace(binding.Thumbprint)) { await ReleaseInviteAsync("Certificate binding thumbprint is required."); - await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "Certificate binding thumbprint is required.", request.ClientId, null, provider.Name, request.AllowedScopes ?? Array.Empty(), request.Confidential, inviteToken).ConfigureAwait(false); + await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "Certificate binding thumbprint is required.", request.ClientId, null, providerMetadata.Name, request.AllowedScopes ?? Array.Empty(), request.Confidential, inviteToken).ConfigureAwait(false); return Results.BadRequest(new { error = "invalid_request", message = "Certificate binding thumbprint is required." }); } @@ -801,7 +814,7 @@ if (authorityOptions.Bootstrap.Enabled) if (!result.Succeeded || result.Value is null) { await ReleaseInviteAsync(result.Message ?? "Client provisioning failed."); - await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, result.Message ?? "Client provisioning failed.", request.ClientId, result.Value?.ClientId, provider.Name, request.AllowedScopes ?? Array.Empty(), request.Confidential, inviteToken).ConfigureAwait(false); + await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, result.Message ?? "Client provisioning failed.", request.ClientId, result.Value?.ClientId, providerMetadata.Name, request.AllowedScopes ?? Array.Empty(), request.Confidential, inviteToken).ConfigureAwait(false); return Results.BadRequest(new { error = result.ErrorCode ?? "bootstrap_failed", message = result.Message ?? "Client provisioning failed." }); } @@ -814,11 +827,11 @@ if (authorityOptions.Bootstrap.Enabled) } } - await WriteBootstrapClientAuditAsync(AuthEventOutcome.Success, null, request.ClientId, result.Value.ClientId, provider.Name, request.AllowedScopes ?? Array.Empty(), request.Confidential, inviteToken).ConfigureAwait(false); + await WriteBootstrapClientAuditAsync(AuthEventOutcome.Success, null, request.ClientId, result.Value.ClientId, providerMetadata.Name, request.AllowedScopes ?? Array.Empty(), request.Confidential, inviteToken).ConfigureAwait(false); return Results.Ok(new { - provider = provider.Name, + provider = providerMetadata.Name, clientId = result.Value.ClientId, confidential = result.Value.Confidential }); @@ -1169,12 +1182,13 @@ app.UseAuthorization(); app.MapGet("/health", async (IAuthorityIdentityProviderRegistry registry, CancellationToken cancellationToken) => { var pluginHealth = new List(); - foreach (var provider in registry.Providers) + foreach (var providerMetadata in registry.Providers) { - var health = await provider.CheckHealthAsync(cancellationToken).ConfigureAwait(false); + await using var handle = await registry.AcquireAsync(providerMetadata.Name, cancellationToken).ConfigureAwait(false); + var health = await handle.Provider.CheckHealthAsync(cancellationToken).ConfigureAwait(false); pluginHealth.Add(new { - provider = provider.Name, + provider = providerMetadata.Name, status = health.Status.ToString().ToLowerInvariant(), message = health.Message }); diff --git a/src/StellaOps.Authority/TASKS.md b/src/StellaOps.Authority/TASKS.md index e3879c0d..f2baed13 100644 --- a/src/StellaOps.Authority/TASKS.md +++ b/src/StellaOps.Authority/TASKS.md @@ -20,13 +20,13 @@ | AUTHCORE-STORAGE-DEVICE-TOKENS | DONE (2025-10-14) | Authority Core, Storage Guild | AUTHCORE-BUILD-OPENIDDICT | Reintroduce `AuthorityTokenDeviceDocument` + projections removed during refactor so storage layer compiles. | ✅ Document type restored with mappings/migrations; ✅ Storage tests cover device artifacts; ✅ Authority solution build green. | | AUTHCORE-BOOTSTRAP-INVITES | DONE (2025-10-14) | Authority Core, DevOps | AUTHCORE-STORAGE-DEVICE-TOKENS | Wire bootstrap invite cleanup service against restored document schema and re-enable lifecycle tests. | ✅ `BootstrapInviteCleanupService` passes integration tests; ✅ Operator guide updated if behavior changes; ✅ Build/test matrices green. | | AUTHSTORAGE-MONGO-08-001 | DONE (2025-10-19) | Authority Core & Storage Guild | — | Harden Mongo session usage with causal consistency for mutations and follow-up reads. | • Scoped middleware/service creates `IClientSessionHandle` with causal consistency + majority read/write concerns
• Stores accept optional session parameter and reuse it for write + immediate reads
• GraphQL/HTTP pipelines updated to flow session through post-mutation queries
• Replica-set integration test exercises primary election and verifies read-your-write guarantees | -| AUTH-PLUGIN-COORD-08-002 | DOING (2025-10-19) | Authority Core, Plugin Platform Guild | PLUGIN-DI-08-001 | Coordinate scoped-service adoption for Authority plug-in registrars and background jobs ahead of PLUGIN-DI-08-002 implementation. | ✅ Workshop locked for 2025-10-20 15:00–16:00 UTC; ✅ Pre-read checklist in `docs/dev/authority-plugin-di-coordination.md`; ✅ Follow-up tasks captured in module backlogs before code changes begin. | -| AUTH-DPOP-11-001 | DOING (2025-10-19) | Authority Core & Security Guild | — | Implement DPoP proof validation + nonce handling for high-value audiences per architecture. | • Proof handler validates method/uri/hash + replay; nonce issuing/consumption implemented for in-memory + Redis stores
• Client credential path stamps `cnf.jkt` and persists sender metadata
• Remaining: finalize Redis configuration surface (docs/sample config), unskip nonce-challenge regression once HTTP pipeline emits high-value audiences, refresh operator docs | -> Remark (2025-10-19): DPoP handler now seeds request resources/audiences from client metadata; nonce challenge integration test re-enabled (still requires full suite once Concelier build restored). +| AUTH-PLUGIN-COORD-08-002 | DONE (2025-10-20) | Authority Core, Plugin Platform Guild | PLUGIN-DI-08-001 | Coordinate scoped-service adoption for Authority plug-in registrars and background jobs ahead of PLUGIN-DI-08-002 implementation. | ✅ Workshop completed 2025-10-20 15:00–16:05 UTC with notes/action log in `docs/dev/authority-plugin-di-coordination.md`; ✅ Follow-up backlog updates assigned via documented action items ahead of PLUGIN-DI-08-002 delivery. | +| AUTH-DPOP-11-001 | DONE (2025-10-20) | Authority Core & Security Guild | — | Implement DPoP proof validation + nonce handling for high-value audiences per architecture. | ✅ Redis-configurable nonce store surfaced via `security.senderConstraints.dpop.nonce` with sample YAML and architecture docs refreshed
✅ High-value audience enforcement uses normalised required audiences to avoid whitespace/case drift
✅ Operator guide updated with Redis-backed nonce snippet and env-var override guidance; integration test already covers nonce challenge | +> Remark (2025-10-20): `etc/authority.yaml.sample` gains senderConstraint sections (rate limits, DPoP, mTLS), docs (`docs/ARCHITECTURE_AUTHORITY.md`, `docs/11_AUTHORITY.md`, plan) refreshed. `ResolveNonceAudience` now relies on `NormalizedAudiences` and options trim persisted values. `dotnet test StellaOps.Authority.sln` attempted (2025-10-20 15:12 UTC) but failed on `NU1900` because the mirrored NuGet service index `https://mirrors.ablera.dev/nuget/nuget-mirror/v3/index.json` was unreachable; no project build executed. | AUTH-MTLS-11-002 | DOING (2025-10-19) | Authority Core & Security Guild | — | Add OAuth mTLS client credential support with certificate-bound tokens and introspection updates. | • Certificate validator scaffold plus cnf stamping present; tokens persist sender thumbprints
• Remaining: provisioning/storage for certificate bindings, SAN/CA validation, introspection propagation, integration tests/docs before marking DONE | > Remark (2025-10-19): Client provisioning accepts certificate bindings; validator enforces SAN types/CA allow-list with rotation grace; mtls integration tests updated (full suite still blocked by upstream build). > Remark (2025-10-19, AUTHSTORAGE-MONGO-08-001): Prerequisites re-checked (none outstanding). Session accessor wired through Authority pipeline; stores accept optional sessions; added replica-set election regression test for read-your-write. -> Remark (2025-10-19, AUTH-DPOP-11-001): Handler, nonce store, and persistence hooks merged; Redis-backed configuration + end-to-end nonce enforcement still open. Full solution test blocked by `StellaOps.Concelier.Storage.Mongo` compile errors. +> Remark (2025-10-19, AUTH-DPOP-11-001): Handler, nonce store, and persistence hooks merged; Redis-backed configuration + end-to-end nonce enforcement still open. (Superseded by 2025-10-20 update above.) > Remark (2025-10-19, AUTH-MTLS-11-002): Certificate validator + cnf stamping delivered; binding storage, CA/SAN validation, integration suites outstanding before status can move to DONE. > Update status columns (TODO / DOING / DONE / BLOCKED) together with code changes. Always run `dotnet test src/StellaOps.Authority.sln` when touching host logic. diff --git a/src/StellaOps.Cli.Tests/Commands/CommandHandlersTests.cs b/src/StellaOps.Cli.Tests/Commands/CommandHandlersTests.cs index 7f08abb9..bb2459ac 100644 --- a/src/StellaOps.Cli.Tests/Commands/CommandHandlersTests.cs +++ b/src/StellaOps.Cli.Tests/Commands/CommandHandlersTests.cs @@ -951,6 +951,15 @@ public sealed class CommandHandlersTests public Task EvaluateRuntimePolicyAsync(RuntimePolicyEvaluationRequest request, CancellationToken cancellationToken) => Task.FromResult(RuntimePolicyResult); + + public Task DownloadOfflineKitAsync(string? bundleId, string destinationDirectory, bool overwrite, bool resume, CancellationToken cancellationToken) + => throw new NotSupportedException(); + + public Task ImportOfflineKitAsync(OfflineKitImportRequest request, CancellationToken cancellationToken) + => throw new NotSupportedException(); + + public Task GetOfflineKitStatusAsync(CancellationToken cancellationToken) + => throw new NotSupportedException(); } private sealed class StubExecutor : IScannerExecutor diff --git a/src/StellaOps.Cli.Tests/Services/BackendOperationsClientTests.cs b/src/StellaOps.Cli.Tests/Services/BackendOperationsClientTests.cs index 541b2273..8927fc81 100644 --- a/src/StellaOps.Cli.Tests/Services/BackendOperationsClientTests.cs +++ b/src/StellaOps.Cli.Tests/Services/BackendOperationsClientTests.cs @@ -1,24 +1,25 @@ using System; using System.Collections.ObjectModel; using System.Globalization; -using System.IO; -using System.Net; -using System.Net.Http; -using System.Net.Http.Json; -using System.Security.Cryptography; -using System.Text; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.IdentityModel.Tokens; -using StellaOps.Auth.Abstractions; -using StellaOps.Auth.Client; -using StellaOps.Cli.Configuration; -using StellaOps.Cli.Services; -using StellaOps.Cli.Services.Models; -using StellaOps.Cli.Services.Models.Transport; -using StellaOps.Cli.Tests.Testing; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Net.Http.Json; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.IdentityModel.Tokens; +using StellaOps.Auth.Abstractions; +using StellaOps.Auth.Client; +using StellaOps.Cli.Configuration; +using StellaOps.Cli.Services; +using StellaOps.Cli.Services.Models; +using StellaOps.Cli.Services.Models.Transport; +using StellaOps.Cli.Tests.Testing; +using System.Linq; namespace StellaOps.Cli.Tests.Services; @@ -481,7 +482,352 @@ public sealed class BackendOperationsClientTests Assert.Equal("manual-override", Assert.IsType(secondary.AdditionalProperties["quietedBy"])); } - private sealed class StubTokenClient : IStellaOpsTokenClient + [Fact] + public async Task DownloadOfflineKitAsync_DownloadsBundleAndWritesMetadata() + { + using var temp = new TempDirectory(); + + var bundleBytes = Encoding.UTF8.GetBytes("bundle-data"); + var manifestBytes = Encoding.UTF8.GetBytes("{\"artifacts\":[]}"); + var bundleDigest = Convert.ToHexString(SHA256.HashData(bundleBytes)).ToLowerInvariant(); + var manifestDigest = Convert.ToHexString(SHA256.HashData(manifestBytes)).ToLowerInvariant(); + + var metadataPayload = JsonSerializer.Serialize(new + { + bundleId = "2025-10-20-full", + bundleName = "stella-ops-offline-kit-2025-10-20.tgz", + bundleSha256 = $"sha256:{bundleDigest}", + bundleSize = (long)bundleBytes.Length, + bundleUrl = "https://mirror.example/stella-ops-offline-kit-2025-10-20.tgz", + bundleSignatureName = "stella-ops-offline-kit-2025-10-20.tgz.sig", + bundleSignatureUrl = "https://mirror.example/stella-ops-offline-kit-2025-10-20.tgz.sig", + manifestName = "offline-manifest-2025-10-20.json", + manifestSha256 = $"sha256:{manifestDigest}", + manifestUrl = "https://mirror.example/offline-manifest-2025-10-20.json", + manifestSignatureName = "offline-manifest-2025-10-20.json.jws", + manifestSignatureUrl = "https://mirror.example/offline-manifest-2025-10-20.json.jws", + capturedAt = DateTimeOffset.UtcNow + }, new JsonSerializerOptions(JsonSerializerDefaults.Web)); + + var handler = new StubHttpMessageHandler( + (request, _) => + { + Assert.Equal("https://backend.example/api/offline-kit/bundles/latest", request.RequestUri!.ToString()); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(metadataPayload) + }; + }, + (request, _) => + { + var absolute = request.RequestUri!.AbsoluteUri; + if (absolute.EndsWith(".tgz", StringComparison.OrdinalIgnoreCase)) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(bundleBytes) + }; + } + + if (absolute.EndsWith(".json", StringComparison.OrdinalIgnoreCase)) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(manifestBytes) + }; + } + + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(Array.Empty()) + }; + }); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://backend.example") + }; + + var options = new StellaOpsCliOptions + { + BackendUrl = "https://backend.example", + Offline = new StellaOpsCliOfflineOptions + { + KitsDirectory = temp.Path + } + }; + + var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); + var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger()); + + var result = await client.DownloadOfflineKitAsync(null, temp.Path, overwrite: false, resume: false, CancellationToken.None); + + Assert.False(result.FromCache); + Assert.True(File.Exists(result.BundlePath)); + Assert.True(File.Exists(result.ManifestPath)); + Assert.NotNull(result.BundleSignaturePath); + Assert.NotNull(result.ManifestSignaturePath); + Assert.True(File.Exists(result.MetadataPath)); + + using var metadata = JsonDocument.Parse(File.ReadAllText(result.MetadataPath)); + Assert.Equal("2025-10-20-full", metadata.RootElement.GetProperty("bundleId").GetString()); + Assert.Equal(bundleDigest, metadata.RootElement.GetProperty("bundleSha256").GetString()); + } + + [Fact] + public async Task DownloadOfflineKitAsync_ResumesPartialDownload() + { + using var temp = new TempDirectory(); + + var bundleBytes = Encoding.UTF8.GetBytes("partial-download-data"); + var manifestBytes = Encoding.UTF8.GetBytes("{\"manifest\":true}"); + var bundleDigest = Convert.ToHexString(SHA256.HashData(bundleBytes)).ToLowerInvariant(); + var manifestDigest = Convert.ToHexString(SHA256.HashData(manifestBytes)).ToLowerInvariant(); + + var metadataJson = JsonSerializer.Serialize(new + { + bundleId = "2025-10-21-full", + bundleName = "kit.tgz", + bundleSha256 = bundleDigest, + bundleSize = (long)bundleBytes.Length, + bundleUrl = "https://mirror.example/kit.tgz", + manifestName = "offline-manifest.json", + manifestSha256 = manifestDigest, + manifestUrl = "https://mirror.example/offline-manifest.json", + capturedAt = DateTimeOffset.UtcNow + }, new JsonSerializerOptions(JsonSerializerDefaults.Web)); + + var partialPath = Path.Combine(temp.Path, "kit.tgz.partial"); + await File.WriteAllBytesAsync(partialPath, bundleBytes.AsSpan(0, bundleBytes.Length / 2).ToArray()); + + var handler = new StubHttpMessageHandler( + (request, _) => new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(metadataJson) + }, + (request, _) => + { + if (request.RequestUri!.AbsoluteUri.EndsWith("kit.tgz", StringComparison.OrdinalIgnoreCase)) + { + Assert.NotNull(request.Headers.Range); + Assert.Equal(bundleBytes.Length / 2, request.Headers.Range!.Ranges.Single().From); + return new HttpResponseMessage(HttpStatusCode.PartialContent) + { + Content = new ByteArrayContent(bundleBytes.AsSpan(bundleBytes.Length / 2).ToArray()) + }; + } + + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(manifestBytes) + }; + }); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://backend.example") + }; + + var options = new StellaOpsCliOptions + { + BackendUrl = "https://backend.example", + Offline = new StellaOpsCliOfflineOptions + { + KitsDirectory = temp.Path + } + }; + + var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); + var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger()); + + var result = await client.DownloadOfflineKitAsync(null, temp.Path, overwrite: false, resume: true, CancellationToken.None); + + Assert.Equal(bundleDigest, result.Descriptor.BundleSha256); + Assert.Equal(bundleBytes.Length, new FileInfo(result.BundlePath).Length); + } + + [Fact] + public async Task ImportOfflineKitAsync_SendsMultipartPayload() + { + using var temp = new TempDirectory(); + var bundlePath = Path.Combine(temp.Path, "kit.tgz"); + var manifestPath = Path.Combine(temp.Path, "offline-manifest.json"); + + var bundleBytes = Encoding.UTF8.GetBytes("bundle-content"); + var manifestBytes = Encoding.UTF8.GetBytes("{\"manifest\":true}"); + await File.WriteAllBytesAsync(bundlePath, bundleBytes); + await File.WriteAllBytesAsync(manifestPath, manifestBytes); + + var bundleDigest = Convert.ToHexString(SHA256.HashData(bundleBytes)).ToLowerInvariant(); + var manifestDigest = Convert.ToHexString(SHA256.HashData(manifestBytes)).ToLowerInvariant(); + + var metadata = new OfflineKitMetadataDocument + { + BundleId = "2025-10-21-full", + BundleName = "kit.tgz", + BundleSha256 = bundleDigest, + BundleSize = bundleBytes.Length, + BundlePath = bundlePath, + CapturedAt = DateTimeOffset.UtcNow, + DownloadedAt = DateTimeOffset.UtcNow, + Channel = "stable", + Kind = "full", + ManifestName = "offline-manifest.json", + ManifestSha256 = manifestDigest, + ManifestSize = manifestBytes.Length, + ManifestPath = manifestPath, + IsDelta = false, + BaseBundleId = null + }; + + await File.WriteAllTextAsync(bundlePath + ".metadata.json", JsonSerializer.Serialize(metadata, new JsonSerializerOptions(JsonSerializerDefaults.Web) { WriteIndented = true })); + + var recordingHandler = new ImportRecordingHandler(); + var httpClient = new HttpClient(recordingHandler) + { + BaseAddress = new Uri("https://backend.example") + }; + + var options = new StellaOpsCliOptions + { + BackendUrl = "https://backend.example" + }; + + var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); + var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger()); + + var request = new OfflineKitImportRequest( + bundlePath, + manifestPath, + null, + null, + metadata.BundleId, + metadata.BundleSha256, + metadata.BundleSize, + metadata.CapturedAt, + metadata.Channel, + metadata.Kind, + metadata.IsDelta, + metadata.BaseBundleId, + metadata.ManifestSha256, + metadata.ManifestSize); + + var result = await client.ImportOfflineKitAsync(request, CancellationToken.None); + + Assert.Equal("imp-1", result.ImportId); + Assert.NotNull(recordingHandler.MetadataJson); + Assert.NotNull(recordingHandler.BundlePayload); + Assert.NotNull(recordingHandler.ManifestPayload); + + using var metadataJson = JsonDocument.Parse(recordingHandler.MetadataJson!); + Assert.Equal(bundleDigest, metadataJson.RootElement.GetProperty("bundleSha256").GetString()); + Assert.Equal(manifestDigest, metadataJson.RootElement.GetProperty("manifestSha256").GetString()); + } + + [Fact] + public async Task GetOfflineKitStatusAsync_ParsesResponse() + { + var captured = DateTimeOffset.UtcNow; + var imported = captured.AddMinutes(5); + + var statusJson = JsonSerializer.Serialize(new + { + current = new + { + bundleId = "2025-10-22-full", + channel = "stable", + kind = "full", + isDelta = false, + baseBundleId = (string?)null, + bundleSha256 = "sha256:abc123", + bundleSize = 42, + capturedAt = captured, + importedAt = imported + }, + components = new[] + { + new + { + name = "concelier-json", + version = "2025-10-22", + digest = "sha256:def456", + capturedAt = captured, + sizeBytes = 1234 + } + } + }, new JsonSerializerOptions(JsonSerializerDefaults.Web)); + + var handler = new StubHttpMessageHandler( + (request, _) => + { + Assert.Equal("https://backend.example/api/offline-kit/status", request.RequestUri!.ToString()); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(statusJson) + }; + }); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://backend.example") + }; + + var options = new StellaOpsCliOptions + { + BackendUrl = "https://backend.example" + }; + + var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); + var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger()); + + var status = await client.GetOfflineKitStatusAsync(CancellationToken.None); + + Assert.Equal("2025-10-22-full", status.BundleId); + Assert.Equal("stable", status.Channel); + Assert.Equal("full", status.Kind); + Assert.False(status.IsDelta); + Assert.Equal(42, status.BundleSize); + Assert.Single(status.Components); + Assert.Equal("concelier-json", status.Components[0].Name); + } + + private sealed class ImportRecordingHandler : HttpMessageHandler + { + public string? MetadataJson { get; private set; } + public byte[]? BundlePayload { get; private set; } + public byte[]? ManifestPayload { get; private set; } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + if (request.RequestUri!.AbsoluteUri.EndsWith("/api/offline-kit/import", StringComparison.OrdinalIgnoreCase)) + { + Assert.IsType(request.Content); + foreach (var part in (MultipartFormDataContent)request.Content!) + { + var name = part.Headers.ContentDisposition?.Name?.Trim('"'); + switch (name) + { + case "metadata": + MetadataJson = await part.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + break; + case "bundle": + BundlePayload = await part.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + break; + case "manifest": + ManifestPayload = await part.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + break; + } + } + } + + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{\"importId\":\"imp-1\",\"status\":\"queued\",\"submittedAt\":\"2025-10-21T00:00:00Z\"}") + }; + } + } + + private sealed class StubTokenClient : IStellaOpsTokenClient { private readonly StellaOpsTokenResult _tokenResult; diff --git a/src/StellaOps.Cli/Commands/CommandFactory.cs b/src/StellaOps.Cli/Commands/CommandFactory.cs index aa0230a5..ef5d6da4 100644 --- a/src/StellaOps.Cli/Commands/CommandFactory.cs +++ b/src/StellaOps.Cli/Commands/CommandFactory.cs @@ -27,6 +27,7 @@ internal static class CommandFactory root.Add(BuildExcititorCommand(services, verboseOption, cancellationToken)); root.Add(BuildRuntimeCommand(services, verboseOption, cancellationToken)); root.Add(BuildAuthCommand(services, options, verboseOption, cancellationToken)); + root.Add(BuildOfflineCommand(services, verboseOption, cancellationToken)); root.Add(BuildConfigCommand(options)); return root; @@ -606,11 +607,102 @@ internal static class CommandFactory return auth; } - private static Command BuildConfigCommand(StellaOpsCliOptions options) - { - var config = new Command("config", "Inspect CLI configuration state."); - var show = new Command("show", "Display resolved configuration values."); - + private static Command BuildOfflineCommand(IServiceProvider services, Option verboseOption, CancellationToken cancellationToken) + { + var offline = new Command("offline", "Offline kit workflows and utilities."); + + var kit = new Command("kit", "Manage offline kit bundles."); + + var pull = new Command("pull", "Download the latest offline kit bundle."); + var bundleIdOption = new Option("--bundle-id") + { + Description = "Optional bundle identifier. Defaults to the latest available." + }; + var destinationOption = new Option("--destination") + { + Description = "Directory to store downloaded bundles (defaults to the configured offline kits directory)." + }; + var overwriteOption = new Option("--overwrite") + { + Description = "Overwrite existing files even if checksums match." + }; + var noResumeOption = new Option("--no-resume") + { + Description = "Disable resuming partial downloads." + }; + + pull.Add(bundleIdOption); + pull.Add(destinationOption); + pull.Add(overwriteOption); + pull.Add(noResumeOption); + pull.SetAction((parseResult, _) => + { + var bundleId = parseResult.GetValue(bundleIdOption); + var destination = parseResult.GetValue(destinationOption); + var overwrite = parseResult.GetValue(overwriteOption); + var resume = !parseResult.GetValue(noResumeOption); + var verbose = parseResult.GetValue(verboseOption); + return CommandHandlers.HandleOfflineKitPullAsync(services, bundleId, destination, overwrite, resume, verbose, cancellationToken); + }); + + var import = new Command("import", "Upload an offline kit bundle to the backend."); + var bundleArgument = new Argument("bundle") + { + Description = "Path to the offline kit tarball (.tgz)." + }; + var manifestOption = new Option("--manifest") + { + Description = "Offline manifest JSON path (defaults to metadata or sibling file)." + }; + var bundleSignatureOption = new Option("--bundle-signature") + { + Description = "Detached signature for the offline bundle (e.g. .sig)." + }; + var manifestSignatureOption = new Option("--manifest-signature") + { + Description = "Detached signature for the offline manifest (e.g. .jws)." + }; + + import.Add(bundleArgument); + import.Add(manifestOption); + import.Add(bundleSignatureOption); + import.Add(manifestSignatureOption); + import.SetAction((parseResult, _) => + { + var bundlePath = parseResult.GetValue(bundleArgument) ?? string.Empty; + var manifest = parseResult.GetValue(manifestOption); + var bundleSignature = parseResult.GetValue(bundleSignatureOption); + var manifestSignature = parseResult.GetValue(manifestSignatureOption); + var verbose = parseResult.GetValue(verboseOption); + return CommandHandlers.HandleOfflineKitImportAsync(services, bundlePath, manifest, bundleSignature, manifestSignature, verbose, cancellationToken); + }); + + var status = new Command("status", "Display offline kit installation status."); + var jsonOption = new Option("--json") + { + Description = "Emit status as JSON." + }; + status.Add(jsonOption); + status.SetAction((parseResult, _) => + { + var asJson = parseResult.GetValue(jsonOption); + var verbose = parseResult.GetValue(verboseOption); + return CommandHandlers.HandleOfflineKitStatusAsync(services, asJson, verbose, cancellationToken); + }); + + kit.Add(pull); + kit.Add(import); + kit.Add(status); + + offline.Add(kit); + return offline; + } + + private static Command BuildConfigCommand(StellaOpsCliOptions options) + { + var config = new Command("config", "Inspect CLI configuration state."); + var show = new Command("show", "Display resolved configuration values."); + show.SetAction((_, _) => { var authority = options.Authority ?? new StellaOpsCliAuthorityOptions(); diff --git a/src/StellaOps.Cli/Commands/CommandHandlers.cs b/src/StellaOps.Cli/Commands/CommandHandlers.cs index 7ac40593..3c412b83 100644 --- a/src/StellaOps.Cli/Commands/CommandHandlers.cs +++ b/src/StellaOps.Cli/Commands/CommandHandlers.cs @@ -1448,17 +1448,415 @@ internal static class CommandHandlers { logger.LogError(ex, "Failed to verify revocation bundle."); Environment.ExitCode = 1; - } - finally - { - loggerFactory.Dispose(); - } - } - - private static bool TryParseDetachedJws(string value, out string encodedHeader, out string encodedSignature) - { - encodedHeader = string.Empty; - encodedSignature = string.Empty; + } + finally + { + loggerFactory.Dispose(); + } + } + + public static async Task HandleOfflineKitPullAsync( + IServiceProvider services, + string? bundleId, + string? destinationDirectory, + bool overwrite, + bool resume, + bool verbose, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var client = scope.ServiceProvider.GetRequiredService(); + var options = scope.ServiceProvider.GetRequiredService(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("offline-kit-pull"); + var verbosity = scope.ServiceProvider.GetRequiredService(); + var previousLevel = verbosity.MinimumLevel; + verbosity.MinimumLevel = verbose ? LogLevel.Debug : LogLevel.Information; + using var activity = CliActivitySource.Instance.StartActivity("cli.offline.kit.pull", ActivityKind.Client); + activity?.SetTag("stellaops.cli.bundle_id", string.IsNullOrWhiteSpace(bundleId) ? "latest" : bundleId); + using var duration = CliMetrics.MeasureCommandDuration("offline kit pull"); + + try + { + var targetDirectory = string.IsNullOrWhiteSpace(destinationDirectory) + ? options.Offline?.KitsDirectory ?? Path.Combine(Environment.CurrentDirectory, "offline-kits") + : destinationDirectory; + + targetDirectory = Path.GetFullPath(targetDirectory); + Directory.CreateDirectory(targetDirectory); + + var result = await client.DownloadOfflineKitAsync(bundleId, targetDirectory, overwrite, resume, cancellationToken).ConfigureAwait(false); + + logger.LogInformation( + "Bundle {BundleId} stored at {Path} (captured {Captured:u}, sha256:{Digest}).", + result.Descriptor.BundleId, + result.BundlePath, + result.Descriptor.CapturedAt, + result.Descriptor.BundleSha256); + + logger.LogInformation("Manifest saved to {Manifest}.", result.ManifestPath); + + if (!string.IsNullOrWhiteSpace(result.MetadataPath)) + { + logger.LogDebug("Metadata recorded at {Metadata}.", result.MetadataPath); + } + + if (result.BundleSignaturePath is not null) + { + logger.LogInformation("Bundle signature saved to {Signature}.", result.BundleSignaturePath); + } + + if (result.ManifestSignaturePath is not null) + { + logger.LogInformation("Manifest signature saved to {Signature}.", result.ManifestSignaturePath); + } + + CliMetrics.RecordOfflineKitDownload(result.Descriptor.Kind ?? "unknown", result.FromCache); + activity?.SetTag("stellaops.cli.bundle_cache", result.FromCache); + Environment.ExitCode = 0; + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to download offline kit bundle."); + Environment.ExitCode = 1; + } + finally + { + verbosity.MinimumLevel = previousLevel; + } + } + + public static async Task HandleOfflineKitImportAsync( + IServiceProvider services, + string bundlePath, + string? manifestPath, + string? bundleSignaturePath, + string? manifestSignaturePath, + bool verbose, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var client = scope.ServiceProvider.GetRequiredService(); + var options = scope.ServiceProvider.GetRequiredService(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("offline-kit-import"); + var verbosity = scope.ServiceProvider.GetRequiredService(); + var previousLevel = verbosity.MinimumLevel; + verbosity.MinimumLevel = verbose ? LogLevel.Debug : LogLevel.Information; + using var activity = CliActivitySource.Instance.StartActivity("cli.offline.kit.import", ActivityKind.Client); + using var duration = CliMetrics.MeasureCommandDuration("offline kit import"); + + try + { + if (string.IsNullOrWhiteSpace(bundlePath)) + { + logger.LogError("Bundle path is required."); + Environment.ExitCode = 1; + return; + } + + bundlePath = Path.GetFullPath(bundlePath); + if (!File.Exists(bundlePath)) + { + logger.LogError("Bundle file {Path} not found.", bundlePath); + Environment.ExitCode = 1; + return; + } + + var metadata = await LoadOfflineKitMetadataAsync(bundlePath, cancellationToken).ConfigureAwait(false); + if (metadata is not null) + { + manifestPath ??= metadata.ManifestPath; + bundleSignaturePath ??= metadata.BundleSignaturePath; + manifestSignaturePath ??= metadata.ManifestSignaturePath; + } + + manifestPath = NormalizeFilePath(manifestPath); + bundleSignaturePath = NormalizeFilePath(bundleSignaturePath); + manifestSignaturePath = NormalizeFilePath(manifestSignaturePath); + + if (manifestPath is null) + { + manifestPath = TryInferManifestPath(bundlePath); + if (manifestPath is not null) + { + logger.LogDebug("Using inferred manifest path {Path}.", manifestPath); + } + } + + if (manifestPath is not null && !File.Exists(manifestPath)) + { + logger.LogError("Manifest file {Path} not found.", manifestPath); + Environment.ExitCode = 1; + return; + } + + if (bundleSignaturePath is not null && !File.Exists(bundleSignaturePath)) + { + logger.LogWarning("Bundle signature {Path} not found; skipping.", bundleSignaturePath); + bundleSignaturePath = null; + } + + if (manifestSignaturePath is not null && !File.Exists(manifestSignaturePath)) + { + logger.LogWarning("Manifest signature {Path} not found; skipping.", manifestSignaturePath); + manifestSignaturePath = null; + } + + if (metadata is not null) + { + var computedBundleDigest = await ComputeSha256Async(bundlePath, cancellationToken).ConfigureAwait(false); + if (!DigestsEqual(computedBundleDigest, metadata.BundleSha256)) + { + logger.LogError("Bundle digest mismatch. Expected sha256:{Expected} but computed sha256:{Actual}.", metadata.BundleSha256, computedBundleDigest); + Environment.ExitCode = 1; + return; + } + + if (manifestPath is not null) + { + var computedManifestDigest = await ComputeSha256Async(manifestPath, cancellationToken).ConfigureAwait(false); + if (!DigestsEqual(computedManifestDigest, metadata.ManifestSha256)) + { + logger.LogError("Manifest digest mismatch. Expected sha256:{Expected} but computed sha256:{Actual}.", metadata.ManifestSha256, computedManifestDigest); + Environment.ExitCode = 1; + return; + } + } + } + + var request = new OfflineKitImportRequest( + bundlePath, + manifestPath, + bundleSignaturePath, + manifestSignaturePath, + metadata?.BundleId, + metadata?.BundleSha256, + metadata?.BundleSize, + metadata?.CapturedAt, + metadata?.Channel, + metadata?.Kind, + metadata?.IsDelta, + metadata?.BaseBundleId, + metadata?.ManifestSha256, + metadata?.ManifestSize); + + var result = await client.ImportOfflineKitAsync(request, cancellationToken).ConfigureAwait(false); + CliMetrics.RecordOfflineKitImport(result.Status); + + logger.LogInformation( + "Import {ImportId} submitted at {Submitted:u} with status {Status}.", + string.IsNullOrWhiteSpace(result.ImportId) ? "" : result.ImportId, + result.SubmittedAt, + string.IsNullOrWhiteSpace(result.Status) ? "queued" : result.Status); + + if (!string.IsNullOrWhiteSpace(result.Message)) + { + logger.LogInformation(result.Message); + } + + Environment.ExitCode = 0; + } + catch (Exception ex) + { + logger.LogError(ex, "Offline kit import failed."); + Environment.ExitCode = 1; + } + finally + { + verbosity.MinimumLevel = previousLevel; + } + } + + public static async Task HandleOfflineKitStatusAsync( + IServiceProvider services, + bool asJson, + bool verbose, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var client = scope.ServiceProvider.GetRequiredService(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("offline-kit-status"); + var verbosity = scope.ServiceProvider.GetRequiredService(); + var previousLevel = verbosity.MinimumLevel; + verbosity.MinimumLevel = verbose ? LogLevel.Debug : LogLevel.Information; + using var activity = CliActivitySource.Instance.StartActivity("cli.offline.kit.status", ActivityKind.Client); + using var duration = CliMetrics.MeasureCommandDuration("offline kit status"); + + try + { + var status = await client.GetOfflineKitStatusAsync(cancellationToken).ConfigureAwait(false); + + if (asJson) + { + var payload = new + { + bundleId = status.BundleId, + channel = status.Channel, + kind = status.Kind, + isDelta = status.IsDelta, + baseBundleId = status.BaseBundleId, + capturedAt = status.CapturedAt, + importedAt = status.ImportedAt, + sha256 = status.BundleSha256, + sizeBytes = status.BundleSize, + components = status.Components.Select(component => new + { + component.Name, + component.Version, + component.Digest, + component.CapturedAt, + component.SizeBytes + }) + }; + + var json = JsonSerializer.Serialize(payload, new JsonSerializerOptions(JsonSerializerDefaults.Web) { WriteIndented = true }); + Console.WriteLine(json); + } + else + { + if (string.IsNullOrWhiteSpace(status.BundleId)) + { + logger.LogInformation("No offline kit bundle has been imported yet."); + } + else + { + logger.LogInformation( + "Current bundle {BundleId} ({Kind}) captured {Captured:u}, imported {Imported:u}, sha256:{Digest}, size {Size}.", + status.BundleId, + status.Kind ?? "unknown", + status.CapturedAt ?? default, + status.ImportedAt ?? default, + status.BundleSha256 ?? "", + status.BundleSize.HasValue ? status.BundleSize.Value.ToString("N0", CultureInfo.InvariantCulture) : ""); + } + + if (status.Components.Count > 0) + { + var table = new Table().AddColumns("Component", "Version", "Digest", "Captured", "Size (bytes)"); + foreach (var component in status.Components) + { + table.AddRow( + component.Name, + string.IsNullOrWhiteSpace(component.Version) ? "-" : component.Version!, + string.IsNullOrWhiteSpace(component.Digest) ? "-" : $"sha256:{component.Digest}", + component.CapturedAt?.ToString("u", CultureInfo.InvariantCulture) ?? "-", + component.SizeBytes.HasValue ? component.SizeBytes.Value.ToString("N0", CultureInfo.InvariantCulture) : "-"); + } + + AnsiConsole.Write(table); + } + } + + Environment.ExitCode = 0; + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to read offline kit status."); + Environment.ExitCode = 1; + } + finally + { + verbosity.MinimumLevel = previousLevel; + } + } + + private static async Task LoadOfflineKitMetadataAsync(string bundlePath, CancellationToken cancellationToken) + { + var metadataPath = bundlePath + ".metadata.json"; + if (!File.Exists(metadataPath)) + { + return null; + } + + try + { + await using var stream = File.OpenRead(metadataPath); + return await JsonSerializer.DeserializeAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false); + } + catch + { + return null; + } + } + + private static string? NormalizeFilePath(string? path) + { + if (string.IsNullOrWhiteSpace(path)) + { + return null; + } + + return Path.GetFullPath(path); + } + + private static string? TryInferManifestPath(string bundlePath) + { + var directory = Path.GetDirectoryName(bundlePath); + if (string.IsNullOrWhiteSpace(directory)) + { + return null; + } + + var baseName = Path.GetFileName(bundlePath); + if (string.IsNullOrWhiteSpace(baseName)) + { + return null; + } + + baseName = Path.GetFileNameWithoutExtension(baseName); + if (baseName.EndsWith(".tar", StringComparison.OrdinalIgnoreCase)) + { + baseName = Path.GetFileNameWithoutExtension(baseName); + } + + var candidates = new[] + { + Path.Combine(directory, $"offline-manifest-{baseName}.json"), + Path.Combine(directory, "offline-manifest.json") + }; + + foreach (var candidate in candidates) + { + if (File.Exists(candidate)) + { + return Path.GetFullPath(candidate); + } + } + + return Directory.EnumerateFiles(directory, "offline-manifest*.json").FirstOrDefault(); + } + + private static bool DigestsEqual(string computed, string? expected) + { + if (string.IsNullOrWhiteSpace(expected)) + { + return true; + } + + return string.Equals(NormalizeDigest(computed), NormalizeDigest(expected), StringComparison.OrdinalIgnoreCase); + } + + private static string NormalizeDigest(string digest) + { + var value = digest.Trim(); + if (value.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)) + { + value = value.Substring("sha256:".Length); + } + + return value.ToLowerInvariant(); + } + + private static async Task ComputeSha256Async(string path, CancellationToken cancellationToken) + { + await using var stream = File.OpenRead(path); + var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + + private static bool TryParseDetachedJws(string value, out string encodedHeader, out string encodedSignature) + { + encodedHeader = string.Empty; + encodedSignature = string.Empty; if (string.IsNullOrWhiteSpace(value)) { diff --git a/src/StellaOps.Cli/Configuration/CliBootstrapper.cs b/src/StellaOps.Cli/Configuration/CliBootstrapper.cs index ffba6a75..d1fa0941 100644 --- a/src/StellaOps.Cli/Configuration/CliBootstrapper.cs +++ b/src/StellaOps.Cli/Configuration/CliBootstrapper.cs @@ -200,6 +200,40 @@ public static class CliBootstrapper { authority.TokenCacheDirectory = Path.GetFullPath(authority.TokenCacheDirectory); } + + cliOptions.Offline ??= new StellaOpsCliOfflineOptions(); + var offline = cliOptions.Offline; + + var kitsDirectory = ResolveWithFallback( + string.Empty, + configuration, + "STELLAOPS_OFFLINE_KITS_DIRECTORY", + "STELLAOPS_OFFLINE_KITS_DIR", + "StellaOps:Offline:KitsDirectory", + "StellaOps:Offline:KitDirectory", + "Offline:KitsDirectory", + "Offline:KitDirectory"); + + if (string.IsNullOrWhiteSpace(kitsDirectory)) + { + kitsDirectory = offline.KitsDirectory ?? "offline-kits"; + } + + offline.KitsDirectory = Path.GetFullPath(kitsDirectory); + if (!Directory.Exists(offline.KitsDirectory)) + { + Directory.CreateDirectory(offline.KitsDirectory); + } + + var mirror = ResolveWithFallback( + string.Empty, + configuration, + "STELLAOPS_OFFLINE_MIRROR_URL", + "StellaOps:Offline:KitMirror", + "Offline:KitMirror", + "Offline:MirrorUrl"); + + offline.MirrorUrl = string.IsNullOrWhiteSpace(mirror) ? null : mirror.Trim(); }; }); diff --git a/src/StellaOps.Cli/Configuration/StellaOpsCliOptions.cs b/src/StellaOps.Cli/Configuration/StellaOpsCliOptions.cs index 54355bbc..01699033 100644 --- a/src/StellaOps.Cli/Configuration/StellaOpsCliOptions.cs +++ b/src/StellaOps.Cli/Configuration/StellaOpsCliOptions.cs @@ -9,7 +9,7 @@ public sealed class StellaOpsCliOptions public string ApiKey { get; set; } = string.Empty; public string BackendUrl { get; set; } = string.Empty; - + public string ScannerCacheDirectory { get; set; } = "scanners"; public string ResultsDirectory { get; set; } = "results"; @@ -23,6 +23,8 @@ public sealed class StellaOpsCliOptions public int ScanUploadAttempts { get; set; } = 3; public StellaOpsCliAuthorityOptions Authority { get; set; } = new(); + + public StellaOpsCliOfflineOptions Offline { get; set; } = new(); } public sealed class StellaOpsCliAuthorityOptions @@ -54,3 +56,10 @@ public sealed class StellaOpsCliAuthorityResilienceOptions public TimeSpan? OfflineCacheTolerance { get; set; } } + +public sealed class StellaOpsCliOfflineOptions +{ + public string KitsDirectory { get; set; } = "offline-kits"; + + public string? MirrorUrl { get; set; } +} diff --git a/src/StellaOps.Cli/Services/BackendOperationsClient.cs b/src/StellaOps.Cli/Services/BackendOperationsClient.cs index 0e387083..9d955fee 100644 --- a/src/StellaOps.Cli/Services/BackendOperationsClient.cs +++ b/src/StellaOps.Cli/Services/BackendOperationsClient.cs @@ -535,7 +535,687 @@ internal sealed class BackendOperationsClient : IBackendOperationsClient return list; } - private static List NormalizeImages(IReadOnlyList images) + public async Task DownloadOfflineKitAsync(string? bundleId, string destinationDirectory, bool overwrite, bool resume, CancellationToken cancellationToken) + { + EnsureBackendConfigured(); + + var rootDirectory = ResolveOfflineDirectory(destinationDirectory); + Directory.CreateDirectory(rootDirectory); + + var descriptor = await FetchOfflineKitDescriptorAsync(bundleId, cancellationToken).ConfigureAwait(false); + + var bundlePath = Path.Combine(rootDirectory, descriptor.BundleName); + var metadataPath = bundlePath + ".metadata.json"; + var manifestPath = Path.Combine(rootDirectory, descriptor.ManifestName); + var bundleSignaturePath = descriptor.BundleSignatureName is not null ? Path.Combine(rootDirectory, descriptor.BundleSignatureName) : null; + var manifestSignaturePath = descriptor.ManifestSignatureName is not null ? Path.Combine(rootDirectory, descriptor.ManifestSignatureName) : null; + + var fromCache = false; + if (!overwrite && File.Exists(bundlePath)) + { + var digest = await ComputeSha256Async(bundlePath, cancellationToken).ConfigureAwait(false); + if (string.Equals(digest, descriptor.BundleSha256, StringComparison.OrdinalIgnoreCase)) + { + fromCache = true; + } + else if (resume) + { + var partial = bundlePath + ".partial"; + File.Move(bundlePath, partial, overwrite: true); + } + else + { + File.Delete(bundlePath); + } + } + + if (!fromCache) + { + await DownloadFileWithResumeAsync(descriptor.BundleDownloadUri, bundlePath, descriptor.BundleSha256, descriptor.BundleSize, resume, cancellationToken).ConfigureAwait(false); + } + + await DownloadFileWithResumeAsync(descriptor.ManifestDownloadUri, manifestPath, descriptor.ManifestSha256, descriptor.ManifestSize ?? 0, resume: false, cancellationToken).ConfigureAwait(false); + + if (descriptor.BundleSignatureDownloadUri is not null && bundleSignaturePath is not null) + { + await DownloadAuxiliaryFileAsync(descriptor.BundleSignatureDownloadUri, bundleSignaturePath, cancellationToken).ConfigureAwait(false); + } + + if (descriptor.ManifestSignatureDownloadUri is not null && manifestSignaturePath is not null) + { + await DownloadAuxiliaryFileAsync(descriptor.ManifestSignatureDownloadUri, manifestSignaturePath, cancellationToken).ConfigureAwait(false); + } + + await WriteOfflineKitMetadataAsync(metadataPath, descriptor, bundlePath, manifestPath, bundleSignaturePath, manifestSignaturePath, cancellationToken).ConfigureAwait(false); + + return new OfflineKitDownloadResult( + descriptor, + bundlePath, + manifestPath, + bundleSignaturePath, + manifestSignaturePath, + metadataPath, + fromCache); + } + + public async Task ImportOfflineKitAsync(OfflineKitImportRequest request, CancellationToken cancellationToken) + { + EnsureBackendConfigured(); + + if (request is null) + { + throw new ArgumentNullException(nameof(request)); + } + + var bundlePath = Path.GetFullPath(request.BundlePath); + if (!File.Exists(bundlePath)) + { + throw new FileNotFoundException("Offline kit bundle not found.", bundlePath); + } + + string? manifestPath = null; + if (!string.IsNullOrWhiteSpace(request.ManifestPath)) + { + manifestPath = Path.GetFullPath(request.ManifestPath); + if (!File.Exists(manifestPath)) + { + throw new FileNotFoundException("Offline kit manifest not found.", manifestPath); + } + } + + string? bundleSignaturePath = null; + if (!string.IsNullOrWhiteSpace(request.BundleSignaturePath)) + { + bundleSignaturePath = Path.GetFullPath(request.BundleSignaturePath); + if (!File.Exists(bundleSignaturePath)) + { + throw new FileNotFoundException("Offline kit bundle signature not found.", bundleSignaturePath); + } + } + + string? manifestSignaturePath = null; + if (!string.IsNullOrWhiteSpace(request.ManifestSignaturePath)) + { + manifestSignaturePath = Path.GetFullPath(request.ManifestSignaturePath); + if (!File.Exists(manifestSignaturePath)) + { + throw new FileNotFoundException("Offline kit manifest signature not found.", manifestSignaturePath); + } + } + + var bundleSize = request.BundleSize ?? new FileInfo(bundlePath).Length; + var bundleSha = string.IsNullOrWhiteSpace(request.BundleSha256) + ? await ComputeSha256Async(bundlePath, cancellationToken).ConfigureAwait(false) + : NormalizeSha(request.BundleSha256) ?? throw new InvalidOperationException("Bundle digest must not be empty."); + + string? manifestSha = null; + long? manifestSize = null; + if (manifestPath is not null) + { + manifestSize = request.ManifestSize ?? new FileInfo(manifestPath).Length; + manifestSha = string.IsNullOrWhiteSpace(request.ManifestSha256) + ? await ComputeSha256Async(manifestPath, cancellationToken).ConfigureAwait(false) + : NormalizeSha(request.ManifestSha256); + } + + var metadata = new OfflineKitImportMetadataPayload + { + BundleId = request.BundleId, + BundleSha256 = bundleSha, + BundleSize = bundleSize, + CapturedAt = request.CapturedAt, + Channel = request.Channel, + Kind = request.Kind, + IsDelta = request.IsDelta, + BaseBundleId = request.BaseBundleId, + ManifestSha256 = manifestSha, + ManifestSize = manifestSize + }; + + using var message = CreateRequest(HttpMethod.Post, "api/offline-kit/import"); + await AuthorizeRequestAsync(message, cancellationToken).ConfigureAwait(false); + + using var content = new MultipartFormDataContent(); + + var metadataOptions = new JsonSerializerOptions(SerializerOptions) + { + WriteIndented = false + }; + var metadataJson = JsonSerializer.Serialize(metadata, metadataOptions); + var metadataContent = new StringContent(metadataJson, Encoding.UTF8, "application/json"); + content.Add(metadataContent, "metadata"); + + var bundleStream = File.OpenRead(bundlePath); + var bundleContent = new StreamContent(bundleStream); + bundleContent.Headers.ContentType = new MediaTypeHeaderValue("application/gzip"); + content.Add(bundleContent, "bundle", Path.GetFileName(bundlePath)); + + if (manifestPath is not null) + { + var manifestStream = File.OpenRead(manifestPath); + var manifestContent = new StreamContent(manifestStream); + manifestContent.Headers.ContentType = new MediaTypeHeaderValue("application/json"); + content.Add(manifestContent, "manifest", Path.GetFileName(manifestPath)); + } + + if (bundleSignaturePath is not null) + { + var signatureStream = File.OpenRead(bundleSignaturePath); + var signatureContent = new StreamContent(signatureStream); + signatureContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + content.Add(signatureContent, "bundleSignature", Path.GetFileName(bundleSignaturePath)); + } + + if (manifestSignaturePath is not null) + { + var manifestSignatureStream = File.OpenRead(manifestSignaturePath); + var manifestSignatureContent = new StreamContent(manifestSignatureStream); + manifestSignatureContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + content.Add(manifestSignatureContent, "manifestSignature", Path.GetFileName(manifestSignaturePath)); + } + + message.Content = content; + + using var response = await _httpClient.SendAsync(message, cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + var failure = await CreateFailureMessageAsync(response, cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException(failure); + } + + OfflineKitImportResponseTransport? document; + try + { + document = await response.Content.ReadFromJsonAsync(SerializerOptions, cancellationToken).ConfigureAwait(false); + } + catch (JsonException ex) + { + var raw = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException($"Failed to parse offline kit import response. {ex.Message}", ex) + { + Data = { ["payload"] = raw } + }; + } + + var submittedAt = document?.SubmittedAt ?? DateTimeOffset.UtcNow; + + return new OfflineKitImportResult( + document?.ImportId, + document?.Status, + submittedAt, + document?.Message); + } + + public async Task GetOfflineKitStatusAsync(CancellationToken cancellationToken) + { + EnsureBackendConfigured(); + + using var request = CreateRequest(HttpMethod.Get, "api/offline-kit/status"); + await AuthorizeRequestAsync(request, cancellationToken).ConfigureAwait(false); + using var response = await _httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + + if (!response.IsSuccessStatusCode) + { + var failure = await CreateFailureMessageAsync(response, cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException(failure); + } + + if (response.Content is null || response.Content.Headers.ContentLength is 0) + { + return new OfflineKitStatus(null, null, null, false, null, null, null, null, null, Array.Empty()); + } + + OfflineKitStatusTransport? document; + try + { + document = await response.Content.ReadFromJsonAsync(SerializerOptions, cancellationToken).ConfigureAwait(false); + } + catch (JsonException ex) + { + var raw = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException($"Failed to parse offline kit status response. {ex.Message}", ex) + { + Data = { ["payload"] = raw } + }; + } + + var current = document?.Current; + var components = MapOfflineComponents(document?.Components); + + if (current is null) + { + return new OfflineKitStatus(null, null, null, false, null, null, null, null, null, components); + } + + return new OfflineKitStatus( + NormalizeOptionalString(current.BundleId), + NormalizeOptionalString(current.Channel), + NormalizeOptionalString(current.Kind), + current.IsDelta ?? false, + NormalizeOptionalString(current.BaseBundleId), + current.CapturedAt?.ToUniversalTime(), + current.ImportedAt?.ToUniversalTime(), + NormalizeSha(current.BundleSha256), + current.BundleSize, + components); + } + + private string ResolveOfflineDirectory(string destinationDirectory) + { + if (!string.IsNullOrWhiteSpace(destinationDirectory)) + { + return Path.GetFullPath(destinationDirectory); + } + + var configured = _options.Offline?.KitsDirectory; + if (!string.IsNullOrWhiteSpace(configured)) + { + return Path.GetFullPath(configured); + } + + return Path.GetFullPath(Path.Combine(Environment.CurrentDirectory, "offline-kits")); + } + + private async Task FetchOfflineKitDescriptorAsync(string? bundleId, CancellationToken cancellationToken) + { + var route = string.IsNullOrWhiteSpace(bundleId) + ? "api/offline-kit/bundles/latest" + : $"api/offline-kit/bundles/{Uri.EscapeDataString(bundleId)}"; + + using var request = CreateRequest(HttpMethod.Get, route); + await AuthorizeRequestAsync(request, cancellationToken).ConfigureAwait(false); + using var response = await _httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + + if (!response.IsSuccessStatusCode) + { + var failure = await CreateFailureMessageAsync(response, cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException(failure); + } + + OfflineKitBundleDescriptorTransport? payload; + try + { + payload = await response.Content.ReadFromJsonAsync(SerializerOptions, cancellationToken).ConfigureAwait(false); + } + catch (JsonException ex) + { + var raw = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException($"Failed to parse offline kit metadata. {ex.Message}", ex) + { + Data = { ["payload"] = raw } + }; + } + + if (payload is null) + { + throw new InvalidOperationException("Offline kit metadata response was empty."); + } + + return MapOfflineKitDescriptor(payload); + } + + private OfflineKitBundleDescriptor MapOfflineKitDescriptor(OfflineKitBundleDescriptorTransport transport) + { + if (transport is null) + { + throw new ArgumentNullException(nameof(transport)); + } + + var bundleName = string.IsNullOrWhiteSpace(transport.BundleName) + ? throw new InvalidOperationException("Offline kit metadata missing bundleName.") + : transport.BundleName!.Trim(); + + var bundleId = string.IsNullOrWhiteSpace(transport.BundleId) ? bundleName : transport.BundleId!.Trim(); + var bundleSha = NormalizeSha(transport.BundleSha256) ?? throw new InvalidOperationException("Offline kit metadata missing bundleSha256."); + + var bundleSize = transport.BundleSize; + if (bundleSize <= 0) + { + throw new InvalidOperationException("Offline kit metadata missing bundle size."); + } + + var manifestName = string.IsNullOrWhiteSpace(transport.ManifestName) ? "offline-manifest.json" : transport.ManifestName!.Trim(); + var manifestSha = NormalizeSha(transport.ManifestSha256) ?? throw new InvalidOperationException("Offline kit metadata missing manifestSha256."); + var capturedAt = transport.CapturedAt?.ToUniversalTime() ?? DateTimeOffset.UtcNow; + + var bundleDownloadUri = ResolveDownloadUri(transport.BundleUrl, transport.BundlePath, bundleName); + var manifestDownloadUri = ResolveDownloadUri(transport.ManifestUrl, transport.ManifestPath, manifestName); + var bundleSignatureUri = ResolveOptionalDownloadUri(transport.BundleSignatureUrl, transport.BundleSignaturePath, transport.BundleSignatureName); + var manifestSignatureUri = ResolveOptionalDownloadUri(transport.ManifestSignatureUrl, transport.ManifestSignaturePath, transport.ManifestSignatureName); + var bundleSignatureName = ResolveArtifactName(transport.BundleSignatureName, bundleSignatureUri); + var manifestSignatureName = ResolveArtifactName(transport.ManifestSignatureName, manifestSignatureUri); + + return new OfflineKitBundleDescriptor( + bundleId, + bundleName, + bundleSha, + bundleSize, + bundleDownloadUri, + manifestName, + manifestSha, + manifestDownloadUri, + capturedAt, + NormalizeOptionalString(transport.Channel), + NormalizeOptionalString(transport.Kind), + transport.IsDelta ?? false, + NormalizeOptionalString(transport.BaseBundleId), + bundleSignatureName, + bundleSignatureUri, + manifestSignatureName, + manifestSignatureUri, + transport.ManifestSize); + } + + private static string? ResolveArtifactName(string? explicitName, Uri? uri) + { + if (!string.IsNullOrWhiteSpace(explicitName)) + { + return explicitName.Trim(); + } + + if (uri is not null) + { + var name = Path.GetFileName(uri.LocalPath); + return string.IsNullOrWhiteSpace(name) ? null : name; + } + + return null; + } + + private Uri ResolveDownloadUri(string? absoluteOrRelativeUrl, string? relativePath, string fallbackFileName) + { + if (!string.IsNullOrWhiteSpace(absoluteOrRelativeUrl)) + { + var candidate = new Uri(absoluteOrRelativeUrl, UriKind.RelativeOrAbsolute); + if (candidate.IsAbsoluteUri) + { + return candidate; + } + + if (_httpClient.BaseAddress is not null) + { + return new Uri(_httpClient.BaseAddress, candidate); + } + + return BuildUriFromRelative(candidate.ToString()); + } + + if (!string.IsNullOrWhiteSpace(relativePath)) + { + return BuildUriFromRelative(relativePath); + } + + if (!string.IsNullOrWhiteSpace(fallbackFileName)) + { + return BuildUriFromRelative(fallbackFileName); + } + + throw new InvalidOperationException("Offline kit metadata did not include a download URL."); + } + + private Uri BuildUriFromRelative(string relative) + { + var normalized = relative.TrimStart('/'); + if (!string.IsNullOrWhiteSpace(_options.Offline?.MirrorUrl) && + Uri.TryCreate(_options.Offline.MirrorUrl, UriKind.Absolute, out var mirrorBase)) + { + if (!mirrorBase.AbsoluteUri.EndsWith("/")) + { + mirrorBase = new Uri(mirrorBase.AbsoluteUri + "/"); + } + + return new Uri(mirrorBase, normalized); + } + + if (_httpClient.BaseAddress is not null) + { + return new Uri(_httpClient.BaseAddress, normalized); + } + + throw new InvalidOperationException($"Cannot resolve offline kit URI for '{relative}' because no mirror or backend base address is configured."); + } + + private Uri? ResolveOptionalDownloadUri(string? absoluteOrRelativeUrl, string? relativePath, string? fallbackName) + { + var hasData = !string.IsNullOrWhiteSpace(absoluteOrRelativeUrl) || + !string.IsNullOrWhiteSpace(relativePath) || + !string.IsNullOrWhiteSpace(fallbackName); + + if (!hasData) + { + return null; + } + + try + { + return ResolveDownloadUri(absoluteOrRelativeUrl, relativePath, fallbackName ?? string.Empty); + } + catch + { + return null; + } + } + + private async Task DownloadFileWithResumeAsync(Uri downloadUri, string targetPath, string expectedSha256, long expectedSize, bool resume, CancellationToken cancellationToken) + { + var directory = Path.GetDirectoryName(targetPath); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + var partialPath = resume ? targetPath + ".partial" : targetPath + ".tmp"; + + if (!resume && File.Exists(targetPath)) + { + File.Delete(targetPath); + } + + if (resume && File.Exists(targetPath)) + { + File.Move(targetPath, partialPath, overwrite: true); + } + + long existingLength = 0; + if (resume && File.Exists(partialPath)) + { + existingLength = new FileInfo(partialPath).Length; + if (expectedSize > 0 && existingLength >= expectedSize) + { + existingLength = expectedSize; + } + } + + while (true) + { + using var request = new HttpRequestMessage(HttpMethod.Get, downloadUri); + if (resume && existingLength > 0 && expectedSize > 0 && existingLength < expectedSize) + { + request.Headers.Range = new RangeHeaderValue(existingLength, null); + } + + using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + + if (resume && existingLength > 0 && expectedSize > 0 && existingLength < expectedSize && response.StatusCode == HttpStatusCode.OK) + { + existingLength = 0; + if (File.Exists(partialPath)) + { + File.Delete(partialPath); + } + + continue; + } + + if (!response.IsSuccessStatusCode && + !(resume && existingLength > 0 && response.StatusCode == HttpStatusCode.PartialContent)) + { + var failure = await CreateFailureMessageAsync(response, cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException(failure); + } + + var destination = resume ? partialPath : targetPath; + var mode = resume && existingLength > 0 ? FileMode.Append : FileMode.Create; + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + await using (var file = new FileStream(destination, mode, FileAccess.Write, FileShare.None, 81920, useAsync: true)) + { + await stream.CopyToAsync(file, cancellationToken).ConfigureAwait(false); + } + + break; + } + + if (resume && File.Exists(partialPath)) + { + File.Move(partialPath, targetPath, overwrite: true); + } + + var digest = await ComputeSha256Async(targetPath, cancellationToken).ConfigureAwait(false); + if (!string.Equals(digest, expectedSha256, StringComparison.OrdinalIgnoreCase)) + { + File.Delete(targetPath); + throw new InvalidOperationException($"Digest mismatch for {Path.GetFileName(targetPath)}. Expected {expectedSha256} but computed {digest}."); + } + + if (expectedSize > 0) + { + var actualSize = new FileInfo(targetPath).Length; + if (actualSize != expectedSize) + { + File.Delete(targetPath); + throw new InvalidOperationException($"Size mismatch for {Path.GetFileName(targetPath)}. Expected {expectedSize:N0} bytes but downloaded {actualSize:N0} bytes."); + } + } + } + + private async Task DownloadAuxiliaryFileAsync(Uri downloadUri, string targetPath, CancellationToken cancellationToken) + { + var directory = Path.GetDirectoryName(targetPath); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + using var request = new HttpRequestMessage(HttpMethod.Get, downloadUri); + using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + + if (!response.IsSuccessStatusCode) + { + var failure = await CreateFailureMessageAsync(response, cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException(failure); + } + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + await using var file = new FileStream(targetPath, FileMode.Create, FileAccess.Write, FileShare.None, 81920, useAsync: true); + await stream.CopyToAsync(file, cancellationToken).ConfigureAwait(false); + } + + private static async Task WriteOfflineKitMetadataAsync( + string metadataPath, + OfflineKitBundleDescriptor descriptor, + string bundlePath, + string manifestPath, + string? bundleSignaturePath, + string? manifestSignaturePath, + CancellationToken cancellationToken) + { + var document = new OfflineKitMetadataDocument + { + BundleId = descriptor.BundleId, + BundleName = descriptor.BundleName, + BundleSha256 = descriptor.BundleSha256, + BundleSize = descriptor.BundleSize, + BundlePath = Path.GetFullPath(bundlePath), + CapturedAt = descriptor.CapturedAt, + DownloadedAt = DateTimeOffset.UtcNow, + Channel = descriptor.Channel, + Kind = descriptor.Kind, + IsDelta = descriptor.IsDelta, + BaseBundleId = descriptor.BaseBundleId, + ManifestName = descriptor.ManifestName, + ManifestSha256 = descriptor.ManifestSha256, + ManifestSize = descriptor.ManifestSize, + ManifestPath = Path.GetFullPath(manifestPath), + BundleSignaturePath = bundleSignaturePath is null ? null : Path.GetFullPath(bundleSignaturePath), + ManifestSignaturePath = manifestSignaturePath is null ? null : Path.GetFullPath(manifestSignaturePath) + }; + + var options = new JsonSerializerOptions(SerializerOptions) + { + WriteIndented = true + }; + + var payload = JsonSerializer.Serialize(document, options); + await File.WriteAllTextAsync(metadataPath, payload, cancellationToken).ConfigureAwait(false); + } + + private static IReadOnlyList MapOfflineComponents(List? transports) + { + if (transports is null || transports.Count == 0) + { + return Array.Empty(); + } + + var list = new List(); + foreach (var transport in transports) + { + if (transport is null || string.IsNullOrWhiteSpace(transport.Name)) + { + continue; + } + + list.Add(new OfflineKitComponentStatus( + transport.Name.Trim(), + NormalizeOptionalString(transport.Version), + NormalizeSha(transport.Digest), + transport.CapturedAt?.ToUniversalTime(), + transport.SizeBytes)); + } + + return list.Count == 0 ? Array.Empty() : list; + } + + private static string? NormalizeSha(string? digest) + { + if (string.IsNullOrWhiteSpace(digest)) + { + return null; + } + + var value = digest.Trim(); + if (value.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)) + { + value = value.Substring("sha256:".Length); + } + + return value.ToLowerInvariant(); + } + + private sealed class OfflineKitImportMetadataPayload + { + public string? BundleId { get; set; } + + public string BundleSha256 { get; set; } = string.Empty; + + public long BundleSize { get; set; } + + public DateTimeOffset? CapturedAt { get; set; } + + public string? Channel { get; set; } + + public string? Kind { get; set; } + + public bool? IsDelta { get; set; } + + public string? BaseBundleId { get; set; } + + public string? ManifestSha256 { get; set; } + + public long? ManifestSize { get; set; } + } + + private static List NormalizeImages(IReadOnlyList images) { var normalized = new List(); if (images is null) diff --git a/src/StellaOps.Cli/Services/IBackendOperationsClient.cs b/src/StellaOps.Cli/Services/IBackendOperationsClient.cs index 1c44f132..fdecf9ee 100644 --- a/src/StellaOps.Cli/Services/IBackendOperationsClient.cs +++ b/src/StellaOps.Cli/Services/IBackendOperationsClient.cs @@ -22,4 +22,10 @@ internal interface IBackendOperationsClient Task> GetExcititorProvidersAsync(bool includeDisabled, CancellationToken cancellationToken); Task EvaluateRuntimePolicyAsync(RuntimePolicyEvaluationRequest request, CancellationToken cancellationToken); + + Task DownloadOfflineKitAsync(string? bundleId, string destinationDirectory, bool overwrite, bool resume, CancellationToken cancellationToken); + + Task ImportOfflineKitAsync(OfflineKitImportRequest request, CancellationToken cancellationToken); + + Task GetOfflineKitStatusAsync(CancellationToken cancellationToken); } diff --git a/src/StellaOps.Cli/Services/Models/OfflineKitModels.cs b/src/StellaOps.Cli/Services/Models/OfflineKitModels.cs new file mode 100644 index 00000000..da408685 --- /dev/null +++ b/src/StellaOps.Cli/Services/Models/OfflineKitModels.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Cli.Services.Models; + +internal sealed record OfflineKitBundleDescriptor( + string BundleId, + string BundleName, + string BundleSha256, + long BundleSize, + Uri BundleDownloadUri, + string ManifestName, + string ManifestSha256, + Uri ManifestDownloadUri, + DateTimeOffset CapturedAt, + string? Channel, + string? Kind, + bool IsDelta, + string? BaseBundleId, + string? BundleSignatureName, + Uri? BundleSignatureDownloadUri, + string? ManifestSignatureName, + Uri? ManifestSignatureDownloadUri, + long? ManifestSize); + +internal sealed record OfflineKitDownloadResult( + OfflineKitBundleDescriptor Descriptor, + string BundlePath, + string ManifestPath, + string? BundleSignaturePath, + string? ManifestSignaturePath, + string MetadataPath, + bool FromCache); + +internal sealed record OfflineKitImportRequest( + string BundlePath, + string? ManifestPath, + string? BundleSignaturePath, + string? ManifestSignaturePath, + string? BundleId, + string? BundleSha256, + long? BundleSize, + DateTimeOffset? CapturedAt, + string? Channel, + string? Kind, + bool? IsDelta, + string? BaseBundleId, + string? ManifestSha256, + long? ManifestSize); + +internal sealed record OfflineKitImportResult( + string? ImportId, + string? Status, + DateTimeOffset SubmittedAt, + string? Message); + +internal sealed record OfflineKitStatus( + string? BundleId, + string? Channel, + string? Kind, + bool IsDelta, + string? BaseBundleId, + DateTimeOffset? CapturedAt, + DateTimeOffset? ImportedAt, + string? BundleSha256, + long? BundleSize, + IReadOnlyList Components); + +internal sealed record OfflineKitComponentStatus( + string Name, + string? Version, + string? Digest, + DateTimeOffset? CapturedAt, + long? SizeBytes); + +internal sealed record OfflineKitMetadataDocument +{ + public string? BundleId { get; init; } + + public string BundleName { get; init; } = string.Empty; + + public string BundleSha256 { get; init; } = string.Empty; + + public long BundleSize { get; init; } + + public string BundlePath { get; init; } = string.Empty; + + public DateTimeOffset CapturedAt { get; init; } + + public DateTimeOffset DownloadedAt { get; init; } + + public string? Channel { get; init; } + + public string? Kind { get; init; } + + public bool IsDelta { get; init; } + + public string? BaseBundleId { get; init; } + + public string ManifestName { get; init; } = string.Empty; + + public string ManifestSha256 { get; init; } = string.Empty; + + public long? ManifestSize { get; init; } + + public string ManifestPath { get; init; } = string.Empty; + + public string? BundleSignaturePath { get; init; } + + public string? ManifestSignaturePath { get; init; } +} diff --git a/src/StellaOps.Cli/Services/Models/Transport/OfflineKitTransport.cs b/src/StellaOps.Cli/Services/Models/Transport/OfflineKitTransport.cs new file mode 100644 index 00000000..d6eac33f --- /dev/null +++ b/src/StellaOps.Cli/Services/Models/Transport/OfflineKitTransport.cs @@ -0,0 +1,103 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Cli.Services.Models.Transport; + +internal sealed class OfflineKitBundleDescriptorTransport +{ + public string? BundleId { get; set; } + + public string? BundleName { get; set; } + + public string? BundleSha256 { get; set; } + + public long BundleSize { get; set; } + + public string? BundleUrl { get; set; } + + public string? BundlePath { get; set; } + + public string? BundleSignatureName { get; set; } + + public string? BundleSignatureUrl { get; set; } + + public string? BundleSignaturePath { get; set; } + + public string? ManifestName { get; set; } + + public string? ManifestSha256 { get; set; } + + public long? ManifestSize { get; set; } + + public string? ManifestUrl { get; set; } + + public string? ManifestPath { get; set; } + + public string? ManifestSignatureName { get; set; } + + public string? ManifestSignatureUrl { get; set; } + + public string? ManifestSignaturePath { get; set; } + + public DateTimeOffset? CapturedAt { get; set; } + + public string? Channel { get; set; } + + public string? Kind { get; set; } + + public bool? IsDelta { get; set; } + + public string? BaseBundleId { get; set; } +} + +internal sealed class OfflineKitStatusBundleTransport +{ + public string? BundleId { get; set; } + + public string? Channel { get; set; } + + public string? Kind { get; set; } + + public bool? IsDelta { get; set; } + + public string? BaseBundleId { get; set; } + + public string? BundleSha256 { get; set; } + + public long? BundleSize { get; set; } + + public DateTimeOffset? CapturedAt { get; set; } + + public DateTimeOffset? ImportedAt { get; set; } +} + +internal sealed class OfflineKitStatusTransport +{ + public OfflineKitStatusBundleTransport? Current { get; set; } + + public List? Components { get; set; } +} + +internal sealed class OfflineKitComponentStatusTransport +{ + public string? Name { get; set; } + + public string? Version { get; set; } + + public string? Digest { get; set; } + + public DateTimeOffset? CapturedAt { get; set; } + + public long? SizeBytes { get; set; } +} + +internal sealed class OfflineKitImportResponseTransport +{ + public string? ImportId { get; set; } + + public string? Status { get; set; } + + public DateTimeOffset? SubmittedAt { get; set; } + + public string? Message { get; set; } +} diff --git a/src/StellaOps.Cli/TASKS.md b/src/StellaOps.Cli/TASKS.md index fbf604b8..b5f56cc1 100644 --- a/src/StellaOps.Cli/TASKS.md +++ b/src/StellaOps.Cli/TASKS.md @@ -18,7 +18,7 @@ If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md |EXCITITOR-CLI-01-002 – Export download & attestation UX|DevEx/CLI|EXCITITOR-CLI-01-001, EXCITITOR-EXPORT-01-001|DONE (2025-10-19) – CLI export prints digest/size/Rekor metadata, `--output` downloads with SHA-256 verification + cache reuse, and unit coverage validated via `dotnet test src/StellaOps.Cli.Tests`.| |EXCITITOR-CLI-01-003 – CLI docs & examples for Excititor|Docs/CLI|EXCITITOR-CLI-01-001|**DOING (2025-10-19)** – Update docs/09_API_CLI_REFERENCE.md and quickstart snippets to cover Excititor verbs, offline guidance, and attestation verification workflow.| |CLI-RUNTIME-13-005 – Runtime policy test verbs|DevEx/CLI|SCANNER-RUNTIME-12-302, ZASTAVA-WEBHOOK-12-102|**DONE (2025-10-19)** – Added `runtime policy test` command (stdin/file support, JSON output), backend client method + typed models, verdict table output, docs/tests updated (`dotnet test src/StellaOps.Cli.Tests`).| -|CLI-OFFLINE-13-006 – Offline kit workflows|DevEx/CLI|DEVOPS-OFFLINE-14-002|TODO – Implement `offline kit pull/import/status` commands with integrity checks, resumable downloads, and doc updates.| +|CLI-OFFLINE-13-006 – Offline kit workflows|DevEx/CLI|DEVOPS-OFFLINE-14-002|**DONE (2025-10-21)** – Added `offline kit pull/import/status` commands with resumable downloads, digest/metadata validation, metrics, docs updates, and regression coverage (`dotnet test src/StellaOps.Cli.Tests`).| |CLI-PLUGIN-13-007 – Plugin packaging|DevEx/CLI|CLI-RUNTIME-13-005, CLI-OFFLINE-13-006|TODO – Package non-core verbs as restart-time plug-ins (manifest + loader updates, tests ensuring no hot reload).| |CLI-RUNTIME-13-008 – Runtime policy contract sync|DevEx/CLI, Scanner WebService Guild|SCANNER-RUNTIME-12-302|**DONE (2025-10-19)** – CLI runtime table/JSON now align with SCANNER-RUNTIME-12-302 (SBOM referrers, quieted provenance, confidence, verified Rekor); docs/09 updated with joint sign-off note.| |CLI-RUNTIME-13-009 – Runtime policy smoke fixture|DevEx/CLI, QA Guild|CLI-RUNTIME-13-005|**DONE (2025-10-19)** – Spectre console harness + regression tests cover table and `--json` output paths for `runtime policy test`, using stubbed backend and integrated into `dotnet test` suite.| diff --git a/src/StellaOps.Cli/Telemetry/CliMetrics.cs b/src/StellaOps.Cli/Telemetry/CliMetrics.cs index 21206108..c745fc4a 100644 --- a/src/StellaOps.Cli/Telemetry/CliMetrics.cs +++ b/src/StellaOps.Cli/Telemetry/CliMetrics.cs @@ -7,14 +7,16 @@ internal static class CliMetrics { private static readonly Meter Meter = new("StellaOps.Cli", "1.0.0"); - private static readonly Counter ScannerDownloadCounter = Meter.CreateCounter("stellaops.cli.scanner.download.count"); - private static readonly Counter ScannerInstallCounter = Meter.CreateCounter("stellaops.cli.scanner.install.count"); - private static readonly Counter ScanRunCounter = Meter.CreateCounter("stellaops.cli.scan.run.count"); - private static readonly Histogram CommandDurationHistogram = Meter.CreateHistogram("stellaops.cli.command.duration.ms"); - - public static void RecordScannerDownload(string channel, bool fromCache) - => ScannerDownloadCounter.Add(1, new KeyValuePair[] - { + private static readonly Counter ScannerDownloadCounter = Meter.CreateCounter("stellaops.cli.scanner.download.count"); + private static readonly Counter ScannerInstallCounter = Meter.CreateCounter("stellaops.cli.scanner.install.count"); + private static readonly Counter ScanRunCounter = Meter.CreateCounter("stellaops.cli.scan.run.count"); + private static readonly Counter OfflineKitDownloadCounter = Meter.CreateCounter("stellaops.cli.offline.kit.download.count"); + private static readonly Counter OfflineKitImportCounter = Meter.CreateCounter("stellaops.cli.offline.kit.import.count"); + private static readonly Histogram CommandDurationHistogram = Meter.CreateHistogram("stellaops.cli.command.duration.ms"); + + public static void RecordScannerDownload(string channel, bool fromCache) + => ScannerDownloadCounter.Add(1, new KeyValuePair[] + { new("channel", channel), new("cache", fromCache ? "hit" : "miss") }); @@ -23,16 +25,29 @@ internal static class CliMetrics => ScannerInstallCounter.Add(1, new KeyValuePair[] { new("channel", channel) }); public static void RecordScanRun(string runner, int exitCode) - => ScanRunCounter.Add(1, new KeyValuePair[] - { - new("runner", runner), - new("exit_code", exitCode) - }); - - public static IDisposable MeasureCommandDuration(string command) - { - var start = DateTime.UtcNow; - return new DurationScope(command, start); + => ScanRunCounter.Add(1, new KeyValuePair[] + { + new("runner", runner), + new("exit_code", exitCode) + }); + + public static void RecordOfflineKitDownload(string kind, bool fromCache) + => OfflineKitDownloadCounter.Add(1, new KeyValuePair[] + { + new("kind", string.IsNullOrWhiteSpace(kind) ? "unknown" : kind), + new("cache", fromCache ? "hit" : "miss") + }); + + public static void RecordOfflineKitImport(string? status) + => OfflineKitImportCounter.Add(1, new KeyValuePair[] + { + new("status", string.IsNullOrWhiteSpace(status) ? "queued" : status) + }); + + public static IDisposable MeasureCommandDuration(string command) + { + var start = DateTime.UtcNow; + return new DurationScope(command, start); } private sealed class DurationScope : IDisposable diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/FixtureLoader.cs b/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/FixtureLoader.cs new file mode 100644 index 00000000..31cbd261 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/FixtureLoader.cs @@ -0,0 +1,33 @@ +using System; +using System.IO; + +namespace StellaOps.Concelier.Connector.StellaOpsMirror.Tests; + +internal static class FixtureLoader +{ + private static readonly string FixturesRoot = Path.Combine(AppContext.BaseDirectory, "Fixtures"); + + public static string Read(string relativePath) + { + if (string.IsNullOrWhiteSpace(relativePath)) + { + throw new ArgumentException("Fixture path must be provided.", nameof(relativePath)); + } + + var normalized = relativePath.Replace('\\', Path.DirectorySeparatorChar).Replace('/', Path.DirectorySeparatorChar); + var path = Path.Combine(FixturesRoot, normalized); + + if (!File.Exists(path)) + { + throw new FileNotFoundException($"Fixture '{relativePath}' not found at '{path}'.", path); + } + + var content = File.ReadAllText(path); + return NormalizeLineEndings(content); + } + + public static string Normalize(string value) => NormalizeLineEndings(value); + + private static string NormalizeLineEndings(string value) + => value.Replace("\r\n", "\n", StringComparison.Ordinal); +} diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/Fixtures/mirror-advisory.expected.json b/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/Fixtures/mirror-advisory.expected.json new file mode 100644 index 00000000..dfd8ab6f --- /dev/null +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/Fixtures/mirror-advisory.expected.json @@ -0,0 +1,212 @@ +{ + "advisoryKey": "CVE-2025-1111", + "affectedPackages": [ + { + "type": "semver", + "identifier": "pkg:npm/example@1.0.0", + "platform": null, + "versionRanges": [ + { + "fixedVersion": "1.2.0", + "introducedVersion": "1.0.0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": false, + "nevra": null, + "semVer": { + "constraintExpression": ">=1.0.0,<1.2.0", + "exactValue": null, + "fixed": "1.2.0", + "fixedInclusive": false, + "introduced": "1.0.0", + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": true, + "style": "range" + }, + "vendorExtensions": null + }, + "provenance": { + "source": "ghsa", + "kind": "map", + "value": "range", + "decisionReason": null, + "recordedAt": "2025-10-19T12:00:00+00:00", + "fieldMask": [ + "affectedpackages[].versionranges[]" + ] + }, + "rangeExpression": ">=1.0.0,<1.2.0", + "rangeKind": "semver" + } + ], + "normalizedVersions": [ + { + "scheme": "semver", + "type": "range", + "min": "1.0.0", + "minInclusive": true, + "max": "1.2.0", + "maxInclusive": false, + "value": null, + "notes": null + } + ], + "statuses": [ + { + "provenance": { + "source": "ghsa", + "kind": "map", + "value": "status", + "decisionReason": null, + "recordedAt": "2025-10-19T12:00:00+00:00", + "fieldMask": [ + "affectedpackages[].statuses[]" + ] + }, + "status": "fixed" + } + ], + "provenance": [ + { + "source": "ghsa", + "kind": "map", + "value": "package", + "decisionReason": null, + "recordedAt": "2025-10-19T12:00:00+00:00", + "fieldMask": [ + "affectedpackages[]" + ] + }, + { + "source": "stellaops-mirror", + "kind": "map", + "value": "domain=primary;repository=mirror-primary;generated=2025-10-19T12:00:00.0000000+00:00;package=pkg:npm/example@1.0.0", + "decisionReason": null, + "recordedAt": "2025-10-19T12:00:00+00:00", + "fieldMask": [ + "affectedpackages[]", + "affectedpackages[].normalizedversions[]", + "affectedpackages[].statuses[]", + "affectedpackages[].versionranges[]" + ] + } + ] + } + ], + "aliases": [ + "CVE-2025-1111", + "GHSA-xxxx-xxxx-xxxx" + ], + "canonicalMetricId": "cvss::ghsa::CVE-2025-1111", + "credits": [ + { + "displayName": "Security Researcher", + "role": "reporter", + "contacts": [ + "mailto:researcher@example.com" + ], + "provenance": { + "source": "ghsa", + "kind": "map", + "value": "credit", + "decisionReason": null, + "recordedAt": "2025-10-19T12:00:00+00:00", + "fieldMask": [ + "credits[]" + ] + } + } + ], + "cvssMetrics": [ + { + "baseScore": 9.8, + "baseSeverity": "critical", + "provenance": { + "source": "ghsa", + "kind": "map", + "value": "cvss", + "decisionReason": null, + "recordedAt": "2025-10-19T12:00:00+00:00", + "fieldMask": [ + "cvssmetrics[]" + ] + }, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "version": "3.1" + } + ], + "cwes": [ + { + "taxonomy": "cwe", + "identifier": "CWE-79", + "name": "Cross-site Scripting", + "uri": "https://cwe.mitre.org/data/definitions/79.html", + "provenance": [ + { + "source": "ghsa", + "kind": "map", + "value": "cwe", + "decisionReason": null, + "recordedAt": "2025-10-19T12:00:00+00:00", + "fieldMask": [ + "cwes[]" + ] + } + ] + } + ], + "description": "Deterministic test payload distributed via mirror.", + "exploitKnown": false, + "language": "en", + "modified": "2025-10-11T00:00:00+00:00", + "provenance": [ + { + "source": "ghsa", + "kind": "map", + "value": "advisory", + "decisionReason": null, + "recordedAt": "2025-10-19T12:00:00+00:00", + "fieldMask": [ + "advisory" + ] + }, + { + "source": "stellaops-mirror", + "kind": "map", + "value": "domain=primary;repository=mirror-primary;generated=2025-10-19T12:00:00.0000000+00:00", + "decisionReason": null, + "recordedAt": "2025-10-19T12:00:00+00:00", + "fieldMask": [ + "advisory", + "credits[]", + "cvssmetrics[]", + "cwes[]", + "references[]" + ] + } + ], + "published": "2025-10-10T00:00:00+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "map", + "value": "reference", + "decisionReason": null, + "recordedAt": "2025-10-19T12:00:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "vendor", + "summary": "Vendor bulletin", + "url": "https://example.com/advisory" + } + ], + "severity": "high", + "summary": "Upstream advisory replicated through StellaOps mirror.", + "title": "Sample Mirror Advisory" +} diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/Fixtures/mirror-bundle.sample.json b/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/Fixtures/mirror-bundle.sample.json new file mode 100644 index 00000000..14945746 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/Fixtures/mirror-bundle.sample.json @@ -0,0 +1,202 @@ +{ + "advisories": [ + { + "advisoryKey": "CVE-2025-1111", + "affectedPackages": [ + { + "type": "semver", + "identifier": "pkg:npm/example@1.0.0", + "platform": null, + "versionRanges": [ + { + "fixedVersion": "1.2.0", + "introducedVersion": "1.0.0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": false, + "nevra": null, + "semVer": { + "constraintExpression": ">=1.0.0,<1.2.0", + "exactValue": null, + "fixed": "1.2.0", + "fixedInclusive": false, + "introduced": "1.0.0", + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": true, + "style": "range" + }, + "vendorExtensions": null + }, + "provenance": { + "source": "ghsa", + "kind": "map", + "value": "range", + "decisionReason": null, + "recordedAt": "2025-10-19T12:00:00+00:00", + "fieldMask": [ + "affectedpackages[].versionranges[]" + ] + }, + "rangeExpression": ">=1.0.0,<1.2.0", + "rangeKind": "semver" + } + ], + "normalizedVersions": [ + { + "scheme": "semver", + "type": "range", + "min": "1.0.0", + "minInclusive": true, + "max": "1.2.0", + "maxInclusive": false, + "value": null, + "notes": null + } + ], + "statuses": [ + { + "provenance": { + "source": "ghsa", + "kind": "map", + "value": "status", + "decisionReason": null, + "recordedAt": "2025-10-19T12:00:00+00:00", + "fieldMask": [ + "affectedpackages[].statuses[]" + ] + }, + "status": "fixed" + } + ], + "provenance": [ + { + "source": "ghsa", + "kind": "map", + "value": "package", + "decisionReason": null, + "recordedAt": "2025-10-19T12:00:00+00:00", + "fieldMask": [ + "affectedpackages[]" + ] + } + ] + } + ], + "aliases": [ + "GHSA-xxxx-xxxx-xxxx" + ], + "canonicalMetricId": "cvss::ghsa::CVE-2025-1111", + "credits": [ + { + "displayName": "Security Researcher", + "role": "reporter", + "contacts": [ + "mailto:researcher@example.com" + ], + "provenance": { + "source": "ghsa", + "kind": "map", + "value": "credit", + "decisionReason": null, + "recordedAt": "2025-10-19T12:00:00+00:00", + "fieldMask": [ + "credits[]" + ] + } + } + ], + "cvssMetrics": [ + { + "baseScore": 9.8, + "baseSeverity": "critical", + "provenance": { + "source": "ghsa", + "kind": "map", + "value": "cvss", + "decisionReason": null, + "recordedAt": "2025-10-19T12:00:00+00:00", + "fieldMask": [ + "cvssmetrics[]" + ] + }, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "version": "3.1" + } + ], + "cwes": [ + { + "taxonomy": "cwe", + "identifier": "CWE-79", + "name": "Cross-site Scripting", + "uri": "https://cwe.mitre.org/data/definitions/79.html", + "provenance": [ + { + "source": "ghsa", + "kind": "map", + "value": "cwe", + "decisionReason": null, + "recordedAt": "2025-10-19T12:00:00+00:00", + "fieldMask": [ + "cwes[]" + ] + } + ] + } + ], + "description": "Deterministic test payload distributed via mirror.", + "exploitKnown": false, + "language": "en", + "modified": "2025-10-11T00:00:00+00:00", + "provenance": [ + { + "source": "ghsa", + "kind": "map", + "value": "advisory", + "decisionReason": null, + "recordedAt": "2025-10-19T12:00:00+00:00", + "fieldMask": [ + "advisory" + ] + } + ], + "published": "2025-10-10T00:00:00+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "map", + "value": "reference", + "decisionReason": null, + "recordedAt": "2025-10-19T12:00:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "vendor", + "summary": "Vendor bulletin", + "url": "https://example.com/advisory" + } + ], + "severity": "high", + "summary": "Upstream advisory replicated through StellaOps mirror.", + "title": "Sample Mirror Advisory" + } + ], + "advisoryCount": 1, + "displayName": "Primary Mirror", + "domainId": "primary", + "generatedAt": "2025-10-19T12:00:00+00:00", + "schemaVersion": 1, + "sources": [ + { + "advisoryCount": 1, + "firstRecordedAt": "2025-10-19T12:00:00+00:00", + "lastRecordedAt": "2025-10-19T12:00:00+00:00", + "source": "ghsa" + } + ], + "targetRepository": "mirror-primary" +} diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/MirrorAdvisoryMapperTests.cs b/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/MirrorAdvisoryMapperTests.cs new file mode 100644 index 00000000..1d4f8d3c --- /dev/null +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/MirrorAdvisoryMapperTests.cs @@ -0,0 +1,47 @@ +using System; +using StellaOps.Concelier.Connector.StellaOpsMirror.Internal; +using StellaOps.Concelier.Models; +using Xunit; + +namespace StellaOps.Concelier.Connector.StellaOpsMirror.Tests; + +public sealed class MirrorAdvisoryMapperTests +{ + [Fact] + public void Map_ProducesCanonicalAdvisoryWithMirrorProvenance() + { + var bundle = SampleData.CreateBundle(); + var bundleJson = CanonicalJsonSerializer.SerializeIndented(bundle); + Assert.Equal( + FixtureLoader.Read(SampleData.BundleFixture).TrimEnd(), + FixtureLoader.Normalize(bundleJson).TrimEnd()); + + var advisories = MirrorAdvisoryMapper.Map(bundle); + + Assert.Single(advisories); + var advisory = advisories[0]; + + var expectedAdvisory = SampleData.CreateExpectedMappedAdvisory(); + var expectedJson = CanonicalJsonSerializer.SerializeIndented(expectedAdvisory); + Assert.Equal( + FixtureLoader.Read(SampleData.AdvisoryFixture).TrimEnd(), + FixtureLoader.Normalize(expectedJson).TrimEnd()); + + var actualJson = CanonicalJsonSerializer.SerializeIndented(advisory); + Assert.Equal( + FixtureLoader.Normalize(expectedJson).TrimEnd(), + FixtureLoader.Normalize(actualJson).TrimEnd()); + + Assert.Contains(advisory.Aliases, alias => string.Equals(alias, advisory.AdvisoryKey, StringComparison.OrdinalIgnoreCase)); + Assert.Contains( + advisory.Provenance, + provenance => string.Equals(provenance.Source, StellaOpsMirrorConnector.Source, StringComparison.Ordinal) && + string.Equals(provenance.Kind, "map", StringComparison.Ordinal)); + + var package = Assert.Single(advisory.AffectedPackages); + Assert.Contains( + package.Provenance, + provenance => string.Equals(provenance.Source, StellaOpsMirrorConnector.Source, StringComparison.Ordinal) && + string.Equals(provenance.Kind, "map", StringComparison.Ordinal)); + } +} diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/MirrorSignatureVerifierTests.cs b/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/MirrorSignatureVerifierTests.cs index a01252a1..548501c5 100644 --- a/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/MirrorSignatureVerifierTests.cs +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/MirrorSignatureVerifierTests.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; +using System.IO; using System.Security.Cryptography; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging.Abstractions; using StellaOps.Concelier.Connector.StellaOpsMirror.Security; using StellaOps.Cryptography; @@ -18,7 +20,7 @@ public sealed class MirrorSignatureVerifierTests provider.UpsertSigningKey(key); var registry = new CryptoProviderRegistry(new[] { provider }); - var verifier = new MirrorSignatureVerifier(registry, NullLogger.Instance); + var verifier = new MirrorSignatureVerifier(registry, NullLogger.Instance, new MemoryCache(new MemoryCacheOptions())); var payloadText = System.Text.Json.JsonSerializer.Serialize(new { advisories = Array.Empty() }); var payload = payloadText.ToUtf8Bytes(); @@ -35,13 +37,13 @@ public sealed class MirrorSignatureVerifierTests provider.UpsertSigningKey(key); var registry = new CryptoProviderRegistry(new[] { provider }); - var verifier = new MirrorSignatureVerifier(registry, NullLogger.Instance); + var verifier = new MirrorSignatureVerifier(registry, NullLogger.Instance, new MemoryCache(new MemoryCacheOptions())); var payloadText = System.Text.Json.JsonSerializer.Serialize(new { advisories = Array.Empty() }); var payload = payloadText.ToUtf8Bytes(); var (signature, _) = await CreateDetachedJwsAsync(provider, key.Reference.KeyId, payload); - var tampered = signature.Replace('a', 'b', StringComparison.Ordinal); + var tampered = signature.Replace('a', 'b'); await Assert.ThrowsAsync(() => verifier.VerifyAsync(payload, tampered, CancellationToken.None)); } @@ -54,7 +56,7 @@ public sealed class MirrorSignatureVerifierTests provider.UpsertSigningKey(key); var registry = new CryptoProviderRegistry(new[] { provider }); - var verifier = new MirrorSignatureVerifier(registry, NullLogger.Instance); + var verifier = new MirrorSignatureVerifier(registry, NullLogger.Instance, new MemoryCache(new MemoryCacheOptions())); var payloadText = System.Text.Json.JsonSerializer.Serialize(new { advisories = Array.Empty() }); var payload = payloadText.ToUtf8Bytes(); @@ -65,6 +67,7 @@ public sealed class MirrorSignatureVerifierTests signature, expectedKeyId: "unexpected-key", expectedProvider: null, + fallbackPublicKeyPath: null, cancellationToken: CancellationToken.None)); } @@ -76,7 +79,7 @@ public sealed class MirrorSignatureVerifierTests provider.UpsertSigningKey(key); var registry = new CryptoProviderRegistry(new[] { provider }); - var verifier = new MirrorSignatureVerifier(registry, NullLogger.Instance); + var verifier = new MirrorSignatureVerifier(registry, NullLogger.Instance, new MemoryCache(new MemoryCacheOptions())); var payloadText = System.Text.Json.JsonSerializer.Serialize(new { advisories = Array.Empty() }); var payload = payloadText.ToUtf8Bytes(); @@ -89,9 +92,42 @@ public sealed class MirrorSignatureVerifierTests signature, expectedKeyId: key.Reference.KeyId, expectedProvider: provider.Name, + fallbackPublicKeyPath: null, cancellationToken: CancellationToken.None)); } + [Fact] + public async Task VerifyAsync_UsesCachedPublicKeyWhenFileRemoved() + { + var provider = new DefaultCryptoProvider(); + var signingKey = CreateSigningKey("mirror-key"); + provider.UpsertSigningKey(signingKey); + var registry = new CryptoProviderRegistry(new[] { provider }); + var memoryCache = new MemoryCache(new MemoryCacheOptions()); + var verifier = new MirrorSignatureVerifier(registry, NullLogger.Instance, memoryCache); + + var payload = "{\"advisories\":[]}"; + var (signature, _) = await CreateDetachedJwsAsync(provider, signingKey.Reference.KeyId, payload.ToUtf8Bytes()); + provider.RemoveSigningKey(signingKey.Reference.KeyId); + var pemPath = WritePublicKeyPem(signingKey); + + try + { + await verifier.VerifyAsync(payload.ToUtf8Bytes(), signature, expectedKeyId: signingKey.Reference.KeyId, expectedProvider: "default", fallbackPublicKeyPath: pemPath, cancellationToken: CancellationToken.None); + + File.Delete(pemPath); + + await verifier.VerifyAsync(payload.ToUtf8Bytes(), signature, expectedKeyId: signingKey.Reference.KeyId, expectedProvider: "default", fallbackPublicKeyPath: pemPath, cancellationToken: CancellationToken.None); + } + finally + { + if (File.Exists(pemPath)) + { + File.Delete(pemPath); + } + } + } + private static CryptoSigningKey CreateSigningKey(string keyId) { using var ecdsa = ECDsa.Create(ECCurve.NamedCurves.nistP256); @@ -99,6 +135,16 @@ public sealed class MirrorSignatureVerifierTests return new CryptoSigningKey(new CryptoKeyReference(keyId), SignatureAlgorithms.Es256, in parameters, DateTimeOffset.UtcNow); } + private static string WritePublicKeyPem(CryptoSigningKey signingKey) + { + using var ecdsa = ECDsa.Create(signingKey.PublicParameters); + var info = ecdsa.ExportSubjectPublicKeyInfo(); + var pem = PemEncoding.Write("PUBLIC KEY", info); + var path = Path.Combine(Path.GetTempPath(), $"stellaops-mirror-{Guid.NewGuid():N}.pem"); + File.WriteAllText(path, pem); + return path; + } + private static async Task<(string Signature, DateTimeOffset SignedAt)> CreateDetachedJwsAsync( DefaultCryptoProvider provider, string keyId, diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/SampleData.cs b/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/SampleData.cs new file mode 100644 index 00000000..72d4c50c --- /dev/null +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/SampleData.cs @@ -0,0 +1,265 @@ +using System; +using System.Globalization; +using StellaOps.Concelier.Connector.StellaOpsMirror.Internal; +using StellaOps.Concelier.Models; + +namespace StellaOps.Concelier.Connector.StellaOpsMirror.Tests; + +internal static class SampleData +{ + public const string BundleFixture = "mirror-bundle.sample.json"; + public const string AdvisoryFixture = "mirror-advisory.expected.json"; + public const string TargetRepository = "mirror-primary"; + public const string DomainId = "primary"; + public const string AdvisoryKey = "CVE-2025-1111"; + public const string GhsaAlias = "GHSA-xxxx-xxxx-xxxx"; + + public static DateTimeOffset GeneratedAt { get; } = new(2025, 10, 19, 12, 0, 0, TimeSpan.Zero); + + public static MirrorBundleDocument CreateBundle() + => new( + SchemaVersion: 1, + GeneratedAt: GeneratedAt, + TargetRepository: TargetRepository, + DomainId: DomainId, + DisplayName: "Primary Mirror", + AdvisoryCount: 1, + Advisories: new[] { CreateSourceAdvisory() }, + Sources: new[] + { + new MirrorSourceSummary("ghsa", GeneratedAt, GeneratedAt, 1) + }); + + public static Advisory CreateExpectedMappedAdvisory() + { + var baseAdvisory = CreateSourceAdvisory(); + var recordedAt = GeneratedAt.ToUniversalTime(); + var mirrorValue = BuildMirrorValue(recordedAt); + + var topProvenance = baseAdvisory.Provenance.Add(new AdvisoryProvenance( + StellaOpsMirrorConnector.Source, + "map", + mirrorValue, + recordedAt, + new[] + { + ProvenanceFieldMasks.Advisory, + ProvenanceFieldMasks.References, + ProvenanceFieldMasks.Credits, + ProvenanceFieldMasks.CvssMetrics, + ProvenanceFieldMasks.Weaknesses, + })); + + var package = baseAdvisory.AffectedPackages[0]; + var packageProvenance = package.Provenance.Add(new AdvisoryProvenance( + StellaOpsMirrorConnector.Source, + "map", + $"{mirrorValue};package={package.Identifier}", + recordedAt, + new[] + { + ProvenanceFieldMasks.AffectedPackages, + ProvenanceFieldMasks.VersionRanges, + ProvenanceFieldMasks.PackageStatuses, + ProvenanceFieldMasks.NormalizedVersions, + })); + var updatedPackage = new AffectedPackage( + package.Type, + package.Identifier, + package.Platform, + package.VersionRanges, + package.Statuses, + packageProvenance, + package.NormalizedVersions); + + return new Advisory( + AdvisoryKey, + baseAdvisory.Title, + baseAdvisory.Summary, + baseAdvisory.Language, + baseAdvisory.Published, + baseAdvisory.Modified, + baseAdvisory.Severity, + baseAdvisory.ExploitKnown, + new[] { AdvisoryKey, GhsaAlias }, + baseAdvisory.Credits, + baseAdvisory.References, + new[] { updatedPackage }, + baseAdvisory.CvssMetrics, + topProvenance, + baseAdvisory.Description, + baseAdvisory.Cwes, + baseAdvisory.CanonicalMetricId); + } + + private static Advisory CreateSourceAdvisory() + { + var recordedAt = GeneratedAt.ToUniversalTime(); + + var reference = new AdvisoryReference( + "https://example.com/advisory", + "advisory", + "vendor", + "Vendor bulletin", + new AdvisoryProvenance( + "ghsa", + "map", + "reference", + recordedAt, + new[] + { + ProvenanceFieldMasks.References, + })); + + var credit = new AdvisoryCredit( + "Security Researcher", + "reporter", + new[] { "mailto:researcher@example.com" }, + new AdvisoryProvenance( + "ghsa", + "map", + "credit", + recordedAt, + new[] + { + ProvenanceFieldMasks.Credits, + })); + + var semVerPrimitive = new SemVerPrimitive( + Introduced: "1.0.0", + IntroducedInclusive: true, + Fixed: "1.2.0", + FixedInclusive: false, + LastAffected: null, + LastAffectedInclusive: true, + ConstraintExpression: ">=1.0.0,<1.2.0", + ExactValue: null); + + var range = new AffectedVersionRange( + rangeKind: "semver", + introducedVersion: "1.0.0", + fixedVersion: "1.2.0", + lastAffectedVersion: null, + rangeExpression: ">=1.0.0,<1.2.0", + provenance: new AdvisoryProvenance( + "ghsa", + "map", + "range", + recordedAt, + new[] + { + ProvenanceFieldMasks.VersionRanges, + }), + primitives: new RangePrimitives(semVerPrimitive, null, null, null)); + + var status = new AffectedPackageStatus( + "fixed", + new AdvisoryProvenance( + "ghsa", + "map", + "status", + recordedAt, + new[] + { + ProvenanceFieldMasks.PackageStatuses, + })); + + var normalizedRule = new NormalizedVersionRule( + scheme: "semver", + type: "range", + min: "1.0.0", + minInclusive: true, + max: "1.2.0", + maxInclusive: false, + value: null, + notes: null); + + var package = new AffectedPackage( + AffectedPackageTypes.SemVer, + "pkg:npm/example@1.0.0", + platform: null, + versionRanges: new[] { range }, + statuses: new[] { status }, + provenance: new[] + { + new AdvisoryProvenance( + "ghsa", + "map", + "package", + recordedAt, + new[] + { + ProvenanceFieldMasks.AffectedPackages, + }) + }, + normalizedVersions: new[] { normalizedRule }); + + var cvss = new CvssMetric( + "3.1", + "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + 9.8, + "critical", + new AdvisoryProvenance( + "ghsa", + "map", + "cvss", + recordedAt, + new[] + { + ProvenanceFieldMasks.CvssMetrics, + })); + + var weakness = new AdvisoryWeakness( + "cwe", + "CWE-79", + "Cross-site Scripting", + "https://cwe.mitre.org/data/definitions/79.html", + new[] + { + new AdvisoryProvenance( + "ghsa", + "map", + "cwe", + recordedAt, + new[] + { + ProvenanceFieldMasks.Weaknesses, + }) + }); + + var advisory = new Advisory( + AdvisoryKey, + "Sample Mirror Advisory", + "Upstream advisory replicated through StellaOps mirror.", + "en", + published: new DateTimeOffset(2025, 10, 10, 0, 0, 0, TimeSpan.Zero), + modified: new DateTimeOffset(2025, 10, 11, 0, 0, 0, TimeSpan.Zero), + severity: "high", + exploitKnown: false, + aliases: new[] { GhsaAlias }, + credits: new[] { credit }, + references: new[] { reference }, + affectedPackages: new[] { package }, + cvssMetrics: new[] { cvss }, + provenance: new[] + { + new AdvisoryProvenance( + "ghsa", + "map", + "advisory", + recordedAt, + new[] + { + ProvenanceFieldMasks.Advisory, + }) + }, + description: "Deterministic test payload distributed via mirror.", + cwes: new[] { weakness }, + canonicalMetricId: "cvss::ghsa::CVE-2025-1111"); + + return CanonicalJsonSerializer.Normalize(advisory); + } + + private static string BuildMirrorValue(DateTimeOffset recordedAt) + => $"domain={DomainId};repository={TargetRepository};generated={recordedAt.ToString("O", CultureInfo.InvariantCulture)}"; +} diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/StellaOps.Concelier.Connector.StellaOpsMirror.Tests.csproj b/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/StellaOps.Concelier.Connector.StellaOpsMirror.Tests.csproj index 339c2eb1..76070cd6 100644 --- a/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/StellaOps.Concelier.Connector.StellaOpsMirror.Tests.csproj +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/StellaOps.Concelier.Connector.StellaOpsMirror.Tests.csproj @@ -4,8 +4,11 @@ enable enable - - - - - + + + + + + + + diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/StellaOpsMirrorConnectorTests.cs b/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/StellaOpsMirrorConnectorTests.cs index 8fb9f211..e1bf4a0a 100644 --- a/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/StellaOpsMirrorConnectorTests.cs +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/StellaOpsMirrorConnectorTests.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.Net; using System.Net.Http; using System.Security.Cryptography; @@ -15,11 +16,15 @@ using MongoDB.Bson; using StellaOps.Concelier.Connector.Common; using StellaOps.Concelier.Connector.Common.Fetch; using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Connector.StellaOpsMirror.Internal; using StellaOps.Concelier.Connector.StellaOpsMirror.Settings; using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; using StellaOps.Concelier.Testing; using StellaOps.Cryptography; +using StellaOps.Concelier.Models; using Xunit; namespace StellaOps.Concelier.Connector.StellaOpsMirror.Tests; @@ -168,6 +173,95 @@ public sealed class StellaOpsMirrorConnectorTests : IAsyncLifetime await Assert.ThrowsAsync(() => connector.FetchAsync(provider, CancellationToken.None)); } + [Fact] + public async Task FetchAsync_VerifiesSignatureUsingFallbackPublicKey() + { + var manifestContent = "{\"domain\":\"primary\"}"; + var bundleContent = "{\"advisories\":[{\"id\":\"CVE-2025-0004\"}]}"; + + var manifestDigest = ComputeDigest(manifestContent); + var bundleDigest = ComputeDigest(bundleContent); + var index = BuildIndex(manifestDigest, Encoding.UTF8.GetByteCount(manifestContent), bundleDigest, Encoding.UTF8.GetByteCount(bundleContent), includeSignature: true); + + var signingKey = CreateSigningKey("mirror-key"); + var (signatureValue, _) = CreateDetachedJws(signingKey, bundleContent); + var publicKeyPath = WritePublicKeyPem(signingKey); + + await using var provider = await BuildServiceProviderAsync(options => + { + options.Signature.Enabled = true; + options.Signature.KeyId = "mirror-key"; + options.Signature.Provider = "default"; + options.Signature.PublicKeyPath = publicKeyPath; + }); + + try + { + SeedResponses(index, manifestContent, bundleContent, signatureValue); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(StellaOpsMirrorConnector.Source, CancellationToken.None); + Assert.NotNull(state); + Assert.Equal(0, state!.FailCount); + } + finally + { + if (File.Exists(publicKeyPath)) + { + File.Delete(publicKeyPath); + } + } + } + + [Fact] + public async Task FetchAsync_DigestMismatchMarksFailure() + { + var manifestExpected = "{\"domain\":\"primary\"}"; + var manifestTampered = "{\"domain\":\"tampered\"}"; + var bundleContent = "{\"advisories\":[{\"id\":\"CVE-2025-0005\"}]}"; + + var manifestDigest = ComputeDigest(manifestExpected); + var bundleDigest = ComputeDigest(bundleContent); + var index = BuildIndex(manifestDigest, Encoding.UTF8.GetByteCount(manifestExpected), bundleDigest, Encoding.UTF8.GetByteCount(bundleContent), includeSignature: false); + + await using var provider = await BuildServiceProviderAsync(); + + SeedResponses(index, manifestTampered, bundleContent, signature: null); + + var connector = provider.GetRequiredService(); + + await Assert.ThrowsAsync(() => connector.FetchAsync(provider, CancellationToken.None)); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(StellaOpsMirrorConnector.Source, CancellationToken.None); + Assert.NotNull(state); + var cursor = state!.Cursor ?? new BsonDocument(); + Assert.True(state.FailCount >= 1); + Assert.False(cursor.Contains("bundleDigest")); + } + + [Fact] + public void ParseAndMap_PersistAdvisoriesFromBundle() + { + var bundleDocument = SampleData.CreateBundle(); + var bundleJson = CanonicalJsonSerializer.SerializeIndented(bundleDocument); + var normalizedFixture = FixtureLoader.Read(SampleData.BundleFixture).TrimEnd(); + Assert.Equal(normalizedFixture, FixtureLoader.Normalize(bundleJson).TrimEnd()); + + var advisories = MirrorAdvisoryMapper.Map(bundleDocument); + Assert.Single(advisories); + var advisory = advisories[0]; + + var expectedAdvisoryJson = FixtureLoader.Read(SampleData.AdvisoryFixture).TrimEnd(); + var mappedJson = CanonicalJsonSerializer.SerializeIndented(advisory); + Assert.Equal(expectedAdvisoryJson, FixtureLoader.Normalize(mappedJson).TrimEnd()); + + // AdvisoryStore integration validated elsewhere; ensure canonical serialization is stable. + } + public Task InitializeAsync() => Task.CompletedTask; public Task DisposeAsync() @@ -323,6 +417,17 @@ public sealed class StellaOpsMirrorConnectorTests : IAsyncLifetime return new CryptoSigningKey(new CryptoKeyReference(keyId), SignatureAlgorithms.Es256, in parameters, DateTimeOffset.UtcNow); } + private static string WritePublicKeyPem(CryptoSigningKey signingKey) + { + ArgumentNullException.ThrowIfNull(signingKey); + var path = Path.Combine(Path.GetTempPath(), $"stellaops-mirror-{Guid.NewGuid():N}.pem"); + using var ecdsa = ECDsa.Create(signingKey.PublicParameters); + var publicKeyInfo = ecdsa.ExportSubjectPublicKeyInfo(); + var pem = PemEncoding.Write("PUBLIC KEY", publicKeyInfo); + File.WriteAllText(path, pem); + return path; + } + private static (string Signature, DateTimeOffset SignedAt) CreateDetachedJws(CryptoSigningKey signingKey, string payload) { var provider = new DefaultCryptoProvider(); diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror/Internal/MirrorAdvisoryMapper.cs b/src/StellaOps.Concelier.Connector.StellaOpsMirror/Internal/MirrorAdvisoryMapper.cs new file mode 100644 index 00000000..3b1cce86 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror/Internal/MirrorAdvisoryMapper.cs @@ -0,0 +1,203 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Globalization; +using StellaOps.Concelier.Models; + +namespace StellaOps.Concelier.Connector.StellaOpsMirror.Internal; + +internal static class MirrorAdvisoryMapper +{ + private const string MirrorProvenanceKind = "map"; + + private static readonly string[] TopLevelFieldMask = + { + ProvenanceFieldMasks.Advisory, + ProvenanceFieldMasks.References, + ProvenanceFieldMasks.Credits, + ProvenanceFieldMasks.CvssMetrics, + ProvenanceFieldMasks.Weaknesses, + }; + + public static ImmutableArray Map(MirrorBundleDocument bundle) + { + if (bundle?.Advisories is null || bundle.Advisories.Count == 0) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(bundle.Advisories.Count); + var recordedAt = bundle.GeneratedAt.ToUniversalTime(); + var mirrorValue = BuildMirrorValue(bundle, recordedAt); + var topLevelProvenance = new AdvisoryProvenance( + StellaOpsMirrorConnector.Source, + MirrorProvenanceKind, + mirrorValue, + recordedAt, + TopLevelFieldMask); + + foreach (var advisory in bundle.Advisories) + { + if (advisory is null) + { + continue; + } + + var normalized = CanonicalJsonSerializer.Normalize(advisory); + var aliases = EnsureAliasCoverage(normalized); + var provenance = EnsureProvenance(normalized.Provenance, topLevelProvenance); + var packages = EnsurePackageProvenance(normalized.AffectedPackages, mirrorValue, recordedAt); + + var updated = new Advisory( + normalized.AdvisoryKey, + normalized.Title, + normalized.Summary, + normalized.Language, + normalized.Published, + normalized.Modified, + normalized.Severity, + normalized.ExploitKnown, + aliases, + normalized.Credits, + normalized.References, + packages, + normalized.CvssMetrics, + provenance, + normalized.Description, + normalized.Cwes, + normalized.CanonicalMetricId); + + builder.Add(updated); + } + + return builder.ToImmutable(); + } + + private static IEnumerable EnsureAliasCoverage(Advisory advisory) + { + var aliases = new List(advisory.Aliases.Length + 1); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var alias in advisory.Aliases) + { + if (seen.Add(alias)) + { + aliases.Add(alias); + } + } + + if (seen.Add(advisory.AdvisoryKey)) + { + aliases.Add(advisory.AdvisoryKey); + } + + return aliases; + } + + private static IEnumerable EnsureProvenance( + ImmutableArray existing, + AdvisoryProvenance mirrorProvenance) + { + if (!existing.IsDefaultOrEmpty + && existing.Any(provenance => + string.Equals(provenance.Source, mirrorProvenance.Source, StringComparison.Ordinal) + && string.Equals(provenance.Kind, mirrorProvenance.Kind, StringComparison.Ordinal) + && string.Equals(provenance.Value, mirrorProvenance.Value, StringComparison.Ordinal))) + { + return existing; + } + + return existing.Add(mirrorProvenance); + } + + private static IEnumerable EnsurePackageProvenance( + ImmutableArray packages, + string mirrorValue, + DateTimeOffset recordedAt) + { + if (packages.IsDefaultOrEmpty || packages.Length == 0) + { + return packages; + } + + var results = new List(packages.Length); + + foreach (var package in packages) + { + var value = $"{mirrorValue};package={package.Identifier}"; + if (!package.Provenance.IsDefaultOrEmpty + && package.Provenance.Any(provenance => + string.Equals(provenance.Source, StellaOpsMirrorConnector.Source, StringComparison.Ordinal) + && string.Equals(provenance.Kind, MirrorProvenanceKind, StringComparison.Ordinal) + && string.Equals(provenance.Value, value, StringComparison.Ordinal))) + { + results.Add(package); + continue; + } + + var masks = BuildPackageFieldMask(package); + var packageProvenance = new AdvisoryProvenance( + StellaOpsMirrorConnector.Source, + MirrorProvenanceKind, + value, + recordedAt, + masks); + + var provenance = package.Provenance.Add(packageProvenance); + var updated = new AffectedPackage( + package.Type, + package.Identifier, + package.Platform, + package.VersionRanges, + package.Statuses, + provenance, + package.NormalizedVersions); + + results.Add(updated); + } + + return results; + } + + private static string[] BuildPackageFieldMask(AffectedPackage package) + { + var masks = new HashSet(StringComparer.Ordinal) + { + ProvenanceFieldMasks.AffectedPackages, + }; + + if (!package.VersionRanges.IsDefaultOrEmpty && package.VersionRanges.Length > 0) + { + masks.Add(ProvenanceFieldMasks.VersionRanges); + } + + if (!package.Statuses.IsDefaultOrEmpty && package.Statuses.Length > 0) + { + masks.Add(ProvenanceFieldMasks.PackageStatuses); + } + + if (!package.NormalizedVersions.IsDefaultOrEmpty && package.NormalizedVersions.Length > 0) + { + masks.Add(ProvenanceFieldMasks.NormalizedVersions); + } + + return masks.ToArray(); + } + + private static string BuildMirrorValue(MirrorBundleDocument bundle, DateTimeOffset recordedAt) + { + var segments = new List + { + $"domain={bundle.DomainId}", + }; + + if (!string.IsNullOrWhiteSpace(bundle.TargetRepository)) + { + segments.Add($"repository={bundle.TargetRepository}"); + } + + segments.Add($"generated={recordedAt.ToString("O", CultureInfo.InvariantCulture)}"); + return string.Join(';', segments); + } +} diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror/Internal/MirrorBundleDocument.cs b/src/StellaOps.Concelier.Connector.StellaOpsMirror/Internal/MirrorBundleDocument.cs new file mode 100644 index 00000000..33aa553c --- /dev/null +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror/Internal/MirrorBundleDocument.cs @@ -0,0 +1,14 @@ +using System.Text.Json.Serialization; +using StellaOps.Concelier.Models; + +namespace StellaOps.Concelier.Connector.StellaOpsMirror.Internal; + +public sealed record MirrorBundleDocument( + [property: JsonPropertyName("schemaVersion")] int SchemaVersion, + [property: JsonPropertyName("generatedAt")] DateTimeOffset GeneratedAt, + [property: JsonPropertyName("targetRepository")] string? TargetRepository, + [property: JsonPropertyName("domainId")] string DomainId, + [property: JsonPropertyName("displayName")] string DisplayName, + [property: JsonPropertyName("advisoryCount")] int AdvisoryCount, + [property: JsonPropertyName("advisories")] IReadOnlyList Advisories, + [property: JsonPropertyName("sources")] IReadOnlyList Sources); diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror/Internal/StellaOpsMirrorCursor.cs b/src/StellaOps.Concelier.Connector.StellaOpsMirror/Internal/StellaOpsMirrorCursor.cs index 47c26135..270056e4 100644 --- a/src/StellaOps.Concelier.Connector.StellaOpsMirror/Internal/StellaOpsMirrorCursor.cs +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror/Internal/StellaOpsMirrorCursor.cs @@ -3,93 +3,107 @@ using MongoDB.Bson; namespace StellaOps.Concelier.Connector.StellaOpsMirror.Internal; -internal sealed record StellaOpsMirrorCursor( - string? ExportId, - string? BundleDigest, - DateTimeOffset? GeneratedAt, - IReadOnlyCollection PendingDocuments, - IReadOnlyCollection PendingMappings) -{ - private static readonly IReadOnlyCollection EmptyGuids = Array.Empty(); - - public static StellaOpsMirrorCursor Empty { get; } = new( - ExportId: null, - BundleDigest: null, - GeneratedAt: null, - PendingDocuments: EmptyGuids, - PendingMappings: EmptyGuids); - - public BsonDocument ToBsonDocument() - { - var document = new BsonDocument - { - ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), - ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), - }; - - if (!string.IsNullOrWhiteSpace(ExportId)) - { - document["exportId"] = ExportId; - } - - if (!string.IsNullOrWhiteSpace(BundleDigest)) - { - document["bundleDigest"] = BundleDigest; - } - - if (GeneratedAt.HasValue) - { - document["generatedAt"] = GeneratedAt.Value.UtcDateTime; - } - - return document; - } - - public static StellaOpsMirrorCursor FromBson(BsonDocument? document) +internal sealed record StellaOpsMirrorCursor( + string? ExportId, + string? BundleDigest, + DateTimeOffset? GeneratedAt, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings, + string? CompletedFingerprint) +{ + private static readonly IReadOnlyCollection EmptyGuids = Array.Empty(); + + public static StellaOpsMirrorCursor Empty { get; } = new( + ExportId: null, + BundleDigest: null, + GeneratedAt: null, + PendingDocuments: EmptyGuids, + PendingMappings: EmptyGuids, + CompletedFingerprint: null); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + }; + + if (!string.IsNullOrWhiteSpace(ExportId)) + { + document["exportId"] = ExportId; + } + + if (!string.IsNullOrWhiteSpace(BundleDigest)) + { + document["bundleDigest"] = BundleDigest; + } + + if (GeneratedAt.HasValue) + { + document["generatedAt"] = GeneratedAt.Value.UtcDateTime; + } + + if (!string.IsNullOrWhiteSpace(CompletedFingerprint)) + { + document["completedFingerprint"] = CompletedFingerprint; + } + + return document; + } + + public static StellaOpsMirrorCursor FromBson(BsonDocument? document) { if (document is null || document.ElementCount == 0) { return Empty; } - var exportId = document.TryGetValue("exportId", out var exportValue) && exportValue.IsString ? exportValue.AsString : null; - var digest = document.TryGetValue("bundleDigest", out var digestValue) && digestValue.IsString ? digestValue.AsString : null; - DateTimeOffset? generatedAt = null; - if (document.TryGetValue("generatedAt", out var generatedValue)) - { - generatedAt = generatedValue.BsonType switch - { + var exportId = document.TryGetValue("exportId", out var exportValue) && exportValue.IsString ? exportValue.AsString : null; + var digest = document.TryGetValue("bundleDigest", out var digestValue) && digestValue.IsString ? digestValue.AsString : null; + DateTimeOffset? generatedAt = null; + if (document.TryGetValue("generatedAt", out var generatedValue)) + { + generatedAt = generatedValue.BsonType switch + { BsonType.DateTime => DateTime.SpecifyKind(generatedValue.ToUniversalTime(), DateTimeKind.Utc), BsonType.String when DateTimeOffset.TryParse(generatedValue.AsString, out var parsed) => parsed.ToUniversalTime(), - _ => null, - }; - } - - var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); - var pendingMappings = ReadGuidArray(document, "pendingMappings"); - - return new StellaOpsMirrorCursor(exportId, digest, generatedAt, pendingDocuments, pendingMappings); - } - - public StellaOpsMirrorCursor WithPendingDocuments(IEnumerable documents) - => this with { PendingDocuments = documents?.Distinct().ToArray() ?? EmptyGuids }; - - public StellaOpsMirrorCursor WithPendingMappings(IEnumerable mappings) - => this with { PendingMappings = mappings?.Distinct().ToArray() ?? EmptyGuids }; - - public StellaOpsMirrorCursor WithBundleSnapshot(string? exportId, string? digest, DateTimeOffset generatedAt) - => this with - { - ExportId = string.IsNullOrWhiteSpace(exportId) ? ExportId : exportId, - BundleDigest = digest, - GeneratedAt = generatedAt, - }; - - private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) - { - if (!document.TryGetValue(field, out var value) || value is not BsonArray array) - { - return EmptyGuids; + _ => null, + }; + } + + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + + var fingerprint = document.TryGetValue("completedFingerprint", out var fingerprintValue) && fingerprintValue.IsString + ? fingerprintValue.AsString + : null; + + return new StellaOpsMirrorCursor(exportId, digest, generatedAt, pendingDocuments, pendingMappings, fingerprint); + } + + public StellaOpsMirrorCursor WithPendingDocuments(IEnumerable documents) + => this with { PendingDocuments = documents?.Distinct().ToArray() ?? EmptyGuids }; + + public StellaOpsMirrorCursor WithPendingMappings(IEnumerable mappings) + => this with { PendingMappings = mappings?.Distinct().ToArray() ?? EmptyGuids }; + + public StellaOpsMirrorCursor WithBundleSnapshot(string? exportId, string? digest, DateTimeOffset generatedAt) + => this with + { + ExportId = string.IsNullOrWhiteSpace(exportId) ? ExportId : exportId, + BundleDigest = digest, + GeneratedAt = generatedAt, + }; + + public StellaOpsMirrorCursor WithCompletedFingerprint(string? fingerprint) + => this with { CompletedFingerprint = string.IsNullOrWhiteSpace(fingerprint) ? null : fingerprint }; + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyGuids; } var results = new List(array.Count); diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror/Properties/AssemblyInfo.cs b/src/StellaOps.Concelier.Connector.StellaOpsMirror/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..98e2ebf6 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.StellaOpsMirror.Tests")] diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror/Security/MirrorSignatureVerifier.cs b/src/StellaOps.Concelier.Connector.StellaOpsMirror/Security/MirrorSignatureVerifier.cs index 1477782d..bb3ddcd7 100644 --- a/src/StellaOps.Concelier.Connector.StellaOpsMirror/Security/MirrorSignatureVerifier.cs +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror/Security/MirrorSignatureVerifier.cs @@ -1,7 +1,12 @@ using System; +using System.IO; +using System.Security.Cryptography; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Microsoft.IdentityModel.Tokens; using StellaOps.Cryptography; @@ -13,6 +18,7 @@ namespace StellaOps.Concelier.Connector.StellaOpsMirror.Security; /// public sealed class MirrorSignatureVerifier { + private const string CachePrefix = "stellaops:mirror:public-key:"; private static readonly JsonSerializerOptions HeaderSerializerOptions = new(JsonSerializerDefaults.Web) { PropertyNameCaseInsensitive = true @@ -20,21 +26,27 @@ public sealed class MirrorSignatureVerifier private readonly ICryptoProviderRegistry _providerRegistry; private readonly ILogger _logger; + private readonly IMemoryCache? _memoryCache; - public MirrorSignatureVerifier(ICryptoProviderRegistry providerRegistry, ILogger logger) + public MirrorSignatureVerifier( + ICryptoProviderRegistry providerRegistry, + ILogger logger, + IMemoryCache? memoryCache = null) { _providerRegistry = providerRegistry ?? throw new ArgumentNullException(nameof(providerRegistry)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _memoryCache = memoryCache; } public Task VerifyAsync(ReadOnlyMemory payload, string signatureValue, CancellationToken cancellationToken) - => VerifyAsync(payload, signatureValue, expectedKeyId: null, expectedProvider: null, cancellationToken); + => VerifyAsync(payload, signatureValue, expectedKeyId: null, expectedProvider: null, fallbackPublicKeyPath: null, cancellationToken); public async Task VerifyAsync( ReadOnlyMemory payload, string signatureValue, string? expectedKeyId, string? expectedProvider, + string? fallbackPublicKeyPath, CancellationToken cancellationToken) { if (payload.IsEmpty) @@ -92,7 +104,8 @@ public sealed class MirrorSignatureVerifier var signatureBytes = Base64UrlEncoder.DecodeBytes(encodedSignature); var keyReference = new CryptoKeyReference(header.KeyId, header.Provider); - CryptoSignerResolution resolution; + CryptoSignerResolution? resolution = null; + bool providerVerified = false; try { resolution = _providerRegistry.ResolveSigner( @@ -100,19 +113,38 @@ public sealed class MirrorSignatureVerifier header.Algorithm, keyReference, header.Provider); + providerVerified = await resolution.Signer.VerifyAsync(signingInput, signatureBytes, cancellationToken).ConfigureAwait(false); + if (providerVerified) + { + return; + } + + _logger.LogWarning( + "Detached JWS verification failed for key {KeyId} via provider {Provider}.", + header.KeyId, + resolution.ProviderName); } catch (Exception ex) when (ex is InvalidOperationException or KeyNotFoundException) { _logger.LogWarning(ex, "Unable to resolve signer for mirror signature key {KeyId} via provider {Provider}.", header.KeyId, header.Provider ?? ""); - throw new InvalidOperationException("Detached JWS signature verification failed.", ex); } - var verified = await resolution.Signer.VerifyAsync(signingInput, signatureBytes, cancellationToken).ConfigureAwait(false); - if (!verified) + if (providerVerified) { - _logger.LogWarning("Detached JWS verification failed for key {KeyId} via provider {Provider}.", header.KeyId, resolution.ProviderName); - throw new InvalidOperationException("Detached JWS signature verification failed."); + return; } + + if (!string.IsNullOrWhiteSpace(fallbackPublicKeyPath) && + await TryVerifyWithFallbackAsync(signingInput, signatureBytes, header.Algorithm, fallbackPublicKeyPath!, cancellationToken).ConfigureAwait(false)) + { + _logger.LogDebug( + "Detached JWS verification succeeded for key {KeyId} using fallback public key at {Path}.", + header.KeyId, + fallbackPublicKeyPath); + return; + } + + throw new InvalidOperationException("Detached JWS signature verification failed."); } private static bool TryParseDetachedJws(string value, out string encodedHeader, out string encodedSignature) @@ -140,6 +172,97 @@ public sealed class MirrorSignatureVerifier return buffer; } + private async Task TryVerifyWithFallbackAsync( + ReadOnlyMemory signingInput, + ReadOnlyMemory signature, + string algorithm, + string fallbackPublicKeyPath, + CancellationToken cancellationToken) + { + try + { + cancellationToken.ThrowIfCancellationRequested(); + var parameters = await GetFallbackPublicKeyAsync(fallbackPublicKeyPath, cancellationToken).ConfigureAwait(false); + if (parameters is null) + { + return false; + } + + using var ecdsa = ECDsa.Create(); + ecdsa.ImportParameters(parameters.Value); + var hashAlgorithm = ResolveHashAlgorithm(algorithm); + return ecdsa.VerifyData(signingInput.Span, signature.Span, hashAlgorithm); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or CryptographicException or ArgumentException) + { + _logger.LogWarning(ex, "Failed to verify mirror signature using fallback public key at {Path}.", fallbackPublicKeyPath); + return false; + } + } + + private Task GetFallbackPublicKeyAsync(string path, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (_memoryCache is null) + { + return Task.FromResult(LoadPublicKey(path)); + } + + if (_memoryCache.TryGetValue>(CachePrefix + path, out var cached)) + { + return Task.FromResult(cached?.Value); + } + + if (!File.Exists(path)) + { + _logger.LogWarning("Mirror signature fallback public key path {Path} was not found.", path); + return Task.FromResult(null); + } + + var lazy = new Lazy( + () => LoadPublicKey(path), + LazyThreadSafetyMode.ExecutionAndPublication); + + var options = new MemoryCacheEntryOptions + { + AbsoluteExpirationRelativeToNow = TimeSpan.FromHours(6), + SlidingExpiration = TimeSpan.FromMinutes(30), + }; + + _memoryCache.Set(CachePrefix + path, lazy, options); + return Task.FromResult(lazy.Value); + } + + private ECParameters? LoadPublicKey(string path) + { + try + { + var pem = File.ReadAllText(path); + using var ecdsa = ECDsa.Create(); + ecdsa.ImportFromPem(pem.AsSpan()); + return ecdsa.ExportParameters(includePrivateParameters: false); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or CryptographicException or ArgumentException) + { + _logger.LogWarning(ex, "Failed to load mirror fallback public key from {Path}.", path); + return null; + } + } + + private static HashAlgorithmName ResolveHashAlgorithm(string algorithmId) + => algorithmId switch + { + { } alg when string.Equals(alg, SignatureAlgorithms.Es256, StringComparison.OrdinalIgnoreCase) => HashAlgorithmName.SHA256, + { } alg when string.Equals(alg, SignatureAlgorithms.Es384, StringComparison.OrdinalIgnoreCase) => HashAlgorithmName.SHA384, + { } alg when string.Equals(alg, SignatureAlgorithms.Es512, StringComparison.OrdinalIgnoreCase) => HashAlgorithmName.SHA512, + _ => throw new InvalidOperationException($"Unsupported mirror signature algorithm '{algorithmId}'."), + }; + private sealed record MirrorSignatureHeader( [property: JsonPropertyName("alg")] string Algorithm, [property: JsonPropertyName("kid")] string KeyId, diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror/StellaOpsMirrorConnector.cs b/src/StellaOps.Concelier.Connector.StellaOpsMirror/StellaOpsMirrorConnector.cs index 47eea368..fda33b07 100644 --- a/src/StellaOps.Concelier.Connector.StellaOpsMirror/StellaOpsMirrorConnector.cs +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror/StellaOpsMirrorConnector.cs @@ -12,8 +12,11 @@ using StellaOps.Concelier.Connector.StellaOpsMirror.Client; using StellaOps.Concelier.Connector.StellaOpsMirror.Internal; using StellaOps.Concelier.Connector.StellaOpsMirror.Security; using StellaOps.Concelier.Connector.StellaOpsMirror.Settings; +using StellaOps.Concelier.Models; using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; using StellaOps.Plugin; namespace StellaOps.Concelier.Connector.StellaOpsMirror; @@ -21,11 +24,14 @@ namespace StellaOps.Concelier.Connector.StellaOpsMirror; public sealed class StellaOpsMirrorConnector : IFeedConnector { public const string Source = "stellaops-mirror"; + private const string BundleDtoSchemaVersion = "stellaops.mirror.bundle.v1"; private readonly MirrorManifestClient _client; private readonly MirrorSignatureVerifier _signatureVerifier; private readonly RawDocumentStorage _rawDocumentStorage; private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; private readonly ISourceStateRepository _stateRepository; private readonly TimeProvider _timeProvider; private readonly ILogger _logger; @@ -36,6 +42,8 @@ public sealed class StellaOpsMirrorConnector : IFeedConnector MirrorSignatureVerifier signatureVerifier, RawDocumentStorage rawDocumentStorage, IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, ISourceStateRepository stateRepository, IOptions options, TimeProvider? timeProvider, @@ -45,6 +53,8 @@ public sealed class StellaOpsMirrorConnector : IFeedConnector _signatureVerifier = signatureVerifier ?? throw new ArgumentNullException(nameof(signatureVerifier)); _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _timeProvider = timeProvider ?? TimeProvider.System; @@ -84,6 +94,15 @@ public sealed class StellaOpsMirrorConnector : IFeedConnector throw new InvalidOperationException(message); } + var fingerprint = CreateFingerprint(index, domain); + var isNewDigest = !string.Equals(domain.Bundle.Digest, cursor.BundleDigest, StringComparison.OrdinalIgnoreCase); + + if (isNewDigest) + { + pendingDocuments.Clear(); + pendingMappings.Clear(); + } + if (string.Equals(domain.Bundle.Digest, cursor.BundleDigest, StringComparison.OrdinalIgnoreCase)) { _logger.LogInformation("Mirror bundle digest {Digest} unchanged; skipping fetch.", domain.Bundle.Digest); @@ -100,19 +119,29 @@ public sealed class StellaOpsMirrorConnector : IFeedConnector throw; } + var completedFingerprint = isNewDigest ? null : cursor.CompletedFingerprint; var updatedCursor = cursor .WithPendingDocuments(pendingDocuments) .WithPendingMappings(pendingMappings) - .WithBundleSnapshot(domain.Bundle.Path, domain.Bundle.Digest, index.GeneratedAt); + .WithBundleSnapshot(domain.Bundle.Path, domain.Bundle.Digest, index.GeneratedAt) + .WithCompletedFingerprint(completedFingerprint); await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); } public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) - => Task.CompletedTask; + { + ArgumentNullException.ThrowIfNull(services); + + return ParseInternalAsync(cancellationToken); + } public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) - => Task.CompletedTask; + { + ArgumentNullException.ThrowIfNull(services); + + return MapInternalAsync(cancellationToken); + } private async Task ProcessDomainAsync( MirrorIndexDocument index, @@ -152,6 +181,7 @@ public sealed class StellaOpsMirrorConnector : IFeedConnector signatureValue, expectedKeyId: _options.Signature.KeyId, expectedProvider: _options.Signature.Provider, + fallbackPublicKeyPath: _options.Signature.PublicKeyPath, cancellationToken).ConfigureAwait(false); } else if (domain.Bundle.Signature is not null) @@ -288,6 +318,20 @@ public sealed class StellaOpsMirrorConnector : IFeedConnector : digest.ToLowerInvariant(); } + private static string? CreateFingerprint(MirrorIndexDocument index, MirrorIndexDomainEntry domain) + => CreateFingerprint(domain.Bundle.Digest, index.GeneratedAt); + + private static string? CreateFingerprint(string? digest, DateTimeOffset? generatedAt) + { + var normalizedDigest = NormalizeDigest(digest ?? string.Empty); + if (string.IsNullOrWhiteSpace(normalizedDigest) || generatedAt is null) + { + return null; + } + + return FormattableString.Invariant($"{normalizedDigest}:{generatedAt.Value.ToUniversalTime():O}"); + } + private static void ValidateOptions(StellaOpsMirrorConnectorOptions options) { if (options.BaseAddress is null || !options.BaseAddress.IsAbsoluteUri) @@ -300,6 +344,226 @@ public sealed class StellaOpsMirrorConnector : IFeedConnector throw new InvalidOperationException("Mirror connector requires domainId to be specified."); } } + + private async Task ParseInternalAsync(CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + var pendingMappings = cursor.PendingMappings.ToHashSet(); + var now = _timeProvider.GetUtcNow(); + var parsed = 0; + var failures = 0; + + foreach (var documentId in cursor.PendingDocuments.ToArray()) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + failures++; + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("Mirror bundle document {DocumentId} missing GridFS payload.", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + failures++; + continue; + } + + byte[] payload; + try + { + payload = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Mirror bundle {DocumentId} failed to download from raw storage.", documentId); + throw; + } + + MirrorBundleDocument? bundle; + string json; + try + { + json = Encoding.UTF8.GetString(payload); + bundle = CanonicalJsonSerializer.Deserialize(json); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Mirror bundle {DocumentId} failed to deserialize.", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + failures++; + continue; + } + + if (bundle is null || bundle.Advisories is null) + { + _logger.LogWarning("Mirror bundle {DocumentId} produced null payload.", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + failures++; + continue; + } + + var dtoBson = BsonDocument.Parse(json); + var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, Source, BundleDtoSchemaVersion, dtoBson, now); + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + pendingDocuments.Remove(documentId); + pendingMappings.Add(document.Id); + parsed++; + + _logger.LogDebug( + "Parsed mirror bundle {DocumentId} domain={DomainId} advisories={AdvisoryCount}.", + document.Id, + bundle.DomainId, + bundle.AdvisoryCount); + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + + if (parsed > 0 || failures > 0) + { + _logger.LogInformation( + "Mirror parse completed parsed={Parsed} failures={Failures} pendingDocuments={PendingDocuments} pendingMappings={PendingMappings}.", + parsed, + failures, + pendingDocuments.Count, + pendingMappings.Count); + } + } + + private async Task MapInternalAsync(CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToHashSet(); + var mapped = 0; + var failures = 0; + var completedFingerprint = cursor.CompletedFingerprint; + + foreach (var documentId in cursor.PendingMappings.ToArray()) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + pendingMappings.Remove(documentId); + failures++; + continue; + } + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + if (dtoRecord is null) + { + _logger.LogWarning("Mirror document {DocumentId} missing DTO payload.", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + failures++; + continue; + } + + MirrorBundleDocument? bundle; + try + { + var json = dtoRecord.Payload.ToJson(); + bundle = CanonicalJsonSerializer.Deserialize(json); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Mirror DTO for document {DocumentId} failed to deserialize.", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + failures++; + continue; + } + + if (bundle is null || bundle.Advisories is null) + { + _logger.LogWarning("Mirror bundle DTO {DocumentId} evaluated to null.", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + failures++; + continue; + } + + try + { + var advisories = MirrorAdvisoryMapper.Map(bundle); + + foreach (var advisory in advisories) + { + cancellationToken.ThrowIfCancellationRequested(); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + } + + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + mapped++; + + _logger.LogDebug( + "Mirror map completed for document {DocumentId} domain={DomainId} advisories={AdvisoryCount}.", + document.Id, + bundle.DomainId, + advisories.Length); + } + catch (Exception ex) + { + _logger.LogError(ex, "Mirror mapping failed for document {DocumentId}.", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + failures++; + } + } + + if (pendingMappings.Count == 0 && failures == 0) + { + var fingerprint = CreateFingerprint(cursor.BundleDigest, cursor.GeneratedAt); + if (!string.IsNullOrWhiteSpace(fingerprint)) + { + completedFingerprint = fingerprint; + } + } + + var updatedCursor = cursor + .WithPendingMappings(pendingMappings) + .WithCompletedFingerprint(completedFingerprint); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + + if (mapped > 0 || failures > 0) + { + _logger.LogInformation( + "Mirror map completed mapped={Mapped} failures={Failures} pendingMappings={PendingMappings}.", + mapped, + failures, + pendingMappings.Count); + } + } } file static class UriExtensions diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror/StellaOpsMirrorDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.StellaOpsMirror/StellaOpsMirrorDependencyInjectionRoutine.cs index 91ef741d..8beea7b3 100644 --- a/src/StellaOps.Concelier.Connector.StellaOpsMirror/StellaOpsMirrorDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror/StellaOpsMirrorDependencyInjectionRoutine.cs @@ -23,23 +23,24 @@ public sealed class StellaOpsMirrorDependencyInjectionRoutine : IDependencyInjec ArgumentNullException.ThrowIfNull(services); ArgumentNullException.ThrowIfNull(configuration); - services.AddOptions() - .Bind(configuration.GetSection(ConfigurationSection)) - .PostConfigure(options => - { - if (options.BaseAddress is null) + services.AddOptions() + .Bind(configuration.GetSection(ConfigurationSection)) + .PostConfigure(options => + { + if (options.BaseAddress is null) { throw new InvalidOperationException("stellaopsMirror.baseAddress must be configured."); } }) - .ValidateOnStart(); - - services.AddSourceCommon(); - - services.AddHttpClient(HttpClientName, (sp, client) => - { - var options = sp.GetRequiredService>().Value; - client.BaseAddress = options.BaseAddress; + .ValidateOnStart(); + + services.AddSourceCommon(); + services.AddMemoryCache(); + + services.AddHttpClient(HttpClientName, (sp, client) => + { + var options = sp.GetRequiredService>().Value; + client.BaseAddress = options.BaseAddress; client.Timeout = options.HttpTimeout; client.DefaultRequestHeaders.Accept.Clear(); client.DefaultRequestHeaders.Accept.Add(new System.Net.Http.Headers.MediaTypeWithQualityHeaderValue("application/json")); diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md b/src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md index 7df65210..403c4340 100644 --- a/src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md @@ -2,6 +2,6 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| FEEDCONN-STELLA-08-001 | DOING (2025-10-19) | BE-Conn-Stella | CONCELIER-EXPORT-08-201 | Implement Concelier mirror fetcher hitting `https://.stella-ops.org/concelier/exports/index.json`, verify signatures/digests, and persist raw documents with provenance. | Fetch job downloads mirror manifest, verifies digest/signature, stores raw docs with tests covering happy-path + tampered manifest. *(In progress: HTTP client + detached JWS verifier scaffolding landed.)* | -| FEEDCONN-STELLA-08-002 | TODO | BE-Conn-Stella | FEEDCONN-STELLA-08-001 | Map mirror payloads into canonical advisory DTOs with provenance referencing mirror domain + original source metadata. | Mapper produces advisories/aliases/affected with mirror provenance; fixtures assert canonical parity with upstream JSON exporters. | -| FEEDCONN-STELLA-08-003 | TODO | BE-Conn-Stella | FEEDCONN-STELLA-08-002 | Add incremental cursor + resume support (per-export fingerprint) and document configuration for downstream Concelier instances. | Connector resumes from last export, handles deletion/delta cases, docs updated with config sample; integration test covers resume + new export scenario. | +| FEEDCONN-STELLA-08-001 | DONE (2025-10-20) | BE-Conn-Stella | CONCELIER-EXPORT-08-201 | Implement Concelier mirror fetcher hitting `https://.stella-ops.org/concelier/exports/index.json`, verify signatures/digests, and persist raw documents with provenance. | Fetch job downloads mirror manifest, verifies digest/signature, stores raw docs with tests covering happy-path + tampered manifest. *(Completed 2025-10-20: detached JWS + digest enforcement, metadata persisted, and regression coverage via `dotnet test src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/StellaOps.Concelier.Connector.StellaOpsMirror.Tests.csproj`.)* | +| FEEDCONN-STELLA-08-002 | DONE (2025-10-20) | BE-Conn-Stella | FEEDCONN-STELLA-08-001 | Map mirror payloads into canonical advisory DTOs with provenance referencing mirror domain + original source metadata. | Mapper produces advisories/aliases/affected with mirror provenance; fixtures assert canonical parity with upstream JSON exporters. | +| FEEDCONN-STELLA-08-003 | DONE (2025-10-20) | BE-Conn-Stella | FEEDCONN-STELLA-08-002 | Add incremental cursor + resume support (per-export fingerprint) and document configuration for downstream Concelier instances. | Connector resumes from last export, handles deletion/delta cases, docs updated with config sample; integration test covers resume + new export scenario. | diff --git a/src/StellaOps.Concelier.Core.Tests/Noise/NoisePriorServiceTests.cs b/src/StellaOps.Concelier.Core.Tests/Noise/NoisePriorServiceTests.cs new file mode 100644 index 00000000..faf1f39c --- /dev/null +++ b/src/StellaOps.Concelier.Core.Tests/Noise/NoisePriorServiceTests.cs @@ -0,0 +1,320 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Concelier.Core.Events; +using StellaOps.Concelier.Core.Noise; +using StellaOps.Concelier.Models; +using Xunit; + +namespace StellaOps.Concelier.Core.Tests.Noise; + +public sealed class NoisePriorServiceTests +{ + [Fact] + public async Task RecomputeAsync_PersistsSummariesWithRules() + { + var statements = ImmutableArray.Create( + CreateStatement( + asOf: DateTimeOffset.Parse("2025-10-10T00:00:00Z"), + CreatePackage( + statuses: new[] + { + CreateStatus(AffectedPackageStatusCatalog.NotAffected, "vendor.redhat"), + }, + platform: "linux"))); + + statements = statements.Add(CreateStatement( + asOf: DateTimeOffset.Parse("2025-10-11T00:00:00Z"), + CreatePackage( + statuses: new[] + { + CreateStatus(AffectedPackageStatusCatalog.KnownNotAffected, "vendor.canonical"), + }, + platform: "linux"))); + + statements = statements.Add(CreateStatement( + asOf: DateTimeOffset.Parse("2025-10-12T00:00:00Z"), + CreatePackage( + statuses: new[] + { + CreateStatus(AffectedPackageStatusCatalog.Affected, "vendor.osv"), + }, + platform: "linux", + versionRanges: new[] + { + new AffectedVersionRange( + rangeKind: "semver", + introducedVersion: "1.0.0", + fixedVersion: null, + lastAffectedVersion: null, + rangeExpression: null, + provenance: CreateProvenance("vendor.osv")), + }))); + + var replay = new AdvisoryReplay( + "cve-9999-0001", + null, + statements, + ImmutableArray.Empty); + + var eventLog = new FakeEventLog(replay); + var repository = new FakeNoisePriorRepository(); + var now = DateTimeOffset.Parse("2025-10-21T12:00:00Z"); + var timeProvider = new FixedTimeProvider(now); + var service = new NoisePriorService(eventLog, repository, timeProvider); + + var result = await service.RecomputeAsync( + new NoisePriorComputationRequest("CVE-9999-0001"), + CancellationToken.None); + + Assert.Equal("cve-9999-0001", result.VulnerabilityKey); + Assert.Single(result.Summaries); + + var summary = result.Summaries[0]; + Assert.Equal("cve-9999-0001", summary.VulnerabilityKey); + Assert.Equal("semver", summary.PackageType); + Assert.Equal("pkg:npm/example", summary.PackageIdentifier); + Assert.Equal("linux", summary.Platform); + Assert.Equal(3, summary.ObservationCount); + Assert.Equal(2, summary.NegativeSignals); + Assert.Equal(1, summary.PositiveSignals); + Assert.Equal(0, summary.NeutralSignals); + Assert.Equal(1, summary.VersionRangeSignals); + Assert.Equal(2, summary.UniqueNegativeSources); + Assert.Equal(0.6, summary.Probability); + Assert.Equal(now, summary.GeneratedAt); + Assert.Equal(DateTimeOffset.Parse("2025-10-10T00:00:00Z"), summary.FirstObserved); + Assert.Equal(DateTimeOffset.Parse("2025-10-12T00:00:00Z"), summary.LastObserved); + + Assert.Equal( + new[] { "conflicting_signals", "multi_source_negative", "positive_evidence" }, + summary.RuleHits.ToArray()); + + Assert.Equal("cve-9999-0001", repository.LastUpsertKey); + Assert.NotNull(repository.LastUpsertSummaries); + Assert.Single(repository.LastUpsertSummaries!); + } + + [Fact] + public async Task RecomputeAsync_AllNegativeSignalsProducesHighPrior() + { + var statements = ImmutableArray.Create( + CreateStatement( + asOf: DateTimeOffset.Parse("2025-10-01T00:00:00Z"), + CreatePackage( + statuses: new[] + { + CreateStatus(AffectedPackageStatusCatalog.NotAffected, "vendor.redhat"), + }), + vulnerabilityKey: "cve-2025-1111")); + + statements = statements.Add(CreateStatement( + asOf: DateTimeOffset.Parse("2025-10-02T00:00:00Z"), + CreatePackage( + statuses: new[] + { + CreateStatus(AffectedPackageStatusCatalog.KnownNotAffected, "vendor.redhat"), + }), + vulnerabilityKey: "cve-2025-1111")); + + var replay = new AdvisoryReplay( + "cve-2025-1111", + null, + statements, + ImmutableArray.Empty); + + var eventLog = new FakeEventLog(replay); + var repository = new FakeNoisePriorRepository(); + var now = DateTimeOffset.Parse("2025-10-21T13:00:00Z"); + var timeProvider = new FixedTimeProvider(now); + var service = new NoisePriorService(eventLog, repository, timeProvider); + + var result = await service.RecomputeAsync( + new NoisePriorComputationRequest("cve-2025-1111"), + CancellationToken.None); + + var summary = Assert.Single(result.Summaries); + Assert.Equal(1.0, summary.Probability); + Assert.Equal( + new[] { "all_negative", "sparse_observations" }, + summary.RuleHits.ToArray()); + } + + [Fact] + public async Task GetByPackageAsync_NormalizesInputs() + { + var statements = ImmutableArray.Create( + CreateStatement( + asOf: DateTimeOffset.Parse("2025-10-03T00:00:00Z"), + CreatePackage( + statuses: new[] + { + CreateStatus(AffectedPackageStatusCatalog.Unknown, "vendor.generic"), + }, + platform: "linux"), + vulnerabilityKey: "cve-2025-2000")); + + var replay = new AdvisoryReplay( + "cve-2025-2000", + null, + statements, + ImmutableArray.Empty); + + var eventLog = new FakeEventLog(replay); + var repository = new FakeNoisePriorRepository(); + var service = new NoisePriorService(eventLog, repository, new FixedTimeProvider(DateTimeOffset.UtcNow)); + + await service.RecomputeAsync( + new NoisePriorComputationRequest("CVE-2025-2000"), + CancellationToken.None); + + var summaries = await service.GetByPackageAsync( + " SemVer ", + "pkg:npm/example", + " linux ", + CancellationToken.None); + + Assert.Single(summaries); + Assert.Equal("semver", summaries[0].PackageType); + Assert.Equal("linux", summaries[0].Platform); + } + + private static AdvisoryStatementSnapshot CreateStatement( + DateTimeOffset asOf, + AffectedPackage package, + string vulnerabilityKey = "cve-9999-0001") + { + var advisory = new Advisory( + advisoryKey: $"adv-{asOf:yyyyMMddHHmmss}", + title: "Example Advisory", + summary: null, + language: "en", + published: null, + modified: asOf, + severity: "high", + exploitKnown: false, + aliases: new[] { "CVE-TEST-0001" }, + references: Array.Empty(), + affectedPackages: new[] { package }, + cvssMetrics: Array.Empty(), + provenance: Array.Empty()); + + return new AdvisoryStatementSnapshot( + Guid.NewGuid(), + vulnerabilityKey, + advisory.AdvisoryKey, + advisory, + StatementHash: ImmutableArray.Empty, + AsOf: asOf, + RecordedAt: asOf, + InputDocumentIds: ImmutableArray.Empty); + } + + private static AffectedPackage CreatePackage( + IEnumerable statuses, + string? platform = null, + IEnumerable? versionRanges = null) + => new( + type: "semver", + identifier: "pkg:npm/example", + platform: platform, + versionRanges: versionRanges, + statuses: statuses, + provenance: new[] { CreateProvenance("vendor.core") }, + normalizedVersions: null); + + private static AffectedPackageStatus CreateStatus(string status, string source) + => new( + status, + CreateProvenance(source)); + + private static AdvisoryProvenance CreateProvenance(string source, string kind = "vendor") + => new( + source, + kind, + value: string.Empty, + recordedAt: DateTimeOffset.Parse("2025-10-01T00:00:00Z"), + fieldMask: null, + decisionReason: null); + + private sealed class FakeEventLog : IAdvisoryEventLog + { + private readonly AdvisoryReplay _replay; + + public FakeEventLog(AdvisoryReplay replay) + { + _replay = replay; + } + + public ValueTask AppendAsync(AdvisoryEventAppendRequest request, CancellationToken cancellationToken) + => throw new NotSupportedException("Append operations are not required for tests."); + + public ValueTask ReplayAsync(string vulnerabilityKey, DateTimeOffset? asOf, CancellationToken cancellationToken) + => ValueTask.FromResult(_replay); + } + + private sealed class FakeNoisePriorRepository : INoisePriorRepository + { + private readonly List _store = new(); + + public string? LastUpsertKey { get; private set; } + + public IReadOnlyCollection? LastUpsertSummaries { get; private set; } + + public ValueTask UpsertAsync( + string vulnerabilityKey, + IReadOnlyCollection summaries, + CancellationToken cancellationToken) + { + LastUpsertKey = vulnerabilityKey; + LastUpsertSummaries = summaries; + + _store.RemoveAll(summary => + string.Equals(summary.VulnerabilityKey, vulnerabilityKey, StringComparison.Ordinal)); + + _store.AddRange(summaries); + return ValueTask.CompletedTask; + } + + public ValueTask> GetByVulnerabilityAsync( + string vulnerabilityKey, + CancellationToken cancellationToken) + { + var matches = _store + .Where(summary => string.Equals(summary.VulnerabilityKey, vulnerabilityKey, StringComparison.Ordinal)) + .ToList(); + return ValueTask.FromResult>(matches); + } + + public ValueTask> GetByPackageAsync( + string packageType, + string packageIdentifier, + string? platform, + CancellationToken cancellationToken) + { + var matches = _store + .Where(summary => + string.Equals(summary.PackageType, packageType, StringComparison.Ordinal) && + string.Equals(summary.PackageIdentifier, packageIdentifier, StringComparison.Ordinal) && + string.Equals(summary.Platform ?? string.Empty, platform ?? string.Empty, StringComparison.Ordinal)) + .ToList(); + + return ValueTask.FromResult>(matches); + } + } + + private sealed class FixedTimeProvider : TimeProvider + { + private readonly DateTimeOffset _now; + + public FixedTimeProvider(DateTimeOffset now) + { + _now = now.ToUniversalTime(); + } + + public override DateTimeOffset GetUtcNow() => _now; + } +} diff --git a/src/StellaOps.Concelier.Core/Noise/INoisePriorRepository.cs b/src/StellaOps.Concelier.Core/Noise/INoisePriorRepository.cs new file mode 100644 index 00000000..27b9f9fc --- /dev/null +++ b/src/StellaOps.Concelier.Core/Noise/INoisePriorRepository.cs @@ -0,0 +1,26 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Concelier.Core.Noise; + +/// +/// Persistence abstraction for storing and retrieving noise prior summaries. +/// +public interface INoisePriorRepository +{ + ValueTask UpsertAsync( + string vulnerabilityKey, + IReadOnlyCollection summaries, + CancellationToken cancellationToken); + + ValueTask> GetByVulnerabilityAsync( + string vulnerabilityKey, + CancellationToken cancellationToken); + + ValueTask> GetByPackageAsync( + string packageType, + string packageIdentifier, + string? platform, + CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Concelier.Core/Noise/INoisePriorService.cs b/src/StellaOps.Concelier.Core/Noise/INoisePriorService.cs new file mode 100644 index 00000000..da45995e --- /dev/null +++ b/src/StellaOps.Concelier.Core/Noise/INoisePriorService.cs @@ -0,0 +1,25 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Concelier.Core.Noise; + +/// +/// Computes and serves false-positive priors for canonical advisories. +/// +public interface INoisePriorService +{ + ValueTask RecomputeAsync( + NoisePriorComputationRequest request, + CancellationToken cancellationToken); + + ValueTask> GetByVulnerabilityAsync( + string vulnerabilityKey, + CancellationToken cancellationToken); + + ValueTask> GetByPackageAsync( + string packageType, + string packageIdentifier, + string? platform, + CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Concelier.Core/Noise/NoisePriorComputationRequest.cs b/src/StellaOps.Concelier.Core/Noise/NoisePriorComputationRequest.cs new file mode 100644 index 00000000..8bf2569e --- /dev/null +++ b/src/StellaOps.Concelier.Core/Noise/NoisePriorComputationRequest.cs @@ -0,0 +1,10 @@ +using System; + +namespace StellaOps.Concelier.Core.Noise; + +/// +/// Options for recomputing noise priors for a single vulnerability key. +/// +public sealed record NoisePriorComputationRequest( + string VulnerabilityKey, + DateTimeOffset? AsOf = null); diff --git a/src/StellaOps.Concelier.Core/Noise/NoisePriorComputationResult.cs b/src/StellaOps.Concelier.Core/Noise/NoisePriorComputationResult.cs new file mode 100644 index 00000000..4fffda41 --- /dev/null +++ b/src/StellaOps.Concelier.Core/Noise/NoisePriorComputationResult.cs @@ -0,0 +1,10 @@ +using System.Collections.Immutable; + +namespace StellaOps.Concelier.Core.Noise; + +/// +/// Results of a recompute operation containing per-package noise prior summaries. +/// +public sealed record NoisePriorComputationResult( + string VulnerabilityKey, + ImmutableArray Summaries); diff --git a/src/StellaOps.Concelier.Core/Noise/NoisePriorService.cs b/src/StellaOps.Concelier.Core/Noise/NoisePriorService.cs new file mode 100644 index 00000000..087860c4 --- /dev/null +++ b/src/StellaOps.Concelier.Core/Noise/NoisePriorService.cs @@ -0,0 +1,400 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Concelier.Core.Events; +using StellaOps.Concelier.Models; + +namespace StellaOps.Concelier.Core.Noise; + +/// +/// Default implementation that derives false-positive priors from advisory statements. +/// +public sealed class NoisePriorService : INoisePriorService +{ + private static readonly HashSet NegativeStatuses = new( + new[] + { + AffectedPackageStatusCatalog.KnownNotAffected, + AffectedPackageStatusCatalog.NotAffected, + AffectedPackageStatusCatalog.NotApplicable, + }, + StringComparer.Ordinal); + + private static readonly HashSet PositiveStatuses = new( + new[] + { + AffectedPackageStatusCatalog.KnownAffected, + AffectedPackageStatusCatalog.Affected, + AffectedPackageStatusCatalog.UnderInvestigation, + AffectedPackageStatusCatalog.Pending, + }, + StringComparer.Ordinal); + + private static readonly HashSet ResolvedStatuses = new( + new[] + { + AffectedPackageStatusCatalog.Fixed, + AffectedPackageStatusCatalog.FirstFixed, + AffectedPackageStatusCatalog.Mitigated, + }, + StringComparer.Ordinal); + + private readonly IAdvisoryEventLog _eventLog; + private readonly INoisePriorRepository _repository; + private readonly TimeProvider _timeProvider; + + public NoisePriorService( + IAdvisoryEventLog eventLog, + INoisePriorRepository repository, + TimeProvider? timeProvider = null) + { + _eventLog = eventLog ?? throw new ArgumentNullException(nameof(eventLog)); + _repository = repository ?? throw new ArgumentNullException(nameof(repository)); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public async ValueTask RecomputeAsync( + NoisePriorComputationRequest request, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + + var normalizedKey = NormalizeKey(request.VulnerabilityKey, nameof(request.VulnerabilityKey)); + var replay = await _eventLog.ReplayAsync(normalizedKey, request.AsOf, cancellationToken).ConfigureAwait(false); + + var generatedAt = _timeProvider.GetUtcNow(); + var summaries = ComputeSummaries(replay, generatedAt); + + await _repository.UpsertAsync(normalizedKey, summaries, cancellationToken).ConfigureAwait(false); + + return new NoisePriorComputationResult( + normalizedKey, + summaries); + } + + public ValueTask> GetByVulnerabilityAsync( + string vulnerabilityKey, + CancellationToken cancellationToken) + { + var normalizedKey = NormalizeKey(vulnerabilityKey, nameof(vulnerabilityKey)); + return _repository.GetByVulnerabilityAsync(normalizedKey, cancellationToken); + } + + public ValueTask> GetByPackageAsync( + string packageType, + string packageIdentifier, + string? platform, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(packageType); + ArgumentException.ThrowIfNullOrWhiteSpace(packageIdentifier); + + var normalizedType = packageType.Trim().ToLowerInvariant(); + var normalizedIdentifier = packageIdentifier.Trim(); + var normalizedPlatform = NormalizePlatform(platform); + + return _repository.GetByPackageAsync( + normalizedType, + normalizedIdentifier, + normalizedPlatform, + cancellationToken); + } + + private ImmutableArray ComputeSummaries( + AdvisoryReplay replay, + DateTimeOffset generatedAt) + { + if (replay is null || replay.Statements.IsDefaultOrEmpty) + { + return ImmutableArray.Empty; + } + + var accumulators = new Dictionary(capacity: replay.Statements.Length); + + foreach (var statement in replay.Statements) + { + if (statement is null) + { + continue; + } + + foreach (var package in statement.Advisory.AffectedPackages) + { + if (package is null || string.IsNullOrWhiteSpace(package.Identifier)) + { + continue; + } + + var platform = NormalizePlatform(package.Platform); + var key = new PackageKey(package.Type, package.Identifier, platform); + + if (!accumulators.TryGetValue(key, out var accumulator)) + { + accumulator = new NoiseAccumulator( + replay.VulnerabilityKey, + package.Type, + package.Identifier, + platform); + accumulators.Add(key, accumulator); + } + + accumulator.Register(statement.AsOf, package); + } + } + + if (accumulators.Count == 0) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(accumulators.Count); + foreach (var accumulator in accumulators.Values + .OrderBy(static a => a.PackageType, StringComparer.Ordinal) + .ThenBy(static a => a.PackageIdentifier, StringComparer.Ordinal) + .ThenBy(static a => a.Platform, StringComparer.Ordinal)) + { + builder.Add(accumulator.ToSummary(generatedAt)); + } + + return builder.ToImmutable(); + } + + private static string NormalizeKey(string value, string parameterName) + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new ArgumentException("Value must be provided.", parameterName); + } + + return value.Trim().ToLowerInvariant(); + } + + private static string? NormalizePlatform(string? value) + => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); + + private sealed record PackageKey( + string PackageType, + string PackageIdentifier, + string? Platform); + + private sealed class NoiseAccumulator + { + private readonly string _vulnerabilityKey; + private readonly HashSet _negativeSources = new(StringComparer.Ordinal); + + public NoiseAccumulator( + string vulnerabilityKey, + string packageType, + string packageIdentifier, + string? platform) + { + _vulnerabilityKey = vulnerabilityKey; + PackageType = packageType; + PackageIdentifier = packageIdentifier; + Platform = platform; + FirstObserved = DateTimeOffset.MaxValue; + LastObserved = DateTimeOffset.MinValue; + } + + public string PackageType { get; } + + public string PackageIdentifier { get; } + + public string? Platform { get; } + + public int ObservationCount { get; private set; } + + public int NegativeSignals { get; private set; } + + public int PositiveSignals { get; private set; } + + public int NeutralSignals { get; private set; } + + public int VersionRangeSignals { get; private set; } + + public bool HasMissingStatus { get; private set; } + + public DateTimeOffset FirstObserved { get; private set; } + + public DateTimeOffset LastObserved { get; private set; } + + public int UniqueNegativeSources => _negativeSources.Count; + + public void Register(DateTimeOffset asOf, AffectedPackage package) + { + ObservationCount++; + + var asOfUtc = asOf.ToUniversalTime(); + if (asOfUtc < FirstObserved) + { + FirstObserved = asOfUtc; + } + + if (asOfUtc > LastObserved) + { + LastObserved = asOfUtc; + } + + var statuses = package.Statuses; + if (statuses.IsDefaultOrEmpty || statuses.Length == 0) + { + HasMissingStatus = true; + } + + foreach (var status in statuses) + { + if (NegativeStatuses.Contains(status.Status)) + { + NegativeSignals++; + if (!string.IsNullOrWhiteSpace(status.Provenance.Source)) + { + _negativeSources.Add(status.Provenance.Source); + } + } + else if (PositiveStatuses.Contains(status.Status) || ResolvedStatuses.Contains(status.Status)) + { + PositiveSignals++; + } + else if (string.Equals(status.Status, AffectedPackageStatusCatalog.Unknown, StringComparison.Ordinal)) + { + NeutralSignals++; + } + else + { + NeutralSignals++; + } + } + + if (!package.VersionRanges.IsDefaultOrEmpty && package.VersionRanges.Length > 0) + { + VersionRangeSignals++; + } + } + + public NoisePriorSummary ToSummary(DateTimeOffset generatedAt) + { + var boundedFirst = FirstObserved == DateTimeOffset.MaxValue ? generatedAt : FirstObserved; + var boundedLast = LastObserved == DateTimeOffset.MinValue ? generatedAt : LastObserved; + + var probability = ComputeProbability(); + var rules = BuildRules(); + + return new NoisePriorSummary( + _vulnerabilityKey, + PackageType, + PackageIdentifier, + Platform, + probability, + ObservationCount, + NegativeSignals, + PositiveSignals, + NeutralSignals, + VersionRangeSignals, + UniqueNegativeSources, + rules, + boundedFirst, + boundedLast, + generatedAt); + } + + private double ComputeProbability() + { + var positiveSignals = PositiveSignals + VersionRangeSignals; + var denominator = NegativeSignals + positiveSignals; + + double score; + if (denominator == 0) + { + if (HasMissingStatus) + { + score = 0.35; + } + else if (NeutralSignals > 0) + { + score = 0.40; + } + else + { + score = 0.0; + } + } + else + { + score = NegativeSignals / (double)denominator; + + if (NegativeSignals > 0 && positiveSignals == 0) + { + score = Math.Min(1.0, score + 0.20); + } + + if (positiveSignals > 0 && NegativeSignals == 0) + { + score = Math.Max(0.0, score - 0.25); + } + + if (PositiveSignals > NegativeSignals) + { + score = Math.Max(0.0, score - 0.10); + } + + if (UniqueNegativeSources >= 2) + { + score = Math.Min(1.0, score + 0.10); + } + + if (NeutralSignals > 0) + { + var neutralBoost = Math.Min(0.10, NeutralSignals * 0.02); + score = Math.Min(1.0, score + neutralBoost); + } + } + + return Math.Round(Math.Clamp(score, 0.0, 1.0), 4, MidpointRounding.ToZero); + } + + private ImmutableArray BuildRules() + { + var rules = new HashSet(StringComparer.Ordinal); + + if (NegativeSignals > 0 && PositiveSignals == 0 && VersionRangeSignals == 0) + { + rules.Add("all_negative"); + } + + if (UniqueNegativeSources >= 2) + { + rules.Add("multi_source_negative"); + } + + if (PositiveSignals > 0 || VersionRangeSignals > 0) + { + rules.Add("positive_evidence"); + } + + if (NegativeSignals > 0 && (PositiveSignals > 0 || VersionRangeSignals > 0)) + { + rules.Add("conflicting_signals"); + } + + if (ObservationCount < 3) + { + rules.Add("sparse_observations"); + } + + if (HasMissingStatus) + { + rules.Add("missing_status"); + } + + if (NeutralSignals > 0 && NegativeSignals == 0 && PositiveSignals == 0 && VersionRangeSignals == 0) + { + rules.Add("neutral_only"); + } + + return rules.OrderBy(static rule => rule, StringComparer.Ordinal).ToImmutableArray(); + } + } +} diff --git a/src/StellaOps.Concelier.Core/Noise/NoisePriorServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Core/Noise/NoisePriorServiceCollectionExtensions.cs new file mode 100644 index 00000000..e3de3795 --- /dev/null +++ b/src/StellaOps.Concelier.Core/Noise/NoisePriorServiceCollectionExtensions.cs @@ -0,0 +1,24 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; + +namespace StellaOps.Concelier.Core.Noise; + +/// +/// Dependency injection helpers for the noise prior service. +/// +public static class NoisePriorServiceCollectionExtensions +{ + public static IServiceCollection AddNoisePriorService(this IServiceCollection services) + { + if (services is null) + { + throw new ArgumentNullException(nameof(services)); + } + + services.TryAddSingleton(TimeProvider.System); + services.AddSingleton(); + + return services; + } +} diff --git a/src/StellaOps.Concelier.Core/Noise/NoisePriorSummary.cs b/src/StellaOps.Concelier.Core/Noise/NoisePriorSummary.cs new file mode 100644 index 00000000..23c21f39 --- /dev/null +++ b/src/StellaOps.Concelier.Core/Noise/NoisePriorSummary.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Immutable; + +namespace StellaOps.Concelier.Core.Noise; + +/// +/// Immutable noise prior summary describing false-positive likelihood signals for a package/environment tuple. +/// +public sealed record NoisePriorSummary( + string VulnerabilityKey, + string PackageType, + string PackageIdentifier, + string? Platform, + double Probability, + int ObservationCount, + int NegativeSignals, + int PositiveSignals, + int NeutralSignals, + int VersionRangeSignals, + int UniqueNegativeSources, + ImmutableArray RuleHits, + DateTimeOffset FirstObserved, + DateTimeOffset LastObserved, + DateTimeOffset GeneratedAt); diff --git a/src/StellaOps.Concelier.Core/TASKS.md b/src/StellaOps.Concelier.Core/TASKS.md index ef2be75d..96cf6bb9 100644 --- a/src/StellaOps.Concelier.Core/TASKS.md +++ b/src/StellaOps.Concelier.Core/TASKS.md @@ -17,5 +17,5 @@ |Canonical merger parity for description/CWE/canonical metric|BE-Core|Models|DONE (2025-10-15) – merger now populates description/CWEs/canonical metric id with provenance and regression tests cover the new decisions.| |Reference normalization & freshness instrumentation cleanup|BE-Core, QA|Models|DONE (2025-10-15) – reference keys normalized, freshness overrides applied to union fields, and new tests assert decision logging.| |FEEDCORE-ENGINE-07-001 – Advisory event log & asOf queries|Team Core Engine & Storage Analytics|FEEDSTORAGE-DATA-07-001|**DONE (2025-10-19)** – Implemented `AdvisoryEventLog` service plus repository contracts, canonical hashing, and lower-cased key normalization with replay support; documented determinism guarantees. Tests: `dotnet test src/StellaOps.Concelier.Core.Tests/StellaOps.Concelier.Core.Tests.csproj`.| -|FEEDCORE-ENGINE-07-002 – Noise prior computation service|Team Core Engine & Data Science|FEEDCORE-ENGINE-07-001|TODO – Build rule-based learner capturing false-positive priors per package/env, persist summaries, and expose APIs for Excititor/scan suppressors with reproducible statistics.| +|FEEDCORE-ENGINE-07-002 – Noise prior computation service|Team Core Engine & Data Science|FEEDCORE-ENGINE-07-001|**DONE (2025-10-21)** – Build rule-based learner capturing false-positive priors per package/env, persist summaries, and expose APIs for Excititor/scan suppressors with reproducible statistics.| |FEEDCORE-ENGINE-07-003 – Unknown state ledger & confidence seeding|Team Core Engine & Storage Analytics|FEEDCORE-ENGINE-07-001|TODO – Persist `unknown_vuln_range/unknown_origin/ambiguous_fix` markers with initial confidence bands, expose query surface for Policy, and add fixtures validating canonical serialization.| diff --git a/src/StellaOps.Concelier.Merge/RANGE_PRIMITIVES_COORDINATION.md b/src/StellaOps.Concelier.Merge/RANGE_PRIMITIVES_COORDINATION.md index 86612721..1e26c7ea 100644 --- a/src/StellaOps.Concelier.Merge/RANGE_PRIMITIVES_COORDINATION.md +++ b/src/StellaOps.Concelier.Merge/RANGE_PRIMITIVES_COORDINATION.md @@ -1,6 +1,6 @@ # Range Primitive Coordination (Sprint 2) -_Status date: 2025-10-11_ +_Status date: 2025-10-20_ ## Why this exists - SemVer range outputs must follow the embedded rule guidance in `../FASTER_MODELING_AND_NORMALIZATION.md` (array of `{scheme,type,min/max/value,notes}`). @@ -16,24 +16,24 @@ _Status date: 2025-10-11_ Until these blocks land, connectors should stage changes behind a feature flag or fixture branch so we can flip on normalized writes in sync. ## Connector adoption matrix -| Connector | Owner team | Current state (2025-10-11) | Required actions for SemVer guidance | Coordination notes | -|-----------|------------|-----------------------------|-------------------------------------|--------------------| -| Acsc | BE-Conn-ACSC | All tasks still TODO | Blocked on initial ingest work; align DTO design with normalized rule array before mapper lands. | Schedule pairing once `SemVerRangeRuleBuilder` API is published; ensure fixtures capture vendor/device taxonomy for provenance notes. | -| Cccs | BE-Conn-CCCS | All tasks still TODO | Same as Acsc; design DTOs with normalized rule hooks from day one. | Provide sample rule snippets in kickoff; share Mongo dual-write plan once storage flag is ready. | -| CertBund | BE-Conn-CERTBUND | All tasks still TODO | Ensure canonical mapper emits vendor range primitives plus normalized rules for product firmware. | Needs language/localisation guidance; coordinate with Localization WG for deterministic casing. | -| CertCc | BE-Conn-CERTCC | Fetch in progress, mapping TODO | Map VINCE vendor/product data into `RangePrimitives` with `certcc.vendor` extensions; build normalized SemVer ranges when version strings surface. | Follow up on 2025-10-14 to review VINCE payload examples and confirm builder requirements. | -| Cve | BE-Conn-CVE | Mapping/tests DONE (legacy SemVer) | Refactor `CveMapper` to call the shared builder and populate `NormalizedVersions` + provenance notes once models land. | Prepare MR behind `ENABLE_NORMALIZED_VERSIONS` flag; regression fixtures already cover version ranges—extend snapshots to cover rule arrays. | -| Ghsa | BE-Conn-GHSA | Normalized rules emitted (2025-10-11) | Maintain SemVer builder integration; share regression diffs if schema shifts occur. | Fixtures refreshed with `ghsa:{identifier}` notes; OSV rollout next in queue—await connector handoff update. | -| Osv | BE-Conn-OSV | Normalized rules emitted (2025-10-11) | Keep SemVer builder wiring current; extend notes if new ecosystems appear. | npm/PyPI parity snapshots updated with `osv:{ecosystem}:{advisoryId}:{identifier}` notes; merge analytics notified. | -| Nvd | BE-Conn-NVD | Normalized rules emitted (2025-10-11) | Maintain SemVer coverage for ecosystem ranges; keep notes aligned with CVE IDs. | CPE ranges now emit semver primitives when versions parse; fixtures refreshed, report sent to FEEDMERGE-COORD-02-900. | -| Cve | BE-Conn-CVE | Normalized rules emitted (2025-10-11) | Maintain SemVer notes for vendor ecosystems; backfill additional fixture coverage as CVE payloads expand. | Connector outputs `cve:{cveId}:{identifier}` notes; npm parity test fixtures updated and merge ping acknowledged. | -| Ics.Cisa | BE-Conn-ICS-CISA | All tasks TODO | When defining product schema, plan for SemVer or vendor version rules (many advisories use firmware revisions). | Gather sample advisories and confirm whether ranges are SemVer or vendor-specific so we can introduce scheme identifiers early. | -| Kisa | BE-Conn-KISA | All tasks TODO | Ensure DTO parsing captures version strings despite localisation; feed into normalized rule builder once ready. | Requires translation samples; request help from Localization WG before mapper implementation. | -| Ru.Bdu | BE-Conn-BDU | All tasks TODO | Map product releases into normalized rules; add provenance notes referencing BDU advisory identifiers. | Verify we have UTF-8 safe handling in builder; share sample sanitized inputs. | -| Ru.Nkcki | BE-Conn-Nkcki | All tasks TODO | Similar to BDU; capture vendor firmware/build numbers and map into normalized rules. | Coordinate with Localization WG for Cyrillic transliteration strategy. | -| Vndr.Apple | BE-Conn-Apple | Mapper/tests/telemetry marked DOING | Continue extending vendor range primitives (`apple.version`, `apple.build`) and adopt normalized rule arrays for OS build spans. | Request builder integration review on 2025-10-16; ensure fixtures cover multi-range tables and include provenance notes. | -| Vndr.Cisco | BE-Conn-Cisco | ✅ Emits SemVer primitives with vendor notes | Parser maps versions into SemVer primitives with `cisco.productId` vendor extensions; sample fixtures landing in `StellaOps.Concelier.Connector.Vndr.Cisco.Tests`. | No custom comparer required; SemVer + vendor metadata suffices. | -| Vndr.Msrc | BE-Conn-MSRC | All tasks TODO | Canonical mapper must output product/build coverage as normalized rules (likely `msrc.patch` scheme) with provenance referencing KB IDs. | Sync with Models on adding scheme identifiers for MSRC packages; plan fixture coverage for monthly rollups. | +| Connector | Owner team | Current state (2025-10-20) | Required actions for normalized rules | Coordination notes | +|-----------|------------|----------------------------|--------------------------------------|--------------------| +| Acsc | BE-Conn-ACSC | ❌ Not started – mapper emits legacy range strings only | Stage `SemVerRangeRuleBuilder` integration once relay HTTP/2 fixes stabilise; target kickoff 2025-10-24. | Pair with Merge on sample payloads; ensure fixtures capture vendor/device taxonomy for provenance notes. | +| Cccs | BE-Conn-CCCS | ⚠️ DOING – helper branch under review (due 2025-10-21) | Wire trailing-version split helper, emit `NormalizedVersions` with `cccs:{serial}:{index}` notes, refresh fixtures/tests. | Share MR link before 2025-10-21 stand-up; Merge to validate counters once fixtures land. | +| CertBund | BE-Conn-CERTBUND | ⚠️ In progress – localisation work pending (due 2025-10-22) | Translate `product.Versions` phrases (`bis`, `alle`) into builder inputs; emit provenance `certbund:{advisoryId}:{vendor}`; update README/tests. | Localization WG drafting deterministic casing guidance; expect sample payloads 2025-10-21. | +| CertCc | BE-Conn-CERTCC | ✅ Complete – emitting `certcc.vendor` rules since 2025-10-12 | Keep builder contract stable; bubble any VINCE payload changes. | Merge verified counters drop on 2025-10-19 run; no follow-up. | +| Cve | BE-Conn-CVE | ✅ Complete – SemVer rules emitted 2025-10-12 | Maintain provenance notes (`cve:{cveId}:{identifier}`) and extend fixtures as schema grows. | Latest nightly confirms normalized counters at expected baseline. | +| Ghsa | BE-Conn-GHSA | ✅ Complete – normalized rollout live 2025-10-11 | Monitor schema diffs; keep fixtures synced with GHSA provenance notes. | Coordinate with OSV on shared ecosystems; no open issues. | +| Osv | BE-Conn-OSV | ✅ Complete – normalized rules shipping 2025-10-11 | Track new ecosystems; ensure notes stay aligned with `osv:{ecosystem}:{advisoryId}:{identifier}`. | Merge analytics watching npm/PyPI parity; no action needed. | +| Nvd | BE-Conn-NVD | ✅ Complete – normalized SemVer output live 2025-10-11 | Maintain CVE-aligned provenance; monitor MR toggles if schema shifts. | Next check: confirm export parity once storage migration flips on 2025-10-23. | +| Kev | BE-Conn-KEV | ✅ Complete – catalog/due-date rules emitted 2025-10-12 | Keep schedule metadata synced with CISA feed. | Acts as flag-only enrich; no additional merge work required. | +| Ics.Cisa | BE-Conn-ICS-CISA | ⚠️ Pending decision (due 2025-10-23) | Promote existing SemVer primitives into normalized rules; open Models ticket if firmware requires new scheme. | Provide sample advisories to Merge by 2025-10-22 for schema review. | +| Kisa | BE-Conn-KISA | ⚠️ Proposal drafting (due 2025-10-24) | Finalise `kisa.build` (or alternate) scheme with Models, then emit normalized rules and update localisation notes/tests. | Localization WG prepping translation samples; Merge to review scheme request immediately. | +| Ru.Bdu | BE-Conn-BDU | ✅ Complete – emitting `ru-bdu.raw` rules since 2025-10-14 | Monitor UTF-8 sanitisation; keep provenance notes aligned with advisory ids. | Storage snapshot verified 2025-10-19; counters green. | +| Ru.Nkcki | BE-Conn-Nkcki | ✅ Complete – SemVer + normalized rules live 2025-10-13 | Maintain Cyrillic provenance fields and SemVer coverage. | Localization WG confirmed transliteration guidance; no open items. | +| Vndr.Apple | BE-Conn-Apple | ✅ Complete – `apple.build` SemVer rules live 2025-10-11 | Keep fixtures covering multi-range tables; notify Merge of schema evolutions. | Prepare follow-up for macOS/iOS beta channels by 2025-10-26. | +| Vndr.Cisco | BE-Conn-Cisco | ⚠️ DOING – normalized promotion branch open (due 2025-10-21) | Use helper to convert SemVer primitives into rule arrays with `cisco:{productId}` notes; refresh tests. | OAuth throttling validated; Merge to rerun counters post-merge. | +| Vndr.Msrc | BE-Conn-MSRC | ✅ Complete – `msrc.build` rules live 2025-10-15 | Monitor monthly rollup coverage and provenance notes. | Merge verified rule ingestion 2025-10-19; no outstanding actions. | ## Storage alignment quick reference (2025-10-11) - `NormalizedVersionDocumentFactory` copies each `NormalizedVersionRule` into Mongo with the shape `{ packageId, packageType, scheme, type, style, min, minInclusive, max, maxInclusive, value, notes, decisionReason, constraint, source, recordedAt }`. `style` is currently a direct echo of `type` but reserved for future vendor comparers—no connector action required. @@ -83,12 +83,14 @@ Until these blocks land, connectors should stage changes behind a feature flag o ``` ## Immediate next steps -- Normalization team to share draft `SemVerRangeRuleBuilder` API by **2025-10-13** for review; Merge will circulate feedback within 24 hours. -- Connector owners to prepare fixture pull requests demonstrating sample normalized rule arrays (even if feature-flagged) by **2025-10-17**. -- Merge team will run a cross-connector review on **2025-10-18** to confirm consistent field usage and provenance tagging before enabling merge union logic. -- Schedule held for **2025-10-14 14:00 UTC** to review the CERT/CC staging VINCE advisory sample once `enableDetailMapping` is flipped; capture findings in `#concelier-merge` with snapshot diffs. +- **2025-10-21** – Cccs and Cisco teams to merge normalized-rule branches, regenerate fixtures, and post counter screenshots. +- **2025-10-22** – CertBund translator review with Localization WG; confirm localisation glossary + deterministic casing before merge. +- **2025-10-23** – ICS-CISA to confirm SemVer vs firmware scheme; escalate Models ticket if new scheme required. +- **2025-10-24** – KISA firmware scheme proposal due; Merge to review immediately and unblock builder integration. +- **2025-10-25** – Merge cross-connector review to validate counters, provenance notes, and storage projections before flipping default union logic. ## Tracking & follow-up +- Track due dates above; if a connector slips past its deadline, flag in `#concelier-merge` stand-up and open a blocker ticket referencing FEEDMERGE-COORD-02-900. - Capture connector progress updates in stand-ups twice per week; link PRs/issues back to this document and the rollout dashboard (`docs/dev/normalized_versions_rollout.md`). - Monitor merge counters `concelier.merge.normalized_rules` and `concelier.merge.normalized_rules_missing` to spot advisories that still lack normalized arrays after precedence merge. - When a connector is ready to emit normalized rules, update its module `TASKS.md` status and ping Merge in `#concelier-merge` with fixture diff screenshots. diff --git a/src/StellaOps.Concelier.Merge/TASKS.md b/src/StellaOps.Concelier.Merge/TASKS.md index 009ef0fb..513437e4 100644 --- a/src/StellaOps.Concelier.Merge/TASKS.md +++ b/src/StellaOps.Concelier.Merge/TASKS.md @@ -16,7 +16,11 @@ |Override audit logging|BE-Merge|Observability|DONE – override audits now emit structured logs plus bounded-tag metrics suitable for prod telemetry.| |Configurable precedence table|BE-Merge|Architecture|DONE – precedence options bind via concelier:merge:precedence:ranks with docs/tests covering operator workflow.| |Range primitives backlog|BE-Merge|Connector WGs|**DOING** – Coordinate remaining connectors (`Acsc`, `Cccs`, `CertBund`, `CertCc`, `Cve`, `Ghsa`, `Ics.Cisa`, `Kisa`, `Ru.Bdu`, `Ru.Nkcki`, `Vndr.Apple`, `Vndr.Cisco`, `Vndr.Msrc`) to emit canonical RangePrimitives with provenance tags; track progress/fixtures here.
2025-10-11: Storage alignment notes + sample normalized rule JSON now captured in `RANGE_PRIMITIVES_COORDINATION.md` (see “Storage alignment quick reference”).
2025-10-11 18:45Z: GHSA normalized rules landed; OSV connector picked up next for rollout.
2025-10-11 21:10Z: `docs/dev/merge_semver_playbook.md` Section 8 now documents the persisted Mongo projection (SemVer + NEVRA) for connector reviewers.
2025-10-11 21:30Z: Added `docs/dev/normalized_versions_rollout.md` dashboard to centralize connector status and upcoming milestones.
2025-10-11 21:55Z: Merge now emits `concelier.merge.normalized_rules*` counters and unions connector-provided normalized arrays; see new test coverage in `AdvisoryPrecedenceMergerTests.Merge_RecordsNormalizedRuleMetrics`.
2025-10-12 17:05Z: CVE + KEV normalized rule verification complete; OSV parity fixtures revalidated—downstream parity/monitoring tasks may proceed.
2025-10-19 14:35Z: Prerequisites reviewed (none outstanding); FEEDMERGE-COORD-02-900 remains in DOING with connector follow-ups unchanged.
2025-10-19 15:25Z: Refreshed `RANGE_PRIMITIVES_COORDINATION.md` matrix + added targeted follow-ups (Cccs, CertBund, ICS-CISA, Kisa, Vndr.Cisco) with delivery dates 2025-10-21 → 2025-10-25; monitoring merge counters for regression.| +|Range primitives backlog|BE-Merge|Connector WGs|**DOING** – Coordinate remaining connectors (`Acsc`, `Cccs`, `CertBund`, `CertCc`, `Cve`, `Ghsa`, `Ics.Cisa`, `Kisa`, `Ru.Bdu`, `Ru.Nkcki`, `Vndr.Apple`, `Vndr.Cisco`, `Vndr.Msrc`) to emit canonical RangePrimitives with provenance tags; track progress/fixtures here.
2025-10-11: Storage alignment notes + sample normalized rule JSON now captured in `RANGE_PRIMITIVES_COORDINATION.md` (see “Storage alignment quick reference”).
2025-10-11 18:45Z: GHSA normalized rules landed; OSV connector picked up next for rollout.
2025-10-11 21:10Z: `docs/dev/merge_semver_playbook.md` Section 8 now documents the persisted Mongo projection (SemVer + NEVRA) for connector reviewers.
2025-10-11 21:30Z: Added `docs/dev/normalized_versions_rollout.md` dashboard to centralize connector status and upcoming milestones.
2025-10-11 21:55Z: Merge now emits `concelier.merge.normalized_rules*` counters and unions connector-provided normalized arrays; see new test coverage in `AdvisoryPrecedenceMergerTests.Merge_RecordsNormalizedRuleMetrics`.
2025-10-12 17:05Z: CVE + KEV normalized rule verification complete; OSV parity fixtures revalidated—downstream parity/monitoring tasks may proceed.
2025-10-19 14:35Z: Prerequisites reviewed (none outstanding); FEEDMERGE-COORD-02-900 remains in DOING with connector follow-ups unchanged.
2025-10-19 15:25Z: Refreshed `RANGE_PRIMITIVES_COORDINATION.md` matrix + added targeted follow-ups (Cccs, CertBund, ICS-CISA, Kisa, Vndr.Cisco) with delivery dates 2025-10-21 → 2025-10-25; monitoring merge counters for regression.
2025-10-20 19:30Z: Coordination matrix + rollout dashboard updated with current connector statuses and due dates; flagged Slack escalation plan if Cccs/Cisco miss 2025-10-21 and documented Acsc kickoff window for 2025-10-24.| |Merge pipeline parity for new advisory fields|BE-Merge|Models, Core|DONE (2025-10-15) – merge service now surfaces description/CWE/canonical metric decisions with updated metrics/tests.| |Connector coordination for new advisory fields|Connector Leads, BE-Merge|Models, Core|**DONE (2025-10-15)** – GHSA, NVD, and OSV connectors now emit advisory descriptions, CWE weaknesses, and canonical metric ids. Fixtures refreshed (GHSA connector regression suite, `conflict-nvd.canonical.json`, OSV parity snapshots) and completion recorded in coordination log.| |FEEDMERGE-ENGINE-07-001 Conflict sets & explainers|BE-Merge|FEEDSTORAGE-DATA-07-001|**DONE (2025-10-20)** – Merge surfaces conflict explainers with replay hashes via `MergeConflictSummary`; API exposes structured payloads and integration tests cover deterministic `asOf` hashes.| > Remark (2025-10-20): `AdvisoryMergeService` now returns conflict summaries with deterministic hashes; WebService replay endpoint emits typed explainers verified by new tests. +|FEEDMERGE-COORD-02-901 Connector deadline check-ins|BE-Merge|FEEDMERGE-COORD-02-900|**TODO (due 2025-10-21)** – Confirm Cccs/Cisco normalized-rule branches land, capture `concelier.merge.normalized_rules*` counter screenshots, and update coordination docs with the results.| +|FEEDMERGE-COORD-02-902 ICS-CISA normalized-rule decision support|BE-Merge, Models|FEEDMERGE-COORD-02-900|**TODO (due 2025-10-23)** – Review ICS-CISA sample advisories, confirm SemVer reuse vs new firmware scheme, pre-stage Models ticket template, and document outcome in coordination docs + tracker files.| +|FEEDMERGE-COORD-02-903 KISA firmware scheme review|BE-Merge, Models|FEEDMERGE-COORD-02-900|**TODO (due 2025-10-24)** – Pair with KISA team on proposed firmware scheme (`kisa.build` or variant), ensure builder alignment, open Models ticket if required, and log decision in coordination docs + tracker files.| diff --git a/src/StellaOps.Concelier.WebService/TASKS.md b/src/StellaOps.Concelier.WebService/TASKS.md index 09754be3..4777f51c 100644 --- a/src/StellaOps.Concelier.WebService/TASKS.md +++ b/src/StellaOps.Concelier.WebService/TASKS.md @@ -16,7 +16,7 @@ |Batch job definition last-run lookup|BE-Base|Core|DONE – definitions endpoint now precomputes kinds array and reuses batched last-run dictionary; manual smoke verified via local GET `/jobs/definitions`.| |Add no-cache headers to health/readiness/jobs APIs|BE-Base|WebService|DONE – helper applies Cache-Control/Pragma/Expires on all health/ready/jobs endpoints; awaiting automated probe tests once connector fixtures stabilize.| |Authority configuration parity (FSR1)|DevEx/Concelier|Authority options schema|**DONE (2025-10-10)** – Options post-config loads clientSecretFile fallback, validators normalize scopes/audiences, and sample config documents issuer/credential/bypass settings.| -|Document authority toggle & scope requirements|Docs/Concelier|Authority integration|**DOING (2025-10-10)** – Quickstart updated with staging flag, client credentials, env overrides; operator guide refresh pending Docs guild review.| +|Document authority toggle & scope requirements|Docs/Concelier|Authority integration|**DONE (2025-10-21)** – Quickstart now documents staging flag, client credentials, env overrides; operator guide refresh merged. Remaining copy polishing is tracked under `DOCS-CONCELIER-07-201` in `docs/TASKS.md`.| |Plumb Authority client resilience options|BE-Base|Auth libraries LIB5|**DONE (2025-10-12)** – `Program.cs` wires `authority.resilience.*` + client scopes into `AddStellaOpsAuthClient`; new integration test asserts binding and retries.| |Author ops guidance for resilience tuning|Docs/Concelier|Plumb Authority client resilience options|**DONE (2025-10-12)** – `docs/21_INSTALL_GUIDE.md` + `docs/ops/concelier-authority-audit-runbook.md` document resilience profiles for connected vs air-gapped installs and reference monitoring cues.| |Document authority bypass logging patterns|Docs/Concelier|FSR3 logging|**DONE (2025-10-12)** – Updated operator guides clarify `Concelier.Authorization.Audit` fields (route/status/subject/clientId/scopes/bypass/remote) and SIEM triggers.| diff --git a/src/StellaOps.Configuration/StellaOpsAuthorityOptions.cs b/src/StellaOps.Configuration/StellaOpsAuthorityOptions.cs index 7e84a69e..a0926128 100644 --- a/src/StellaOps.Configuration/StellaOpsAuthorityOptions.cs +++ b/src/StellaOps.Configuration/StellaOpsAuthorityOptions.cs @@ -376,17 +376,25 @@ public sealed class AuthorityDpopNonceOptions throw new InvalidOperationException("Dpop.Nonce.RedisConnectionString must be provided when using the 'redis' store."); } - NormalizedAudiences = requiredAudiences - .Select(static aud => aud.Trim()) - .Where(static aud => aud.Length > 0) - .ToHashSet(StringComparer.OrdinalIgnoreCase); - - if (NormalizedAudiences.Count == 0) - { - throw new InvalidOperationException("Dpop.Nonce.RequiredAudiences must include at least one audience."); - } - } -} + var normalizedAudiences = requiredAudiences + .Select(static aud => aud.Trim()) + .Where(static aud => aud.Length > 0) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + if (normalizedAudiences.Count == 0) + { + throw new InvalidOperationException("Dpop.Nonce.RequiredAudiences must include at least one audience."); + } + + requiredAudiences.Clear(); + foreach (var audience in normalizedAudiences) + { + requiredAudiences.Add(audience); + } + + NormalizedAudiences = normalizedAudiences; + } +} public sealed class AuthorityMtlsOptions { diff --git a/src/StellaOps.Excititor.ArtifactStores.S3/StellaOps.Excititor.ArtifactStores.S3.csproj b/src/StellaOps.Excititor.ArtifactStores.S3/StellaOps.Excititor.ArtifactStores.S3.csproj index 3e4e5d38..ee590291 100644 --- a/src/StellaOps.Excititor.ArtifactStores.S3/StellaOps.Excititor.ArtifactStores.S3.csproj +++ b/src/StellaOps.Excititor.ArtifactStores.S3/StellaOps.Excititor.ArtifactStores.S3.csproj @@ -1,17 +1,17 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Attestation/StellaOps.Excititor.Attestation.csproj b/src/StellaOps.Excititor.Attestation/StellaOps.Excititor.Attestation.csproj index 0294085d..312a9f5a 100644 --- a/src/StellaOps.Excititor.Attestation/StellaOps.Excititor.Attestation.csproj +++ b/src/StellaOps.Excititor.Attestation/StellaOps.Excititor.Attestation.csproj @@ -1,17 +1,17 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Connectors.Abstractions/StellaOps.Excititor.Connectors.Abstractions.csproj b/src/StellaOps.Excititor.Connectors.Abstractions/StellaOps.Excititor.Connectors.Abstractions.csproj index 733276ad..1222a9c0 100644 --- a/src/StellaOps.Excititor.Connectors.Abstractions/StellaOps.Excititor.Connectors.Abstractions.csproj +++ b/src/StellaOps.Excititor.Connectors.Abstractions/StellaOps.Excititor.Connectors.Abstractions.csproj @@ -1,17 +1,17 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Connectors.Cisco.CSAF.Tests/Connectors/CiscoCsafConnectorTests.cs b/src/StellaOps.Excititor.Connectors.Cisco.CSAF.Tests/Connectors/CiscoCsafConnectorTests.cs index f6b529e9..f454b839 100644 --- a/src/StellaOps.Excititor.Connectors.Cisco.CSAF.Tests/Connectors/CiscoCsafConnectorTests.cs +++ b/src/StellaOps.Excititor.Connectors.Cisco.CSAF.Tests/Connectors/CiscoCsafConnectorTests.cs @@ -1,16 +1,16 @@ -using System.Collections.Generic; +using System.Collections.Generic; using System.Net; using System.Net.Http; using System.Text; using FluentAssertions; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using Microsoft.Extensions.DependencyInjection; -using StellaOps.Excititor.Connectors.Abstractions; -using StellaOps.Excititor.Connectors.Cisco.CSAF; -using StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration; -using StellaOps.Excititor.Connectors.Cisco.CSAF.Metadata; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.Cisco.CSAF; +using StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration; +using StellaOps.Excititor.Connectors.Cisco.CSAF.Metadata; using StellaOps.Excititor.Core; using StellaOps.Excititor.Storage.Mongo; using System.Collections.Immutable; @@ -18,148 +18,148 @@ using System.IO.Abstractions.TestingHelpers; using Xunit; using System.Threading; using MongoDB.Driver; - -namespace StellaOps.Excititor.Connectors.Cisco.CSAF.Tests.Connectors; - -public sealed class CiscoCsafConnectorTests -{ - [Fact] - public async Task FetchAsync_NewAdvisory_StoresDocumentAndUpdatesState() - { - var responses = new Dictionary> - { - [new Uri("https://api.cisco.test/.well-known/csaf/provider-metadata.json")] = QueueResponses(""" - { - "metadata": { - "publisher": { - "name": "Cisco", - "category": "vendor", - "contact_details": { "id": "excititor:cisco" } - } - }, - "distributions": { - "directories": [ "https://api.cisco.test/csaf/" ] - } - } - """), - [new Uri("https://api.cisco.test/csaf/index.json")] = QueueResponses(""" - { - "advisories": [ - { - "id": "cisco-sa-2025", - "url": "https://api.cisco.test/csaf/cisco-sa-2025.json", - "published": "2025-10-01T00:00:00Z", - "lastModified": "2025-10-02T00:00:00Z", - "sha256": "cafebabe" - } - ] - } - """), - [new Uri("https://api.cisco.test/csaf/cisco-sa-2025.json")] = QueueResponses("{ \"document\": \"payload\" }") - }; - - var handler = new RoutingHttpMessageHandler(responses); - var httpClient = new HttpClient(handler); - var factory = new SingleHttpClientFactory(httpClient); - var metadataLoader = new CiscoProviderMetadataLoader( - factory, - new MemoryCache(new MemoryCacheOptions()), - Options.Create(new CiscoConnectorOptions - { - MetadataUri = "https://api.cisco.test/.well-known/csaf/provider-metadata.json", - PersistOfflineSnapshot = false, - }), - NullLogger.Instance, - new MockFileSystem()); - - var stateRepository = new InMemoryConnectorStateRepository(); - var connector = new CiscoCsafConnector( - metadataLoader, - factory, - stateRepository, - new[] { new CiscoConnectorOptionsValidator() }, - NullLogger.Instance, - TimeProvider.System); - - var settings = new VexConnectorSettings(ImmutableDictionary.Empty); - await connector.ValidateAsync(settings, CancellationToken.None); - - var sink = new InMemoryRawSink(); - var context = new VexConnectorContext(null, VexConnectorSettings.Empty, sink, new NoopSignatureVerifier(), new NoopNormalizerRouter(), new ServiceCollection().BuildServiceProvider()); - - var documents = new List(); - await foreach (var doc in connector.FetchAsync(context, CancellationToken.None)) - { - documents.Add(doc); - } - - documents.Should().HaveCount(1); - sink.Documents.Should().HaveCount(1); - stateRepository.CurrentState.Should().NotBeNull(); - stateRepository.CurrentState!.DocumentDigests.Should().HaveCount(1); - - // second run should not refetch documents - sink.Documents.Clear(); - documents.Clear(); - - await foreach (var doc in connector.FetchAsync(context, CancellationToken.None)) - { - documents.Add(doc); - } - - documents.Should().BeEmpty(); - sink.Documents.Should().BeEmpty(); - } - - private static Queue QueueResponses(string payload) - => new(new[] - { - new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(payload, Encoding.UTF8, "application/json"), - } - }); - - private sealed class RoutingHttpMessageHandler : HttpMessageHandler - { - private readonly Dictionary> _responses; - - public RoutingHttpMessageHandler(Dictionary> responses) - { - _responses = responses; - } - - protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - if (request.RequestUri is not null && _responses.TryGetValue(request.RequestUri, out var queue) && queue.Count > 0) - { - var response = queue.Peek(); - return Task.FromResult(response.Clone()); - } - - return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound) - { - Content = new StringContent($"No response configured for {request.RequestUri}"), - }); - } - } - - private sealed class SingleHttpClientFactory : IHttpClientFactory - { - private readonly HttpClient _client; - - public SingleHttpClientFactory(HttpClient client) - { - _client = client; - } - - public HttpClient CreateClient(string name) => _client; - } - - private sealed class InMemoryConnectorStateRepository : IVexConnectorStateRepository - { - public VexConnectorState? CurrentState { get; private set; } - + +namespace StellaOps.Excititor.Connectors.Cisco.CSAF.Tests.Connectors; + +public sealed class CiscoCsafConnectorTests +{ + [Fact] + public async Task FetchAsync_NewAdvisory_StoresDocumentAndUpdatesState() + { + var responses = new Dictionary> + { + [new Uri("https://api.cisco.test/.well-known/csaf/provider-metadata.json")] = QueueResponses(""" + { + "metadata": { + "publisher": { + "name": "Cisco", + "category": "vendor", + "contact_details": { "id": "excititor:cisco" } + } + }, + "distributions": { + "directories": [ "https://api.cisco.test/csaf/" ] + } + } + """), + [new Uri("https://api.cisco.test/csaf/index.json")] = QueueResponses(""" + { + "advisories": [ + { + "id": "cisco-sa-2025", + "url": "https://api.cisco.test/csaf/cisco-sa-2025.json", + "published": "2025-10-01T00:00:00Z", + "lastModified": "2025-10-02T00:00:00Z", + "sha256": "cafebabe" + } + ] + } + """), + [new Uri("https://api.cisco.test/csaf/cisco-sa-2025.json")] = QueueResponses("{ \"document\": \"payload\" }") + }; + + var handler = new RoutingHttpMessageHandler(responses); + var httpClient = new HttpClient(handler); + var factory = new SingleHttpClientFactory(httpClient); + var metadataLoader = new CiscoProviderMetadataLoader( + factory, + new MemoryCache(new MemoryCacheOptions()), + Options.Create(new CiscoConnectorOptions + { + MetadataUri = "https://api.cisco.test/.well-known/csaf/provider-metadata.json", + PersistOfflineSnapshot = false, + }), + NullLogger.Instance, + new MockFileSystem()); + + var stateRepository = new InMemoryConnectorStateRepository(); + var connector = new CiscoCsafConnector( + metadataLoader, + factory, + stateRepository, + new[] { new CiscoConnectorOptionsValidator() }, + NullLogger.Instance, + TimeProvider.System); + + var settings = new VexConnectorSettings(ImmutableDictionary.Empty); + await connector.ValidateAsync(settings, CancellationToken.None); + + var sink = new InMemoryRawSink(); + var context = new VexConnectorContext(null, VexConnectorSettings.Empty, sink, new NoopSignatureVerifier(), new NoopNormalizerRouter(), new ServiceCollection().BuildServiceProvider(), ImmutableDictionary.Empty); + + var documents = new List(); + await foreach (var doc in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(doc); + } + + documents.Should().HaveCount(1); + sink.Documents.Should().HaveCount(1); + stateRepository.CurrentState.Should().NotBeNull(); + stateRepository.CurrentState!.DocumentDigests.Should().HaveCount(1); + + // second run should not refetch documents + sink.Documents.Clear(); + documents.Clear(); + + await foreach (var doc in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(doc); + } + + documents.Should().BeEmpty(); + sink.Documents.Should().BeEmpty(); + } + + private static Queue QueueResponses(string payload) + => new(new[] + { + new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(payload, Encoding.UTF8, "application/json"), + } + }); + + private sealed class RoutingHttpMessageHandler : HttpMessageHandler + { + private readonly Dictionary> _responses; + + public RoutingHttpMessageHandler(Dictionary> responses) + { + _responses = responses; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + if (request.RequestUri is not null && _responses.TryGetValue(request.RequestUri, out var queue) && queue.Count > 0) + { + var response = queue.Peek(); + return Task.FromResult(response.Clone()); + } + + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound) + { + Content = new StringContent($"No response configured for {request.RequestUri}"), + }); + } + } + + private sealed class SingleHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleHttpClientFactory(HttpClient client) + { + _client = client; + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class InMemoryConnectorStateRepository : IVexConnectorStateRepository + { + public VexConnectorState? CurrentState { get; private set; } + public ValueTask GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null) => ValueTask.FromResult(CurrentState); @@ -168,48 +168,48 @@ public sealed class CiscoCsafConnectorTests CurrentState = state; return ValueTask.CompletedTask; } - } - - private sealed class InMemoryRawSink : IVexRawDocumentSink - { - public List Documents { get; } = new(); - - public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken) - { - Documents.Add(document); - return ValueTask.CompletedTask; - } - } - - private sealed class NoopSignatureVerifier : IVexSignatureVerifier - { - public ValueTask VerifyAsync(VexRawDocument document, CancellationToken cancellationToken) - => ValueTask.FromResult(null); - } - - private sealed class NoopNormalizerRouter : IVexNormalizerRouter - { - public ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) - => ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray.Empty, ImmutableDictionary.Empty)); - } -} - -internal static class HttpResponseMessageExtensions -{ - public static HttpResponseMessage Clone(this HttpResponseMessage response) - { - var clone = new HttpResponseMessage(response.StatusCode); - foreach (var header in response.Headers) - { - clone.Headers.TryAddWithoutValidation(header.Key, header.Value); - } - - if (response.Content is not null) - { - var payload = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); - clone.Content = new StringContent(payload, Encoding.UTF8, response.Content.Headers.ContentType?.MediaType); - } - - return clone; - } -} + } + + private sealed class InMemoryRawSink : IVexRawDocumentSink + { + public List Documents { get; } = new(); + + public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken) + { + Documents.Add(document); + return ValueTask.CompletedTask; + } + } + + private sealed class NoopSignatureVerifier : IVexSignatureVerifier + { + public ValueTask VerifyAsync(VexRawDocument document, CancellationToken cancellationToken) + => ValueTask.FromResult(null); + } + + private sealed class NoopNormalizerRouter : IVexNormalizerRouter + { + public ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray.Empty, ImmutableDictionary.Empty)); + } +} + +internal static class HttpResponseMessageExtensions +{ + public static HttpResponseMessage Clone(this HttpResponseMessage response) + { + var clone = new HttpResponseMessage(response.StatusCode); + foreach (var header in response.Headers) + { + clone.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + + if (response.Content is not null) + { + var payload = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); + clone.Content = new StringContent(payload, Encoding.UTF8, response.Content.Headers.ContentType?.MediaType); + } + + return clone; + } +} diff --git a/src/StellaOps.Excititor.Connectors.Cisco.CSAF/StellaOps.Excititor.Connectors.Cisco.CSAF.csproj b/src/StellaOps.Excititor.Connectors.Cisco.CSAF/StellaOps.Excititor.Connectors.Cisco.CSAF.csproj index 4a15c1f6..fc88a048 100644 --- a/src/StellaOps.Excititor.Connectors.Cisco.CSAF/StellaOps.Excititor.Connectors.Cisco.CSAF.csproj +++ b/src/StellaOps.Excititor.Connectors.Cisco.CSAF/StellaOps.Excititor.Connectors.Cisco.CSAF.csproj @@ -1,20 +1,20 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests/Connectors/MsrcCsafConnectorTests.cs b/src/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests/Connectors/MsrcCsafConnectorTests.cs index 66073403..7431c3ba 100644 --- a/src/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests/Connectors/MsrcCsafConnectorTests.cs +++ b/src/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests/Connectors/MsrcCsafConnectorTests.cs @@ -1,322 +1,325 @@ -using System.Collections.Generic; -using System.Collections.Immutable; -using System.IO.Compression; -using System.Net; -using System.Net.Http; -using System.Text; -using FluentAssertions; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using StellaOps.Excititor.Connectors.Abstractions; -using StellaOps.Excititor.Connectors.MSRC.CSAF; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO.Compression; +using System.Net; +using System.Net.Http; +using System.Text; +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.MSRC.CSAF; using StellaOps.Excititor.Connectors.MSRC.CSAF.Authentication; using StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration; using StellaOps.Excititor.Core; using StellaOps.Excititor.Storage.Mongo; using Xunit; using MongoDB.Driver; - -namespace StellaOps.Excititor.Connectors.MSRC.CSAF.Tests.Connectors; - -public sealed class MsrcCsafConnectorTests -{ - private static readonly VexConnectorDescriptor Descriptor = new("excititor:msrc", VexProviderKind.Vendor, "MSRC CSAF"); - - [Fact] - public async Task FetchAsync_EmitsDocumentAndPersistsState() - { - var summary = """ - { - "value": [ - { - "id": "ADV-0001", - "vulnerabilityId": "ADV-0001", - "severity": "Critical", - "releaseDate": "2025-10-17T00:00:00Z", - "lastModifiedDate": "2025-10-18T00:00:00Z", - "cvrfUrl": "https://example.com/csaf/ADV-0001.json" - } - ] - } - """; - - var csaf = """{"document":{"title":"Example"}}"""; - var handler = TestHttpMessageHandler.Create( - _ => Response(HttpStatusCode.OK, summary, "application/json"), - _ => Response(HttpStatusCode.OK, csaf, "application/json")); - - var httpClient = new HttpClient(handler) - { - BaseAddress = new Uri("https://example.com/"), - }; - - var factory = new SingleClientHttpClientFactory(httpClient); - var stateRepository = new InMemoryConnectorStateRepository(); - var options = Options.Create(CreateOptions()); - var connector = new MsrcCsafConnector( - factory, - new StubTokenProvider(), - stateRepository, - options, - NullLogger.Instance, - TimeProvider.System); - - await connector.ValidateAsync(VexConnectorSettings.Empty, CancellationToken.None); - - var sink = new CapturingRawSink(); - var context = new VexConnectorContext( - Since: new DateTimeOffset(2025, 10, 15, 0, 0, 0, TimeSpan.Zero), - Settings: VexConnectorSettings.Empty, - RawSink: sink, - SignatureVerifier: new NoopSignatureVerifier(), - Normalizers: new NoopNormalizerRouter(), - Services: new ServiceCollection().BuildServiceProvider()); - - var documents = new List(); - await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) - { - documents.Add(document); - } - - documents.Should().HaveCount(1); - sink.Documents.Should().HaveCount(1); - var emitted = documents[0]; - emitted.SourceUri.Should().Be(new Uri("https://example.com/csaf/ADV-0001.json")); - emitted.Metadata["msrc.vulnerabilityId"].Should().Be("ADV-0001"); - emitted.Metadata["msrc.csaf.format"].Should().Be("json"); - emitted.Metadata.Should().NotContainKey("excititor.quarantine.reason"); - - stateRepository.State.Should().NotBeNull(); - stateRepository.State!.LastUpdated.Should().Be(new DateTimeOffset(2025, 10, 18, 0, 0, 0, TimeSpan.Zero)); - stateRepository.State.DocumentDigests.Should().HaveCount(1); - } - - [Fact] - public async Task FetchAsync_SkipsDocumentsWithExistingDigest() - { - var summary = """ - { - "value": [ - { - "id": "ADV-0001", - "vulnerabilityId": "ADV-0001", - "lastModifiedDate": "2025-10-18T00:00:00Z", - "cvrfUrl": "https://example.com/csaf/ADV-0001.json" - } - ] - } - """; - - var csaf = """{"document":{"title":"Example"}}"""; - var handler = TestHttpMessageHandler.Create( - _ => Response(HttpStatusCode.OK, summary, "application/json"), - _ => Response(HttpStatusCode.OK, csaf, "application/json")); - - var httpClient = new HttpClient(handler) - { - BaseAddress = new Uri("https://example.com/"), - }; - - var factory = new SingleClientHttpClientFactory(httpClient); - var stateRepository = new InMemoryConnectorStateRepository(); - var options = Options.Create(CreateOptions()); - var connector = new MsrcCsafConnector( - factory, - new StubTokenProvider(), - stateRepository, - options, - NullLogger.Instance, - TimeProvider.System); - - await connector.ValidateAsync(VexConnectorSettings.Empty, CancellationToken.None); - - var sink = new CapturingRawSink(); - var context = new VexConnectorContext( - Since: new DateTimeOffset(2025, 10, 15, 0, 0, 0, TimeSpan.Zero), - Settings: VexConnectorSettings.Empty, - RawSink: sink, - SignatureVerifier: new NoopSignatureVerifier(), - Normalizers: new NoopNormalizerRouter(), - Services: new ServiceCollection().BuildServiceProvider()); - - var firstPass = new List(); - await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) - { - firstPass.Add(document); - } - - firstPass.Should().HaveCount(1); - stateRepository.State.Should().NotBeNull(); - var persistedState = stateRepository.State!; - - handler.Reset( - _ => Response(HttpStatusCode.OK, summary, "application/json"), - _ => Response(HttpStatusCode.OK, csaf, "application/json")); - - sink.Documents.Clear(); - var secondPass = new List(); - await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) - { - secondPass.Add(document); - } - - secondPass.Should().BeEmpty(); - sink.Documents.Should().BeEmpty(); - stateRepository.State.Should().NotBeNull(); - stateRepository.State!.DocumentDigests.Should().Equal(persistedState.DocumentDigests); - } - - [Fact] - public async Task FetchAsync_QuarantinesInvalidCsafPayload() - { - var summary = """ - { - "value": [ - { - "id": "ADV-0002", - "vulnerabilityId": "ADV-0002", - "lastModifiedDate": "2025-10-19T00:00:00Z", - "cvrfUrl": "https://example.com/csaf/ADV-0002.zip" - } - ] - } - """; - - var csafZip = CreateZip("document.json", "{ invalid json "); - var handler = TestHttpMessageHandler.Create( - _ => Response(HttpStatusCode.OK, summary, "application/json"), - _ => Response(HttpStatusCode.OK, csafZip, "application/zip")); - - var httpClient = new HttpClient(handler) - { - BaseAddress = new Uri("https://example.com/"), - }; - - var factory = new SingleClientHttpClientFactory(httpClient); - var stateRepository = new InMemoryConnectorStateRepository(); - var options = Options.Create(CreateOptions()); - var connector = new MsrcCsafConnector( - factory, - new StubTokenProvider(), - stateRepository, - options, - NullLogger.Instance, - TimeProvider.System); - - await connector.ValidateAsync(VexConnectorSettings.Empty, CancellationToken.None); - - var sink = new CapturingRawSink(); - var context = new VexConnectorContext( - Since: new DateTimeOffset(2025, 10, 17, 0, 0, 0, TimeSpan.Zero), - Settings: VexConnectorSettings.Empty, - RawSink: sink, - SignatureVerifier: new NoopSignatureVerifier(), - Normalizers: new NoopNormalizerRouter(), - Services: new ServiceCollection().BuildServiceProvider()); - - var documents = new List(); - await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) - { - documents.Add(document); - } - - documents.Should().BeEmpty(); - sink.Documents.Should().HaveCount(1); - sink.Documents[0].Metadata["excititor.quarantine.reason"].Should().Contain("JSON parse failed"); - sink.Documents[0].Metadata["msrc.csaf.format"].Should().Be("zip"); - - stateRepository.State.Should().NotBeNull(); - stateRepository.State!.DocumentDigests.Should().HaveCount(1); - } - - private static HttpResponseMessage Response(HttpStatusCode statusCode, string content, string contentType) - => new(statusCode) - { - Content = new StringContent(content, Encoding.UTF8, contentType), - }; - - private static HttpResponseMessage Response(HttpStatusCode statusCode, byte[] content, string contentType) - { - var response = new HttpResponseMessage(statusCode); - response.Content = new ByteArrayContent(content); - response.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue(contentType); - return response; - } - - private static MsrcConnectorOptions CreateOptions() - => new() - { - BaseUri = new Uri("https://example.com/", UriKind.Absolute), - TenantId = Guid.NewGuid().ToString(), - ClientId = "client-id", - ClientSecret = "secret", - Scope = MsrcConnectorOptions.DefaultScope, - PageSize = 5, - MaxAdvisoriesPerFetch = 5, - RequestDelay = TimeSpan.Zero, - RetryBaseDelay = TimeSpan.FromMilliseconds(10), - MaxRetryAttempts = 2, - }; - - private static byte[] CreateZip(string entryName, string content) - { - using var buffer = new MemoryStream(); - using (var archive = new ZipArchive(buffer, ZipArchiveMode.Create, leaveOpen: true)) - { - var entry = archive.CreateEntry(entryName); - using var writer = new StreamWriter(entry.Open(), Encoding.UTF8); - writer.Write(content); - } - - return buffer.ToArray(); - } - - private sealed class StubTokenProvider : IMsrcTokenProvider - { - public ValueTask GetAccessTokenAsync(CancellationToken cancellationToken) - => ValueTask.FromResult(new MsrcAccessToken("token", "Bearer", DateTimeOffset.MaxValue)); - } - - private sealed class CapturingRawSink : IVexRawDocumentSink - { - public List Documents { get; } = new(); - - public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken) - { - Documents.Add(document); - return ValueTask.CompletedTask; - } - } - - private sealed class NoopSignatureVerifier : IVexSignatureVerifier - { - public ValueTask VerifyAsync(VexRawDocument document, CancellationToken cancellationToken) - => ValueTask.FromResult(null); - } - - private sealed class NoopNormalizerRouter : IVexNormalizerRouter - { - public ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) - => ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray.Empty, ImmutableDictionary.Empty)); - } - - private sealed class SingleClientHttpClientFactory : IHttpClientFactory - { - private readonly HttpClient _client; - - public SingleClientHttpClientFactory(HttpClient client) - { - _client = client; - } - - public HttpClient CreateClient(string name) => _client; - } - - private sealed class InMemoryConnectorStateRepository : IVexConnectorStateRepository - { - public VexConnectorState? State { get; private set; } - + +namespace StellaOps.Excititor.Connectors.MSRC.CSAF.Tests.Connectors; + +public sealed class MsrcCsafConnectorTests +{ + private static readonly VexConnectorDescriptor Descriptor = new("excititor:msrc", VexProviderKind.Vendor, "MSRC CSAF"); + + [Fact] + public async Task FetchAsync_EmitsDocumentAndPersistsState() + { + var summary = """ + { + "value": [ + { + "id": "ADV-0001", + "vulnerabilityId": "ADV-0001", + "severity": "Critical", + "releaseDate": "2025-10-17T00:00:00Z", + "lastModifiedDate": "2025-10-18T00:00:00Z", + "cvrfUrl": "https://example.com/csaf/ADV-0001.json" + } + ] + } + """; + + var csaf = """{"document":{"title":"Example"}}"""; + var handler = TestHttpMessageHandler.Create( + _ => Response(HttpStatusCode.OK, summary, "application/json"), + _ => Response(HttpStatusCode.OK, csaf, "application/json")); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://example.com/"), + }; + + var factory = new SingleClientHttpClientFactory(httpClient); + var stateRepository = new InMemoryConnectorStateRepository(); + var options = Options.Create(CreateOptions()); + var connector = new MsrcCsafConnector( + factory, + new StubTokenProvider(), + stateRepository, + options, + NullLogger.Instance, + TimeProvider.System); + + await connector.ValidateAsync(VexConnectorSettings.Empty, CancellationToken.None); + + var sink = new CapturingRawSink(); + var context = new VexConnectorContext( + Since: new DateTimeOffset(2025, 10, 15, 0, 0, 0, TimeSpan.Zero), + Settings: VexConnectorSettings.Empty, + RawSink: sink, + SignatureVerifier: new NoopSignatureVerifier(), + Normalizers: new NoopNormalizerRouter(), + Services: new ServiceCollection().BuildServiceProvider(), + ResumeTokens: ImmutableDictionary.Empty); + + var documents = new List(); + await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(document); + } + + documents.Should().HaveCount(1); + sink.Documents.Should().HaveCount(1); + var emitted = documents[0]; + emitted.SourceUri.Should().Be(new Uri("https://example.com/csaf/ADV-0001.json")); + emitted.Metadata["msrc.vulnerabilityId"].Should().Be("ADV-0001"); + emitted.Metadata["msrc.csaf.format"].Should().Be("json"); + emitted.Metadata.Should().NotContainKey("excititor.quarantine.reason"); + + stateRepository.State.Should().NotBeNull(); + stateRepository.State!.LastUpdated.Should().Be(new DateTimeOffset(2025, 10, 18, 0, 0, 0, TimeSpan.Zero)); + stateRepository.State.DocumentDigests.Should().HaveCount(1); + } + + [Fact] + public async Task FetchAsync_SkipsDocumentsWithExistingDigest() + { + var summary = """ + { + "value": [ + { + "id": "ADV-0001", + "vulnerabilityId": "ADV-0001", + "lastModifiedDate": "2025-10-18T00:00:00Z", + "cvrfUrl": "https://example.com/csaf/ADV-0001.json" + } + ] + } + """; + + var csaf = """{"document":{"title":"Example"}}"""; + var handler = TestHttpMessageHandler.Create( + _ => Response(HttpStatusCode.OK, summary, "application/json"), + _ => Response(HttpStatusCode.OK, csaf, "application/json")); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://example.com/"), + }; + + var factory = new SingleClientHttpClientFactory(httpClient); + var stateRepository = new InMemoryConnectorStateRepository(); + var options = Options.Create(CreateOptions()); + var connector = new MsrcCsafConnector( + factory, + new StubTokenProvider(), + stateRepository, + options, + NullLogger.Instance, + TimeProvider.System); + + await connector.ValidateAsync(VexConnectorSettings.Empty, CancellationToken.None); + + var sink = new CapturingRawSink(); + var context = new VexConnectorContext( + Since: new DateTimeOffset(2025, 10, 15, 0, 0, 0, TimeSpan.Zero), + Settings: VexConnectorSettings.Empty, + RawSink: sink, + SignatureVerifier: new NoopSignatureVerifier(), + Normalizers: new NoopNormalizerRouter(), + Services: new ServiceCollection().BuildServiceProvider(), + ResumeTokens: ImmutableDictionary.Empty); + + var firstPass = new List(); + await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) + { + firstPass.Add(document); + } + + firstPass.Should().HaveCount(1); + stateRepository.State.Should().NotBeNull(); + var persistedState = stateRepository.State!; + + handler.Reset( + _ => Response(HttpStatusCode.OK, summary, "application/json"), + _ => Response(HttpStatusCode.OK, csaf, "application/json")); + + sink.Documents.Clear(); + var secondPass = new List(); + await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) + { + secondPass.Add(document); + } + + secondPass.Should().BeEmpty(); + sink.Documents.Should().BeEmpty(); + stateRepository.State.Should().NotBeNull(); + stateRepository.State!.DocumentDigests.Should().Equal(persistedState.DocumentDigests); + } + + [Fact] + public async Task FetchAsync_QuarantinesInvalidCsafPayload() + { + var summary = """ + { + "value": [ + { + "id": "ADV-0002", + "vulnerabilityId": "ADV-0002", + "lastModifiedDate": "2025-10-19T00:00:00Z", + "cvrfUrl": "https://example.com/csaf/ADV-0002.zip" + } + ] + } + """; + + var csafZip = CreateZip("document.json", "{ invalid json "); + var handler = TestHttpMessageHandler.Create( + _ => Response(HttpStatusCode.OK, summary, "application/json"), + _ => Response(HttpStatusCode.OK, csafZip, "application/zip")); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://example.com/"), + }; + + var factory = new SingleClientHttpClientFactory(httpClient); + var stateRepository = new InMemoryConnectorStateRepository(); + var options = Options.Create(CreateOptions()); + var connector = new MsrcCsafConnector( + factory, + new StubTokenProvider(), + stateRepository, + options, + NullLogger.Instance, + TimeProvider.System); + + await connector.ValidateAsync(VexConnectorSettings.Empty, CancellationToken.None); + + var sink = new CapturingRawSink(); + var context = new VexConnectorContext( + Since: new DateTimeOffset(2025, 10, 17, 0, 0, 0, TimeSpan.Zero), + Settings: VexConnectorSettings.Empty, + RawSink: sink, + SignatureVerifier: new NoopSignatureVerifier(), + Normalizers: new NoopNormalizerRouter(), + Services: new ServiceCollection().BuildServiceProvider(), + ResumeTokens: ImmutableDictionary.Empty); + + var documents = new List(); + await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(document); + } + + documents.Should().BeEmpty(); + sink.Documents.Should().HaveCount(1); + sink.Documents[0].Metadata["excititor.quarantine.reason"].Should().Contain("JSON parse failed"); + sink.Documents[0].Metadata["msrc.csaf.format"].Should().Be("zip"); + + stateRepository.State.Should().NotBeNull(); + stateRepository.State!.DocumentDigests.Should().HaveCount(1); + } + + private static HttpResponseMessage Response(HttpStatusCode statusCode, string content, string contentType) + => new(statusCode) + { + Content = new StringContent(content, Encoding.UTF8, contentType), + }; + + private static HttpResponseMessage Response(HttpStatusCode statusCode, byte[] content, string contentType) + { + var response = new HttpResponseMessage(statusCode); + response.Content = new ByteArrayContent(content); + response.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue(contentType); + return response; + } + + private static MsrcConnectorOptions CreateOptions() + => new() + { + BaseUri = new Uri("https://example.com/", UriKind.Absolute), + TenantId = Guid.NewGuid().ToString(), + ClientId = "client-id", + ClientSecret = "secret", + Scope = MsrcConnectorOptions.DefaultScope, + PageSize = 5, + MaxAdvisoriesPerFetch = 5, + RequestDelay = TimeSpan.Zero, + RetryBaseDelay = TimeSpan.FromMilliseconds(10), + MaxRetryAttempts = 2, + }; + + private static byte[] CreateZip(string entryName, string content) + { + using var buffer = new MemoryStream(); + using (var archive = new ZipArchive(buffer, ZipArchiveMode.Create, leaveOpen: true)) + { + var entry = archive.CreateEntry(entryName); + using var writer = new StreamWriter(entry.Open(), Encoding.UTF8); + writer.Write(content); + } + + return buffer.ToArray(); + } + + private sealed class StubTokenProvider : IMsrcTokenProvider + { + public ValueTask GetAccessTokenAsync(CancellationToken cancellationToken) + => ValueTask.FromResult(new MsrcAccessToken("token", "Bearer", DateTimeOffset.MaxValue)); + } + + private sealed class CapturingRawSink : IVexRawDocumentSink + { + public List Documents { get; } = new(); + + public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken) + { + Documents.Add(document); + return ValueTask.CompletedTask; + } + } + + private sealed class NoopSignatureVerifier : IVexSignatureVerifier + { + public ValueTask VerifyAsync(VexRawDocument document, CancellationToken cancellationToken) + => ValueTask.FromResult(null); + } + + private sealed class NoopNormalizerRouter : IVexNormalizerRouter + { + public ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray.Empty, ImmutableDictionary.Empty)); + } + + private sealed class SingleClientHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleClientHttpClientFactory(HttpClient client) + { + _client = client; + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class InMemoryConnectorStateRepository : IVexConnectorStateRepository + { + public VexConnectorState? State { get; private set; } + public ValueTask GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null) => ValueTask.FromResult(State); @@ -325,40 +328,40 @@ public sealed class MsrcCsafConnectorTests State = state; return ValueTask.CompletedTask; } - } - - private sealed class TestHttpMessageHandler : HttpMessageHandler - { - private readonly Queue> _responders; - - private TestHttpMessageHandler(IEnumerable> responders) - { - _responders = new Queue>(responders); - } - - public static TestHttpMessageHandler Create(params Func[] responders) - => new(responders); - - public void Reset(params Func[] responders) - { - _responders.Clear(); - foreach (var responder in responders) - { - _responders.Enqueue(responder); - } - } - - protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - if (_responders.Count == 0) - { - throw new InvalidOperationException("No responder configured for MSRC connector test request."); - } - - var responder = _responders.Count > 1 ? _responders.Dequeue() : _responders.Peek(); - var response = responder(request); - response.RequestMessage = request; - return Task.FromResult(response); - } - } -} + } + + private sealed class TestHttpMessageHandler : HttpMessageHandler + { + private readonly Queue> _responders; + + private TestHttpMessageHandler(IEnumerable> responders) + { + _responders = new Queue>(responders); + } + + public static TestHttpMessageHandler Create(params Func[] responders) + => new(responders); + + public void Reset(params Func[] responders) + { + _responders.Clear(); + foreach (var responder in responders) + { + _responders.Enqueue(responder); + } + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + if (_responders.Count == 0) + { + throw new InvalidOperationException("No responder configured for MSRC connector test request."); + } + + var responder = _responders.Count > 1 ? _responders.Dequeue() : _responders.Peek(); + var response = responder(request); + response.RequestMessage = request; + return Task.FromResult(response); + } + } +} diff --git a/src/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests.csproj b/src/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests.csproj index 15b12b14..96553da9 100644 --- a/src/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests.csproj +++ b/src/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests.csproj @@ -1,18 +1,18 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Connectors.MSRC.CSAF/StellaOps.Excititor.Connectors.MSRC.CSAF.csproj b/src/StellaOps.Excititor.Connectors.MSRC.CSAF/StellaOps.Excititor.Connectors.MSRC.CSAF.csproj index b3e1398a..036c860d 100644 --- a/src/StellaOps.Excititor.Connectors.MSRC.CSAF/StellaOps.Excititor.Connectors.MSRC.CSAF.csproj +++ b/src/StellaOps.Excititor.Connectors.MSRC.CSAF/StellaOps.Excititor.Connectors.MSRC.CSAF.csproj @@ -1,19 +1,19 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests/Connector/OciOpenVexAttestationConnectorTests.cs b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests/Connector/OciOpenVexAttestationConnectorTests.cs index 134c0fc8..5445c197 100644 --- a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests/Connector/OciOpenVexAttestationConnectorTests.cs +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests/Connector/OciOpenVexAttestationConnectorTests.cs @@ -1,213 +1,215 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Net; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using FluentAssertions; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging.Abstractions; -using StellaOps.Excititor.Connectors.Abstractions; -using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest; -using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration; -using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.DependencyInjection; -using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery; -using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Fetch; -using StellaOps.Excititor.Core; -using System.IO.Abstractions.TestingHelpers; -using Xunit; - -namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests.Connector; - -public sealed class OciOpenVexAttestationConnectorTests -{ - [Fact] - public async Task FetchAsync_WithOfflineBundle_EmitsRawDocument() - { - var fileSystem = new MockFileSystem(new Dictionary - { - ["/bundles/attestation.json"] = new MockFileData("{\"payload\":\"\",\"payloadType\":\"application/vnd.in-toto+json\",\"signatures\":[{\"sig\":\"\"}]}"), - }); - - using var cache = new MemoryCache(new MemoryCacheOptions()); - var httpClient = new HttpClient(new StubHttpMessageHandler()) - { - BaseAddress = new System.Uri("https://registry.example.com/") - }; - - var httpFactory = new SingleClientHttpClientFactory(httpClient); - var discovery = new OciAttestationDiscoveryService(cache, fileSystem, NullLogger.Instance); - var fetcher = new OciAttestationFetcher(httpFactory, fileSystem, NullLogger.Instance); - - var connector = new OciOpenVexAttestationConnector( - discovery, - fetcher, - NullLogger.Instance, - TimeProvider.System); - - var settingsValues = ImmutableDictionary.Empty - .Add("Images:0:Reference", "registry.example.com/repo/image:latest") - .Add("Images:0:OfflineBundlePath", "/bundles/attestation.json") - .Add("Offline:PreferOffline", "true") - .Add("Offline:AllowNetworkFallback", "false") - .Add("Cosign:Mode", "None"); - - var settings = new VexConnectorSettings(settingsValues); - await connector.ValidateAsync(settings, CancellationToken.None); - - var sink = new CapturingRawSink(); - var verifier = new CapturingSignatureVerifier(); - var context = new VexConnectorContext( - Since: null, - Settings: VexConnectorSettings.Empty, - RawSink: sink, - SignatureVerifier: verifier, - Normalizers: new NoopNormalizerRouter(), - Services: new Microsoft.Extensions.DependencyInjection.ServiceCollection().BuildServiceProvider()); - - var documents = new List(); - await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) - { - documents.Add(document); - } - - documents.Should().HaveCount(1); - sink.Documents.Should().HaveCount(1); - documents[0].Format.Should().Be(VexDocumentFormat.OciAttestation); - documents[0].Metadata.Should().ContainKey("oci.attestation.sourceKind").WhoseValue.Should().Be("offline"); - documents[0].Metadata.Should().ContainKey("vex.provenance.sourceKind").WhoseValue.Should().Be("offline"); - documents[0].Metadata.Should().ContainKey("vex.provenance.registryAuthMode").WhoseValue.Should().Be("Anonymous"); - verifier.VerifyCalls.Should().Be(1); - } - - [Fact] - public async Task FetchAsync_WithSignatureMetadata_EnrichesProvenance() - { - var fileSystem = new MockFileSystem(new Dictionary - { - ["/bundles/attestation.json"] = new MockFileData("{\"payload\":\"\",\"payloadType\":\"application/vnd.in-toto+json\",\"signatures\":[{\"sig\":\"\"}]}"), - }); - - using var cache = new MemoryCache(new MemoryCacheOptions()); - var httpClient = new HttpClient(new StubHttpMessageHandler()) - { - BaseAddress = new System.Uri("https://registry.example.com/") - }; - - var httpFactory = new SingleClientHttpClientFactory(httpClient); - var discovery = new OciAttestationDiscoveryService(cache, fileSystem, NullLogger.Instance); - var fetcher = new OciAttestationFetcher(httpFactory, fileSystem, NullLogger.Instance); - - var connector = new OciOpenVexAttestationConnector( - discovery, - fetcher, - NullLogger.Instance, - TimeProvider.System); - - var settingsValues = ImmutableDictionary.Empty - .Add("Images:0:Reference", "registry.example.com/repo/image:latest") - .Add("Images:0:OfflineBundlePath", "/bundles/attestation.json") - .Add("Offline:PreferOffline", "true") - .Add("Offline:AllowNetworkFallback", "false") - .Add("Cosign:Mode", "Keyless") - .Add("Cosign:Keyless:Issuer", "https://issuer.example.com") - .Add("Cosign:Keyless:Subject", "subject@example.com"); - - var settings = new VexConnectorSettings(settingsValues); - await connector.ValidateAsync(settings, CancellationToken.None); - - var sink = new CapturingRawSink(); - var verifier = new CapturingSignatureVerifier - { - Result = new VexSignatureMetadata( - type: "cosign", - subject: "sig-subject", - issuer: "sig-issuer", - keyId: "key-id", - verifiedAt: DateTimeOffset.UtcNow, - transparencyLogReference: "rekor://entry/123") - }; - - var context = new VexConnectorContext( - Since: null, - Settings: VexConnectorSettings.Empty, - RawSink: sink, - SignatureVerifier: verifier, - Normalizers: new NoopNormalizerRouter(), - Services: new ServiceCollection().BuildServiceProvider()); - - var documents = new List(); - await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) - { - documents.Add(document); - } - - documents.Should().HaveCount(1); - var metadata = documents[0].Metadata; - metadata.Should().Contain("vex.signature.type", "cosign"); - metadata.Should().Contain("vex.signature.subject", "sig-subject"); - metadata.Should().Contain("vex.signature.issuer", "sig-issuer"); - metadata.Should().Contain("vex.signature.keyId", "key-id"); - metadata.Should().ContainKey("vex.signature.verifiedAt"); - metadata.Should().Contain("vex.signature.transparencyLogReference", "rekor://entry/123"); - metadata.Should().Contain("vex.provenance.cosign.mode", "Keyless"); - metadata.Should().Contain("vex.provenance.cosign.issuer", "https://issuer.example.com"); - metadata.Should().Contain("vex.provenance.cosign.subject", "subject@example.com"); - verifier.VerifyCalls.Should().Be(1); - } - - private sealed class CapturingRawSink : IVexRawDocumentSink - { - public List Documents { get; } = new(); - - public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken) - { - Documents.Add(document); - return ValueTask.CompletedTask; - } - } - - private sealed class CapturingSignatureVerifier : IVexSignatureVerifier - { - public int VerifyCalls { get; private set; } - - public VexSignatureMetadata? Result { get; set; } - - public ValueTask VerifyAsync(VexRawDocument document, CancellationToken cancellationToken) - { - VerifyCalls++; - return ValueTask.FromResult(Result); - } - } - - private sealed class NoopNormalizerRouter : IVexNormalizerRouter - { - public ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) - => ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray.Empty, ImmutableDictionary.Empty)); - } - - private sealed class SingleClientHttpClientFactory : IHttpClientFactory - { - private readonly HttpClient _client; - - public SingleClientHttpClientFactory(HttpClient client) - { - _client = client; - } - - public HttpClient CreateClient(string name) => _client; - } - - private sealed class StubHttpMessageHandler : HttpMessageHandler - { - protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound) - { - RequestMessage = request - }); - } - } -} +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.DependencyInjection; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Fetch; +using StellaOps.Excititor.Core; +using System.IO.Abstractions.TestingHelpers; +using Xunit; + +namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests.Connector; + +public sealed class OciOpenVexAttestationConnectorTests +{ + [Fact] + public async Task FetchAsync_WithOfflineBundle_EmitsRawDocument() + { + var fileSystem = new MockFileSystem(new Dictionary + { + ["/bundles/attestation.json"] = new MockFileData("{\"payload\":\"\",\"payloadType\":\"application/vnd.in-toto+json\",\"signatures\":[{\"sig\":\"\"}]}"), + }); + + using var cache = new MemoryCache(new MemoryCacheOptions()); + var httpClient = new HttpClient(new StubHttpMessageHandler()) + { + BaseAddress = new System.Uri("https://registry.example.com/") + }; + + var httpFactory = new SingleClientHttpClientFactory(httpClient); + var discovery = new OciAttestationDiscoveryService(cache, fileSystem, NullLogger.Instance); + var fetcher = new OciAttestationFetcher(httpFactory, fileSystem, NullLogger.Instance); + + var connector = new OciOpenVexAttestationConnector( + discovery, + fetcher, + NullLogger.Instance, + TimeProvider.System); + + var settingsValues = ImmutableDictionary.Empty + .Add("Images:0:Reference", "registry.example.com/repo/image:latest") + .Add("Images:0:OfflineBundlePath", "/bundles/attestation.json") + .Add("Offline:PreferOffline", "true") + .Add("Offline:AllowNetworkFallback", "false") + .Add("Cosign:Mode", "None"); + + var settings = new VexConnectorSettings(settingsValues); + await connector.ValidateAsync(settings, CancellationToken.None); + + var sink = new CapturingRawSink(); + var verifier = new CapturingSignatureVerifier(); + var context = new VexConnectorContext( + Since: null, + Settings: VexConnectorSettings.Empty, + RawSink: sink, + SignatureVerifier: verifier, + Normalizers: new NoopNormalizerRouter(), + Services: new Microsoft.Extensions.DependencyInjection.ServiceCollection().BuildServiceProvider(), + ResumeTokens: ImmutableDictionary.Empty); + + var documents = new List(); + await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(document); + } + + documents.Should().HaveCount(1); + sink.Documents.Should().HaveCount(1); + documents[0].Format.Should().Be(VexDocumentFormat.OciAttestation); + documents[0].Metadata.Should().ContainKey("oci.attestation.sourceKind").WhoseValue.Should().Be("offline"); + documents[0].Metadata.Should().ContainKey("vex.provenance.sourceKind").WhoseValue.Should().Be("offline"); + documents[0].Metadata.Should().ContainKey("vex.provenance.registryAuthMode").WhoseValue.Should().Be("Anonymous"); + verifier.VerifyCalls.Should().Be(1); + } + + [Fact] + public async Task FetchAsync_WithSignatureMetadata_EnrichesProvenance() + { + var fileSystem = new MockFileSystem(new Dictionary + { + ["/bundles/attestation.json"] = new MockFileData("{\"payload\":\"\",\"payloadType\":\"application/vnd.in-toto+json\",\"signatures\":[{\"sig\":\"\"}]}"), + }); + + using var cache = new MemoryCache(new MemoryCacheOptions()); + var httpClient = new HttpClient(new StubHttpMessageHandler()) + { + BaseAddress = new System.Uri("https://registry.example.com/") + }; + + var httpFactory = new SingleClientHttpClientFactory(httpClient); + var discovery = new OciAttestationDiscoveryService(cache, fileSystem, NullLogger.Instance); + var fetcher = new OciAttestationFetcher(httpFactory, fileSystem, NullLogger.Instance); + + var connector = new OciOpenVexAttestationConnector( + discovery, + fetcher, + NullLogger.Instance, + TimeProvider.System); + + var settingsValues = ImmutableDictionary.Empty + .Add("Images:0:Reference", "registry.example.com/repo/image:latest") + .Add("Images:0:OfflineBundlePath", "/bundles/attestation.json") + .Add("Offline:PreferOffline", "true") + .Add("Offline:AllowNetworkFallback", "false") + .Add("Cosign:Mode", "Keyless") + .Add("Cosign:Keyless:Issuer", "https://issuer.example.com") + .Add("Cosign:Keyless:Subject", "subject@example.com"); + + var settings = new VexConnectorSettings(settingsValues); + await connector.ValidateAsync(settings, CancellationToken.None); + + var sink = new CapturingRawSink(); + var verifier = new CapturingSignatureVerifier + { + Result = new VexSignatureMetadata( + type: "cosign", + subject: "sig-subject", + issuer: "sig-issuer", + keyId: "key-id", + verifiedAt: DateTimeOffset.UtcNow, + transparencyLogReference: "rekor://entry/123") + }; + + var context = new VexConnectorContext( + Since: null, + Settings: VexConnectorSettings.Empty, + RawSink: sink, + SignatureVerifier: verifier, + Normalizers: new NoopNormalizerRouter(), + Services: new ServiceCollection().BuildServiceProvider(), + ResumeTokens: ImmutableDictionary.Empty); + + var documents = new List(); + await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(document); + } + + documents.Should().HaveCount(1); + var metadata = documents[0].Metadata; + metadata.Should().Contain("vex.signature.type", "cosign"); + metadata.Should().Contain("vex.signature.subject", "sig-subject"); + metadata.Should().Contain("vex.signature.issuer", "sig-issuer"); + metadata.Should().Contain("vex.signature.keyId", "key-id"); + metadata.Should().ContainKey("vex.signature.verifiedAt"); + metadata.Should().Contain("vex.signature.transparencyLogReference", "rekor://entry/123"); + metadata.Should().Contain("vex.provenance.cosign.mode", "Keyless"); + metadata.Should().Contain("vex.provenance.cosign.issuer", "https://issuer.example.com"); + metadata.Should().Contain("vex.provenance.cosign.subject", "subject@example.com"); + verifier.VerifyCalls.Should().Be(1); + } + + private sealed class CapturingRawSink : IVexRawDocumentSink + { + public List Documents { get; } = new(); + + public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken) + { + Documents.Add(document); + return ValueTask.CompletedTask; + } + } + + private sealed class CapturingSignatureVerifier : IVexSignatureVerifier + { + public int VerifyCalls { get; private set; } + + public VexSignatureMetadata? Result { get; set; } + + public ValueTask VerifyAsync(VexRawDocument document, CancellationToken cancellationToken) + { + VerifyCalls++; + return ValueTask.FromResult(Result); + } + } + + private sealed class NoopNormalizerRouter : IVexNormalizerRouter + { + public ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray.Empty, ImmutableDictionary.Empty)); + } + + private sealed class SingleClientHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleClientHttpClientFactory(HttpClient client) + { + _client = client; + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class StubHttpMessageHandler : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound) + { + RequestMessage = request + }); + } + } +} diff --git a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests.csproj b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests.csproj index 9955c490..623f5e8f 100644 --- a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests.csproj +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests.csproj @@ -1,18 +1,17 @@ - - - net10.0 - preview - enable - enable - true - NU1903 - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.csproj b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.csproj index 4aea0c00..6b331924 100644 --- a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.csproj +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.csproj @@ -1,20 +1,19 @@ - - - net10.0 - preview - enable - enable - true - NU1903 - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests/Connectors/OracleCsafConnectorTests.cs b/src/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests/Connectors/OracleCsafConnectorTests.cs index 2f28532b..1e7dceba 100644 --- a/src/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests/Connectors/OracleCsafConnectorTests.cs +++ b/src/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests/Connectors/OracleCsafConnectorTests.cs @@ -1,260 +1,262 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Net; -using System.Net.Http; -using System.Security.Cryptography; -using System.Text; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using FluentAssertions; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging.Abstractions; -using StellaOps.Excititor.Connectors.Abstractions; -using StellaOps.Excititor.Connectors.Oracle.CSAF; -using StellaOps.Excititor.Connectors.Oracle.CSAF.Configuration; +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Net; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.Oracle.CSAF; +using StellaOps.Excititor.Connectors.Oracle.CSAF.Configuration; using StellaOps.Excititor.Connectors.Oracle.CSAF.Metadata; using StellaOps.Excititor.Core; using StellaOps.Excititor.Storage.Mongo; using System.IO.Abstractions.TestingHelpers; using Xunit; using MongoDB.Driver; - -namespace StellaOps.Excititor.Connectors.Oracle.CSAF.Tests.Connectors; - -public sealed class OracleCsafConnectorTests -{ - [Fact] - public async Task FetchAsync_NewEntry_PersistsDocumentAndUpdatesState() - { - var documentUri = new Uri("https://oracle.example/security/csaf/cpu2025oct.json"); - var payload = Encoding.UTF8.GetBytes("{\"document\":\"payload\"}"); - var payloadDigest = ComputeDigest(payload); - var snapshotPath = "/snapshots/oracle-catalog.json"; - var fileSystem = new MockFileSystem(); - fileSystem.AddFile(snapshotPath, new MockFileData(BuildOfflineSnapshot(documentUri, payloadDigest, "2025-10-15T00:00:00Z"))); - - var handler = new StubHttpMessageHandler(new Dictionary - { - [documentUri] = CreateResponse(payload), - }); - var httpClient = new HttpClient(handler); - var httpFactory = new SingleHttpClientFactory(httpClient); - var loader = new OracleCatalogLoader( - httpFactory, - new MemoryCache(new MemoryCacheOptions()), - fileSystem, - NullLogger.Instance, - TimeProvider.System); - - var stateRepository = new InMemoryConnectorStateRepository(); - var connector = new OracleCsafConnector( - loader, - httpFactory, - stateRepository, - new[] { new OracleConnectorOptionsValidator(fileSystem) }, - NullLogger.Instance, - TimeProvider.System); - - var settingsValues = ImmutableDictionary.Empty - .Add(nameof(OracleConnectorOptions.PreferOfflineSnapshot), "true") - .Add(nameof(OracleConnectorOptions.OfflineSnapshotPath), snapshotPath) - .Add(nameof(OracleConnectorOptions.PersistOfflineSnapshot), "false"); - var settings = new VexConnectorSettings(settingsValues); - - await connector.ValidateAsync(settings, CancellationToken.None); - - var sink = new InMemoryRawSink(); - var context = new VexConnectorContext( - Since: null, - Settings: settings, - RawSink: sink, - SignatureVerifier: new NoopSignatureVerifier(), - Normalizers: new NoopNormalizerRouter(), - Services: new ServiceCollection().BuildServiceProvider()); - - var documents = new List(); - await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) - { - documents.Add(document); - } - - documents.Should().HaveCount(1); - sink.Documents.Should().HaveCount(1); - documents[0].Digest.Should().Be(payloadDigest); - documents[0].Metadata["oracle.csaf.entryId"].Should().Be("CPU2025Oct"); - documents[0].Metadata["oracle.csaf.sha256"].Should().Be(payloadDigest); - - stateRepository.State.Should().NotBeNull(); - stateRepository.State!.DocumentDigests.Should().ContainSingle().Which.Should().Be(payloadDigest); - - handler.GetCallCount(documentUri).Should().Be(1); - - // second run should short-circuit without downloading again - sink.Documents.Clear(); - documents.Clear(); - - await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) - { - documents.Add(document); - } - - documents.Should().BeEmpty(); - sink.Documents.Should().BeEmpty(); - handler.GetCallCount(documentUri).Should().Be(1); - } - - [Fact] - public async Task FetchAsync_ChecksumMismatch_SkipsDocument() - { - var documentUri = new Uri("https://oracle.example/security/csaf/cpu2025oct.json"); - var payload = Encoding.UTF8.GetBytes("{\"document\":\"payload\"}"); - var snapshotPath = "/snapshots/oracle-catalog.json"; - var fileSystem = new MockFileSystem(); - fileSystem.AddFile(snapshotPath, new MockFileData(BuildOfflineSnapshot(documentUri, "deadbeef", "2025-10-15T00:00:00Z"))); - - var handler = new StubHttpMessageHandler(new Dictionary - { - [documentUri] = CreateResponse(payload), - }); - var httpClient = new HttpClient(handler); - var httpFactory = new SingleHttpClientFactory(httpClient); - var loader = new OracleCatalogLoader( - httpFactory, - new MemoryCache(new MemoryCacheOptions()), - fileSystem, - NullLogger.Instance, - TimeProvider.System); - - var stateRepository = new InMemoryConnectorStateRepository(); - var connector = new OracleCsafConnector( - loader, - httpFactory, - stateRepository, - new[] { new OracleConnectorOptionsValidator(fileSystem) }, - NullLogger.Instance, - TimeProvider.System); - - var settingsValues = ImmutableDictionary.Empty - .Add(nameof(OracleConnectorOptions.PreferOfflineSnapshot), "true") - .Add(nameof(OracleConnectorOptions.OfflineSnapshotPath), snapshotPath) - .Add(nameof(OracleConnectorOptions.PersistOfflineSnapshot), "false"); - var settings = new VexConnectorSettings(settingsValues); - - await connector.ValidateAsync(settings, CancellationToken.None); - - var sink = new InMemoryRawSink(); - var context = new VexConnectorContext( - Since: null, - Settings: settings, - RawSink: sink, - SignatureVerifier: new NoopSignatureVerifier(), - Normalizers: new NoopNormalizerRouter(), - Services: new ServiceCollection().BuildServiceProvider()); - - var documents = new List(); - await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) - { - documents.Add(document); - } - - documents.Should().BeEmpty(); - sink.Documents.Should().BeEmpty(); - stateRepository.State.Should().BeNull(); - handler.GetCallCount(documentUri).Should().Be(1); - } - - private static HttpResponseMessage CreateResponse(byte[] payload) - => new(HttpStatusCode.OK) - { - Content = new ByteArrayContent(payload) - { - Headers = - { - ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json"), - } - } - }; - - private static string ComputeDigest(byte[] payload) - { - Span buffer = stackalloc byte[32]; - SHA256.HashData(payload, buffer); - return "sha256:" + Convert.ToHexString(buffer).ToLowerInvariant(); - } - - private static string BuildOfflineSnapshot(Uri documentUri, string sha256, string publishedAt) - { - var snapshot = new - { - metadata = new - { - generatedAt = "2025-10-14T12:00:00Z", - entries = new[] - { - new - { - id = "CPU2025Oct", - title = "Oracle Critical Patch Update Advisory - October 2025", - documentUri = documentUri.ToString(), - publishedAt, - revision = publishedAt, - sha256, - size = 1024, - products = new[] { "Oracle Database" } - } - }, - cpuSchedule = Array.Empty() - }, - fetchedAt = "2025-10-14T12:00:00Z" - }; - - return JsonSerializer.Serialize(snapshot, new JsonSerializerOptions(JsonSerializerDefaults.Web)); - } - - private sealed class StubHttpMessageHandler : HttpMessageHandler - { - private readonly Dictionary _responses; - private readonly Dictionary _callCounts = new(); - - public StubHttpMessageHandler(Dictionary responses) - { - _responses = responses; - } - - public int GetCallCount(Uri uri) => _callCounts.TryGetValue(uri, out var count) ? count : 0; - - protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - if (request.RequestUri is null || !_responses.TryGetValue(request.RequestUri, out var response)) - { - return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound)); - } - - _callCounts.TryGetValue(request.RequestUri, out var count); - _callCounts[request.RequestUri] = count + 1; - return Task.FromResult(response.Clone()); - } - } - - private sealed class SingleHttpClientFactory : IHttpClientFactory - { - private readonly HttpClient _client; - - public SingleHttpClientFactory(HttpClient client) - { - _client = client; - } - - public HttpClient CreateClient(string name) => _client; - } - - private sealed class InMemoryConnectorStateRepository : IVexConnectorStateRepository - { - public VexConnectorState? State { get; private set; } - + +namespace StellaOps.Excititor.Connectors.Oracle.CSAF.Tests.Connectors; + +public sealed class OracleCsafConnectorTests +{ + [Fact] + public async Task FetchAsync_NewEntry_PersistsDocumentAndUpdatesState() + { + var documentUri = new Uri("https://oracle.example/security/csaf/cpu2025oct.json"); + var payload = Encoding.UTF8.GetBytes("{\"document\":\"payload\"}"); + var payloadDigest = ComputeDigest(payload); + var snapshotPath = "/snapshots/oracle-catalog.json"; + var fileSystem = new MockFileSystem(); + fileSystem.AddFile(snapshotPath, new MockFileData(BuildOfflineSnapshot(documentUri, payloadDigest, "2025-10-15T00:00:00Z"))); + + var handler = new StubHttpMessageHandler(new Dictionary + { + [documentUri] = CreateResponse(payload), + }); + var httpClient = new HttpClient(handler); + var httpFactory = new SingleHttpClientFactory(httpClient); + var loader = new OracleCatalogLoader( + httpFactory, + new MemoryCache(new MemoryCacheOptions()), + fileSystem, + NullLogger.Instance, + TimeProvider.System); + + var stateRepository = new InMemoryConnectorStateRepository(); + var connector = new OracleCsafConnector( + loader, + httpFactory, + stateRepository, + new[] { new OracleConnectorOptionsValidator(fileSystem) }, + NullLogger.Instance, + TimeProvider.System); + + var settingsValues = ImmutableDictionary.Empty + .Add(nameof(OracleConnectorOptions.PreferOfflineSnapshot), "true") + .Add(nameof(OracleConnectorOptions.OfflineSnapshotPath), snapshotPath) + .Add(nameof(OracleConnectorOptions.PersistOfflineSnapshot), "false"); + var settings = new VexConnectorSettings(settingsValues); + + await connector.ValidateAsync(settings, CancellationToken.None); + + var sink = new InMemoryRawSink(); + var context = new VexConnectorContext( + Since: null, + Settings: settings, + RawSink: sink, + SignatureVerifier: new NoopSignatureVerifier(), + Normalizers: new NoopNormalizerRouter(), + Services: new ServiceCollection().BuildServiceProvider(), + ResumeTokens: ImmutableDictionary.Empty); + + var documents = new List(); + await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(document); + } + + documents.Should().HaveCount(1); + sink.Documents.Should().HaveCount(1); + documents[0].Digest.Should().Be(payloadDigest); + documents[0].Metadata["oracle.csaf.entryId"].Should().Be("CPU2025Oct"); + documents[0].Metadata["oracle.csaf.sha256"].Should().Be(payloadDigest); + + stateRepository.State.Should().NotBeNull(); + stateRepository.State!.DocumentDigests.Should().ContainSingle().Which.Should().Be(payloadDigest); + + handler.GetCallCount(documentUri).Should().Be(1); + + // second run should short-circuit without downloading again + sink.Documents.Clear(); + documents.Clear(); + + await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(document); + } + + documents.Should().BeEmpty(); + sink.Documents.Should().BeEmpty(); + handler.GetCallCount(documentUri).Should().Be(1); + } + + [Fact] + public async Task FetchAsync_ChecksumMismatch_SkipsDocument() + { + var documentUri = new Uri("https://oracle.example/security/csaf/cpu2025oct.json"); + var payload = Encoding.UTF8.GetBytes("{\"document\":\"payload\"}"); + var snapshotPath = "/snapshots/oracle-catalog.json"; + var fileSystem = new MockFileSystem(); + fileSystem.AddFile(snapshotPath, new MockFileData(BuildOfflineSnapshot(documentUri, "deadbeef", "2025-10-15T00:00:00Z"))); + + var handler = new StubHttpMessageHandler(new Dictionary + { + [documentUri] = CreateResponse(payload), + }); + var httpClient = new HttpClient(handler); + var httpFactory = new SingleHttpClientFactory(httpClient); + var loader = new OracleCatalogLoader( + httpFactory, + new MemoryCache(new MemoryCacheOptions()), + fileSystem, + NullLogger.Instance, + TimeProvider.System); + + var stateRepository = new InMemoryConnectorStateRepository(); + var connector = new OracleCsafConnector( + loader, + httpFactory, + stateRepository, + new[] { new OracleConnectorOptionsValidator(fileSystem) }, + NullLogger.Instance, + TimeProvider.System); + + var settingsValues = ImmutableDictionary.Empty + .Add(nameof(OracleConnectorOptions.PreferOfflineSnapshot), "true") + .Add(nameof(OracleConnectorOptions.OfflineSnapshotPath), snapshotPath) + .Add(nameof(OracleConnectorOptions.PersistOfflineSnapshot), "false"); + var settings = new VexConnectorSettings(settingsValues); + + await connector.ValidateAsync(settings, CancellationToken.None); + + var sink = new InMemoryRawSink(); + var context = new VexConnectorContext( + Since: null, + Settings: settings, + RawSink: sink, + SignatureVerifier: new NoopSignatureVerifier(), + Normalizers: new NoopNormalizerRouter(), + Services: new ServiceCollection().BuildServiceProvider(), + ResumeTokens: ImmutableDictionary.Empty); + + var documents = new List(); + await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(document); + } + + documents.Should().BeEmpty(); + sink.Documents.Should().BeEmpty(); + stateRepository.State.Should().BeNull(); + handler.GetCallCount(documentUri).Should().Be(1); + } + + private static HttpResponseMessage CreateResponse(byte[] payload) + => new(HttpStatusCode.OK) + { + Content = new ByteArrayContent(payload) + { + Headers = + { + ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json"), + } + } + }; + + private static string ComputeDigest(byte[] payload) + { + Span buffer = stackalloc byte[32]; + SHA256.HashData(payload, buffer); + return "sha256:" + Convert.ToHexString(buffer).ToLowerInvariant(); + } + + private static string BuildOfflineSnapshot(Uri documentUri, string sha256, string publishedAt) + { + var snapshot = new + { + metadata = new + { + generatedAt = "2025-10-14T12:00:00Z", + entries = new[] + { + new + { + id = "CPU2025Oct", + title = "Oracle Critical Patch Update Advisory - October 2025", + documentUri = documentUri.ToString(), + publishedAt, + revision = publishedAt, + sha256, + size = 1024, + products = new[] { "Oracle Database" } + } + }, + cpuSchedule = Array.Empty() + }, + fetchedAt = "2025-10-14T12:00:00Z" + }; + + return JsonSerializer.Serialize(snapshot, new JsonSerializerOptions(JsonSerializerDefaults.Web)); + } + + private sealed class StubHttpMessageHandler : HttpMessageHandler + { + private readonly Dictionary _responses; + private readonly Dictionary _callCounts = new(); + + public StubHttpMessageHandler(Dictionary responses) + { + _responses = responses; + } + + public int GetCallCount(Uri uri) => _callCounts.TryGetValue(uri, out var count) ? count : 0; + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + if (request.RequestUri is null || !_responses.TryGetValue(request.RequestUri, out var response)) + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound)); + } + + _callCounts.TryGetValue(request.RequestUri, out var count); + _callCounts[request.RequestUri] = count + 1; + return Task.FromResult(response.Clone()); + } + } + + private sealed class SingleHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleHttpClientFactory(HttpClient client) + { + _client = client; + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class InMemoryConnectorStateRepository : IVexConnectorStateRepository + { + public VexConnectorState? State { get; private set; } + public ValueTask GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null) => ValueTask.FromResult(State); @@ -263,50 +265,50 @@ public sealed class OracleCsafConnectorTests State = state; return ValueTask.CompletedTask; } - } - - private sealed class InMemoryRawSink : IVexRawDocumentSink - { - public List Documents { get; } = new(); - - public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken) - { - Documents.Add(document); - return ValueTask.CompletedTask; - } - } - - private sealed class NoopSignatureVerifier : IVexSignatureVerifier - { - public ValueTask VerifyAsync(VexRawDocument document, CancellationToken cancellationToken) - => ValueTask.FromResult(null); - } - - private sealed class NoopNormalizerRouter : IVexNormalizerRouter - { - public ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) - => ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray.Empty, ImmutableDictionary.Empty)); - } -} - -internal static class HttpResponseMessageExtensions -{ - public static HttpResponseMessage Clone(this HttpResponseMessage response) - { - var clone = new HttpResponseMessage(response.StatusCode); - foreach (var header in response.Headers) - { - clone.Headers.TryAddWithoutValidation(header.Key, header.Value); - } - - if (response.Content is not null) - { - var payload = response.Content.ReadAsByteArrayAsync().GetAwaiter().GetResult(); - var mediaType = response.Content.Headers.ContentType?.MediaType ?? "application/json"; - clone.Content = new ByteArrayContent(payload); - clone.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue(mediaType); - } - - return clone; - } -} + } + + private sealed class InMemoryRawSink : IVexRawDocumentSink + { + public List Documents { get; } = new(); + + public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken) + { + Documents.Add(document); + return ValueTask.CompletedTask; + } + } + + private sealed class NoopSignatureVerifier : IVexSignatureVerifier + { + public ValueTask VerifyAsync(VexRawDocument document, CancellationToken cancellationToken) + => ValueTask.FromResult(null); + } + + private sealed class NoopNormalizerRouter : IVexNormalizerRouter + { + public ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray.Empty, ImmutableDictionary.Empty)); + } +} + +internal static class HttpResponseMessageExtensions +{ + public static HttpResponseMessage Clone(this HttpResponseMessage response) + { + var clone = new HttpResponseMessage(response.StatusCode); + foreach (var header in response.Headers) + { + clone.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + + if (response.Content is not null) + { + var payload = response.Content.ReadAsByteArrayAsync().GetAwaiter().GetResult(); + var mediaType = response.Content.Headers.ContentType?.MediaType ?? "application/json"; + clone.Content = new ByteArrayContent(payload); + clone.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue(mediaType); + } + + return clone; + } +} diff --git a/src/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests.csproj b/src/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests.csproj index d100d43e..6048a328 100644 --- a/src/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests.csproj +++ b/src/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests.csproj @@ -1,17 +1,17 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Connectors.Oracle.CSAF/StellaOps.Excititor.Connectors.Oracle.CSAF.csproj b/src/StellaOps.Excititor.Connectors.Oracle.CSAF/StellaOps.Excititor.Connectors.Oracle.CSAF.csproj index 4a15c1f6..fc88a048 100644 --- a/src/StellaOps.Excititor.Connectors.Oracle.CSAF/StellaOps.Excititor.Connectors.Oracle.CSAF.csproj +++ b/src/StellaOps.Excititor.Connectors.Oracle.CSAF/StellaOps.Excititor.Connectors.Oracle.CSAF.csproj @@ -1,20 +1,20 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Connectors.RedHat.CSAF.Tests/Connectors/RedHatCsafConnectorTests.cs b/src/StellaOps.Excititor.Connectors.RedHat.CSAF.Tests/Connectors/RedHatCsafConnectorTests.cs index e73d03cb..a47917db 100644 --- a/src/StellaOps.Excititor.Connectors.RedHat.CSAF.Tests/Connectors/RedHatCsafConnectorTests.cs +++ b/src/StellaOps.Excititor.Connectors.RedHat.CSAF.Tests/Connectors/RedHatCsafConnectorTests.cs @@ -73,13 +73,14 @@ public sealed class RedHatCsafConnectorTests var connector = new RedHatCsafConnector(Descriptor, metadataLoader, factory, stateRepository, NullLogger.Instance, TimeProvider.System); var rawSink = new CapturingRawSink(); - var context = new VexConnectorContext( - new DateTimeOffset(2025, 10, 16, 12, 0, 0, TimeSpan.Zero), - VexConnectorSettings.Empty, - rawSink, - new NoopSignatureVerifier(), - new NoopNormalizerRouter(), - new ServiceCollection().BuildServiceProvider()); + var context = new VexConnectorContext( + new DateTimeOffset(2025, 10, 16, 12, 0, 0, TimeSpan.Zero), + VexConnectorSettings.Empty, + rawSink, + new NoopSignatureVerifier(), + new NoopNormalizerRouter(), + new ServiceCollection().BuildServiceProvider(), + ImmutableDictionary.Empty); var results = new List(); await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) @@ -238,13 +239,14 @@ public sealed class RedHatCsafConnectorTests var connector = new RedHatCsafConnector(Descriptor, metadataLoader, factory, stateRepository, NullLogger.Instance, TimeProvider.System); var rawSink = new CapturingRawSink(); - var context = new VexConnectorContext( - null, - VexConnectorSettings.Empty, - rawSink, - new NoopSignatureVerifier(), - new NoopNormalizerRouter(), - new ServiceCollection().BuildServiceProvider()); + var context = new VexConnectorContext( + null, + VexConnectorSettings.Empty, + rawSink, + new NoopSignatureVerifier(), + new NoopNormalizerRouter(), + new ServiceCollection().BuildServiceProvider(), + ImmutableDictionary.Empty); var documents = new List(); await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) diff --git a/src/StellaOps.Excititor.Connectors.RedHat.CSAF/StellaOps.Excititor.Connectors.RedHat.CSAF.csproj b/src/StellaOps.Excititor.Connectors.RedHat.CSAF/StellaOps.Excititor.Connectors.RedHat.CSAF.csproj index b3e1398a..036c860d 100644 --- a/src/StellaOps.Excititor.Connectors.RedHat.CSAF/StellaOps.Excititor.Connectors.RedHat.CSAF.csproj +++ b/src/StellaOps.Excititor.Connectors.RedHat.CSAF/StellaOps.Excititor.Connectors.RedHat.CSAF.csproj @@ -1,19 +1,19 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.csproj b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.csproj index b3e1398a..036c860d 100644 --- a/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.csproj +++ b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.csproj @@ -1,19 +1,19 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests/Connectors/UbuntuCsafConnectorTests.cs b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests/Connectors/UbuntuCsafConnectorTests.cs index 277b6417..47b10d49 100644 --- a/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests/Connectors/UbuntuCsafConnectorTests.cs +++ b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests/Connectors/UbuntuCsafConnectorTests.cs @@ -61,7 +61,7 @@ public sealed class UbuntuCsafConnectorTests await connector.ValidateAsync(settings, CancellationToken.None); var sink = new InMemoryRawSink(); - var context = new VexConnectorContext(null, VexConnectorSettings.Empty, sink, new NoopSignatureVerifier(), new NoopNormalizerRouter(), new ServiceCollection().BuildServiceProvider()); + var context = new VexConnectorContext(null, VexConnectorSettings.Empty, sink, new NoopSignatureVerifier(), new NoopNormalizerRouter(), new ServiceCollection().BuildServiceProvider(), ImmutableDictionary.Empty); var documents = new List(); await foreach (var doc in connector.FetchAsync(context, CancellationToken.None)) @@ -130,7 +130,7 @@ public sealed class UbuntuCsafConnectorTests await connector.ValidateAsync(new VexConnectorSettings(ImmutableDictionary.Empty), CancellationToken.None); var sink = new InMemoryRawSink(); - var context = new VexConnectorContext(null, VexConnectorSettings.Empty, sink, new NoopSignatureVerifier(), new NoopNormalizerRouter(), new ServiceCollection().BuildServiceProvider()); + var context = new VexConnectorContext(null, VexConnectorSettings.Empty, sink, new NoopSignatureVerifier(), new NoopNormalizerRouter(), new ServiceCollection().BuildServiceProvider(), ImmutableDictionary.Empty); var documents = new List(); await foreach (var doc in connector.FetchAsync(context, CancellationToken.None)) diff --git a/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests.csproj b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests.csproj index ce08adbf..f6efcd05 100644 --- a/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests.csproj +++ b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests.csproj @@ -1,17 +1,17 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/StellaOps.Excititor.Connectors.Ubuntu.CSAF.csproj b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/StellaOps.Excititor.Connectors.Ubuntu.CSAF.csproj index 4a15c1f6..fc88a048 100644 --- a/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/StellaOps.Excititor.Connectors.Ubuntu.CSAF.csproj +++ b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/StellaOps.Excititor.Connectors.Ubuntu.CSAF.csproj @@ -1,20 +1,20 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Core/VexConnectorAbstractions.cs b/src/StellaOps.Excititor.Core/VexConnectorAbstractions.cs index 5a54673d..f8689363 100644 --- a/src/StellaOps.Excititor.Core/VexConnectorAbstractions.cs +++ b/src/StellaOps.Excititor.Core/VexConnectorAbstractions.cs @@ -1,7 +1,7 @@ using System; -using System.Collections.Immutable; -using System.Threading; -using System.Threading.Tasks; +using System.Threading; +using System.Threading.Tasks; +using System.Collections.Immutable; namespace StellaOps.Excititor.Core; @@ -24,13 +24,14 @@ public interface IVexConnector /// /// Connector context populated by the orchestrator/worker. /// -public sealed record VexConnectorContext( - DateTimeOffset? Since, - VexConnectorSettings Settings, - IVexRawDocumentSink RawSink, - IVexSignatureVerifier SignatureVerifier, - IVexNormalizerRouter Normalizers, - IServiceProvider Services); +public sealed record VexConnectorContext( + DateTimeOffset? Since, + VexConnectorSettings Settings, + IVexRawDocumentSink RawSink, + IVexSignatureVerifier SignatureVerifier, + IVexNormalizerRouter Normalizers, + IServiceProvider Services, + ImmutableDictionary ResumeTokens); /// /// Normalized connector configuration values. diff --git a/src/StellaOps.Excititor.Core/VexConsensusHold.cs b/src/StellaOps.Excititor.Core/VexConsensusHold.cs new file mode 100644 index 00000000..e5d429b9 --- /dev/null +++ b/src/StellaOps.Excititor.Core/VexConsensusHold.cs @@ -0,0 +1,47 @@ +namespace StellaOps.Excititor.Core; + +public sealed record VexConsensusHold +{ + public VexConsensusHold( + string vulnerabilityId, + string productKey, + VexConsensus candidate, + DateTimeOffset requestedAt, + DateTimeOffset eligibleAt, + string reason) + { + if (string.IsNullOrWhiteSpace(vulnerabilityId)) + { + throw new ArgumentException("Vulnerability id must be provided.", nameof(vulnerabilityId)); + } + + if (string.IsNullOrWhiteSpace(productKey)) + { + throw new ArgumentException("Product key must be provided.", nameof(productKey)); + } + + if (eligibleAt < requestedAt) + { + throw new ArgumentOutOfRangeException(nameof(eligibleAt), "EligibleAt cannot be earlier than RequestedAt."); + } + + VulnerabilityId = vulnerabilityId.Trim(); + ProductKey = productKey.Trim(); + Candidate = candidate ?? throw new ArgumentNullException(nameof(candidate)); + RequestedAt = requestedAt; + EligibleAt = eligibleAt; + Reason = string.IsNullOrWhiteSpace(reason) ? "unspecified" : reason.Trim(); + } + + public string VulnerabilityId { get; } + + public string ProductKey { get; } + + public VexConsensus Candidate { get; } + + public DateTimeOffset RequestedAt { get; } + + public DateTimeOffset EligibleAt { get; } + + public string Reason { get; } +} diff --git a/src/StellaOps.Excititor.Export/StellaOps.Excititor.Export.csproj b/src/StellaOps.Excititor.Export/StellaOps.Excititor.Export.csproj index 66bdedf8..64e918ce 100644 --- a/src/StellaOps.Excititor.Export/StellaOps.Excititor.Export.csproj +++ b/src/StellaOps.Excititor.Export/StellaOps.Excititor.Export.csproj @@ -1,19 +1,19 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Formats.CSAF/StellaOps.Excititor.Formats.CSAF.csproj b/src/StellaOps.Excititor.Formats.CSAF/StellaOps.Excititor.Formats.CSAF.csproj index a94d7788..8455f3fc 100644 --- a/src/StellaOps.Excititor.Formats.CSAF/StellaOps.Excititor.Formats.CSAF.csproj +++ b/src/StellaOps.Excititor.Formats.CSAF/StellaOps.Excititor.Formats.CSAF.csproj @@ -1,16 +1,16 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + diff --git a/src/StellaOps.Excititor.Formats.CycloneDX/StellaOps.Excititor.Formats.CycloneDX.csproj b/src/StellaOps.Excititor.Formats.CycloneDX/StellaOps.Excititor.Formats.CycloneDX.csproj index a94d7788..8455f3fc 100644 --- a/src/StellaOps.Excititor.Formats.CycloneDX/StellaOps.Excititor.Formats.CycloneDX.csproj +++ b/src/StellaOps.Excititor.Formats.CycloneDX/StellaOps.Excititor.Formats.CycloneDX.csproj @@ -1,16 +1,16 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + diff --git a/src/StellaOps.Excititor.Formats.OpenVEX/StellaOps.Excititor.Formats.OpenVEX.csproj b/src/StellaOps.Excititor.Formats.OpenVEX/StellaOps.Excititor.Formats.OpenVEX.csproj index a94d7788..8455f3fc 100644 --- a/src/StellaOps.Excititor.Formats.OpenVEX/StellaOps.Excititor.Formats.OpenVEX.csproj +++ b/src/StellaOps.Excititor.Formats.OpenVEX/StellaOps.Excititor.Formats.OpenVEX.csproj @@ -1,16 +1,16 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + diff --git a/src/StellaOps.Excititor.Policy/StellaOps.Excititor.Policy.csproj b/src/StellaOps.Excititor.Policy/StellaOps.Excititor.Policy.csproj index b4fef087..2433659d 100644 --- a/src/StellaOps.Excititor.Policy/StellaOps.Excititor.Policy.csproj +++ b/src/StellaOps.Excititor.Policy/StellaOps.Excititor.Policy.csproj @@ -1,17 +1,17 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Storage.Mongo/IVexStorageContracts.cs b/src/StellaOps.Excititor.Storage.Mongo/IVexStorageContracts.cs index 35e00099..7309254a 100644 --- a/src/StellaOps.Excititor.Storage.Mongo/IVexStorageContracts.cs +++ b/src/StellaOps.Excititor.Storage.Mongo/IVexStorageContracts.cs @@ -17,14 +17,17 @@ public interface IVexProviderStore ValueTask SaveAsync(VexProvider provider, CancellationToken cancellationToken, IClientSessionHandle? session = null); } -public interface IVexConsensusStore -{ - ValueTask FindAsync(string vulnerabilityId, string productKey, CancellationToken cancellationToken, IClientSessionHandle? session = null); - - ValueTask> FindByVulnerabilityAsync(string vulnerabilityId, CancellationToken cancellationToken, IClientSessionHandle? session = null); - - ValueTask SaveAsync(VexConsensus consensus, CancellationToken cancellationToken, IClientSessionHandle? session = null); -} +public interface IVexConsensusStore +{ + ValueTask FindAsync(string vulnerabilityId, string productKey, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask> FindByVulnerabilityAsync(string vulnerabilityId, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask SaveAsync(VexConsensus consensus, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + IAsyncEnumerable FindCalculatedBeforeAsync(DateTimeOffset cutoff, int batchSize, CancellationToken cancellationToken, IClientSessionHandle? session = null) + => throw new NotSupportedException(); +} public interface IVexClaimStore { @@ -60,12 +63,23 @@ public sealed record VexConnectorState( } } -public interface IVexConnectorStateRepository -{ - ValueTask GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null); - - ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null); -} +public interface IVexConnectorStateRepository +{ + ValueTask GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null); +} + +public interface IVexConsensusHoldStore +{ + ValueTask FindAsync(string vulnerabilityId, string productKey, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask SaveAsync(VexConsensusHold hold, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask RemoveAsync(string vulnerabilityId, string productKey, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + IAsyncEnumerable FindEligibleAsync(DateTimeOffset asOf, int batchSize, CancellationToken cancellationToken, IClientSessionHandle? session = null); +} public interface IVexCacheIndex { diff --git a/src/StellaOps.Excititor.Storage.Mongo/Migrations/VexConsensusHoldMigration.cs b/src/StellaOps.Excititor.Storage.Mongo/Migrations/VexConsensusHoldMigration.cs new file mode 100644 index 00000000..f8ac75a1 --- /dev/null +++ b/src/StellaOps.Excititor.Storage.Mongo/Migrations/VexConsensusHoldMigration.cs @@ -0,0 +1,29 @@ +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Driver; + +namespace StellaOps.Excititor.Storage.Mongo.Migrations; + +internal sealed class VexConsensusHoldMigration : IVexMongoMigration +{ + public string Id => "20251021-consensus-holds"; + + public async ValueTask ExecuteAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(database); + + var collection = database.GetCollection(VexMongoCollectionNames.ConsensusHolds); + + var eligibleIndex = Builders.IndexKeys + .Ascending(x => x.EligibleAt); + + var keyIndex = Builders.IndexKeys + .Ascending(x => x.VulnerabilityId) + .Ascending(x => x.ProductKey); + + await Task.WhenAll( + collection.Indexes.CreateOneAsync(new CreateIndexModel(eligibleIndex), cancellationToken: cancellationToken), + collection.Indexes.CreateOneAsync(new CreateIndexModel(keyIndex), cancellationToken: cancellationToken)) + .ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Excititor.Storage.Mongo/MongoVexConsensusHoldStore.cs b/src/StellaOps.Excititor.Storage.Mongo/MongoVexConsensusHoldStore.cs new file mode 100644 index 00000000..951644ed --- /dev/null +++ b/src/StellaOps.Excititor.Storage.Mongo/MongoVexConsensusHoldStore.cs @@ -0,0 +1,88 @@ +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Driver; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Storage.Mongo; + +public sealed class MongoVexConsensusHoldStore : IVexConsensusHoldStore +{ + private readonly IMongoCollection _collection; + + public MongoVexConsensusHoldStore(IMongoDatabase database) + { + ArgumentNullException.ThrowIfNull(database); + VexMongoMappingRegistry.Register(); + _collection = database.GetCollection(VexMongoCollectionNames.ConsensusHolds); + } + + public async ValueTask FindAsync(string vulnerabilityId, string productKey, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(vulnerabilityId); + ArgumentException.ThrowIfNullOrWhiteSpace(productKey); + var id = VexConsensusRecord.CreateId(vulnerabilityId, productKey); + var filter = Builders.Filter.Eq(x => x.Id, id); + var record = session is null + ? await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false) + : await _collection.Find(session, filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return record?.ToDomain(); + } + + public async ValueTask SaveAsync(VexConsensusHold hold, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentNullException.ThrowIfNull(hold); + var record = VexConsensusHoldRecord.FromDomain(hold); + var filter = Builders.Filter.Eq(x => x.Id, record.Id); + if (session is null) + { + await _collection.ReplaceOneAsync(filter, record, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false); + } + else + { + await _collection.ReplaceOneAsync(session, filter, record, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false); + } + } + + public async ValueTask RemoveAsync(string vulnerabilityId, string productKey, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(vulnerabilityId); + ArgumentException.ThrowIfNullOrWhiteSpace(productKey); + var id = VexConsensusRecord.CreateId(vulnerabilityId, productKey); + var filter = Builders.Filter.Eq(x => x.Id, id); + if (session is null) + { + await _collection.DeleteOneAsync(filter, cancellationToken).ConfigureAwait(false); + } + else + { + await _collection.DeleteOneAsync(session, filter, options: null, cancellationToken).ConfigureAwait(false); + } + } + + public async IAsyncEnumerable FindEligibleAsync(DateTimeOffset asOf, int batchSize, [EnumeratorCancellation] CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + var cutoff = asOf.UtcDateTime; + var filter = Builders.Filter.Lte(x => x.EligibleAt, cutoff); + var find = session is null + ? _collection.Find(filter) + : _collection.Find(session, filter); + + find = find.SortBy(x => x.EligibleAt); + + if (batchSize > 0) + { + find = find.Limit(batchSize); + } + + using var cursor = await find.ToCursorAsync(cancellationToken).ConfigureAwait(false); + while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false)) + { + foreach (var record in cursor.Current) + { + yield return record.ToDomain(); + } + } + } +} diff --git a/src/StellaOps.Excititor.Storage.Mongo/MongoVexConsensusStore.cs b/src/StellaOps.Excititor.Storage.Mongo/MongoVexConsensusStore.cs index 472fba3b..38567a42 100644 --- a/src/StellaOps.Excititor.Storage.Mongo/MongoVexConsensusStore.cs +++ b/src/StellaOps.Excititor.Storage.Mongo/MongoVexConsensusStore.cs @@ -40,19 +40,43 @@ public sealed class MongoVexConsensusStore : IVexConsensusStore return records.ConvertAll(static record => record.ToDomain()); } - public async ValueTask SaveAsync(VexConsensus consensus, CancellationToken cancellationToken, IClientSessionHandle? session = null) - { - ArgumentNullException.ThrowIfNull(consensus); - var record = VexConsensusRecord.FromDomain(consensus); - var filter = Builders.Filter.Eq(x => x.Id, record.Id); - if (session is null) - { - await _collection.ReplaceOneAsync(filter, record, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false); - } - else - { - await _collection.ReplaceOneAsync(session, filter, record, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false); - } - } - -} + public async ValueTask SaveAsync(VexConsensus consensus, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentNullException.ThrowIfNull(consensus); + var record = VexConsensusRecord.FromDomain(consensus); + var filter = Builders.Filter.Eq(x => x.Id, record.Id); + if (session is null) + { + await _collection.ReplaceOneAsync(filter, record, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false); + } + else + { + await _collection.ReplaceOneAsync(session, filter, record, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false); + } + } + + public async IAsyncEnumerable FindCalculatedBeforeAsync(DateTimeOffset cutoff, int batchSize, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + var filter = Builders.Filter.Lt(x => x.CalculatedAt, cutoff.UtcDateTime); + var find = session is null + ? _collection.Find(filter) + : _collection.Find(session, filter); + + find = find.SortBy(x => x.CalculatedAt); + + if (batchSize > 0) + { + find = find.Limit(batchSize); + } + + using var cursor = await find.ToCursorAsync(cancellationToken).ConfigureAwait(false); + while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false)) + { + foreach (var record in cursor.Current) + { + yield return record.ToDomain(); + } + } + } + +} diff --git a/src/StellaOps.Excititor.Storage.Mongo/ServiceCollectionExtensions.cs b/src/StellaOps.Excititor.Storage.Mongo/ServiceCollectionExtensions.cs index aebb4409..52828f73 100644 --- a/src/StellaOps.Excititor.Storage.Mongo/ServiceCollectionExtensions.cs +++ b/src/StellaOps.Excititor.Storage.Mongo/ServiceCollectionExtensions.cs @@ -48,18 +48,20 @@ public static class VexMongoServiceCollectionExtensions services.AddScoped(); services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddHostedService(); - return services; - } -} + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddHostedService(); + return services; + } +} diff --git a/src/StellaOps.Excititor.Storage.Mongo/StellaOps.Excititor.Storage.Mongo.csproj b/src/StellaOps.Excititor.Storage.Mongo/StellaOps.Excititor.Storage.Mongo.csproj index 246185f0..a2181f1e 100644 --- a/src/StellaOps.Excititor.Storage.Mongo/StellaOps.Excititor.Storage.Mongo.csproj +++ b/src/StellaOps.Excititor.Storage.Mongo/StellaOps.Excititor.Storage.Mongo.csproj @@ -1,18 +1,18 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Storage.Mongo/VexMongoMappingRegistry.cs b/src/StellaOps.Excititor.Storage.Mongo/VexMongoMappingRegistry.cs index 14e57bdd..ec085b93 100644 --- a/src/StellaOps.Excititor.Storage.Mongo/VexMongoMappingRegistry.cs +++ b/src/StellaOps.Excititor.Storage.Mongo/VexMongoMappingRegistry.cs @@ -43,8 +43,9 @@ public static class VexMongoMappingRegistry RegisterClassMap(); RegisterClassMap(); RegisterClassMap(); - RegisterClassMap(); - RegisterClassMap(); + RegisterClassMap(); + RegisterClassMap(); + RegisterClassMap(); } private static void RegisterClassMap() @@ -71,7 +72,8 @@ public static class VexMongoCollectionNames public const string Statements = "vex.statements"; public const string Claims = Statements; public const string Consensus = "vex.consensus"; - public const string Exports = "vex.exports"; - public const string Cache = "vex.cache"; - public const string ConnectorState = "vex.connector_state"; -} + public const string Exports = "vex.exports"; + public const string Cache = "vex.cache"; + public const string ConnectorState = "vex.connector_state"; + public const string ConsensusHolds = "vex.consensus_holds"; +} diff --git a/src/StellaOps.Excititor.Storage.Mongo/VexMongoModels.cs b/src/StellaOps.Excititor.Storage.Mongo/VexMongoModels.cs index a07c95f4..c3e12d48 100644 --- a/src/StellaOps.Excititor.Storage.Mongo/VexMongoModels.cs +++ b/src/StellaOps.Excititor.Storage.Mongo/VexMongoModels.cs @@ -329,8 +329,8 @@ internal sealed class VexCosignTrustDocument } [BsonIgnoreExtraElements] -internal sealed class VexConsensusRecord -{ +internal sealed class VexConsensusRecord +{ [BsonId] public string Id { get; set; } = default!; @@ -395,7 +395,115 @@ internal sealed class VexConsensusRecord Summary, PolicyRevisionId, PolicyDigest); -} +} + +[BsonIgnoreExtraElements] +internal sealed class VexConsensusHoldRecord +{ + [BsonId] + public string Id { get; set; } = default!; + + public string VulnerabilityId { get; set; } = default!; + + public string ProductKey { get; set; } = default!; + + public HeldConsensusDocument Candidate { get; set; } = default!; + + public DateTime RequestedAt { get; set; } + = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc); + + public DateTime EligibleAt { get; set; } + = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc); + + public string Reason { get; set; } = "unspecified"; + + public static VexConsensusHoldRecord FromDomain(VexConsensusHold hold) + => new() + { + Id = VexConsensusRecord.CreateId(hold.VulnerabilityId, hold.ProductKey), + VulnerabilityId = hold.VulnerabilityId, + ProductKey = hold.ProductKey, + Candidate = HeldConsensusDocument.FromDomain(hold.Candidate), + RequestedAt = hold.RequestedAt.UtcDateTime, + EligibleAt = hold.EligibleAt.UtcDateTime, + Reason = hold.Reason, + }; + + public VexConsensusHold ToDomain() + { + var requestedAt = DateTime.SpecifyKind(RequestedAt, DateTimeKind.Utc); + var eligibleAt = DateTime.SpecifyKind(EligibleAt, DateTimeKind.Utc); + return new VexConsensusHold( + VulnerabilityId, + ProductKey, + Candidate.ToDomain(), + new DateTimeOffset(requestedAt, TimeSpan.Zero), + new DateTimeOffset(eligibleAt, TimeSpan.Zero), + Reason); + } +} + +[BsonIgnoreExtraElements] +internal sealed class HeldConsensusDocument +{ + public string VulnerabilityId { get; set; } = default!; + + public VexProductDocument Product { get; set; } = default!; + + public string Status { get; set; } = default!; + + public DateTime CalculatedAt { get; set; } + = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc); + + public List Sources { get; set; } = new(); + + public List Conflicts { get; set; } = new(); + + public VexSignalDocument? Signals { get; set; } + = null; + + public string? PolicyVersion { get; set; } + = null; + + public string? PolicyRevisionId { get; set; } + = null; + + public string? PolicyDigest { get; set; } + = null; + + public string? Summary { get; set; } + = null; + + public static HeldConsensusDocument FromDomain(VexConsensus consensus) + => new() + { + VulnerabilityId = consensus.VulnerabilityId, + Product = VexProductDocument.FromDomain(consensus.Product), + Status = consensus.Status.ToString().ToLowerInvariant(), + CalculatedAt = consensus.CalculatedAt.UtcDateTime, + Sources = consensus.Sources.Select(VexConsensusSourceDocument.FromDomain).ToList(), + Conflicts = consensus.Conflicts.Select(VexConsensusConflictDocument.FromDomain).ToList(), + Signals = VexSignalDocument.FromDomain(consensus.Signals), + PolicyVersion = consensus.PolicyVersion, + PolicyRevisionId = consensus.PolicyRevisionId, + PolicyDigest = consensus.PolicyDigest, + Summary = consensus.Summary, + }; + + public VexConsensus ToDomain() + => new( + VulnerabilityId, + Product.ToDomain(), + Enum.Parse(Status, ignoreCase: true), + new DateTimeOffset(DateTime.SpecifyKind(CalculatedAt, DateTimeKind.Utc), TimeSpan.Zero), + Sources.Select(static source => source.ToDomain()), + Conflicts.Select(static conflict => conflict.ToDomain()), + Signals?.ToDomain(), + PolicyVersion, + Summary, + PolicyRevisionId, + PolicyDigest); +} [BsonIgnoreExtraElements] internal sealed class VexProductDocument diff --git a/src/StellaOps.Excititor.WebService.Tests/IngestEndpointsTests.cs b/src/StellaOps.Excititor.WebService.Tests/IngestEndpointsTests.cs new file mode 100644 index 00000000..eb91502b --- /dev/null +++ b/src/StellaOps.Excititor.WebService.Tests/IngestEndpointsTests.cs @@ -0,0 +1,274 @@ +using System.Collections.Immutable; +using System.IO; +using System.Security.Claims; +using System.Text.Json; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Http.HttpResults; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Excititor.WebService.Endpoints; +using StellaOps.Excititor.WebService.Services; + +namespace StellaOps.Excititor.WebService.Tests; + +public sealed class IngestEndpointsTests +{ + private readonly FakeIngestOrchestrator _orchestrator = new(); + private readonly TimeProvider _timeProvider = TimeProvider.System; + + [Fact] + public async Task InitEndpoint_ReturnsUnauthorized_WhenMissingToken() + { + var httpContext = CreateHttpContext(); + var request = new IngestEndpoints.ExcititorInitRequest(null, false); + + var result = await IngestEndpoints.HandleInitAsync(httpContext, request, _orchestrator, _timeProvider, CancellationToken.None); + Assert.IsType(result); + } + + [Fact] + public async Task InitEndpoint_ReturnsForbidden_WhenScopeMissing() + { + var httpContext = CreateHttpContext("vex.read"); + var request = new IngestEndpoints.ExcititorInitRequest(null, false); + + var result = await IngestEndpoints.HandleInitAsync(httpContext, request, _orchestrator, _timeProvider, CancellationToken.None); + Assert.IsType(result); + } + + [Fact] + public async Task InitEndpoint_NormalizesProviders_AndReturnsSummary() + { + var httpContext = CreateHttpContext("vex.admin"); + var request = new IngestEndpoints.ExcititorInitRequest(new[] { " suse ", "redhat", "REDHAT" }, true); + var started = DateTimeOffset.Parse("2025-10-20T12:00:00Z"); + var completed = started.AddMinutes(2); + _orchestrator.InitFactory = options => new InitSummary( + Guid.Parse("9a5eb53c-3118-4f78-991e-7d2c1af92a14"), + started, + completed, + ImmutableArray.Create( + new InitProviderResult("redhat", "Red Hat", "succeeded", TimeSpan.FromSeconds(12), null), + new InitProviderResult("suse", "SUSE", "failed", TimeSpan.FromSeconds(7), "unreachable"))); + + var result = await IngestEndpoints.HandleInitAsync(httpContext, request, _orchestrator, _timeProvider, CancellationToken.None); + var ok = Assert.IsType>(result); + Assert.Equal(new[] { "redhat", "suse" }, _orchestrator.LastInitOptions?.Providers); + Assert.True(_orchestrator.LastInitOptions?.Resume); + + using var document = JsonDocument.Parse(JsonSerializer.Serialize(ok.Value)); + Assert.Equal("Initialized 2 provider(s); 1 succeeded, 1 failed.", document.RootElement.GetProperty("message").GetString()); + } + + [Fact] + public async Task RunEndpoint_ReturnsBadRequest_WhenSinceInvalid() + { + var httpContext = CreateHttpContext("vex.admin"); + var request = new IngestEndpoints.ExcititorIngestRunRequest(new[] { "redhat" }, "not-a-date", null, false); + + var result = await IngestEndpoints.HandleRunAsync(httpContext, request, _orchestrator, _timeProvider, CancellationToken.None); + var bad = Assert.IsType>(result); + using var document = JsonDocument.Parse(JsonSerializer.Serialize(bad.Value)); + Assert.Contains("Invalid 'since'", document.RootElement.GetProperty("message").GetString()); + } + + [Fact] + public async Task RunEndpoint_ReturnsBadRequest_WhenWindowInvalid() + { + var httpContext = CreateHttpContext("vex.admin"); + var request = new IngestEndpoints.ExcititorIngestRunRequest(Array.Empty(), null, "-01:00:00", false); + + var result = await IngestEndpoints.HandleRunAsync(httpContext, request, _orchestrator, _timeProvider, CancellationToken.None); + var bad = Assert.IsType>(result); + using var document = JsonDocument.Parse(JsonSerializer.Serialize(bad.Value)); + Assert.Contains("Invalid duration", document.RootElement.GetProperty("message").GetString()); + } + + [Fact] + public async Task RunEndpoint_PassesOptionsToOrchestrator() + { + var httpContext = CreateHttpContext("vex.admin"); + var started = DateTimeOffset.Parse("2025-10-20T14:00:00Z"); + var completed = started.AddMinutes(5); + _orchestrator.RunFactory = options => new IngestRunSummary( + Guid.Parse("65bbfa25-82fd-41da-8b6b-9d8bb1e2bb5f"), + started, + completed, + ImmutableArray.Create( + new ProviderRunResult( + "redhat", + "succeeded", + 12, + 42, + started, + completed, + completed - started, + "sha256:abc", + completed.AddHours(-1), + "cp1", + null, + options.Since))); + + var request = new IngestEndpoints.ExcititorIngestRunRequest(new[] { "redhat" }, "2025-10-19T00:00:00Z", "1.00:00:00", true); + var result = await IngestEndpoints.HandleRunAsync(httpContext, request, _orchestrator, _timeProvider, CancellationToken.None); + var ok = Assert.IsType>(result); + + Assert.NotNull(_orchestrator.LastRunOptions); + Assert.Equal(new[] { "redhat" }, _orchestrator.LastRunOptions!.Providers); + Assert.True(_orchestrator.LastRunOptions.Force); + Assert.Equal(TimeSpan.FromDays(1), _orchestrator.LastRunOptions.Window); + + using var document = JsonDocument.Parse(JsonSerializer.Serialize(ok.Value)); + Assert.Equal("cp1", document.RootElement.GetProperty("providers")[0].GetProperty("checkpoint").GetString()); + } + + [Fact] + public async Task ResumeEndpoint_PassesCheckpointToOrchestrator() + { + var httpContext = CreateHttpContext("vex.admin"); + var started = DateTimeOffset.Parse("2025-10-20T16:00:00Z"); + var completed = started.AddMinutes(2); + _orchestrator.ResumeFactory = options => new IngestRunSummary( + Guid.Parse("88407f25-4b3f-434d-8f8e-1c7f4925c37b"), + started, + completed, + ImmutableArray.Create( + new ProviderRunResult( + "suse", + "succeeded", + 5, + 10, + started, + completed, + completed - started, + null, + null, + options.Checkpoint, + null, + DateTimeOffset.UtcNow.AddDays(-1)))); + + var request = new IngestEndpoints.ExcititorIngestResumeRequest(new[] { "suse" }, "resume-token"); + var result = await IngestEndpoints.HandleResumeAsync(httpContext, request, _orchestrator, _timeProvider, CancellationToken.None); + Assert.IsType>(result); + Assert.Equal("resume-token", _orchestrator.LastResumeOptions?.Checkpoint); + } + + [Fact] + public async Task ReconcileEndpoint_ReturnsBadRequest_WhenMaxAgeInvalid() + { + var httpContext = CreateHttpContext("vex.admin"); + var request = new IngestEndpoints.ExcititorReconcileRequest(Array.Empty(), "invalid"); + + var result = await IngestEndpoints.HandleReconcileAsync(httpContext, request, _orchestrator, _timeProvider, CancellationToken.None); + var bad = Assert.IsType>(result); + using var document = JsonDocument.Parse(JsonSerializer.Serialize(bad.Value)); + Assert.Contains("Invalid duration", document.RootElement.GetProperty("message").GetString()); + } + + [Fact] + public async Task ReconcileEndpoint_PassesOptionsAndReturnsSummary() + { + var httpContext = CreateHttpContext("vex.admin"); + var started = DateTimeOffset.Parse("2025-10-20T18:00:00Z"); + var completed = started.AddMinutes(4); + _orchestrator.ReconcileFactory = options => new ReconcileSummary( + Guid.Parse("a2c2cfe6-c21a-4a62-9db7-2ed2792f4e2d"), + started, + completed, + ImmutableArray.Create( + new ReconcileProviderResult( + "ubuntu", + "succeeded", + "reconciled", + started.AddDays(-2), + started - TimeSpan.FromDays(3), + 20, + 18, + null))); + + var request = new IngestEndpoints.ExcititorReconcileRequest(new[] { "ubuntu" }, "2.00:00:00"); + var result = await IngestEndpoints.HandleReconcileAsync(httpContext, request, _orchestrator, _timeProvider, CancellationToken.None); + var ok = Assert.IsType>(result); + + Assert.Equal(TimeSpan.FromDays(2), _orchestrator.LastReconcileOptions?.MaxAge); + using var document = JsonDocument.Parse(JsonSerializer.Serialize(ok.Value)); + Assert.Equal("reconciled", document.RootElement.GetProperty("providers")[0].GetProperty("action").GetString()); + } + + private static DefaultHttpContext CreateHttpContext(params string[] scopes) + { + var context = new DefaultHttpContext + { + RequestServices = new ServiceCollection().BuildServiceProvider(), + Response = { Body = new MemoryStream() } + }; + + if (scopes.Length > 0) + { + var claims = new List { new Claim(ClaimTypes.NameIdentifier, "test-user") }; + claims.Add(new Claim("scope", string.Join(' ', scopes))); + var identity = new ClaimsIdentity(claims, "Test"); + context.User = new ClaimsPrincipal(identity); + } + else + { + context.User = new ClaimsPrincipal(new ClaimsIdentity()); + } + + return context; + } + + private sealed class FakeIngestOrchestrator : IVexIngestOrchestrator + { + public IngestInitOptions? LastInitOptions { get; private set; } + public IngestRunOptions? LastRunOptions { get; private set; } + public IngestResumeOptions? LastResumeOptions { get; private set; } + public ReconcileOptions? LastReconcileOptions { get; private set; } + + public Func? InitFactory { get; set; } + public Func? RunFactory { get; set; } + public Func? ResumeFactory { get; set; } + public Func? ReconcileFactory { get; set; } + + public Task InitializeAsync(IngestInitOptions options, CancellationToken cancellationToken) + { + LastInitOptions = options; + return Task.FromResult(InitFactory is null ? CreateDefaultInitSummary() : InitFactory(options)); + } + + public Task RunAsync(IngestRunOptions options, CancellationToken cancellationToken) + { + LastRunOptions = options; + return Task.FromResult(RunFactory is null ? CreateDefaultRunSummary() : RunFactory(options)); + } + + public Task ResumeAsync(IngestResumeOptions options, CancellationToken cancellationToken) + { + LastResumeOptions = options; + return Task.FromResult(ResumeFactory is null ? CreateDefaultRunSummary() : ResumeFactory(options)); + } + + public Task ReconcileAsync(ReconcileOptions options, CancellationToken cancellationToken) + { + LastReconcileOptions = options; + return Task.FromResult(ReconcileFactory is null ? CreateDefaultReconcileSummary() : ReconcileFactory(options)); + } + + private static InitSummary CreateDefaultInitSummary() + { + var now = DateTimeOffset.UtcNow; + return new InitSummary(Guid.Empty, now, now, ImmutableArray.Empty); + } + + private static IngestRunSummary CreateDefaultRunSummary() + { + var now = DateTimeOffset.UtcNow; + return new IngestRunSummary(Guid.Empty, now, now, ImmutableArray.Empty); + } + + private static ReconcileSummary CreateDefaultReconcileSummary() + { + var now = DateTimeOffset.UtcNow; + return new ReconcileSummary(Guid.Empty, now, now, ImmutableArray.Empty); + } + } +} diff --git a/src/StellaOps.Excititor.WebService.Tests/MirrorEndpointsTests.cs b/src/StellaOps.Excititor.WebService.Tests/MirrorEndpointsTests.cs index ca91090f..367e75be 100644 --- a/src/StellaOps.Excititor.WebService.Tests/MirrorEndpointsTests.cs +++ b/src/StellaOps.Excititor.WebService.Tests/MirrorEndpointsTests.cs @@ -1,225 +1,213 @@ -using System.Collections.Concurrent; -using System.Collections.Immutable; -using System.Net; -using System.Net.Http.Json; -using System.Text.Json; -using Microsoft.AspNetCore.Mvc.Testing; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; -using MongoDB.Driver; -using StellaOps.Excititor.Core; -using StellaOps.Excititor.Connectors.Abstractions; -using StellaOps.Excititor.Export; -using StellaOps.Excititor.Policy; -using StellaOps.Excititor.Storage.Mongo; -using StellaOps.Excititor.WebService.Options; - -namespace StellaOps.Excititor.WebService.Tests; - -public sealed class MirrorEndpointsTests : IClassFixture>, IDisposable -{ - private readonly WebApplicationFactory _factory; - private readonly Mongo2Go.MongoDbRunner _runner; - - public MirrorEndpointsTests(WebApplicationFactory factory) - { - _runner = Mongo2Go.MongoDbRunner.Start(); - _factory = factory.WithWebHostBuilder(builder => - { - builder.ConfigureAppConfiguration((_, configuration) => - { - var data = new Dictionary - { - [$"{MirrorDistributionOptions.SectionName}:Domains:0:Id"] = "primary", - [$"{MirrorDistributionOptions.SectionName}:Domains:0:DisplayName"] = "Primary Mirror", - [$"{MirrorDistributionOptions.SectionName}:Domains:0:MaxIndexRequestsPerHour"] = "1000", - [$"{MirrorDistributionOptions.SectionName}:Domains:0:MaxDownloadRequestsPerHour"] = "1000", - [$"{MirrorDistributionOptions.SectionName}:Domains:0:Exports:0:Key"] = "consensus", - [$"{MirrorDistributionOptions.SectionName}:Domains:0:Exports:0:Format"] = "json", - [$"{MirrorDistributionOptions.SectionName}:Domains:0:Exports:0:Filters:vulnId"] = "CVE-2025-0001", - [$"{MirrorDistributionOptions.SectionName}:Domains:0:Exports:0:Filters:productKey"] = "pkg:test/demo", - }; - - configuration.AddInMemoryCollection(data!); - }); - - builder.ConfigureServices(services => - { - services.RemoveAll(); - services.AddSingleton(_ => new MongoClient(_runner.ConnectionString)); - services.RemoveAll(); - services.AddSingleton(provider => provider.GetRequiredService().GetDatabase("mirror-tests")); - - services.RemoveAll(); - services.AddSingleton(provider => - { - var timeProvider = provider.GetRequiredService(); - return new FakeExportStore(timeProvider); - }); - - services.RemoveAll(); - services.AddSingleton(_ => new FakeArtifactStore()); - services.AddSingleton(new VexConnectorDescriptor("excititor:redhat", VexProviderKind.Distro, "Red Hat CSAF")); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - }); - }); - } - - [Fact] - public async Task ListDomains_ReturnsConfiguredDomain() - { - var client = _factory.CreateClient(); - var response = await client.GetAsync("/excititor/mirror/domains"); - response.EnsureSuccessStatusCode(); - - using var document = JsonDocument.Parse(await response.Content.ReadAsStringAsync()); - var domains = document.RootElement.GetProperty("domains"); - Assert.Equal(1, domains.GetArrayLength()); - Assert.Equal("primary", domains[0].GetProperty("id").GetString()); - } - - [Fact] - public async Task DomainIndex_ReturnsManifestMetadata() - { - var client = _factory.CreateClient(); - var response = await client.GetAsync("/excititor/mirror/domains/primary/index"); - response.EnsureSuccessStatusCode(); - - using var document = JsonDocument.Parse(await response.Content.ReadAsStringAsync()); - var exports = document.RootElement.GetProperty("exports"); - Assert.Equal(1, exports.GetArrayLength()); - var entry = exports[0]; - Assert.Equal("consensus", entry.GetProperty("exportKey").GetString()); - Assert.Equal("exports/20251019T000000000Z/abcdef", entry.GetProperty("exportId").GetString()); - var artifact = entry.GetProperty("artifact"); - Assert.Equal("sha256", artifact.GetProperty("algorithm").GetString()); - Assert.Equal("deadbeef", artifact.GetProperty("digest").GetString()); - } - - [Fact] - public async Task Download_ReturnsArtifactContent() - { - var client = _factory.CreateClient(); - var response = await client.GetAsync("/excititor/mirror/domains/primary/exports/consensus/download"); - response.EnsureSuccessStatusCode(); - Assert.Equal("application/json", response.Content.Headers.ContentType?.MediaType); - var payload = await response.Content.ReadAsStringAsync(); - Assert.Equal("{\"status\":\"ok\"}", payload); - } - - public void Dispose() - { - _runner.Dispose(); - } - - private sealed class FakeExportStore : IVexExportStore - { - private readonly ConcurrentDictionary<(string Signature, VexExportFormat Format), VexExportManifest> _manifests = new(); - - public FakeExportStore(TimeProvider timeProvider) - { - var filters = new[] - { - new VexQueryFilter("vulnId", "CVE-2025-0001"), - new VexQueryFilter("productKey", "pkg:test/demo"), - }; - - var query = VexQuery.Create(filters, Enumerable.Empty()); - var signature = VexQuerySignature.FromQuery(query); - var createdAt = new DateTimeOffset(2025, 10, 19, 0, 0, 0, TimeSpan.Zero); - - var manifest = new VexExportManifest( - "exports/20251019T000000000Z/abcdef", - signature, - VexExportFormat.Json, - createdAt, - new VexContentAddress("sha256", "deadbeef"), - 1, - new[] { "primary" }, - fromCache: false, - consensusRevision: "rev-1", - attestation: new VexAttestationMetadata("https://stella-ops.org/attestations/vex-export"), - sizeBytes: 16); - - _manifests.TryAdd((signature.Value, VexExportFormat.Json), manifest); - - // Seed artifact content for download test. - FakeArtifactStore.Seed(manifest.Artifact, "{\"status\":\"ok\"}"); - } - - public ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null) - { - _manifests.TryGetValue((signature.Value, format), out var manifest); - return ValueTask.FromResult(manifest); - } - - public ValueTask SaveAsync(VexExportManifest manifest, CancellationToken cancellationToken, IClientSessionHandle? session = null) - => ValueTask.CompletedTask; - } - - private sealed class FakeArtifactStore : IVexArtifactStore - { - private static readonly ConcurrentDictionary Content = new(); - - public static void Seed(VexContentAddress contentAddress, string payload) - { - var bytes = System.Text.Encoding.UTF8.GetBytes(payload); - Content[contentAddress] = bytes; - } - - public ValueTask SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken) - { - Content[artifact.ContentAddress] = artifact.Content.ToArray(); - return ValueTask.FromResult(new VexStoredArtifact(artifact.ContentAddress, "memory://artifact", artifact.Content.Length, artifact.Metadata)); - } - - public ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) - { - Content.TryRemove(contentAddress, out _); - return ValueTask.CompletedTask; - } - - public ValueTask OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) - { - if (!Content.TryGetValue(contentAddress, out var bytes)) - { - return ValueTask.FromResult(null); - } - - return ValueTask.FromResult(new MemoryStream(bytes, writable: false)); - } - } - - private sealed class FakeSigner : StellaOps.Excititor.Attestation.Signing.IVexSigner - { - public ValueTask SignAsync(ReadOnlyMemory payload, CancellationToken cancellationToken) - => ValueTask.FromResult(new StellaOps.Excititor.Attestation.Signing.VexSignedPayload("signature", "key")); - } - - private sealed class FakePolicyEvaluator : StellaOps.Excititor.Policy.IVexPolicyEvaluator - { - public string Version => "test"; - - public VexPolicySnapshot Snapshot => VexPolicySnapshot.Default; - - public double GetProviderWeight(VexProvider provider) => 1.0; - - public bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason) - { - rejectionReason = null; - return true; - } - } - - private sealed class FakeExportDataSource : IVexExportDataSource - { - public ValueTask FetchAsync(VexQuery query, CancellationToken cancellationToken) - { - var dataset = new VexExportDataSet(ImmutableArray.Empty, ImmutableArray.Empty, ImmutableArray.Empty); - return ValueTask.FromResult(dataset); - } - } -} +using System.Collections.Concurrent; +using System.Collections.Immutable; +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using EphemeralMongo; +using MongoRunner = EphemeralMongo.MongoRunner; +using MongoRunnerOptions = EphemeralMongo.MongoRunnerOptions; +using MongoDB.Driver; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Export; +using StellaOps.Excititor.Policy; +using StellaOps.Excititor.Storage.Mongo; +using StellaOps.Excititor.WebService.Options; + +namespace StellaOps.Excititor.WebService.Tests; + +public sealed class MirrorEndpointsTests : IDisposable +{ + private readonly TestWebApplicationFactory _factory; + private readonly IMongoRunner _runner; + + public MirrorEndpointsTests() + { + _runner = MongoRunner.Run(new MongoRunnerOptions { UseSingleNodeReplicaSet = true }); + _factory = new TestWebApplicationFactory( + configureConfiguration: configuration => + { + var data = new Dictionary + { + ["Excititor:Storage:Mongo:ConnectionString"] = _runner.ConnectionString, + ["Excititor:Storage:Mongo:DatabaseName"] = "mirror-tests", + [$"{MirrorDistributionOptions.SectionName}:Domains:0:Id"] = "primary", + [$"{MirrorDistributionOptions.SectionName}:Domains:0:DisplayName"] = "Primary Mirror", + [$"{MirrorDistributionOptions.SectionName}:Domains:0:MaxIndexRequestsPerHour"] = "1000", + [$"{MirrorDistributionOptions.SectionName}:Domains:0:MaxDownloadRequestsPerHour"] = "1000", + [$"{MirrorDistributionOptions.SectionName}:Domains:0:Exports:0:Key"] = "consensus", + [$"{MirrorDistributionOptions.SectionName}:Domains:0:Exports:0:Format"] = "json", + [$"{MirrorDistributionOptions.SectionName}:Domains:0:Exports:0:Filters:vulnId"] = "CVE-2025-0001", + [$"{MirrorDistributionOptions.SectionName}:Domains:0:Exports:0:Filters:productKey"] = "pkg:test/demo", + }; + + configuration.AddInMemoryCollection(data!); + }, + configureServices: services => + { + TestServiceOverrides.Apply(services); + services.RemoveAll(); + services.AddSingleton(provider => + { + var timeProvider = provider.GetRequiredService(); + return new FakeExportStore(timeProvider); + }); + services.RemoveAll(); + services.AddSingleton(_ => new FakeArtifactStore()); + services.AddSingleton(new VexConnectorDescriptor("excititor:redhat", VexProviderKind.Distro, "Red Hat CSAF")); + services.AddSingleton(); + services.AddSingleton(); + }); + } + + [Fact] + public async Task ListDomains_ReturnsConfiguredDomain() + { + var client = _factory.CreateClient(); + var response = await client.GetAsync("/excititor/mirror/domains"); + response.EnsureSuccessStatusCode(); + + using var document = JsonDocument.Parse(await response.Content.ReadAsStringAsync()); + var domains = document.RootElement.GetProperty("domains"); + Assert.Equal(1, domains.GetArrayLength()); + Assert.Equal("primary", domains[0].GetProperty("id").GetString()); + } + + [Fact] + public async Task DomainIndex_ReturnsManifestMetadata() + { + var client = _factory.CreateClient(); + var response = await client.GetAsync("/excititor/mirror/domains/primary/index"); + response.EnsureSuccessStatusCode(); + + using var document = JsonDocument.Parse(await response.Content.ReadAsStringAsync()); + var exports = document.RootElement.GetProperty("exports"); + Assert.Equal(1, exports.GetArrayLength()); + var entry = exports[0]; + Assert.Equal("consensus", entry.GetProperty("exportKey").GetString()); + Assert.Equal("exports/20251019T000000000Z/abcdef", entry.GetProperty("exportId").GetString()); + var artifact = entry.GetProperty("artifact"); + Assert.Equal("sha256", artifact.GetProperty("algorithm").GetString()); + Assert.Equal("deadbeef", artifact.GetProperty("digest").GetString()); + } + + [Fact] + public async Task Download_ReturnsArtifactContent() + { + var client = _factory.CreateClient(); + var response = await client.GetAsync("/excititor/mirror/domains/primary/exports/consensus/download"); + response.EnsureSuccessStatusCode(); + Assert.Equal("application/json", response.Content.Headers.ContentType?.MediaType); + var payload = await response.Content.ReadAsStringAsync(); + Assert.Equal("{\"status\":\"ok\"}", payload); + } + + public void Dispose() + { + _factory.Dispose(); + _runner.Dispose(); + } + + private sealed class FakeExportStore : IVexExportStore + { + private readonly ConcurrentDictionary<(string Signature, VexExportFormat Format), VexExportManifest> _manifests = new(); + + public FakeExportStore(TimeProvider timeProvider) + { + var filters = new[] + { + new VexQueryFilter("vulnId", "CVE-2025-0001"), + new VexQueryFilter("productKey", "pkg:test/demo"), + }; + + var query = VexQuery.Create(filters, Enumerable.Empty()); + var signature = VexQuerySignature.FromQuery(query); + var createdAt = new DateTimeOffset(2025, 10, 19, 0, 0, 0, TimeSpan.Zero); + + var manifest = new VexExportManifest( + "exports/20251019T000000000Z/abcdef", + signature, + VexExportFormat.Json, + createdAt, + new VexContentAddress("sha256", "deadbeef"), + 1, + new[] { "primary" }, + fromCache: false, + consensusRevision: "rev-1", + attestation: new VexAttestationMetadata("https://stella-ops.org/attestations/vex-export"), + sizeBytes: 16); + + _manifests.TryAdd((signature.Value, VexExportFormat.Json), manifest); + + // Seed artifact content for download test. + FakeArtifactStore.Seed(manifest.Artifact, "{\"status\":\"ok\"}"); + } + + public ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _manifests.TryGetValue((signature.Value, format), out var manifest); + return ValueTask.FromResult(manifest); + } + + public ValueTask SaveAsync(VexExportManifest manifest, CancellationToken cancellationToken, IClientSessionHandle? session = null) + => ValueTask.CompletedTask; + } + + private sealed class FakeArtifactStore : IVexArtifactStore + { + private static readonly ConcurrentDictionary Content = new(); + + public static void Seed(VexContentAddress contentAddress, string payload) + { + var bytes = System.Text.Encoding.UTF8.GetBytes(payload); + Content[contentAddress] = bytes; + } + + public ValueTask SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken) + { + Content[artifact.ContentAddress] = artifact.Content.ToArray(); + return ValueTask.FromResult(new VexStoredArtifact(artifact.ContentAddress, "memory://artifact", artifact.Content.Length, artifact.Metadata)); + } + + public ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) + { + Content.TryRemove(contentAddress, out _); + return ValueTask.CompletedTask; + } + + public ValueTask OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) + { + if (!Content.TryGetValue(contentAddress, out var bytes)) + { + return ValueTask.FromResult(null); + } + + return ValueTask.FromResult(new MemoryStream(bytes, writable: false)); + } + } + + private sealed class FakeSigner : StellaOps.Excititor.Attestation.Signing.IVexSigner + { + public ValueTask SignAsync(ReadOnlyMemory payload, CancellationToken cancellationToken) + => ValueTask.FromResult(new StellaOps.Excititor.Attestation.Signing.VexSignedPayload("signature", "key")); + } + + private sealed class FakePolicyEvaluator : StellaOps.Excititor.Policy.IVexPolicyEvaluator + { + public string Version => "test"; + + public VexPolicySnapshot Snapshot => VexPolicySnapshot.Default; + + public double GetProviderWeight(VexProvider provider) => 1.0; + + public bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason) + { + rejectionReason = null; + return true; + } + } + +} diff --git a/src/StellaOps.Excititor.WebService.Tests/ResolveEndpointTests.cs b/src/StellaOps.Excititor.WebService.Tests/ResolveEndpointTests.cs index ca0e44d9..35c5d415 100644 --- a/src/StellaOps.Excititor.WebService.Tests/ResolveEndpointTests.cs +++ b/src/StellaOps.Excititor.WebService.Tests/ResolveEndpointTests.cs @@ -1,342 +1,375 @@ -using System.Collections.Immutable; -using System.Net; -using System.Net.Http.Json; -using Microsoft.AspNetCore.Mvc.Testing; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Mongo2Go; -using MongoDB.Driver; -using StellaOps.Excititor.Attestation.Signing; -using StellaOps.Excititor.Connectors.Abstractions; -using StellaOps.Excititor.Core; -using StellaOps.Excititor.Export; -using StellaOps.Excititor.Policy; -using StellaOps.Excititor.Storage.Mongo; - -namespace StellaOps.Excititor.WebService.Tests; - -public sealed class ResolveEndpointTests : IClassFixture>, IDisposable -{ - private readonly WebApplicationFactory _factory; - private readonly MongoDbRunner _runner; - - public ResolveEndpointTests(WebApplicationFactory factory) - { - _runner = MongoDbRunner.Start(); - _factory = factory.WithWebHostBuilder(builder => - { - builder.ConfigureAppConfiguration((_, config) => - { - var rootPath = Path.Combine(Path.GetTempPath(), "excititor-resolve-tests"); - Directory.CreateDirectory(rootPath); - var settings = new Dictionary - { - ["Excititor:Storage:Mongo:RawBucketName"] = "vex.raw", - ["Excititor:Storage:Mongo:GridFsInlineThresholdBytes"] = "256", - ["Excititor:Artifacts:FileSystem:RootPath"] = rootPath, - }; - config.AddInMemoryCollection(settings!); - }); - - builder.ConfigureServices(services => - { - services.AddSingleton(_ => new MongoClient(_runner.ConnectionString)); - services.AddSingleton(provider => provider.GetRequiredService().GetDatabase("excititor-resolve-tests")); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(new VexConnectorDescriptor("excititor:redhat", VexProviderKind.Distro, "Red Hat CSAF")); - }); - }); - } - - [Fact] - public async Task ResolveEndpoint_ReturnsBadRequest_WhenInputsMissing() - { - var client = _factory.CreateClient(); - var response = await client.PostAsJsonAsync("/excititor/resolve", new { vulnerabilityIds = new[] { "CVE-2025-0001" } }); - Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); - } - - [Fact] - public async Task ResolveEndpoint_ComputesConsensusAndAttestation() - { - const string vulnerabilityId = "CVE-2025-2222"; - const string productKey = "pkg:nuget/StellaOps.Demo@1.0.0"; - const string providerId = "redhat"; - - await SeedProviderAsync(providerId); - await SeedClaimAsync(vulnerabilityId, productKey, providerId); - - var client = _factory.CreateClient(); - var request = new ResolveRequest( - new[] { productKey }, - null, - new[] { vulnerabilityId }, - null); - - var response = await client.PostAsJsonAsync("/excititor/resolve", request); - response.EnsureSuccessStatusCode(); - - var payload = await response.Content.ReadFromJsonAsync(); - Assert.NotNull(payload); - Assert.NotNull(payload!.Policy); - - var result = Assert.Single(payload.Results); - Assert.Equal(vulnerabilityId, result.VulnerabilityId); - Assert.Equal(productKey, result.ProductKey); - Assert.Equal("not_affected", result.Status); - Assert.NotNull(result.Envelope); - Assert.Equal("signature", result.Envelope!.ContentSignature!.Value); - Assert.Equal("key", result.Envelope.ContentSignature.KeyId); - Assert.NotEqual(default, result.CalculatedAt); - - Assert.NotNull(result.Signals); - Assert.True(result.Signals!.Kev); - Assert.NotNull(result.Envelope.AttestationSignature); - Assert.False(string.IsNullOrWhiteSpace(result.Envelope.AttestationEnvelope)); - Assert.Equal(payload.Policy.ActiveRevisionId, result.PolicyRevisionId); - Assert.Equal(payload.Policy.Version, result.PolicyVersion); - Assert.Equal(payload.Policy.Digest, result.PolicyDigest); - - var decision = Assert.Single(result.Decisions); - Assert.True(decision.Included); - Assert.Equal(providerId, decision.ProviderId); - } - - [Fact] - public async Task ResolveEndpoint_ReturnsConflict_WhenPolicyRevisionMismatch() - { - const string vulnerabilityId = "CVE-2025-3333"; - const string productKey = "pkg:docker/demo@sha256:abcd"; - - var client = _factory.CreateClient(); - var request = new ResolveRequest( - new[] { productKey }, - null, - new[] { vulnerabilityId }, - "rev-0"); - - var response = await client.PostAsJsonAsync("/excititor/resolve", request); - Assert.Equal(HttpStatusCode.Conflict, response.StatusCode); - } - - private async Task SeedProviderAsync(string providerId) - { - await using var scope = _factory.Services.CreateAsyncScope(); - var store = scope.ServiceProvider.GetRequiredService(); - var provider = new VexProvider(providerId, "Red Hat", VexProviderKind.Distro); - await store.SaveAsync(provider, CancellationToken.None); - } - - private async Task SeedClaimAsync(string vulnerabilityId, string productKey, string providerId) - { - await using var scope = _factory.Services.CreateAsyncScope(); - var store = scope.ServiceProvider.GetRequiredService(); - var timeProvider = scope.ServiceProvider.GetRequiredService(); - var observedAt = timeProvider.GetUtcNow(); - - var claim = new VexClaim( - vulnerabilityId, - providerId, - new VexProduct(productKey, "Demo Component", version: "1.0.0", purl: productKey), - VexClaimStatus.NotAffected, - new VexClaimDocument(VexDocumentFormat.Csaf, "sha256:deadbeef", new Uri("https://example.org/vex/csaf.json")), - observedAt.AddDays(-1), - observedAt, - VexJustification.ProtectedByMitigatingControl, - detail: "Test justification", - confidence: new VexConfidence("high", 0.9, "unit-test"), - signals: new VexSignalSnapshot( - new VexSeveritySignal("cvss:v3.1", 5.5, "medium"), - kev: true, - epss: 0.25)); - - await store.AppendAsync(new[] { claim }, observedAt, CancellationToken.None); - } - - public void Dispose() - { - _runner.Dispose(); - } - - private sealed class ResolveRequest - { - public ResolveRequest( - IReadOnlyList? productKeys, - IReadOnlyList? purls, - IReadOnlyList? vulnerabilityIds, - string? policyRevisionId) - { - ProductKeys = productKeys; - Purls = purls; - VulnerabilityIds = vulnerabilityIds; - PolicyRevisionId = policyRevisionId; - } - - public IReadOnlyList? ProductKeys { get; } - - public IReadOnlyList? Purls { get; } - - public IReadOnlyList? VulnerabilityIds { get; } - - public string? PolicyRevisionId { get; } - } - - private sealed class ResolveResponse - { - public required DateTimeOffset ResolvedAt { get; init; } - - public required ResolvePolicy Policy { get; init; } - - public required List Results { get; init; } - } - - private sealed class ResolvePolicy - { - public required string ActiveRevisionId { get; init; } - - public required string Version { get; init; } - - public required string Digest { get; init; } - - public string? RequestedRevisionId { get; init; } - } - - private sealed class ResolveResult - { - public required string VulnerabilityId { get; init; } - - public required string ProductKey { get; init; } - - public required string Status { get; init; } - - public required DateTimeOffset CalculatedAt { get; init; } - - public required List Sources { get; init; } - - public required List Conflicts { get; init; } - - public ResolveSignals? Signals { get; init; } - - public string? Summary { get; init; } - - public required string PolicyRevisionId { get; init; } - - public required string PolicyVersion { get; init; } - - public required string PolicyDigest { get; init; } - - public required List Decisions { get; init; } - - public ResolveEnvelope? Envelope { get; init; } - } - - private sealed class ResolveSource - { - public required string ProviderId { get; init; } - } - - private sealed class ResolveConflict - { - public string? ProviderId { get; init; } - } - - private sealed class ResolveSignals - { - public ResolveSeverity? Severity { get; init; } - - public bool? Kev { get; init; } - - public double? Epss { get; init; } - } - - private sealed class ResolveSeverity - { - public string? Scheme { get; init; } - - public double? Score { get; init; } - } - - private sealed class ResolveDecision - { - public required string ProviderId { get; init; } - - public required bool Included { get; init; } - - public string? Reason { get; init; } - } - - private sealed class ResolveEnvelope - { - public required ResolveArtifact Artifact { get; init; } - - public ResolveSignature? ContentSignature { get; init; } - - public ResolveAttestationMetadata? Attestation { get; init; } - - public string? AttestationEnvelope { get; init; } - - public ResolveSignature? AttestationSignature { get; init; } - } - - private sealed class ResolveArtifact - { - public required string Algorithm { get; init; } - - public required string Digest { get; init; } - } - - private sealed class ResolveSignature - { - public required string Value { get; init; } - - public string? KeyId { get; init; } - } - - private sealed class ResolveAttestationMetadata - { - public required string PredicateType { get; init; } - - public ResolveRekorReference? Rekor { get; init; } - - public string? EnvelopeDigest { get; init; } - - public DateTimeOffset? SignedAt { get; init; } - } - - private sealed class ResolveRekorReference - { - public string? Location { get; init; } - } - - private sealed class FakeSigner : IVexSigner - { - public ValueTask SignAsync(ReadOnlyMemory payload, CancellationToken cancellationToken) - => ValueTask.FromResult(new VexSignedPayload("signature", "key")); - } - - private sealed class FakePolicyEvaluator : IVexPolicyEvaluator - { - public string Version => "test"; - - public VexPolicySnapshot Snapshot => VexPolicySnapshot.Default; - - public double GetProviderWeight(VexProvider provider) => 1.0; - - public bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason) - { - rejectionReason = null; - return true; - } - } - - private sealed class FakeExportDataSource : IVexExportDataSource - { - public ValueTask FetchAsync(VexQuery query, CancellationToken cancellationToken) - { - var dataset = new VexExportDataSet(ImmutableArray.Empty, ImmutableArray.Empty, ImmutableArray.Empty); - return ValueTask.FromResult(dataset); - } - } -} +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Net; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using EphemeralMongo; +using MongoRunner = EphemeralMongo.MongoRunner; +using MongoRunnerOptions = EphemeralMongo.MongoRunnerOptions; +using StellaOps.Excititor.Attestation.Signing; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Export; +using StellaOps.Excititor.Policy; +using StellaOps.Excititor.Storage.Mongo; + +namespace StellaOps.Excititor.WebService.Tests; + +public sealed class ResolveEndpointTests : IDisposable +{ + private readonly TestWebApplicationFactory _factory; + private readonly IMongoRunner _runner; + + public ResolveEndpointTests() + { + _runner = MongoRunner.Run(new MongoRunnerOptions { UseSingleNodeReplicaSet = true }); + + _factory = new TestWebApplicationFactory( + configureConfiguration: config => + { + var rootPath = Path.Combine(Path.GetTempPath(), "excititor-resolve-tests"); + Directory.CreateDirectory(rootPath); + var settings = new Dictionary + { + ["Excititor:Storage:Mongo:ConnectionString"] = _runner.ConnectionString, + ["Excititor:Storage:Mongo:DatabaseName"] = "excititor-resolve-tests", + ["Excititor:Storage:Mongo:RawBucketName"] = "vex.raw", + ["Excititor:Storage:Mongo:GridFsInlineThresholdBytes"] = "256", + ["Excititor:Artifacts:FileSystem:RootPath"] = rootPath, + }; + config.AddInMemoryCollection(settings!); + }, + configureServices: services => + { + services.AddTestAuthentication(); + TestServiceOverrides.Apply(services); + services.AddSingleton(); + services.AddSingleton(); + }); + } + + [Fact] + public async Task ResolveEndpoint_ReturnsBadRequest_WhenInputsMissing() + { + var client = CreateClient("vex.read"); + var response = await client.PostAsJsonAsync("/excititor/resolve", new { vulnerabilityIds = new[] { "CVE-2025-0001" } }); + Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); + } + + [Fact] + public async Task ResolveEndpoint_ComputesConsensusAndAttestation() + { + const string vulnerabilityId = "CVE-2025-2222"; + const string productKey = "pkg:nuget/StellaOps.Demo@1.0.0"; + const string providerId = "redhat"; + + await SeedProviderAsync(providerId); + await SeedClaimAsync(vulnerabilityId, productKey, providerId); + + var client = CreateClient("vex.read"); + var request = new ResolveRequest( + new[] { productKey }, + null, + new[] { vulnerabilityId }, + null); + + var response = await client.PostAsJsonAsync("/excititor/resolve", request); + response.EnsureSuccessStatusCode(); + + var payload = await response.Content.ReadFromJsonAsync(); + Assert.NotNull(payload); + Assert.NotNull(payload!.Policy); + + var result = Assert.Single(payload.Results); + Assert.Equal(vulnerabilityId, result.VulnerabilityId); + Assert.Equal(productKey, result.ProductKey); + Assert.Equal("not_affected", result.Status); + Assert.NotNull(result.Envelope); + Assert.Equal("signature", result.Envelope!.ContentSignature!.Value); + Assert.Equal("key", result.Envelope.ContentSignature.KeyId); + Assert.NotEqual(default, result.CalculatedAt); + + Assert.NotNull(result.Signals); + Assert.True(result.Signals!.Kev); + Assert.NotNull(result.Envelope.AttestationSignature); + Assert.False(string.IsNullOrWhiteSpace(result.Envelope.AttestationEnvelope)); + Assert.Equal(payload.Policy.ActiveRevisionId, result.PolicyRevisionId); + Assert.Equal(payload.Policy.Version, result.PolicyVersion); + Assert.Equal(payload.Policy.Digest, result.PolicyDigest); + + var decision = Assert.Single(result.Decisions); + Assert.True(decision.Included); + Assert.Equal(providerId, decision.ProviderId); + } + + [Fact] + public async Task ResolveEndpoint_ReturnsConflict_WhenPolicyRevisionMismatch() + { + const string vulnerabilityId = "CVE-2025-3333"; + const string productKey = "pkg:docker/demo@sha256:abcd"; + + var client = CreateClient("vex.read"); + var request = new ResolveRequest( + new[] { productKey }, + null, + new[] { vulnerabilityId }, + "rev-0"); + + var response = await client.PostAsJsonAsync("/excititor/resolve", request); + Assert.Equal(HttpStatusCode.Conflict, response.StatusCode); + } + + [Fact] + public async Task ResolveEndpoint_ReturnsUnauthorized_WhenMissingToken() + { + var client = CreateClient(); + var request = new ResolveRequest( + new[] { "pkg:test/demo" }, + null, + new[] { "CVE-2025-0001" }, + null); + + var response = await client.PostAsJsonAsync("/excititor/resolve", request); + Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode); + } + + [Fact] + public async Task ResolveEndpoint_ReturnsForbidden_WhenScopeMissing() + { + var client = CreateClient("vex.admin"); + var request = new ResolveRequest( + new[] { "pkg:test/demo" }, + null, + new[] { "CVE-2025-0001" }, + null); + + var response = await client.PostAsJsonAsync("/excititor/resolve", request); + Assert.Equal(HttpStatusCode.Forbidden, response.StatusCode); + } + + private async Task SeedProviderAsync(string providerId) + { + await using var scope = _factory.Services.CreateAsyncScope(); + var store = scope.ServiceProvider.GetRequiredService(); + var provider = new VexProvider(providerId, "Red Hat", VexProviderKind.Distro); + await store.SaveAsync(provider, CancellationToken.None); + } + + private async Task SeedClaimAsync(string vulnerabilityId, string productKey, string providerId) + { + await using var scope = _factory.Services.CreateAsyncScope(); + var store = scope.ServiceProvider.GetRequiredService(); + var timeProvider = scope.ServiceProvider.GetRequiredService(); + var observedAt = timeProvider.GetUtcNow(); + + var claim = new VexClaim( + vulnerabilityId, + providerId, + new VexProduct(productKey, "Demo Component", version: "1.0.0", purl: productKey), + VexClaimStatus.NotAffected, + new VexClaimDocument(VexDocumentFormat.Csaf, "sha256:deadbeef", new Uri("https://example.org/vex/csaf.json")), + observedAt.AddDays(-1), + observedAt, + VexJustification.ProtectedByMitigatingControl, + detail: "Test justification", + confidence: new VexConfidence("high", 0.9, "unit-test"), + signals: new VexSignalSnapshot( + new VexSeveritySignal("cvss:v3.1", 5.5, "medium"), + kev: true, + epss: 0.25)); + + await store.AppendAsync(new[] { claim }, observedAt, CancellationToken.None); + } + + private HttpClient CreateClient(params string[] scopes) + { + var client = _factory.CreateClient(); + if (scopes.Length > 0) + { + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", string.Join(' ', scopes)); + } + + return client; + } + + public void Dispose() + { + _factory.Dispose(); + _runner.Dispose(); + } + + private sealed class ResolveRequest + { + public ResolveRequest( + IReadOnlyList? productKeys, + IReadOnlyList? purls, + IReadOnlyList? vulnerabilityIds, + string? policyRevisionId) + { + ProductKeys = productKeys; + Purls = purls; + VulnerabilityIds = vulnerabilityIds; + PolicyRevisionId = policyRevisionId; + } + + public IReadOnlyList? ProductKeys { get; } + + public IReadOnlyList? Purls { get; } + + public IReadOnlyList? VulnerabilityIds { get; } + + public string? PolicyRevisionId { get; } + } + + private sealed class ResolveResponse + { + public required DateTimeOffset ResolvedAt { get; init; } + + public required ResolvePolicy Policy { get; init; } + + public required List Results { get; init; } + } + + private sealed class ResolvePolicy + { + public required string ActiveRevisionId { get; init; } + + public required string Version { get; init; } + + public required string Digest { get; init; } + + public string? RequestedRevisionId { get; init; } + } + + private sealed class ResolveResult + { + public required string VulnerabilityId { get; init; } + + public required string ProductKey { get; init; } + + public required string Status { get; init; } + + public required DateTimeOffset CalculatedAt { get; init; } + + public required List Sources { get; init; } + + public required List Conflicts { get; init; } + + public ResolveSignals? Signals { get; init; } + + public string? Summary { get; init; } + + public required string PolicyRevisionId { get; init; } + + public required string PolicyVersion { get; init; } + + public required string PolicyDigest { get; init; } + + public required List Decisions { get; init; } + + public ResolveEnvelope? Envelope { get; init; } + } + + private sealed class ResolveSource + { + public required string ProviderId { get; init; } + } + + private sealed class ResolveConflict + { + public string? ProviderId { get; init; } + } + + private sealed class ResolveSignals + { + public ResolveSeverity? Severity { get; init; } + + public bool? Kev { get; init; } + + public double? Epss { get; init; } + } + + private sealed class ResolveSeverity + { + public string? Scheme { get; init; } + + public double? Score { get; init; } + } + + private sealed class ResolveDecision + { + public required string ProviderId { get; init; } + + public required bool Included { get; init; } + + public string? Reason { get; init; } + } + + private sealed class ResolveEnvelope + { + public required ResolveArtifact Artifact { get; init; } + + public ResolveSignature? ContentSignature { get; init; } + + public ResolveAttestationMetadata? Attestation { get; init; } + + public string? AttestationEnvelope { get; init; } + + public ResolveSignature? AttestationSignature { get; init; } + } + + private sealed class ResolveArtifact + { + public required string Algorithm { get; init; } + + public required string Digest { get; init; } + } + + private sealed class ResolveSignature + { + public required string Value { get; init; } + + public string? KeyId { get; init; } + } + + private sealed class ResolveAttestationMetadata + { + public required string PredicateType { get; init; } + + public ResolveRekorReference? Rekor { get; init; } + + public string? EnvelopeDigest { get; init; } + + public DateTimeOffset? SignedAt { get; init; } + } + + private sealed class ResolveRekorReference + { + public string? Location { get; init; } + } + + private sealed class FakeSigner : IVexSigner + { + public ValueTask SignAsync(ReadOnlyMemory payload, CancellationToken cancellationToken) + => ValueTask.FromResult(new VexSignedPayload("signature", "key")); + } + + private sealed class FakePolicyEvaluator : IVexPolicyEvaluator + { + public string Version => "test"; + + public VexPolicySnapshot Snapshot => VexPolicySnapshot.Default; + + public double GetProviderWeight(VexProvider provider) => 1.0; + + public bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason) + { + rejectionReason = null; + return true; + } + } + + +} diff --git a/src/StellaOps.Excititor.WebService.Tests/StatusEndpointTests.cs b/src/StellaOps.Excititor.WebService.Tests/StatusEndpointTests.cs index 42845ef8..3e990957 100644 --- a/src/StellaOps.Excititor.WebService.Tests/StatusEndpointTests.cs +++ b/src/StellaOps.Excititor.WebService.Tests/StatusEndpointTests.cs @@ -1,107 +1,97 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Net.Http.Json; -using System.IO; -using Microsoft.AspNetCore.Mvc.Testing; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Mongo2Go; -using MongoDB.Driver; -using StellaOps.Excititor.Attestation.Signing; -using StellaOps.Excititor.Connectors.Abstractions; -using StellaOps.Excititor.Policy; -using StellaOps.Excititor.Core; -using StellaOps.Excititor.Export; -using StellaOps.Excititor.WebService; - -namespace StellaOps.Excititor.WebService.Tests; - -public sealed class StatusEndpointTests : IClassFixture>, IDisposable -{ - private readonly WebApplicationFactory _factory; - private readonly MongoDbRunner _runner; - - public StatusEndpointTests(WebApplicationFactory factory) - { - _runner = MongoDbRunner.Start(); - _factory = factory.WithWebHostBuilder(builder => - { - builder.ConfigureAppConfiguration((_, config) => - { - var rootPath = Path.Combine(Path.GetTempPath(), "excititor-offline-tests"); - Directory.CreateDirectory(rootPath); - var settings = new Dictionary - { - ["Excititor:Storage:Mongo:RawBucketName"] = "vex.raw", - ["Excititor:Storage:Mongo:GridFsInlineThresholdBytes"] = "256", - ["Excititor:Artifacts:FileSystem:RootPath"] = rootPath, - }; - config.AddInMemoryCollection(settings!); - }); - - builder.ConfigureServices(services => - { - services.AddSingleton(_ => new MongoClient(_runner.ConnectionString)); - services.AddSingleton(provider => provider.GetRequiredService().GetDatabase("excititor-web-tests")); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(new VexConnectorDescriptor("excititor:redhat", VexProviderKind.Distro, "Red Hat CSAF")); - }); - }); - } - - [Fact] - public async Task StatusEndpoint_ReturnsArtifactStores() - { - var client = _factory.CreateClient(); - var response = await client.GetAsync("/excititor/status"); - var raw = await response.Content.ReadAsStringAsync(); - Assert.True(response.IsSuccessStatusCode, raw); - - var payload = System.Text.Json.JsonSerializer.Deserialize(raw); - Assert.NotNull(payload); - Assert.NotEmpty(payload!.ArtifactStores); - } - - public void Dispose() - { - _runner.Dispose(); - } - - private sealed class StatusResponse - { - public string[] ArtifactStores { get; set; } = Array.Empty(); - } - - private sealed class FakeSigner : IVexSigner - { - public ValueTask SignAsync(ReadOnlyMemory payload, CancellationToken cancellationToken) - => ValueTask.FromResult(new VexSignedPayload("signature", "key")); - } - - private sealed class FakePolicyEvaluator : IVexPolicyEvaluator - { - public string Version => "test"; - - public VexPolicySnapshot Snapshot => VexPolicySnapshot.Default; - - public double GetProviderWeight(VexProvider provider) => 1.0; - - public bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason) - { - rejectionReason = null; - return true; - } - } - - private sealed class FakeExportDataSource : IVexExportDataSource - { - public ValueTask FetchAsync(VexQuery query, CancellationToken cancellationToken) - { - var dataset = new VexExportDataSet(ImmutableArray.Empty, ImmutableArray.Empty, ImmutableArray.Empty); - return ValueTask.FromResult(dataset); - } - } -} +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Net.Http.Json; +using System.IO; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using EphemeralMongo; +using MongoRunner = EphemeralMongo.MongoRunner; +using MongoRunnerOptions = EphemeralMongo.MongoRunnerOptions; +using StellaOps.Excititor.Attestation.Signing; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Policy; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Export; +using StellaOps.Excititor.WebService; + +namespace StellaOps.Excititor.WebService.Tests; + +public sealed class StatusEndpointTests : IDisposable +{ + private readonly TestWebApplicationFactory _factory; + private readonly IMongoRunner _runner; + + public StatusEndpointTests() + { + _runner = MongoRunner.Run(new MongoRunnerOptions { UseSingleNodeReplicaSet = true }); + _factory = new TestWebApplicationFactory( + configureConfiguration: config => + { + var rootPath = Path.Combine(Path.GetTempPath(), "excititor-offline-tests"); + Directory.CreateDirectory(rootPath); + var settings = new Dictionary + { + ["Excititor:Storage:Mongo:ConnectionString"] = _runner.ConnectionString, + ["Excititor:Storage:Mongo:DatabaseName"] = "excititor-web-tests", + ["Excititor:Storage:Mongo:RawBucketName"] = "vex.raw", + ["Excititor:Storage:Mongo:GridFsInlineThresholdBytes"] = "256", + ["Excititor:Artifacts:FileSystem:RootPath"] = rootPath, + }; + config.AddInMemoryCollection(settings!); + }, + configureServices: services => + { + TestServiceOverrides.Apply(services); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(new VexConnectorDescriptor("excititor:redhat", VexProviderKind.Distro, "Red Hat CSAF")); + }); + } + + [Fact] + public async Task StatusEndpoint_ReturnsArtifactStores() + { + var client = _factory.CreateClient(); + var response = await client.GetAsync("/excititor/status"); + var raw = await response.Content.ReadAsStringAsync(); + Assert.True(response.IsSuccessStatusCode, raw); + + var payload = System.Text.Json.JsonSerializer.Deserialize(raw); + Assert.NotNull(payload); + Assert.NotEmpty(payload!.ArtifactStores); + } + + public void Dispose() + { + _factory.Dispose(); + _runner.Dispose(); + } + + private sealed class StatusResponse + { + public string[] ArtifactStores { get; set; } = Array.Empty(); + } + + private sealed class FakeSigner : IVexSigner + { + public ValueTask SignAsync(ReadOnlyMemory payload, CancellationToken cancellationToken) + => ValueTask.FromResult(new VexSignedPayload("signature", "key")); + } + + private sealed class FakePolicyEvaluator : IVexPolicyEvaluator + { + public string Version => "test"; + + public VexPolicySnapshot Snapshot => VexPolicySnapshot.Default; + + public double GetProviderWeight(VexProvider provider) => 1.0; + + public bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason) + { + rejectionReason = null; + return true; + } + } + +} diff --git a/src/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj b/src/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj index 097a85d0..d8823043 100644 --- a/src/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj +++ b/src/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj @@ -1,16 +1,25 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - + + + net10.0 + preview + enable + enable + true + false + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Excititor.WebService.Tests/TestAuthentication.cs b/src/StellaOps.Excititor.WebService.Tests/TestAuthentication.cs new file mode 100644 index 00000000..40500197 --- /dev/null +++ b/src/StellaOps.Excititor.WebService.Tests/TestAuthentication.cs @@ -0,0 +1,61 @@ +using System.Security.Claims; +using System.Text.Encodings.Web; +using Microsoft.AspNetCore.Authentication; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace StellaOps.Excititor.WebService.Tests; + +internal static class TestAuthenticationExtensions +{ + public const string SchemeName = "TestBearer"; + + public static AuthenticationBuilder AddTestAuthentication(this IServiceCollection services) + { + return services.AddAuthentication(options => + { + options.DefaultAuthenticateScheme = SchemeName; + options.DefaultChallengeScheme = SchemeName; + }).AddScheme(SchemeName, _ => { }); + } + + private sealed class TestAuthenticationHandler : AuthenticationHandler + { + public TestAuthenticationHandler( + IOptionsMonitor options, + ILoggerFactory logger, + UrlEncoder encoder) + : base(options, logger, encoder) + { + } + + protected override Task HandleAuthenticateAsync() + { + if (!Request.Headers.TryGetValue("Authorization", out var authorization) || authorization.Count == 0) + { + return Task.FromResult(AuthenticateResult.NoResult()); + } + + var header = authorization[0]; + if (string.IsNullOrWhiteSpace(header) || !header.StartsWith("Bearer ", StringComparison.OrdinalIgnoreCase)) + { + return Task.FromResult(AuthenticateResult.Fail("Invalid authentication scheme.")); + } + + var scopeSegment = header.Substring("Bearer ".Length); + var scopes = scopeSegment.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + var claims = new List { new Claim(ClaimTypes.NameIdentifier, "test-user") }; + if (scopes.Length > 0) + { + claims.Add(new Claim("scope", string.Join(' ', scopes))); + } + + var identity = new ClaimsIdentity(claims, SchemeName); + var principal = new ClaimsPrincipal(identity); + var ticket = new AuthenticationTicket(principal, SchemeName); + return Task.FromResult(AuthenticateResult.Success(ticket)); + } + } +} diff --git a/src/StellaOps.Excititor.WebService.Tests/TestServiceOverrides.cs b/src/StellaOps.Excititor.WebService.Tests/TestServiceOverrides.cs new file mode 100644 index 00000000..5646d5c4 --- /dev/null +++ b/src/StellaOps.Excititor.WebService.Tests/TestServiceOverrides.cs @@ -0,0 +1,203 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Immutable; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Export; +using StellaOps.Excititor.Storage.Mongo; +using StellaOps.Excititor.WebService.Services; +using MongoDB.Driver; +using StellaOps.Excititor.Attestation.Dsse; + +namespace StellaOps.Excititor.WebService.Tests; + +internal static class TestServiceOverrides +{ + public static void Apply(IServiceCollection services) + { + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.RemoveAll(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + + services.RemoveAll(); + services.AddSingleton(); + } + + private sealed class StubExportCacheService : IVexExportCacheService + { + public ValueTask InvalidateAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken) + => ValueTask.CompletedTask; + + public ValueTask PruneDanglingAsync(CancellationToken cancellationToken) + => ValueTask.FromResult(0); + + public ValueTask PruneExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken) + => ValueTask.FromResult(0); + } + + private sealed class StubExportEngine : IExportEngine + { + public ValueTask ExportAsync(VexExportRequestContext context, CancellationToken cancellationToken) + { + var manifest = new VexExportManifest( + exportId: "stub/export", + querySignature: VexQuerySignature.FromQuery(context.Query), + format: context.Format, + createdAt: DateTimeOffset.UtcNow, + artifact: new VexContentAddress("sha256", "stub"), + claimCount: 0, + sourceProviders: Array.Empty()); + + return ValueTask.FromResult(manifest); + } + } + + private sealed class StubExportDataSource : IVexExportDataSource + { + public ValueTask FetchAsync(VexQuery query, CancellationToken cancellationToken) + { + return ValueTask.FromResult(new VexExportDataSet( + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty)); + } + } + + private sealed class StubExportStore : IVexExportStore + { + private readonly ConcurrentDictionary<(string Signature, VexExportFormat Format), VexExportManifest> _store = new(); + + public ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _store.TryGetValue((signature.Value, format), out var manifest); + return ValueTask.FromResult(manifest); + } + + public ValueTask SaveAsync(VexExportManifest manifest, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _store[(manifest.QuerySignature.Value, manifest.Format)] = manifest; + return ValueTask.CompletedTask; + } + } + + private sealed class StubCacheIndex : IVexCacheIndex + { + private readonly ConcurrentDictionary<(string Signature, VexExportFormat Format), VexCacheEntry> _entries = new(); + + public ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _entries.TryGetValue((signature.Value, format), out var entry); + return ValueTask.FromResult(entry); + } + + public ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _entries.TryRemove((signature.Value, format), out _); + return ValueTask.CompletedTask; + } + + public ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _entries[(entry.QuerySignature.Value, entry.Format)] = entry; + return ValueTask.CompletedTask; + } + } + + private sealed class StubCacheMaintenance : IVexCacheMaintenance + { + public ValueTask RemoveExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken, IClientSessionHandle? session = null) + => ValueTask.FromResult(0); + + public ValueTask RemoveMissingManifestReferencesAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null) + => ValueTask.FromResult(0); + } + + private sealed class StubAttestationClient : IVexAttestationClient + { + public ValueTask SignAsync(VexAttestationRequest request, CancellationToken cancellationToken) + { + var envelope = new DsseEnvelope( + Convert.ToBase64String(Encoding.UTF8.GetBytes("{\"stub\":\"payload\"}")), + "application/vnd.stellaops.resolve+json", + new[] + { + new DsseSignature("attestation-signature", "attestation-key"), + }); + + var diagnostics = ImmutableDictionary.Empty + .Add("envelope", JsonSerializer.Serialize(envelope)); + + var response = new VexAttestationResponse( + new VexAttestationMetadata("stub"), + diagnostics); + return ValueTask.FromResult(response); + } + + public ValueTask VerifyAsync(VexAttestationRequest request, CancellationToken cancellationToken) + { + var verification = new VexAttestationVerification(true, ImmutableDictionary.Empty); + return ValueTask.FromResult(verification); + } + } + + private sealed class StubConnectorStateRepository : IVexConnectorStateRepository + { + private readonly ConcurrentDictionary _states = new(StringComparer.Ordinal); + + public ValueTask GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _states.TryGetValue(connectorId, out var state); + return ValueTask.FromResult(state); + } + + public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _states[state.ConnectorId] = state; + return ValueTask.CompletedTask; + } + } + + private sealed class StubIngestOrchestrator : IVexIngestOrchestrator + { + public Task InitializeAsync(IngestInitOptions options, CancellationToken cancellationToken) + => Task.FromResult(new InitSummary(Guid.Empty, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, ImmutableArray.Empty)); + + public Task RunAsync(IngestRunOptions options, CancellationToken cancellationToken) + => Task.FromResult(new IngestRunSummary(Guid.Empty, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, ImmutableArray.Empty)); + + public Task ResumeAsync(IngestResumeOptions options, CancellationToken cancellationToken) + => Task.FromResult(new IngestRunSummary(Guid.Empty, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, ImmutableArray.Empty)); + + public Task ReconcileAsync(ReconcileOptions options, CancellationToken cancellationToken) + => Task.FromResult(new ReconcileSummary(Guid.Empty, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, ImmutableArray.Empty)); + } + + private sealed class NoopHostedService : IHostedService + { + public Task StartAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; + } +} diff --git a/src/StellaOps.Excititor.WebService.Tests/TestWebApplicationFactory.cs b/src/StellaOps.Excititor.WebService.Tests/TestWebApplicationFactory.cs new file mode 100644 index 00000000..39d80b24 --- /dev/null +++ b/src/StellaOps.Excititor.WebService.Tests/TestWebApplicationFactory.cs @@ -0,0 +1,42 @@ +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +namespace StellaOps.Excititor.WebService.Tests; + +internal sealed class TestWebApplicationFactory : WebApplicationFactory +{ + private readonly Action? _configureConfiguration; + private readonly Action? _configureServices; + + public TestWebApplicationFactory( + Action? configureConfiguration, + Action? configureServices) + { + _configureConfiguration = configureConfiguration; + _configureServices = configureServices; + } + + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + builder.UseEnvironment("Production"); + if (_configureConfiguration is not null) + { + builder.ConfigureAppConfiguration((_, config) => _configureConfiguration(config)); + } + + if (_configureServices is not null) + { + builder.ConfigureServices(services => _configureServices(services)); + } + } + + protected override IHost CreateHost(IHostBuilder builder) + { + builder.UseEnvironment("Production"); + builder.UseDefaultServiceProvider(options => options.ValidateScopes = false); + return base.CreateHost(builder); + } +} diff --git a/src/StellaOps.Excititor.WebService/Endpoints/IngestEndpoints.cs b/src/StellaOps.Excititor.WebService/Endpoints/IngestEndpoints.cs index 5566e137..a86c43d2 100644 --- a/src/StellaOps.Excititor.WebService/Endpoints/IngestEndpoints.cs +++ b/src/StellaOps.Excititor.WebService/Endpoints/IngestEndpoints.cs @@ -20,48 +20,49 @@ internal static class IngestEndpoints group.MapPost("/reconcile", HandleReconcileAsync); } - private static async Task HandleInitAsync( - HttpContext httpContext, - ExcititorInitRequest request, - IVexIngestOrchestrator orchestrator, - TimeProvider timeProvider, - CancellationToken cancellationToken) + internal static async Task HandleInitAsync( + HttpContext httpContext, + ExcititorInitRequest request, + IVexIngestOrchestrator orchestrator, + TimeProvider timeProvider, + CancellationToken cancellationToken) { var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope); if (scopeResult is not null) - { - return scopeResult; - } - - var providerIds = NormalizeProviders(request.Providers); - var options = new IngestInitOptions(providerIds, request.Resume ?? false, timeProvider); + { + return scopeResult; + } + + var providerIds = NormalizeProviders(request.Providers); + _ = timeProvider; + var options = new IngestInitOptions(providerIds, request.Resume ?? false); var summary = await orchestrator.InitializeAsync(options, cancellationToken).ConfigureAwait(false); var message = $"Initialized {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed."; - return Results.Ok(new - { - message, - runId = summary.RunId, - startedAt = summary.StartedAt, - completedAt = summary.CompletedAt, - providers = summary.Providers.Select(static provider => new - { - provider.providerId, - provider.displayName, - provider.status, - provider.durationMs, - provider.error - }) - }); - } + return TypedResults.Ok(new + { + message, + runId = summary.RunId, + startedAt = summary.StartedAt, + completedAt = summary.CompletedAt, + providers = summary.Providers.Select(static provider => new + { + providerId = provider.ProviderId, + displayName = provider.DisplayName, + status = provider.Status, + durationMs = provider.Duration.TotalMilliseconds, + error = provider.Error + }) + }); + } - private static async Task HandleRunAsync( - HttpContext httpContext, - ExcititorIngestRunRequest request, - IVexIngestOrchestrator orchestrator, - TimeProvider timeProvider, - CancellationToken cancellationToken) + internal static async Task HandleRunAsync( + HttpContext httpContext, + ExcititorIngestRunRequest request, + IVexIngestOrchestrator orchestrator, + TimeProvider timeProvider, + CancellationToken cancellationToken) { var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope); if (scopeResult is not null) @@ -71,55 +72,98 @@ internal static class IngestEndpoints if (!TryParseDateTimeOffset(request.Since, out var since, out var sinceError)) { - return Results.BadRequest(new { message = sinceError }); - } - - if (!TryParseTimeSpan(request.Window, out var window, out var windowError)) - { - return Results.BadRequest(new { message = windowError }); - } - - var providerIds = NormalizeProviders(request.Providers); - var options = new IngestRunOptions( - providerIds, - since, - window, - request.Force ?? false, - timeProvider); - - var summary = await orchestrator.RunAsync(options, cancellationToken).ConfigureAwait(false); - var message = $"Ingest run completed for {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed."; - - return Results.Ok(new - { - message, - runId = summary.RunId, - startedAt = summary.StartedAt, + return TypedResults.BadRequest(new { message = sinceError }); + } + + if (!TryParseTimeSpan(request.Window, out var window, out var windowError)) + { + return TypedResults.BadRequest(new { message = windowError }); + } + + _ = timeProvider; + var providerIds = NormalizeProviders(request.Providers); + var options = new IngestRunOptions( + providerIds, + since, + window, + request.Force ?? false); + + var summary = await orchestrator.RunAsync(options, cancellationToken).ConfigureAwait(false); + var message = $"Ingest run completed for {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed."; + + return TypedResults.Ok(new + { + message, + runId = summary.RunId, + startedAt = summary.StartedAt, completedAt = summary.CompletedAt, - durationMs = summary.Duration.TotalMilliseconds, - providers = summary.Providers.Select(static provider => new - { - provider.providerId, - provider.status, - provider.documents, - provider.claims, - provider.startedAt, - provider.completedAt, - provider.durationMs, - provider.lastDigest, - provider.lastUpdated, - provider.checkpoint, - provider.error - }) - }); - } + durationMs = summary.Duration.TotalMilliseconds, + providers = summary.Providers.Select(static provider => new + { + providerId = provider.ProviderId, + status = provider.Status, + documents = provider.Documents, + claims = provider.Claims, + startedAt = provider.StartedAt, + completedAt = provider.CompletedAt, + durationMs = provider.Duration.TotalMilliseconds, + lastDigest = provider.LastDigest, + lastUpdated = provider.LastUpdated, + checkpoint = provider.Checkpoint, + error = provider.Error + }) + }); + } - private static async Task HandleResumeAsync( - HttpContext httpContext, - ExcititorIngestResumeRequest request, - IVexIngestOrchestrator orchestrator, - TimeProvider timeProvider, - CancellationToken cancellationToken) + internal static async Task HandleResumeAsync( + HttpContext httpContext, + ExcititorIngestResumeRequest request, + IVexIngestOrchestrator orchestrator, + TimeProvider timeProvider, + CancellationToken cancellationToken) + { + var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope); + if (scopeResult is not null) + { + return scopeResult; + } + + _ = timeProvider; + var providerIds = NormalizeProviders(request.Providers); + var options = new IngestResumeOptions(providerIds, request.Checkpoint); + + var summary = await orchestrator.ResumeAsync(options, cancellationToken).ConfigureAwait(false); + var message = $"Resume run completed for {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed."; + + return TypedResults.Ok(new + { + message, + runId = summary.RunId, + startedAt = summary.StartedAt, + completedAt = summary.CompletedAt, + durationMs = summary.Duration.TotalMilliseconds, + providers = summary.Providers.Select(static provider => new + { + providerId = provider.ProviderId, + status = provider.Status, + documents = provider.Documents, + claims = provider.Claims, + startedAt = provider.StartedAt, + completedAt = provider.CompletedAt, + durationMs = provider.Duration.TotalMilliseconds, + since = provider.Since, + checkpoint = provider.Checkpoint, + error = provider.Error + }) + }); + } + + internal static async Task HandleReconcileAsync( + HttpContext httpContext, + ExcititorReconcileRequest request, + IVexIngestOrchestrator orchestrator, + TimeProvider timeProvider, + CancellationToken cancellationToken) { var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope); if (scopeResult is not null) @@ -127,81 +171,40 @@ internal static class IngestEndpoints return scopeResult; } - var providerIds = NormalizeProviders(request.Providers); - var options = new IngestResumeOptions(providerIds, request.Checkpoint, timeProvider); - - var summary = await orchestrator.ResumeAsync(options, cancellationToken).ConfigureAwait(false); - var message = $"Resume run completed for {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed."; - - return Results.Ok(new - { - message, - runId = summary.RunId, - startedAt = summary.StartedAt, + if (!TryParseTimeSpan(request.MaxAge, out var maxAge, out var error)) + { + return TypedResults.BadRequest(new { message = error }); + } + + _ = timeProvider; + var providerIds = NormalizeProviders(request.Providers); + var options = new ReconcileOptions(providerIds, maxAge); + + var summary = await orchestrator.ReconcileAsync(options, cancellationToken).ConfigureAwait(false); + var message = $"Reconcile completed for {summary.ProviderCount} provider(s); {summary.ReconciledCount} reconciled, {summary.SkippedCount} skipped, {summary.FailureCount} failed."; + + return TypedResults.Ok(new + { + message, + runId = summary.RunId, + startedAt = summary.StartedAt, completedAt = summary.CompletedAt, - durationMs = summary.Duration.TotalMilliseconds, - providers = summary.Providers.Select(static provider => new - { - provider.providerId, - provider.status, - provider.documents, - provider.claims, - provider.startedAt, - provider.completedAt, - provider.durationMs, - provider.since, - provider.checkpoint, - provider.error - }) - }); - } + durationMs = summary.Duration.TotalMilliseconds, + providers = summary.Providers.Select(static provider => new + { + providerId = provider.ProviderId, + status = provider.Status, + action = provider.Action, + lastUpdated = provider.LastUpdated, + threshold = provider.Threshold, + documents = provider.Documents, + claims = provider.Claims, + error = provider.Error + }) + }); + } - private static async Task HandleReconcileAsync( - HttpContext httpContext, - ExcititorReconcileRequest request, - IVexIngestOrchestrator orchestrator, - TimeProvider timeProvider, - CancellationToken cancellationToken) - { - var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope); - if (scopeResult is not null) - { - return scopeResult; - } - - if (!TryParseTimeSpan(request.MaxAge, out var maxAge, out var error)) - { - return Results.BadRequest(new { message = error }); - } - - var providerIds = NormalizeProviders(request.Providers); - var options = new ReconcileOptions(providerIds, maxAge, timeProvider); - - var summary = await orchestrator.ReconcileAsync(options, cancellationToken).ConfigureAwait(false); - var message = $"Reconcile completed for {summary.ProviderCount} provider(s); {summary.ReconciledCount} reconciled, {summary.SkippedCount} skipped, {summary.FailureCount} failed."; - - return Results.Ok(new - { - message, - runId = summary.RunId, - startedAt = summary.StartedAt, - completedAt = summary.CompletedAt, - durationMs = summary.Duration.TotalMilliseconds, - providers = summary.Providers.Select(static provider => new - { - provider.providerId, - provider.status, - provider.action, - provider.lastUpdated, - provider.threshold, - provider.documents, - provider.claims, - provider.error - }) - }); - } - - private static ImmutableArray NormalizeProviders(IReadOnlyCollection? providers) + internal static ImmutableArray NormalizeProviders(IReadOnlyCollection? providers) { if (providers is null || providers.Count == 0) { @@ -222,7 +225,7 @@ internal static class IngestEndpoints return set.ToImmutableArray(); } - private static bool TryParseDateTimeOffset(string? value, out DateTimeOffset? result, out string? error) + internal static bool TryParseDateTimeOffset(string? value, out DateTimeOffset? result, out string? error) { result = null; error = null; @@ -246,7 +249,7 @@ internal static class IngestEndpoints return false; } - private static bool TryParseTimeSpan(string? value, out TimeSpan? result, out string? error) + internal static bool TryParseTimeSpan(string? value, out TimeSpan? result, out string? error) { result = null; error = null; @@ -266,19 +269,19 @@ internal static class IngestEndpoints return false; } - private sealed record ExcititorInitRequest(IReadOnlyList? Providers, bool? Resume); - - private sealed record ExcititorIngestRunRequest( - IReadOnlyList? Providers, - string? Since, - string? Window, - bool? Force); - - private sealed record ExcititorIngestResumeRequest( - IReadOnlyList? Providers, - string? Checkpoint); - - private sealed record ExcititorReconcileRequest( - IReadOnlyList? Providers, - string? MaxAge); -} + internal sealed record ExcititorInitRequest(IReadOnlyList? Providers, bool? Resume); + + internal sealed record ExcititorIngestRunRequest( + IReadOnlyList? Providers, + string? Since, + string? Window, + bool? Force); + + internal sealed record ExcititorIngestResumeRequest( + IReadOnlyList? Providers, + string? Checkpoint); + + internal sealed record ExcititorReconcileRequest( + IReadOnlyList? Providers, + string? MaxAge); +} diff --git a/src/StellaOps.Excititor.WebService/Endpoints/ResolveEndpoint.cs b/src/StellaOps.Excititor.WebService/Endpoints/ResolveEndpoint.cs index d64811f4..6da2d31b 100644 --- a/src/StellaOps.Excititor.WebService/Endpoints/ResolveEndpoint.cs +++ b/src/StellaOps.Excititor.WebService/Endpoints/ResolveEndpoint.cs @@ -15,16 +15,18 @@ using StellaOps.Excititor.Attestation.Dsse; using StellaOps.Excititor.Attestation.Signing; using StellaOps.Excititor.Core; using StellaOps.Excititor.Policy; -using StellaOps.Excititor.Storage.Mongo; - -internal static class ResolveEndpoint -{ - private const int MaxSubjectPairs = 256; - - public static void MapResolveEndpoint(WebApplication app) - { - app.MapPost("/excititor/resolve", HandleResolveAsync); - } +using StellaOps.Excititor.Storage.Mongo; +using StellaOps.Excititor.WebService.Services; + +internal static class ResolveEndpoint +{ + private const int MaxSubjectPairs = 256; + private const string ReadScope = "vex.read"; + + public static void MapResolveEndpoint(WebApplication app) + { + app.MapPost("/excititor/resolve", HandleResolveAsync); + } private static async Task HandleResolveAsync( VexResolveRequest request, @@ -38,13 +40,19 @@ internal static class ResolveEndpoint IVexAttestationClient? attestationClient, IVexSigner? signer, CancellationToken cancellationToken) - { - if (request is null) - { - return Results.BadRequest("Request payload is required."); - } - - var logger = loggerFactory.CreateLogger("ResolveEndpoint"); + { + var scopeResult = ScopeAuthorization.RequireScope(httpContext, ReadScope); + if (scopeResult is not null) + { + return scopeResult; + } + + if (request is null) + { + return Results.BadRequest("Request payload is required."); + } + + var logger = loggerFactory.CreateLogger("ResolveEndpoint"); var productKeys = NormalizeValues(request.ProductKeys, request.Purls); var vulnerabilityIds = NormalizeValues(request.VulnerabilityIds); diff --git a/src/StellaOps.Excititor.WebService/Properties/AssemblyInfo.cs b/src/StellaOps.Excititor.WebService/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..b440af32 --- /dev/null +++ b/src/StellaOps.Excititor.WebService/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Excititor.WebService.Tests")] diff --git a/src/StellaOps.Excititor.WebService/Services/VexIngestOrchestrator.cs b/src/StellaOps.Excititor.WebService/Services/VexIngestOrchestrator.cs index 465ddef6..ede256a5 100644 --- a/src/StellaOps.Excititor.WebService/Services/VexIngestOrchestrator.cs +++ b/src/StellaOps.Excititor.WebService/Services/VexIngestOrchestrator.cs @@ -94,12 +94,12 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator await EnsureProviderRegistrationAsync(handle.Descriptor, session, cancellationToken).ConfigureAwait(false); stopwatch.Stop(); - results.Add(new InitProviderResult( - handle.Descriptor.Id, - handle.Descriptor.DisplayName, - "succeeded", - stopwatch.Elapsed, - error: null)); + results.Add(new InitProviderResult( + handle.Descriptor.Id, + handle.Descriptor.DisplayName, + "succeeded", + stopwatch.Elapsed, + Error: null)); _logger.LogInformation("Excititor init validated provider {ProviderId} in {Duration}ms.", handle.Descriptor.Id, stopwatch.Elapsed.TotalMilliseconds); } @@ -223,15 +223,15 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator } else { - results.Add(new ReconcileProviderResult( - handle.Descriptor.Id, - "succeeded", - "skipped", - lastUpdated, - threshold, - documents: 0, - claims: 0, - error: null)); + results.Add(new ReconcileProviderResult( + handle.Descriptor.Id, + "succeeded", + "skipped", + lastUpdated, + threshold, + Documents: 0, + Claims: 0, + Error: null)); } } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) @@ -299,19 +299,23 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator await ValidateConnectorAsync(handle, cancellationToken).ConfigureAwait(false); await EnsureProviderRegistrationAsync(handle.Descriptor, session, cancellationToken).ConfigureAwait(false); - if (force) - { - var resetState = new VexConnectorState(providerId, null, ImmutableArray.Empty); - await _stateRepository.SaveAsync(resetState, cancellationToken, session).ConfigureAwait(false); - } - - var context = new VexConnectorContext( - since, - VexConnectorSettings.Empty, - _rawStore, - _signatureVerifier, - _normalizerRouter, - _serviceProvider); + if (force) + { + var resetState = new VexConnectorState(providerId, null, ImmutableArray.Empty); + await _stateRepository.SaveAsync(resetState, cancellationToken, session).ConfigureAwait(false); + } + + var stateBeforeRun = await _stateRepository.GetAsync(providerId, cancellationToken, session).ConfigureAwait(false); + var resumeTokens = stateBeforeRun?.ResumeTokens ?? ImmutableDictionary.Empty; + + var context = new VexConnectorContext( + since, + VexConnectorSettings.Empty, + _rawStore, + _signatureVerifier, + _normalizerRouter, + _serviceProvider, + resumeTokens); var documents = 0; var claims = 0; @@ -332,25 +336,25 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator stopwatch.Stop(); var completedAt = _timeProvider.GetUtcNow(); - var state = await _stateRepository.GetAsync(providerId, cancellationToken, session).ConfigureAwait(false); - - var checkpoint = state?.DocumentDigests.IsDefaultOrEmpty == false - ? state.DocumentDigests[^1] - : lastDigest; - - var result = new ProviderRunResult( - providerId, - "succeeded", + var stateAfterRun = await _stateRepository.GetAsync(providerId, cancellationToken, session).ConfigureAwait(false); + + var checkpoint = stateAfterRun?.DocumentDigests.IsDefaultOrEmpty == false + ? stateAfterRun.DocumentDigests[^1] + : lastDigest; + + var result = new ProviderRunResult( + providerId, + "succeeded", documents, claims, startedAt, completedAt, stopwatch.Elapsed, - lastDigest, - state?.LastUpdated, - checkpoint, - null, - since); + lastDigest, + stateAfterRun?.LastUpdated, + checkpoint, + null, + since); _logger.LogInformation( "Excititor ingest provider {ProviderId} completed: documents={Documents} claims={Claims} since={Since} duration={Duration}ms", diff --git a/src/StellaOps.Excititor.WebService/TASKS.md b/src/StellaOps.Excititor.WebService/TASKS.md index 01778400..fd9736eb 100644 --- a/src/StellaOps.Excititor.WebService/TASKS.md +++ b/src/StellaOps.Excititor.WebService/TASKS.md @@ -3,7 +3,7 @@ If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md | Task | Owner(s) | Depends on | Notes | |---|---|---|---| |EXCITITOR-WEB-01-001 – Minimal API bootstrap & DI|Team Excititor WebService|EXCITITOR-CORE-01-003, EXCITITOR-STORAGE-01-003|**DONE (2025-10-17)** – Minimal API host composes storage/export/attestation/artifact stores, binds Mongo/attestation options, and exposes `/excititor/status` + health endpoints with regression coverage in `StatusEndpointTests`.| -|EXCITITOR-WEB-01-002 – Ingest & reconcile endpoints|Team Excititor WebService|EXCITITOR-WEB-01-001|**DOING (2025-10-19)** – Prereqs EXCITITOR-WEB-01-001, EXCITITOR-EXPORT-01-001, and EXCITITOR-ATTEST-01-001 verified DONE; drafting `/excititor/init`, `/excititor/ingest/run`, `/excititor/ingest/resume`, `/excititor/reconcile` with scope enforcement & structured telemetry plan.| +|EXCITITOR-WEB-01-002 – Ingest & reconcile endpoints|Team Excititor WebService|EXCITITOR-WEB-01-001|**DONE (2025-10-20)** – `/excititor/init`, `/excititor/ingest/run`, `/excititor/ingest/resume`, `/excititor/reconcile` enforce `vex.admin`, normalize provider lists, and return deterministic summaries; covered via unit tests (`dotnet test src/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj --filter FullyQualifiedName~IngestEndpointsTests`).| |EXCITITOR-WEB-01-003 – Export & verify endpoints|Team Excititor WebService|EXCITITOR-WEB-01-001, EXCITITOR-EXPORT-01-001, EXCITITOR-ATTEST-01-001|**DOING (2025-10-19)** – Prereqs confirmed (EXCITITOR-WEB-01-001, EXCITITOR-EXPORT-01-001, EXCITITOR-ATTEST-01-001); preparing `/excititor/export*` surfaces and `/excititor/verify` with artifact/attestation metadata caching strategy.| -|EXCITITOR-WEB-01-004 – Resolve API & signed responses|Team Excititor WebService|EXCITITOR-WEB-01-001, EXCITITOR-ATTEST-01-002|**DOING (2025-10-19)** – Prereqs EXCITITOR-WEB-01-001, EXCITITOR-ATTEST-01-001, and EXCITITOR-ATTEST-01-002 verified DONE; planning `/excititor/resolve` signed response flow with consensus envelope + attestation metadata wiring.| +|EXCITITOR-WEB-01-004 – Resolve API & signed responses|Team Excititor WebService|EXCITITOR-WEB-01-001, EXCITITOR-ATTEST-01-002|**DONE (2025-10-20)** – Added `vex.read` scope enforcement, signed consensus/attestation envelopes, docs updates, and expanded tests (auth, unauthorized/forbidden). Mirror/ingest DTO casing fixed to restore builds.| |EXCITITOR-WEB-01-005 – Mirror distribution endpoints|Team Excititor WebService|EXCITITOR-EXPORT-01-007, DEVOPS-MIRROR-08-001|**DONE (2025-10-19)** – `/excititor/mirror` surfaces domain listings, indices, metadata, and downloads with quota/auth checks; tests cover Happy-path listing/download (`dotnet test src/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj`).| diff --git a/src/StellaOps.Excititor.Worker.Tests/DefaultVexProviderRunnerTests.cs b/src/StellaOps.Excititor.Worker.Tests/DefaultVexProviderRunnerTests.cs new file mode 100644 index 00000000..ed2cfd01 --- /dev/null +++ b/src/StellaOps.Excititor.Worker.Tests/DefaultVexProviderRunnerTests.cs @@ -0,0 +1,436 @@ +using System.Collections.Concurrent; +using System.Collections.Immutable; +using System.Linq; +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using MongoDB.Driver; +using StellaOps.Plugin; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Storage.Mongo; +using StellaOps.Excititor.Worker.Options; +using StellaOps.Excititor.Worker.Scheduling; +using Xunit; + +namespace StellaOps.Excititor.Worker.Tests; + +public sealed class DefaultVexProviderRunnerTests +{ + private static readonly VexConnectorSettings EmptySettings = VexConnectorSettings.Empty; + + [Fact] + public async Task RunAsync_Skips_WhenNextEligibleRunInFuture() + { + var time = new FixedTimeProvider(new DateTimeOffset(2025, 10, 21, 15, 0, 0, TimeSpan.Zero)); + var connector = TestConnector.Success("excititor:test"); + var stateRepository = new InMemoryStateRepository(); + stateRepository.Save(new VexConnectorState( + "excititor:test", + LastUpdated: null, + DocumentDigests: ImmutableArray.Empty, + ResumeTokens: ImmutableDictionary.Empty, + LastSuccessAt: null, + FailureCount: 1, + NextEligibleRun: time.GetUtcNow().AddHours(1), + LastFailureReason: "previous failure")); + + var services = CreateServiceProvider(connector, stateRepository); + var runner = CreateRunner(services, time, options => + { + options.Retry.BaseDelay = TimeSpan.FromMinutes(5); + options.Retry.MaxDelay = TimeSpan.FromMinutes(30); + options.Retry.JitterRatio = 0; + }); + + await runner.RunAsync(new VexWorkerSchedule(connector.Id, TimeSpan.FromMinutes(10), TimeSpan.Zero, EmptySettings), CancellationToken.None); + + connector.FetchInvoked.Should().BeFalse(); + var state = stateRepository.Get("excititor:test"); + state.Should().NotBeNull(); + state!.FailureCount.Should().Be(1); + state.NextEligibleRun.Should().Be(time.GetUtcNow().AddHours(1)); + } + + [Fact] + public async Task RunAsync_Success_ResetsFailureCounters() + { + var now = new DateTimeOffset(2025, 10, 21, 16, 0, 0, TimeSpan.Zero); + var time = new FixedTimeProvider(now); + var connector = TestConnector.Success("excititor:test"); + var stateRepository = new InMemoryStateRepository(); + stateRepository.Save(new VexConnectorState( + "excititor:test", + LastUpdated: now.AddDays(-1), + DocumentDigests: ImmutableArray.Empty, + ResumeTokens: ImmutableDictionary.Empty, + LastSuccessAt: now.AddHours(-4), + FailureCount: 2, + NextEligibleRun: null, + LastFailureReason: "failure")); + + var services = CreateServiceProvider(connector, stateRepository); + var runner = CreateRunner(services, time, options => + { + options.Retry.BaseDelay = TimeSpan.FromMinutes(2); + options.Retry.MaxDelay = TimeSpan.FromMinutes(30); + options.Retry.JitterRatio = 0; + }); + + await runner.RunAsync(new VexWorkerSchedule(connector.Id, TimeSpan.FromMinutes(10), TimeSpan.Zero, EmptySettings), CancellationToken.None); + + connector.FetchInvoked.Should().BeTrue(); + var state = stateRepository.Get("excititor:test"); + state.Should().NotBeNull(); + state!.FailureCount.Should().Be(0); + state.NextEligibleRun.Should().BeNull(); + state.LastFailureReason.Should().BeNull(); + state.LastSuccessAt.Should().Be(now); + } + + [Fact] + public async Task RunAsync_UsesStoredResumeTokens() + { + var now = new DateTimeOffset(2025, 10, 21, 18, 0, 0, TimeSpan.Zero); + var time = new FixedTimeProvider(now); + var resumeTokens = ImmutableDictionary.Empty + .Add("cursor", "abc123"); + var stateRepository = new InMemoryStateRepository(); + stateRepository.Save(new VexConnectorState( + "excititor:resume", + LastUpdated: now.AddHours(-6), + DocumentDigests: ImmutableArray.Empty, + ResumeTokens: resumeTokens, + LastSuccessAt: now.AddHours(-7), + FailureCount: 0, + NextEligibleRun: null, + LastFailureReason: null)); + + var connector = TestConnector.SuccessWithCapture("excititor:resume"); + var services = CreateServiceProvider(connector, stateRepository); + var runner = CreateRunner(services, time, options => + { + options.Retry.BaseDelay = TimeSpan.FromMinutes(2); + options.Retry.MaxDelay = TimeSpan.FromMinutes(10); + options.Retry.JitterRatio = 0; + }); + + await runner.RunAsync(new VexWorkerSchedule(connector.Id, TimeSpan.FromMinutes(10), TimeSpan.Zero, EmptySettings), CancellationToken.None); + + connector.LastContext.Should().NotBeNull(); + connector.LastContext!.Since.Should().Be(now.AddHours(-6)); + connector.LastContext.ResumeTokens.Should().BeEquivalentTo(resumeTokens); + } + +[Fact] + public async Task RunAsync_SchedulesRefresh_ForUniqueClaims() + { + var now = new DateTimeOffset(2025, 10, 21, 19, 0, 0, TimeSpan.Zero); + var time = new FixedTimeProvider(now); + var rawDocument = new VexRawDocument( + "provider-a", + VexDocumentFormat.Csaf, + new Uri("https://example.org/vex.json"), + now, + "sha256:raw", + ReadOnlyMemory.Empty, + ImmutableDictionary.Empty); + + var claimDocument = new VexClaimDocument( + VexDocumentFormat.Csaf, + "sha256:claim", + new Uri("https://example.org/vex.json")); + + var primaryProduct = new VexProduct("pkg:test/app", "Test App", componentIdentifiers: new[] { "fingerprint:base" }); + var secondaryProduct = new VexProduct("pkg:test/other", "Other App", componentIdentifiers: new[] { "fingerprint:other" }); + + var claims = new[] + { + new VexClaim("CVE-2025-0001", "provider-a", primaryProduct, VexClaimStatus.Affected, claimDocument, now.AddHours(-3), now.AddHours(-2)), + new VexClaim("CVE-2025-0001", "provider-b", primaryProduct, VexClaimStatus.NotAffected, claimDocument, now.AddHours(-3), now.AddHours(-2)), + new VexClaim("CVE-2025-0002", "provider-a", secondaryProduct, VexClaimStatus.Affected, claimDocument, now.AddHours(-2), now.AddHours(-1)), + }; + + var connector = TestConnector.WithDocuments("excititor:test", rawDocument); + var stateRepository = new InMemoryStateRepository(); + var scheduler = new RecordingRefreshScheduler(); + var normalizer = new StubNormalizerRouter(claims); + var services = CreateServiceProvider(connector, stateRepository, normalizer); + var runner = CreateRunner(services, time, options => + { + options.Retry.BaseDelay = TimeSpan.FromMinutes(1); + options.Retry.MaxDelay = TimeSpan.FromMinutes(5); + options.Retry.JitterRatio = 0; + }, scheduler); + + await runner.RunAsync(new VexWorkerSchedule(connector.Id, TimeSpan.FromMinutes(10), TimeSpan.Zero, EmptySettings), CancellationToken.None); + + scheduler.Entries.Should().BeEquivalentTo(new[] + { + ("CVE-2025-0001", "pkg:test/app"), + ("CVE-2025-0002", "pkg:test/other"), + }); + } + +[Fact] + public async Task RunAsync_Failure_AppliesBackoff() + { + var now = new DateTimeOffset(2025, 10, 21, 17, 0, 0, TimeSpan.Zero); + var time = new FixedTimeProvider(now); + var connector = TestConnector.Failure("excititor:test", new InvalidOperationException("boom")); + var stateRepository = new InMemoryStateRepository(); + stateRepository.Save(new VexConnectorState( + "excititor:test", + LastUpdated: now.AddDays(-2), + DocumentDigests: ImmutableArray.Empty, + ResumeTokens: ImmutableDictionary.Empty, + LastSuccessAt: now.AddDays(-1), + FailureCount: 1, + NextEligibleRun: null, + LastFailureReason: null)); + + var services = CreateServiceProvider(connector, stateRepository); + var runner = CreateRunner(services, time, options => + { + options.Retry.BaseDelay = TimeSpan.FromMinutes(5); + options.Retry.MaxDelay = TimeSpan.FromMinutes(60); + options.Retry.FailureThreshold = 3; + options.Retry.QuarantineDuration = TimeSpan.FromHours(12); + options.Retry.JitterRatio = 0; + }); + + await Assert.ThrowsAsync(async () => await runner.RunAsync(new VexWorkerSchedule(connector.Id, TimeSpan.FromMinutes(10), TimeSpan.Zero, EmptySettings), CancellationToken.None).AsTask()); + + var state = stateRepository.Get("excititor:test"); + state.Should().NotBeNull(); + state!.FailureCount.Should().Be(2); + state.LastFailureReason.Should().Be("boom"); + state.NextEligibleRun.Should().Be(now + TimeSpan.FromMinutes(10)); + } + + private static ServiceProvider CreateServiceProvider( + IVexConnector connector, + InMemoryStateRepository stateRepository, + IVexNormalizerRouter? normalizerRouter = null) + { + var services = new ServiceCollection(); + services.AddSingleton(connector); + services.AddSingleton(new NoopRawStore()); + services.AddSingleton(new NoopClaimStore()); + services.AddSingleton(new NoopProviderStore()); + services.AddSingleton(stateRepository); + services.AddSingleton(normalizerRouter ?? new NoopNormalizerRouter()); + services.AddSingleton(new NoopSignatureVerifier()); + return services.BuildServiceProvider(); + } + + private static DefaultVexProviderRunner CreateRunner( + IServiceProvider serviceProvider, + TimeProvider timeProvider, + Action configure, + RecordingRefreshScheduler? scheduler = null) + { + var options = new VexWorkerOptions(); + configure(options); + scheduler ??= new RecordingRefreshScheduler(); + return new DefaultVexProviderRunner( + serviceProvider, + new PluginCatalog(), + NullLogger.Instance, + timeProvider, + Microsoft.Extensions.Options.Options.Create(options), + scheduler); + } + + private sealed class RecordingRefreshScheduler : IVexConsensusRefreshScheduler + { + private readonly ConcurrentDictionary<(string VulnerabilityId, string ProductKey), byte> _entries = new(); + + public IReadOnlyCollection<(string VulnerabilityId, string ProductKey)> Entries + => _entries.Keys.ToArray(); + + public void ScheduleRefresh(string vulnerabilityId, string productKey) + { + _entries.TryAdd((vulnerabilityId, productKey), 0); + } + } + + private sealed class FixedTimeProvider : TimeProvider + { + private DateTimeOffset _utcNow; + + public FixedTimeProvider(DateTimeOffset utcNow) => _utcNow = utcNow; + + public override DateTimeOffset GetUtcNow() => _utcNow; + + public void Advance(TimeSpan delta) => _utcNow += delta; + } + + private sealed class NoopRawStore : IVexRawStore + { + public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken) => ValueTask.CompletedTask; + + public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken, IClientSessionHandle? session) => ValueTask.CompletedTask; + + public ValueTask FindByDigestAsync(string digest, CancellationToken cancellationToken, IClientSessionHandle? session = null) + => ValueTask.FromResult(null); + } + + private sealed class NoopClaimStore : IVexClaimStore + { + public ValueTask AppendAsync(IEnumerable claims, DateTimeOffset observedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null) + => ValueTask.CompletedTask; + + public ValueTask> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken, IClientSessionHandle? session = null) + => ValueTask.FromResult>(Array.Empty()); + } + + private sealed class NoopProviderStore : IVexProviderStore + { + private readonly ConcurrentDictionary _providers = new(StringComparer.Ordinal); + + public ValueTask FindAsync(string id, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _providers.TryGetValue(id, out var provider); + return ValueTask.FromResult(provider); + } + + public ValueTask> ListAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null) + => ValueTask.FromResult>(_providers.Values.ToList()); + + public ValueTask SaveAsync(VexProvider provider, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _providers[provider.Id] = provider; + return ValueTask.CompletedTask; + } + } + + private sealed class NoopNormalizerRouter : IVexNormalizerRouter + { + public ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray.Empty, ImmutableDictionary.Empty)); + } + + private sealed class StubNormalizerRouter : IVexNormalizerRouter + { + private readonly ImmutableArray _claims; + + public StubNormalizerRouter(IEnumerable claims) + { + _claims = claims.ToImmutableArray(); + } + + public ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + { + return ValueTask.FromResult(new VexClaimBatch(document, _claims, ImmutableDictionary.Empty)); + } + } + + private sealed class NoopSignatureVerifier : IVexSignatureVerifier + { + public ValueTask VerifyAsync(VexRawDocument document, CancellationToken cancellationToken) + => ValueTask.FromResult(null); + } + + private sealed class InMemoryStateRepository : IVexConnectorStateRepository + { + private readonly ConcurrentDictionary _states = new(StringComparer.Ordinal); + + public VexConnectorState? Get(string connectorId) + => _states.TryGetValue(connectorId, out var state) ? state : null; + + public void Save(VexConnectorState state) + => _states[state.ConnectorId] = state; + + public ValueTask GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null) + => ValueTask.FromResult(Get(connectorId)); + + public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + Save(state); + return ValueTask.CompletedTask; + } + } + + private sealed class TestConnector : IVexConnector + { + private readonly Func> _fetch; + private readonly Exception? _normalizeException; + private readonly List? _capturedContexts; + + private TestConnector(string id, Func> fetch, Exception? normalizeException = null, List? capturedContexts = null) + { + Id = id; + _fetch = fetch; + _normalizeException = normalizeException; + _capturedContexts = capturedContexts; + } + + public static TestConnector Success(string id) => new(id, (_, _) => AsyncEnumerable.Empty()); + + public static TestConnector SuccessWithCapture(string id) + { + var contexts = new List(); + return new TestConnector(id, (_, _) => AsyncEnumerable.Empty(), capturedContexts: contexts); + } + + public static TestConnector WithDocuments(string id, params VexRawDocument[] documents) + { + return new TestConnector(id, (_, _) => documents.ToAsyncEnumerable()); + } + + public static TestConnector Failure(string id, Exception exception) + { + return new TestConnector(id, (_, _) => new ThrowingAsyncEnumerable(exception)); + } + + public string Id { get; } + + public VexProviderKind Kind => VexProviderKind.Vendor; + + public bool ValidateInvoked { get; private set; } + + public bool FetchInvoked { get; private set; } + + public VexConnectorContext? LastContext => _capturedContexts is { Count: > 0 } ? _capturedContexts[^1] : null; + + public ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken) + { + ValidateInvoked = true; + return ValueTask.CompletedTask; + } + + public IAsyncEnumerable FetchAsync(VexConnectorContext context, CancellationToken cancellationToken) + { + FetchInvoked = true; + _capturedContexts?.Add(context); + return _fetch(context, cancellationToken); + } + + public ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + { + if (_normalizeException is not null) + { + throw _normalizeException; + } + + return ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray.Empty, ImmutableDictionary.Empty)); + } + } + + private sealed class ThrowingAsyncEnumerable : IAsyncEnumerable, IAsyncEnumerator + { + private readonly Exception _exception; + + public ThrowingAsyncEnumerable(Exception exception) => _exception = exception; + + public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) => this; + + public ValueTask MoveNextAsync() => ValueTask.FromException(_exception); + + public VexRawDocument Current => throw new InvalidOperationException(); + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } +} diff --git a/src/StellaOps.Excititor.Worker.Tests/VexWorkerOptionsTests.cs b/src/StellaOps.Excititor.Worker.Tests/VexWorkerOptionsTests.cs index d4f357f9..f19dde6e 100644 --- a/src/StellaOps.Excititor.Worker.Tests/VexWorkerOptionsTests.cs +++ b/src/StellaOps.Excititor.Worker.Tests/VexWorkerOptionsTests.cs @@ -57,11 +57,11 @@ public sealed class VexWorkerOptionsTests } [Fact] - public void ResolveSchedules_UsesProviderIntervalOverride() - { - var options = new VexWorkerOptions - { - DefaultInterval = TimeSpan.FromMinutes(15), + public void ResolveSchedules_UsesProviderIntervalOverride() + { + var options = new VexWorkerOptions + { + DefaultInterval = TimeSpan.FromMinutes(15), }; options.Providers.Add(new VexWorkerProviderOptions { @@ -72,8 +72,35 @@ public sealed class VexWorkerOptionsTests var schedules = options.ResolveSchedules(); - schedules.Should().ContainSingle(); - schedules[0].Interval.Should().Be(TimeSpan.FromMinutes(5)); - schedules[0].InitialDelay.Should().Be(TimeSpan.FromSeconds(10)); - } -} + schedules.Should().ContainSingle(); + schedules[0].Interval.Should().Be(TimeSpan.FromMinutes(5)); + schedules[0].InitialDelay.Should().Be(TimeSpan.FromSeconds(10)); + } + + [Fact] + public void RefreshOptions_DefaultsAlignWithExpectedValues() + { + var options = new VexWorkerRefreshOptions(); + + options.Enabled.Should().BeTrue(); + options.ConsensusTtl.Should().Be(TimeSpan.FromHours(2)); + options.Damper.ResolveDuration(0.95).Should().Be(TimeSpan.FromHours(24)); + options.Damper.ResolveDuration(0.6).Should().Be(TimeSpan.FromHours(36)); + } + + [Fact] + public void DamperOptions_ClampDurationWithinBounds() + { + var options = new VexStabilityDamperOptions + { + Minimum = TimeSpan.FromHours(10), + Maximum = TimeSpan.FromHours(20), + DefaultDuration = TimeSpan.FromHours(30), + }; + options.Rules.Clear(); + options.Rules.Add(new VexStabilityDamperRule { MinWeight = 0.8, Duration = TimeSpan.FromHours(40) }); + + options.ClampDuration(TimeSpan.FromHours(5)).Should().Be(TimeSpan.FromHours(10)); + options.ResolveDuration(0.85).Should().Be(TimeSpan.FromHours(20)); + } +} diff --git a/src/StellaOps.Excititor.Worker/Options/VexWorkerOptions.cs b/src/StellaOps.Excititor.Worker/Options/VexWorkerOptions.cs index 13fe3d83..9538616f 100644 --- a/src/StellaOps.Excititor.Worker/Options/VexWorkerOptions.cs +++ b/src/StellaOps.Excititor.Worker/Options/VexWorkerOptions.cs @@ -15,9 +15,11 @@ public sealed class VexWorkerOptions public bool OfflineMode { get; set; } - public IList Providers { get; } = new List(); - - public VexWorkerRetryOptions Retry { get; } = new(); + public IList Providers { get; } = new List(); + + public VexWorkerRetryOptions Retry { get; } = new(); + + public VexWorkerRefreshOptions Refresh { get; } = new(); internal IReadOnlyList ResolveSchedules() { diff --git a/src/StellaOps.Excititor.Worker/Options/VexWorkerOptionsValidator.cs b/src/StellaOps.Excititor.Worker/Options/VexWorkerOptionsValidator.cs index b2749e52..a37b9cdb 100644 --- a/src/StellaOps.Excititor.Worker/Options/VexWorkerOptionsValidator.cs +++ b/src/StellaOps.Excititor.Worker/Options/VexWorkerOptionsValidator.cs @@ -54,15 +54,64 @@ internal sealed class VexWorkerOptionsValidator : IValidateOptions options.Refresh.Damper.Maximum) + { + failures.Add("Excititor.Worker.Refresh.Damper.DefaultDuration must be within [Minimum, Maximum]."); + } + + for (var i = 0; i < options.Refresh.Damper.Rules.Count; i++) + { + var rule = options.Refresh.Damper.Rules[i]; + if (rule.MinWeight < 0) + { + failures.Add($"Excititor.Worker.Refresh.Damper.Rules[{i}].MinWeight must be non-negative."); + } + + if (rule.Duration <= TimeSpan.Zero) + { + failures.Add($"Excititor.Worker.Refresh.Damper.Rules[{i}].Duration must be greater than zero."); + } + + if (rule.Duration < options.Refresh.Damper.Minimum || rule.Duration > options.Refresh.Damper.Maximum) + { + failures.Add($"Excititor.Worker.Refresh.Damper.Rules[{i}].Duration must be within [Minimum, Maximum]."); + } + } + + for (var i = 0; i < options.Providers.Count; i++) + { + var provider = options.Providers[i]; + if (string.IsNullOrWhiteSpace(provider.ProviderId)) { failures.Add($"Excititor.Worker.Providers[{i}].ProviderId must be set."); } diff --git a/src/StellaOps.Excititor.Worker/Options/VexWorkerRefreshOptions.cs b/src/StellaOps.Excititor.Worker/Options/VexWorkerRefreshOptions.cs new file mode 100644 index 00000000..e15a01c8 --- /dev/null +++ b/src/StellaOps.Excititor.Worker/Options/VexWorkerRefreshOptions.cs @@ -0,0 +1,90 @@ +using System.Collections.Generic; +using System.Linq; + +namespace StellaOps.Excititor.Worker.Options; + +public sealed class VexWorkerRefreshOptions +{ + private static readonly TimeSpan DefaultScanInterval = TimeSpan.FromMinutes(10); + private static readonly TimeSpan DefaultConsensusTtl = TimeSpan.FromHours(2); + + public bool Enabled { get; set; } = true; + + public TimeSpan ScanInterval { get; set; } = DefaultScanInterval; + + public TimeSpan ConsensusTtl { get; set; } = DefaultConsensusTtl; + + public int ScanBatchSize { get; set; } = 250; + + public VexStabilityDamperOptions Damper { get; } = new(); +} + +public sealed class VexStabilityDamperOptions +{ + private static readonly TimeSpan DefaultMinimum = TimeSpan.FromHours(24); + private static readonly TimeSpan DefaultMaximum = TimeSpan.FromHours(48); + private static readonly TimeSpan DefaultDurationBaseline = TimeSpan.FromHours(36); + + public TimeSpan Minimum { get; set; } = DefaultMinimum; + + public TimeSpan Maximum { get; set; } = DefaultMaximum; + + public TimeSpan DefaultDuration { get; set; } = DefaultDurationBaseline; + + public IList Rules { get; } = new List + { + new() { MinWeight = 0.9, Duration = TimeSpan.FromHours(24) }, + new() { MinWeight = 0.75, Duration = TimeSpan.FromHours(30) }, + new() { MinWeight = 0.5, Duration = TimeSpan.FromHours(36) }, + }; + + internal TimeSpan ClampDuration(TimeSpan duration) + { + if (duration < Minimum) + { + return Minimum; + } + + if (duration > Maximum) + { + return Maximum; + } + + return duration; + } + + public TimeSpan ResolveDuration(double weight) + { + if (double.IsNaN(weight) || double.IsInfinity(weight) || weight < 0) + { + return ClampDuration(DefaultDuration); + } + + if (Rules.Count == 0) + { + return ClampDuration(DefaultDuration); + } + + // Evaluate highest weight threshold first. + TimeSpan? selected = null; + foreach (var rule in Rules.OrderByDescending(static r => r.MinWeight)) + { + if (weight >= rule.MinWeight) + { + selected = rule.Duration; + break; + } + } + + return ClampDuration(selected ?? DefaultDuration); + } +} + +public sealed class VexStabilityDamperRule +{ + public double MinWeight { get; set; } + = 1.0; + + public TimeSpan Duration { get; set; } + = TimeSpan.FromHours(24); +} diff --git a/src/StellaOps.Excititor.Worker/Program.cs b/src/StellaOps.Excititor.Worker/Program.cs index 8ca6adac..fb27154d 100644 --- a/src/StellaOps.Excititor.Worker/Program.cs +++ b/src/StellaOps.Excititor.Worker/Program.cs @@ -46,11 +46,11 @@ services.PostConfigure(options => }); } }); -services.AddSingleton(provider => -{ - var pluginOptions = provider.GetRequiredService>().Value; - var catalog = new PluginCatalog(); - +services.AddSingleton(provider => +{ + var pluginOptions = provider.GetRequiredService>().Value; + var catalog = new PluginCatalog(); + var directory = pluginOptions.ResolveDirectory(); if (Directory.Exists(directory)) { @@ -63,12 +63,15 @@ services.AddSingleton(provider => } return catalog; -}); - -services.AddSingleton(); -services.AddHostedService(); - -var host = builder.Build(); -await host.RunAsync(); +}); + +services.AddSingleton(); +services.AddSingleton(); +services.AddSingleton(static provider => provider.GetRequiredService()); +services.AddHostedService(); +services.AddHostedService(static provider => provider.GetRequiredService()); + +var host = builder.Build(); +await host.RunAsync(); public partial class Program; diff --git a/src/StellaOps.Excititor.Worker/Scheduling/DefaultVexProviderRunner.cs b/src/StellaOps.Excititor.Worker/Scheduling/DefaultVexProviderRunner.cs index e6811c97..e61b4652 100644 --- a/src/StellaOps.Excititor.Worker/Scheduling/DefaultVexProviderRunner.cs +++ b/src/StellaOps.Excititor.Worker/Scheduling/DefaultVexProviderRunner.cs @@ -1,12 +1,17 @@ using System; +using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; +using System.Security.Cryptography; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Driver; using StellaOps.Plugin; using StellaOps.Excititor.Connectors.Abstractions; using StellaOps.Excititor.Core; using StellaOps.Excititor.Storage.Mongo; +using StellaOps.Excititor.Worker.Options; namespace StellaOps.Excititor.Worker.Scheduling; @@ -16,17 +21,28 @@ internal sealed class DefaultVexProviderRunner : IVexProviderRunner private readonly PluginCatalog _pluginCatalog; private readonly ILogger _logger; private readonly TimeProvider _timeProvider; + private readonly VexWorkerRetryOptions _retryOptions; + private readonly IVexConsensusRefreshScheduler _refreshScheduler; public DefaultVexProviderRunner( IServiceProvider serviceProvider, PluginCatalog pluginCatalog, ILogger logger, - TimeProvider timeProvider) + TimeProvider timeProvider, + IOptions workerOptions, + IVexConsensusRefreshScheduler refreshScheduler) { _serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider)); _pluginCatalog = pluginCatalog ?? throw new ArgumentNullException(nameof(pluginCatalog)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _refreshScheduler = refreshScheduler ?? throw new ArgumentNullException(nameof(refreshScheduler)); + if (workerOptions is null) + { + throw new ArgumentNullException(nameof(workerOptions)); + } + + _retryOptions = workerOptions.Value?.Retry ?? throw new InvalidOperationException("VexWorkerOptions.Retry must be configured."); } public async ValueTask RunAsync(VexWorkerSchedule schedule, CancellationToken cancellationToken) @@ -69,10 +85,15 @@ internal sealed class DefaultVexProviderRunner : IVexProviderRunner var rawStore = scopeProvider.GetRequiredService(); var claimStore = scopeProvider.GetRequiredService(); var providerStore = scopeProvider.GetRequiredService(); + var stateRepository = scopeProvider.GetRequiredService(); var normalizerRouter = scopeProvider.GetRequiredService(); var signatureVerifier = scopeProvider.GetRequiredService(); - var sessionProvider = scopeProvider.GetRequiredService(); - var session = await sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false); + var sessionProvider = scopeProvider.GetService(); + IClientSessionHandle? session = null; + if (sessionProvider is not null) + { + session = await sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false); + } var descriptor = connector switch { @@ -85,35 +106,202 @@ internal sealed class DefaultVexProviderRunner : IVexProviderRunner await providerStore.SaveAsync(provider, cancellationToken, session).ConfigureAwait(false); + var stateBeforeRun = await stateRepository.GetAsync(descriptor.Id, cancellationToken, session).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + + if (stateBeforeRun?.NextEligibleRun is { } nextEligible && nextEligible > now) + { + _logger.LogInformation( + "Connector {ConnectorId} is in backoff until {NextEligible:O}; skipping run.", + connector.Id, + nextEligible); + return; + } + await connector.ValidateAsync(effectiveSettings, cancellationToken).ConfigureAwait(false); var context = new VexConnectorContext( - Since: null, + Since: stateBeforeRun?.LastUpdated, Settings: effectiveSettings, RawSink: rawStore, SignatureVerifier: signatureVerifier, Normalizers: normalizerRouter, - Services: scopeProvider); + Services: scopeProvider, + ResumeTokens: stateBeforeRun?.ResumeTokens ?? ImmutableDictionary.Empty); var documentCount = 0; var claimCount = 0; - await foreach (var document in connector.FetchAsync(context, cancellationToken)) - { - documentCount++; + var refreshKeys = new HashSet(StringComparer.Ordinal); - var batch = await normalizerRouter.NormalizeAsync(document, cancellationToken).ConfigureAwait(false); - if (!batch.Claims.IsDefaultOrEmpty && batch.Claims.Length > 0) + try + { + await foreach (var document in connector.FetchAsync(context, cancellationToken).ConfigureAwait(false)) { - claimCount += batch.Claims.Length; - await claimStore.AppendAsync(batch.Claims, _timeProvider.GetUtcNow(), cancellationToken, session).ConfigureAwait(false); + documentCount++; + + var batch = await normalizerRouter.NormalizeAsync(document, cancellationToken).ConfigureAwait(false); + if (!batch.Claims.IsDefaultOrEmpty && batch.Claims.Length > 0) + { + claimCount += batch.Claims.Length; + await claimStore.AppendAsync(batch.Claims, _timeProvider.GetUtcNow(), cancellationToken, session).ConfigureAwait(false); + + foreach (var claim in batch.Claims) + { + if (string.IsNullOrWhiteSpace(claim.VulnerabilityId) || string.IsNullOrWhiteSpace(claim.Product?.Key)) + { + continue; + } + + var refreshKey = string.Create( + claim.VulnerabilityId.Length + claim.Product.Key.Length + 1, + (claim.VulnerabilityId, claim.Product.Key), + static (span, value) => + { + value.Item1.AsSpan().CopyTo(span); + span[value.Item1.Length] = '|'; + value.Item2.AsSpan().CopyTo(span[(value.Item1.Length + 1)..]); + }); + + if (refreshKeys.Add(refreshKey)) + { + _refreshScheduler.ScheduleRefresh(claim.VulnerabilityId, claim.Product.Key); + } + } + } + } + + _logger.LogInformation( + "Connector {ConnectorId} persisted {DocumentCount} document(s) and {ClaimCount} claim(s) this run.", + connector.Id, + documentCount, + claimCount); + + await UpdateSuccessStateAsync(stateRepository, descriptor.Id, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + await UpdateFailureStateAsync(stateRepository, descriptor.Id, _timeProvider.GetUtcNow(), ex, cancellationToken).ConfigureAwait(false); + throw; + } + } + + private async Task UpdateSuccessStateAsync( + IVexConnectorStateRepository stateRepository, + string connectorId, + DateTimeOffset completedAt, + CancellationToken cancellationToken) + { + var current = await stateRepository.GetAsync(connectorId, cancellationToken).ConfigureAwait(false) + ?? new VexConnectorState(connectorId, null, ImmutableArray.Empty); + + var updated = current with + { + LastSuccessAt = completedAt, + FailureCount = 0, + NextEligibleRun = null, + LastFailureReason = null + }; + + await stateRepository.SaveAsync(updated, cancellationToken).ConfigureAwait(false); + } + + private async Task UpdateFailureStateAsync( + IVexConnectorStateRepository stateRepository, + string connectorId, + DateTimeOffset failureTime, + Exception exception, + CancellationToken cancellationToken) + { + var current = await stateRepository.GetAsync(connectorId, cancellationToken).ConfigureAwait(false) + ?? new VexConnectorState(connectorId, null, ImmutableArray.Empty); + + var failureCount = current.FailureCount + 1; + var delay = CalculateDelayWithJitter(failureCount); + var nextEligible = failureTime + delay; + + if (failureCount >= _retryOptions.FailureThreshold) + { + var quarantineUntil = failureTime + _retryOptions.QuarantineDuration; + if (quarantineUntil > nextEligible) + { + nextEligible = quarantineUntil; } } - _logger.LogInformation( - "Connector {ConnectorId} persisted {DocumentCount} document(s) and {ClaimCount} claim(s) this run.", - connector.Id, - documentCount, - claimCount); + var retryCap = failureTime + _retryOptions.RetryCap; + if (nextEligible > retryCap) + { + nextEligible = retryCap; + } + + if (nextEligible < failureTime) + { + nextEligible = failureTime; + } + + var updated = current with + { + FailureCount = failureCount, + NextEligibleRun = nextEligible, + LastFailureReason = Truncate(exception.Message, 512) + }; + + await stateRepository.SaveAsync(updated, cancellationToken).ConfigureAwait(false); + + _logger.LogWarning( + exception, + "Connector {ConnectorId} failed (attempt {Attempt}). Next eligible run at {NextEligible:O}.", + connectorId, + failureCount, + nextEligible); + } + + private TimeSpan CalculateDelayWithJitter(int failureCount) + { + var exponent = Math.Max(0, failureCount - 1); + var factor = Math.Pow(2, exponent); + var baselineTicks = (long)Math.Min(_retryOptions.BaseDelay.Ticks * factor, _retryOptions.MaxDelay.Ticks); + + if (_retryOptions.JitterRatio <= 0) + { + return TimeSpan.FromTicks(baselineTicks); + } + + var minFactor = 1.0 - _retryOptions.JitterRatio; + var maxFactor = 1.0 + _retryOptions.JitterRatio; + Span buffer = stackalloc byte[8]; + RandomNumberGenerator.Fill(buffer); + var sample = BitConverter.ToUInt64(buffer) / (double)ulong.MaxValue; + var jitterFactor = minFactor + (maxFactor - minFactor) * sample; + var jitteredTicks = (long)Math.Round(baselineTicks * jitterFactor); + + if (jitteredTicks < _retryOptions.BaseDelay.Ticks) + { + jitteredTicks = _retryOptions.BaseDelay.Ticks; + } + + if (jitteredTicks > _retryOptions.MaxDelay.Ticks) + { + jitteredTicks = _retryOptions.MaxDelay.Ticks; + } + + return TimeSpan.FromTicks(jitteredTicks); + } + + private static string Truncate(string? value, int maxLength) + { + if (string.IsNullOrEmpty(value)) + { + return string.Empty; + } + + return value.Length <= maxLength + ? value + : value[..maxLength]; } } diff --git a/src/StellaOps.Excititor.Worker/Scheduling/IVexConsensusRefreshScheduler.cs b/src/StellaOps.Excititor.Worker/Scheduling/IVexConsensusRefreshScheduler.cs new file mode 100644 index 00000000..0c16298c --- /dev/null +++ b/src/StellaOps.Excititor.Worker/Scheduling/IVexConsensusRefreshScheduler.cs @@ -0,0 +1,6 @@ +namespace StellaOps.Excititor.Worker.Scheduling; + +public interface IVexConsensusRefreshScheduler +{ + void ScheduleRefresh(string vulnerabilityId, string productKey); +} diff --git a/src/StellaOps.Excititor.Worker/Scheduling/VexConsensusRefreshService.cs b/src/StellaOps.Excititor.Worker/Scheduling/VexConsensusRefreshService.cs new file mode 100644 index 00000000..4cda1385 --- /dev/null +++ b/src/StellaOps.Excititor.Worker/Scheduling/VexConsensusRefreshService.cs @@ -0,0 +1,622 @@ +using System.Collections.Concurrent; +using System.Collections.Immutable; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading.Channels; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Policy; +using StellaOps.Excititor.Storage.Mongo; +using StellaOps.Excititor.Worker.Options; + +namespace StellaOps.Excititor.Worker.Scheduling; + +internal sealed class VexConsensusRefreshService : BackgroundService, IVexConsensusRefreshScheduler +{ + private readonly IServiceScopeFactory _scopeFactory; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly Channel _refreshRequests; + private readonly ConcurrentDictionary _scheduledKeys = new(StringComparer.Ordinal); + private readonly IDisposable? _optionsSubscription; + private RefreshState _refreshState; + + public VexConsensusRefreshService( + IServiceScopeFactory scopeFactory, + IOptionsMonitor optionsMonitor, + ILogger logger, + TimeProvider timeProvider) + { + _scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _refreshRequests = Channel.CreateUnbounded(new UnboundedChannelOptions + { + AllowSynchronousContinuations = false, + SingleReader = true, + SingleWriter = false, + }); + + if (optionsMonitor is null) + { + throw new ArgumentNullException(nameof(optionsMonitor)); + } + + var options = optionsMonitor.CurrentValue; + _refreshState = RefreshState.FromOptions(options.Refresh); + _optionsSubscription = optionsMonitor.OnChange(o => + { + var state = RefreshState.FromOptions((o?.Refresh) ?? new VexWorkerRefreshOptions()); + Volatile.Write(ref _refreshState, state); + _logger.LogInformation( + "Consensus refresh options updated: enabled={Enabled}, interval={Interval}, ttl={Ttl}, batch={Batch}", + state.Enabled, + state.ScanInterval, + state.ConsensusTtl, + state.ScanBatchSize); + }); + } + + public override void Dispose() + { + _optionsSubscription?.Dispose(); + base.Dispose(); + } + + public void ScheduleRefresh(string vulnerabilityId, string productKey) + { + if (string.IsNullOrWhiteSpace(vulnerabilityId) || string.IsNullOrWhiteSpace(productKey)) + { + return; + } + + var key = BuildKey(vulnerabilityId, productKey); + if (!_scheduledKeys.TryAdd(key, 0)) + { + return; + } + + var request = new RefreshRequest(vulnerabilityId.Trim(), productKey.Trim()); + if (!_refreshRequests.Writer.TryWrite(request)) + { + _scheduledKeys.TryRemove(key, out _); + } + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + var queueTask = ProcessQueueAsync(stoppingToken); + + try + { + while (!stoppingToken.IsCancellationRequested) + { + var options = CurrentOptions; + + try + { + await ProcessEligibleHoldsAsync(options, stoppingToken).ConfigureAwait(false); + if (options.Enabled) + { + await ProcessTtlRefreshAsync(options, stoppingToken).ConfigureAwait(false); + } + else + { + _logger.LogDebug("Consensus refresh disabled; skipping TTL sweep."); + } + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + _logger.LogError(ex, "Consensus refresh loop failed."); + } + + try + { + await Task.Delay(options.ScanInterval, stoppingToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + } + } + finally + { + _refreshRequests.Writer.TryComplete(); + try + { + await queueTask.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + } + } + } + + private RefreshState CurrentOptions => Volatile.Read(ref _refreshState); + + private async Task ProcessQueueAsync(CancellationToken cancellationToken) + { + try + { + while (await _refreshRequests.Reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false)) + { + while (_refreshRequests.Reader.TryRead(out var request)) + { + var key = BuildKey(request.VulnerabilityId, request.ProductKey); + try + { + await ProcessCandidateAsync(request.VulnerabilityId, request.ProductKey, existingConsensus: null, CurrentOptions, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return; + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to refresh consensus for {VulnerabilityId}/{ProductKey} from queue.", request.VulnerabilityId, request.ProductKey); + } + finally + { + _scheduledKeys.TryRemove(key, out _); + } + } + } + } + catch (OperationCanceledException) + { + } + } + + private async Task ProcessEligibleHoldsAsync(RefreshState options, CancellationToken cancellationToken) + { + using var scope = _scopeFactory.CreateScope(); + var holdStore = scope.ServiceProvider.GetRequiredService(); + var consensusStore = scope.ServiceProvider.GetRequiredService(); + + var now = _timeProvider.GetUtcNow(); + await foreach (var hold in holdStore.FindEligibleAsync(now, options.ScanBatchSize, cancellationToken).ConfigureAwait(false)) + { + var key = BuildKey(hold.VulnerabilityId, hold.ProductKey); + if (!_scheduledKeys.TryAdd(key, 0)) + { + continue; + } + + try + { + await consensusStore.SaveAsync(hold.Candidate with { }, cancellationToken).ConfigureAwait(false); + await holdStore.RemoveAsync(hold.VulnerabilityId, hold.ProductKey, cancellationToken).ConfigureAwait(false); + _logger.LogInformation( + "Promoted consensus hold for {VulnerabilityId}/{ProductKey}; status={Status}, reason={Reason}", + hold.VulnerabilityId, + hold.ProductKey, + hold.Candidate.Status, + hold.Reason); + } + catch (Exception ex) when (!cancellationToken.IsCancellationRequested) + { + _logger.LogError( + ex, + "Failed to promote consensus hold for {VulnerabilityId}/{ProductKey}.", + hold.VulnerabilityId, + hold.ProductKey); + } + finally + { + _scheduledKeys.TryRemove(key, out _); + } + } + } + + private async Task ProcessTtlRefreshAsync(RefreshState options, CancellationToken cancellationToken) + { + var now = _timeProvider.GetUtcNow(); + var cutoff = now - options.ConsensusTtl; + + using var scope = _scopeFactory.CreateScope(); + var consensusStore = scope.ServiceProvider.GetRequiredService(); + + await foreach (var consensus in consensusStore.FindCalculatedBeforeAsync(cutoff, options.ScanBatchSize, cancellationToken).ConfigureAwait(false)) + { + var key = BuildKey(consensus.VulnerabilityId, consensus.Product.Key); + if (!_scheduledKeys.TryAdd(key, 0)) + { + continue; + } + + try + { + await ProcessCandidateAsync(consensus.VulnerabilityId, consensus.Product.Key, consensus, options, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (!cancellationToken.IsCancellationRequested) + { + _logger.LogError( + ex, + "Failed to refresh consensus for {VulnerabilityId}/{ProductKey} during TTL sweep.", + consensus.VulnerabilityId, + consensus.Product.Key); + } + finally + { + _scheduledKeys.TryRemove(key, out _); + } + } + } + + private async Task ProcessCandidateAsync( + string vulnerabilityId, + string productKey, + VexConsensus? existingConsensus, + RefreshState options, + CancellationToken cancellationToken) + { + using var scope = _scopeFactory.CreateScope(); + var consensusStore = scope.ServiceProvider.GetRequiredService(); + var holdStore = scope.ServiceProvider.GetRequiredService(); + var claimStore = scope.ServiceProvider.GetRequiredService(); + var providerStore = scope.ServiceProvider.GetRequiredService(); + var policyProvider = scope.ServiceProvider.GetRequiredService(); + + existingConsensus ??= await consensusStore.FindAsync(vulnerabilityId, productKey, cancellationToken).ConfigureAwait(false); + + var claims = await claimStore.FindAsync(vulnerabilityId, productKey, since: null, cancellationToken).ConfigureAwait(false); + if (claims.Count == 0) + { + _logger.LogDebug("No claims found for {VulnerabilityId}/{ProductKey}; skipping consensus refresh.", vulnerabilityId, productKey); + await holdStore.RemoveAsync(vulnerabilityId, productKey, cancellationToken).ConfigureAwait(false); + return; + } + + var claimList = claims as IReadOnlyList ?? claims.ToList(); + + var snapshot = policyProvider.GetSnapshot(); + var providerCache = new Dictionary(StringComparer.Ordinal); + var providers = await LoadProvidersAsync(claimList, providerStore, providerCache, cancellationToken).ConfigureAwait(false); + var product = ResolveProduct(claimList, productKey); + var calculatedAt = _timeProvider.GetUtcNow(); + + var resolver = new VexConsensusResolver(snapshot.ConsensusPolicy); + var request = new VexConsensusRequest( + vulnerabilityId, + product, + claimList.ToArray(), + providers, + calculatedAt, + snapshot.ConsensusOptions.WeightCeiling, + AggregateSignals(claimList), + snapshot.RevisionId, + snapshot.Digest); + + var resolution = resolver.Resolve(request); + var candidate = NormalizePolicyMetadata(resolution.Consensus, snapshot); + + await ApplyConsensusAsync( + candidate, + existingConsensus, + holdStore, + consensusStore, + options.Damper, + options, + cancellationToken).ConfigureAwait(false); + } + + private async Task ApplyConsensusAsync( + VexConsensus candidate, + VexConsensus? existing, + IVexConsensusHoldStore holdStore, + IVexConsensusStore consensusStore, + DamperState damper, + RefreshState options, + CancellationToken cancellationToken) + { + var vulnerabilityId = candidate.VulnerabilityId; + var productKey = candidate.Product.Key; + + var componentChanged = HasComponentChange(existing, candidate); + var statusChanged = existing is not null && existing.Status != candidate.Status; + + if (existing is null) + { + await consensusStore.SaveAsync(candidate, cancellationToken).ConfigureAwait(false); + await holdStore.RemoveAsync(vulnerabilityId, productKey, cancellationToken).ConfigureAwait(false); + _logger.LogInformation("Stored initial consensus for {VulnerabilityId}/{ProductKey} with status {Status}.", vulnerabilityId, productKey, candidate.Status); + return; + } + + TimeSpan duration = TimeSpan.Zero; + if (statusChanged) + { + if (componentChanged) + { + duration = TimeSpan.Zero; + } + else + { + var mappedStatus = MapConsensusStatus(candidate.Status); + var supportingWeight = mappedStatus is null + ? 0d + : candidate.Sources + .Where(source => source.Status == mappedStatus.Value) + .Sum(source => source.Weight); + duration = damper.ResolveDuration(supportingWeight); + } + } + + var requestedAt = _timeProvider.GetUtcNow(); + + if (statusChanged && duration > TimeSpan.Zero) + { + var eligibleAt = requestedAt + duration; + var reason = componentChanged ? "component_change" : "status_change"; + var newHold = new VexConsensusHold(vulnerabilityId, productKey, candidate, requestedAt, eligibleAt, reason); + var existingHold = await holdStore.FindAsync(vulnerabilityId, productKey, cancellationToken).ConfigureAwait(false); + + if (existingHold is null || existingHold.Candidate != candidate || existingHold.EligibleAt != newHold.EligibleAt) + { + await holdStore.SaveAsync(newHold, cancellationToken).ConfigureAwait(false); + _logger.LogInformation( + "Deferred consensus update for {VulnerabilityId}/{ProductKey} until {EligibleAt:O}; status {Status} pending (reason={Reason}).", + vulnerabilityId, + productKey, + eligibleAt, + candidate.Status, + reason); + } + return; + } + + await consensusStore.SaveAsync(candidate, cancellationToken).ConfigureAwait(false); + await holdStore.RemoveAsync(vulnerabilityId, productKey, cancellationToken).ConfigureAwait(false); + _logger.LogInformation( + "Updated consensus for {VulnerabilityId}/{ProductKey}; status={Status}, componentChange={ComponentChanged}.", + vulnerabilityId, + productKey, + candidate.Status, + componentChanged); + } + + private static bool HasComponentChange(VexConsensus? existing, VexConsensus candidate) + { + if (existing is null) + { + return false; + } + + var previous = existing.Product.ComponentIdentifiers; + var current = candidate.Product.ComponentIdentifiers; + + if (previous.IsDefaultOrEmpty && current.IsDefaultOrEmpty) + { + return false; + } + + if (previous.Length != current.Length) + { + return true; + } + + for (var i = 0; i < previous.Length; i++) + { + if (!string.Equals(previous[i], current[i], StringComparison.Ordinal)) + { + return true; + } + } + + return false; + } + + private static VexConsensus NormalizePolicyMetadata(VexConsensus consensus, VexPolicySnapshot snapshot) + { + if (string.Equals(consensus.PolicyVersion, snapshot.Version, StringComparison.Ordinal) && + string.Equals(consensus.PolicyRevisionId, snapshot.RevisionId, StringComparison.Ordinal) && + string.Equals(consensus.PolicyDigest, snapshot.Digest, StringComparison.Ordinal)) + { + return consensus; + } + + return new VexConsensus( + consensus.VulnerabilityId, + consensus.Product, + consensus.Status, + consensus.CalculatedAt, + consensus.Sources, + consensus.Conflicts, + consensus.Signals, + snapshot.Version, + consensus.Summary, + snapshot.RevisionId, + snapshot.Digest); + } + + private static VexClaimStatus? MapConsensusStatus(VexConsensusStatus status) + => status switch + { + VexConsensusStatus.Affected => VexClaimStatus.Affected, + VexConsensusStatus.NotAffected => VexClaimStatus.NotAffected, + VexConsensusStatus.Fixed => VexClaimStatus.Fixed, + _ => null, + }; + + private static string BuildKey(string vulnerabilityId, string productKey) + => string.Create( + vulnerabilityId.Length + productKey.Length + 1, + (vulnerabilityId, productKey), + static (span, tuple) => + { + tuple.vulnerabilityId.AsSpan().CopyTo(span); + span[tuple.vulnerabilityId.Length] = '|'; + tuple.productKey.AsSpan().CopyTo(span[(tuple.vulnerabilityId.Length + 1)..]); + }); + + private static VexProduct ResolveProduct(IReadOnlyList claims, string productKey) + { + if (claims.Count > 0) + { + return claims[0].Product; + } + + var inferredPurl = productKey.StartsWith("pkg:", StringComparison.OrdinalIgnoreCase) ? productKey : null; + return new VexProduct(productKey, name: null, version: null, purl: inferredPurl); + } + + private static VexSignalSnapshot? AggregateSignals(IReadOnlyList claims) + { + if (claims.Count == 0) + { + return null; + } + + VexSeveritySignal? bestSeverity = null; + double? bestScore = null; + bool kevPresent = false; + bool kevTrue = false; + double? bestEpss = null; + + foreach (var claim in claims) + { + if (claim.Signals is null) + { + continue; + } + + var severity = claim.Signals.Severity; + if (severity is not null) + { + var score = severity.Score; + if (bestSeverity is null || + (score is not null && (bestScore is null || score.Value > bestScore.Value)) || + (score is null && bestScore is null && !string.IsNullOrWhiteSpace(severity.Label) && string.IsNullOrWhiteSpace(bestSeverity.Label))) + { + bestSeverity = severity; + bestScore = severity.Score; + } + } + + if (claim.Signals.Kev is { } kevValue) + { + kevPresent = true; + if (kevValue) + { + kevTrue = true; + } + } + + if (claim.Signals.Epss is { } epss) + { + if (bestEpss is null || epss > bestEpss.Value) + { + bestEpss = epss; + } + } + } + + if (bestSeverity is null && !kevPresent && bestEpss is null) + { + return null; + } + + bool? kev = kevTrue ? true : (kevPresent ? false : null); + return new VexSignalSnapshot(bestSeverity, kev, bestEpss); + } + + private static async Task> LoadProvidersAsync( + IReadOnlyList claims, + IVexProviderStore providerStore, + IDictionary cache, + CancellationToken cancellationToken) + { + if (claims.Count == 0) + { + return ImmutableDictionary.Empty; + } + + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + var seen = new HashSet(StringComparer.Ordinal); + + foreach (var providerId in claims.Select(claim => claim.ProviderId)) + { + if (!seen.Add(providerId)) + { + continue; + } + + if (cache.TryGetValue(providerId, out var cached)) + { + builder[providerId] = cached; + continue; + } + + var provider = await providerStore.FindAsync(providerId, cancellationToken).ConfigureAwait(false); + if (provider is not null) + { + cache[providerId] = provider; + builder[providerId] = provider; + } + } + + return builder.ToImmutable(); + } + + private readonly record struct RefreshRequest(string VulnerabilityId, string ProductKey); + + private sealed record RefreshState( + bool Enabled, + TimeSpan ScanInterval, + TimeSpan ConsensusTtl, + int ScanBatchSize, + DamperState Damper) + { + public static RefreshState FromOptions(VexWorkerRefreshOptions options) + { + var interval = options.ScanInterval > TimeSpan.Zero ? options.ScanInterval : TimeSpan.FromMinutes(10); + var ttl = options.ConsensusTtl > TimeSpan.Zero ? options.ConsensusTtl : TimeSpan.FromHours(2); + var batchSize = options.ScanBatchSize > 0 ? options.ScanBatchSize : 250; + var damper = DamperState.FromOptions(options.Damper); + return new RefreshState(options.Enabled, interval, ttl, batchSize, damper); + } + } + + private sealed record DamperState(TimeSpan Minimum, TimeSpan Maximum, TimeSpan DefaultDuration, ImmutableArray Rules) + { + public static DamperState FromOptions(VexStabilityDamperOptions options) + { + var minimum = options.Minimum < TimeSpan.Zero ? TimeSpan.Zero : options.Minimum; + var maximum = options.Maximum > minimum ? options.Maximum : minimum + TimeSpan.FromHours(1); + var defaultDuration = options.ClampDuration(options.DefaultDuration); + var rules = options.Rules + .Select(rule => new DamperRuleState(Math.Max(0, rule.MinWeight), options.ClampDuration(rule.Duration))) + .OrderByDescending(rule => rule.MinWeight) + .ToImmutableArray(); + return new DamperState(minimum, maximum, defaultDuration, rules); + } + + public TimeSpan ResolveDuration(double weight) + { + if (double.IsNaN(weight) || double.IsInfinity(weight) || weight < 0) + { + return DefaultDuration; + } + + foreach (var rule in Rules) + { + if (weight >= rule.MinWeight) + { + return rule.Duration; + } + } + + return DefaultDuration; + } + } + + private sealed record DamperRuleState(double MinWeight, TimeSpan Duration); +} diff --git a/src/StellaOps.Excititor.Worker/StellaOps.Excititor.Worker.csproj b/src/StellaOps.Excititor.Worker/StellaOps.Excititor.Worker.csproj index 2c694928..82c79177 100644 --- a/src/StellaOps.Excititor.Worker/StellaOps.Excititor.Worker.csproj +++ b/src/StellaOps.Excititor.Worker/StellaOps.Excititor.Worker.csproj @@ -1,22 +1,23 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Worker/TASKS.md b/src/StellaOps.Excititor.Worker/TASKS.md index 871dde4f..b15cf300 100644 --- a/src/StellaOps.Excititor.Worker/TASKS.md +++ b/src/StellaOps.Excititor.Worker/TASKS.md @@ -3,7 +3,8 @@ If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md | Task | Owner(s) | Depends on | Notes | |---|---|---|---| |EXCITITOR-WORKER-01-001 – Worker host & scheduling|Team Excititor Worker|EXCITITOR-STORAGE-01-003, EXCITITOR-WEB-01-001|**DONE (2025-10-17)** – Worker project bootstraps provider schedules from configuration, integrates plugin catalog discovery, and emits structured logs/metrics-ready events via `VexWorkerHostedService`; scheduling logic covered by `VexWorkerOptionsTests`.| -|EXCITITOR-WORKER-01-002 – Resume tokens & retry policy|Team Excititor Worker|EXCITITOR-WORKER-01-001|DOING (2025-10-19) – Prereq EXCITITOR-WORKER-01-001 closed 2025-10-17; implementing durable resume markers, jittered backoff, and failure quarantine flow.| +|EXCITITOR-WORKER-01-002 – Resume tokens & retry policy|Team Excititor Worker|EXCITITOR-WORKER-01-001|DONE (2025-10-21) – Worker flows resume tokens through `VexConnectorContext`, persists success/failure metadata with jittered exponential backoff and quarantine scheduling, and ships unit coverage for skip/backoff/resume behaviour.| |EXCITITOR-WORKER-01-003 – Verification & cache GC loops|Team Excititor Worker|EXCITITOR-WORKER-01-001, EXCITITOR-ATTEST-01-003, EXCITITOR-EXPORT-01-002|TODO – Add scheduled attestation re-verification and cache pruning routines, surfacing metrics for export reuse ratios.| -|EXCITITOR-WORKER-01-004 – TTL refresh & stability damper|Team Excititor Worker|EXCITITOR-WORKER-01-001, EXCITITOR-CORE-02-001|DOING (2025-10-19) – Prereqs EXCITITOR-WORKER-01-001 (closed 2025-10-17) and EXCITITOR-CORE-02-001 (closed 2025-10-19) verified; building TTL monitor with dampers and re-resolve triggers.| -|EXCITITOR-WORKER-02-001 – Resolve Microsoft.Extensions.Caching.Memory advisory|Team Excititor Worker|EXCITITOR-WORKER-01-001|DOING (2025-10-19) – Prereq EXCITITOR-WORKER-01-001 closed 2025-10-17; upgrading `Microsoft.Extensions.Caching.Memory` stack and refreshing lockfiles/tests to clear NU1903.| +|EXCITITOR-WORKER-01-004 – TTL refresh & stability damper|Team Excititor Worker|EXCITITOR-WORKER-01-001, EXCITITOR-CORE-02-001|DONE (2025-10-21) – Added configurable TTL refresh service with trust-weighted dampers, component fingerprint bypass, and Mongo-backed hold promotion to stabilize consensus updates.| +|EXCITITOR-WORKER-02-001 – Resolve Microsoft.Extensions.Caching.Memory advisory|Team Excititor Worker|EXCITITOR-WORKER-01-001|DONE (2025-10-21) – Upgraded Excititor dependencies to `Microsoft.Extensions.*` 10.0.0-preview.7.25380.108, re-enabled attestation fixtures, and reran worker/webservice regression suites without NU1903 warnings.| +|EXCITITOR-WORKER-02-001-REVIEW – Review Microsoft.Extensions.* upgrade|Team Excititor Worker (Review)|EXCITITOR-WORKER-02-001|TODO – Peer review for dependency bump/attestation fixture changes; verify connector coverage updates and approve release note entry.| diff --git a/src/StellaOps.Notify.Connectors.Email.Tests/EmailChannelHealthProviderTests.cs b/src/StellaOps.Notify.Connectors.Email.Tests/EmailChannelHealthProviderTests.cs new file mode 100644 index 00000000..984ea24a --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Email.Tests/EmailChannelHealthProviderTests.cs @@ -0,0 +1,100 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Notify.Engine; +using StellaOps.Notify.Models; +using Xunit; + +namespace StellaOps.Notify.Connectors.Email.Tests; + +public sealed class EmailChannelHealthProviderTests +{ + private static readonly EmailChannelHealthProvider Provider = new(); + + [Fact] + public async Task CheckAsync_ReturnsHealthy() + { + var channel = CreateChannel(enabled: true, target: "ops@example.com"); + + var context = new ChannelHealthContext( + channel.TenantId, + channel, + channel.Config.Target!, + new DateTimeOffset(2025, 10, 20, 15, 0, 0, TimeSpan.Zero), + "trace-email-001"); + + var result = await Provider.CheckAsync(context, CancellationToken.None); + + Assert.Equal(ChannelHealthStatus.Healthy, result.Status); + Assert.Equal("true", result.Metadata["email.channel.enabled"]); + Assert.Equal("true", result.Metadata["email.validation.targetPresent"]); + Assert.Equal("ops@example.com", result.Metadata["email.target"]); + } + + [Fact] + public async Task CheckAsync_ReturnsDegradedWhenDisabled() + { + var channel = CreateChannel(enabled: false, target: "ops@example.com"); + + var context = new ChannelHealthContext( + channel.TenantId, + channel, + channel.Config.Target!, + DateTimeOffset.UtcNow, + "trace-email-002"); + + var result = await Provider.CheckAsync(context, CancellationToken.None); + + Assert.Equal(ChannelHealthStatus.Degraded, result.Status); + Assert.Equal("false", result.Metadata["email.channel.enabled"]); + } + + [Fact] + public async Task CheckAsync_ReturnsUnhealthyWhenTargetMissing() + { + var channel = NotifyChannel.Create( + channelId: "channel-email-ops", + tenantId: "tenant-sec", + name: "email:ops", + type: NotifyChannelType.Email, + config: NotifyChannelConfig.Create( + secretRef: "ref://notify/channels/email/ops", + target: null, + properties: new Dictionary + { + ["smtpHost"] = "smtp.ops.example.com" + }), + enabled: true); + + var context = new ChannelHealthContext( + channel.TenantId, + channel, + channel.Name, + DateTimeOffset.UtcNow, + "trace-email-003"); + + var result = await Provider.CheckAsync(context, CancellationToken.None); + + Assert.Equal(ChannelHealthStatus.Unhealthy, result.Status); + Assert.Equal("false", result.Metadata["email.validation.targetPresent"]); + } + + private static NotifyChannel CreateChannel(bool enabled, string? target) + { + return NotifyChannel.Create( + channelId: "channel-email-ops", + tenantId: "tenant-sec", + name: "email:ops", + type: NotifyChannelType.Email, + config: NotifyChannelConfig.Create( + secretRef: "ref://notify/channels/email/ops", + target: target, + properties: new Dictionary + { + ["smtpHost"] = "smtp.ops.example.com", + ["password"] = "super-secret" + }), + enabled: enabled); + } +} diff --git a/src/StellaOps.Notify.Connectors.Email.Tests/StellaOps.Notify.Connectors.Email.Tests.csproj b/src/StellaOps.Notify.Connectors.Email.Tests/StellaOps.Notify.Connectors.Email.Tests.csproj new file mode 100644 index 00000000..b155d003 --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Email.Tests/StellaOps.Notify.Connectors.Email.Tests.csproj @@ -0,0 +1,21 @@ + + + net10.0 + enable + enable + false + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Notify.Connectors.Email/EmailChannelHealthProvider.cs b/src/StellaOps.Notify.Connectors.Email/EmailChannelHealthProvider.cs new file mode 100644 index 00000000..92fa6ae6 --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Email/EmailChannelHealthProvider.cs @@ -0,0 +1,59 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Notify.Engine; +using StellaOps.Notify.Models; + +namespace StellaOps.Notify.Connectors.Email; + +[ServiceBinding(typeof(INotifyChannelHealthProvider), ServiceLifetime.Singleton)] +public sealed class EmailChannelHealthProvider : INotifyChannelHealthProvider +{ + public NotifyChannelType ChannelType => NotifyChannelType.Email; + + public Task CheckAsync(ChannelHealthContext context, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + cancellationToken.ThrowIfCancellationRequested(); + + var builder = EmailMetadataBuilder.CreateBuilder(context) + .Add("email.channel.enabled", context.Channel.Enabled ? "true" : "false") + .Add("email.validation.targetPresent", HasConfiguredTarget(context.Channel) ? "true" : "false"); + + var metadata = builder.Build(); + var status = ResolveStatus(context.Channel); + var message = status switch + { + ChannelHealthStatus.Healthy => "Email channel configuration validated.", + ChannelHealthStatus.Degraded => "Email channel is disabled; enable it to resume deliveries.", + ChannelHealthStatus.Unhealthy => "Email channel target/configuration incomplete.", + _ => "Email channel diagnostics completed." + }; + + return Task.FromResult(new ChannelHealthResult(status, message, metadata)); + } + + private static ChannelHealthStatus ResolveStatus(NotifyChannel channel) + { + if (!HasConfiguredTarget(channel)) + { + return ChannelHealthStatus.Unhealthy; + } + + if (!channel.Enabled) + { + return ChannelHealthStatus.Degraded; + } + + return ChannelHealthStatus.Healthy; + } + + private static bool HasConfiguredTarget(NotifyChannel channel) + => !string.IsNullOrWhiteSpace(channel.Config.Target) || + (channel.Config.Properties is not null && + channel.Config.Properties.TryGetValue("fromAddress", out var from) && + !string.IsNullOrWhiteSpace(from)); +} diff --git a/src/StellaOps.Notify.Connectors.Email/EmailChannelTestProvider.cs b/src/StellaOps.Notify.Connectors.Email/EmailChannelTestProvider.cs index 96ae3e6b..8d148099 100644 --- a/src/StellaOps.Notify.Connectors.Email/EmailChannelTestProvider.cs +++ b/src/StellaOps.Notify.Connectors.Email/EmailChannelTestProvider.cs @@ -37,11 +37,8 @@ public sealed class EmailChannelTestProvider : INotifyChannelTestProvider ChannelTestPreviewUtilities.ComputeBodyHash(htmlBody), context.Request.Attachments); - var metadata = new Dictionary(StringComparer.Ordinal) - { - ["email.to"] = context.Target - }; - - return Task.FromResult(new ChannelTestPreviewResult(preview, metadata)); - } -} + var metadata = EmailMetadataBuilder.Build(context); + + return Task.FromResult(new ChannelTestPreviewResult(preview, metadata)); + } +} diff --git a/src/StellaOps.Notify.Connectors.Email/EmailMetadataBuilder.cs b/src/StellaOps.Notify.Connectors.Email/EmailMetadataBuilder.cs new file mode 100644 index 00000000..8db44a9b --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Email/EmailMetadataBuilder.cs @@ -0,0 +1,54 @@ +using System; +using System.Collections.Generic; +using StellaOps.Notify.Connectors.Shared; +using StellaOps.Notify.Engine; +using StellaOps.Notify.Models; + +namespace StellaOps.Notify.Connectors.Email; + +/// +/// Builds metadata for Email previews and health diagnostics with redacted secrets. +/// +internal static class EmailMetadataBuilder +{ + private const int SecretHashLengthBytes = 8; + + public static ConnectorMetadataBuilder CreateBuilder(ChannelTestPreviewContext context) + => CreateBaseBuilder( + channel: context.Channel, + target: context.Target, + timestamp: context.Timestamp, + properties: context.Channel.Config.Properties, + secretRef: context.Channel.Config.SecretRef); + + public static ConnectorMetadataBuilder CreateBuilder(ChannelHealthContext context) + => CreateBaseBuilder( + channel: context.Channel, + target: context.Target, + timestamp: context.Timestamp, + properties: context.Channel.Config.Properties, + secretRef: context.Channel.Config.SecretRef); + + public static IReadOnlyDictionary Build(ChannelTestPreviewContext context) + => CreateBuilder(context).Build(); + + public static IReadOnlyDictionary Build(ChannelHealthContext context) + => CreateBuilder(context).Build(); + + private static ConnectorMetadataBuilder CreateBaseBuilder( + NotifyChannel channel, + string target, + DateTimeOffset timestamp, + IReadOnlyDictionary? properties, + string secretRef) + { + var builder = new ConnectorMetadataBuilder(); + + builder.AddTarget("email.target", target) + .AddTimestamp("email.preview.generatedAt", timestamp) + .AddSecretRefHash("email.secretRef.hash", secretRef, SecretHashLengthBytes) + .AddConfigProperties("email.config.", properties); + + return builder; + } +} diff --git a/src/StellaOps.Notify.Connectors.Email/StellaOps.Notify.Connectors.Email.csproj b/src/StellaOps.Notify.Connectors.Email/StellaOps.Notify.Connectors.Email.csproj index 54540bcb..37a32ab8 100644 --- a/src/StellaOps.Notify.Connectors.Email/StellaOps.Notify.Connectors.Email.csproj +++ b/src/StellaOps.Notify.Connectors.Email/StellaOps.Notify.Connectors.Email.csproj @@ -5,9 +5,16 @@ enable - - - - - - + + + + + + + + + + PreserveNewest + + + diff --git a/src/StellaOps.Notify.Connectors.Email/TASKS.md b/src/StellaOps.Notify.Connectors.Email/TASKS.md index 9fb0f18b..e3754233 100644 --- a/src/StellaOps.Notify.Connectors.Email/TASKS.md +++ b/src/StellaOps.Notify.Connectors.Email/TASKS.md @@ -3,5 +3,5 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| | NOTIFY-CONN-EMAIL-15-701 | TODO | Notify Connectors Guild | NOTIFY-ENGINE-15-303 | Implement SMTP connector with STARTTLS/implicit TLS support, HTML+text rendering, attachment policy enforcement. | Integration tests with SMTP stub pass; TLS enforced; attachments blocked per policy. | -| NOTIFY-CONN-EMAIL-15-702 | DOING (2025-10-19) | Notify Connectors Guild | NOTIFY-CONN-EMAIL-15-701 | Add DKIM signing optional support and health/test-send flows. | DKIM optional config verified; test-send passes; secrets handled securely. | -| NOTIFY-CONN-EMAIL-15-703 | TODO | Notify Connectors Guild | NOTIFY-CONN-EMAIL-15-702 | Package Email connector as restart-time plug-in (manifest + host registration). | Plugin manifest added; host loads connector from `plugins/notify/email/`; restart validation passes. | +| NOTIFY-CONN-EMAIL-15-702 | BLOCKED (2025-10-20) | Notify Connectors Guild | NOTIFY-CONN-EMAIL-15-701 | Add DKIM signing optional support and health/test-send flows. | DKIM optional config verified; test-send passes; secrets handled securely. | +| NOTIFY-CONN-EMAIL-15-703 | DONE (2025-10-20) | Notify Connectors Guild | NOTIFY-CONN-EMAIL-15-702 | Package Email connector as restart-time plug-in (manifest + host registration). | Plugin manifest added; host loads connector from `plugins/notify/email/`; restart validation passes. | diff --git a/src/StellaOps.Notify.Connectors.Email/notify-plugin.json b/src/StellaOps.Notify.Connectors.Email/notify-plugin.json new file mode 100644 index 00000000..56407f5f --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Email/notify-plugin.json @@ -0,0 +1,18 @@ +{ + "schemaVersion": "1.0", + "id": "stellaops.notify.connector.email", + "displayName": "StellaOps Email Notify Connector", + "version": "0.1.0-alpha", + "requiresRestart": true, + "entryPoint": { + "type": "dotnet", + "assembly": "StellaOps.Notify.Connectors.Email.dll" + }, + "capabilities": [ + "notify-connector", + "email" + ], + "metadata": { + "org.stellaops.notify.channel.type": "email" + } +} diff --git a/src/StellaOps.Notify.Connectors.Shared/ConnectorHashing.cs b/src/StellaOps.Notify.Connectors.Shared/ConnectorHashing.cs new file mode 100644 index 00000000..acc7ab0e --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Shared/ConnectorHashing.cs @@ -0,0 +1,31 @@ +using System; +using System.Security.Cryptography; +using System.Text; + +namespace StellaOps.Notify.Connectors.Shared; + +/// +/// Common hashing helpers for Notify connector metadata. +/// +public static class ConnectorHashing +{ + /// + /// Computes a lowercase hex SHA-256 hash and truncates it to the requested number of bytes. + /// + public static string ComputeSha256Hash(string value, int lengthBytes = 8) + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new ArgumentException("Value must not be null or whitespace.", nameof(value)); + } + + if (lengthBytes <= 0 || lengthBytes > 32) + { + throw new ArgumentOutOfRangeException(nameof(lengthBytes), "Length must be between 1 and 32 bytes."); + } + + var bytes = Encoding.UTF8.GetBytes(value.Trim()); + var hash = SHA256.HashData(bytes); + return Convert.ToHexString(hash.AsSpan(0, lengthBytes)).ToLowerInvariant(); + } +} diff --git a/src/StellaOps.Notify.Connectors.Shared/ConnectorMetadataBuilder.cs b/src/StellaOps.Notify.Connectors.Shared/ConnectorMetadataBuilder.cs new file mode 100644 index 00000000..829a9045 --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Shared/ConnectorMetadataBuilder.cs @@ -0,0 +1,147 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Globalization; + +namespace StellaOps.Notify.Connectors.Shared; + +/// +/// Utility for constructing connector metadata payloads with consistent redaction rules. +/// +public sealed class ConnectorMetadataBuilder +{ + private readonly Dictionary _metadata; + + public ConnectorMetadataBuilder(StringComparer? comparer = null) + { + _metadata = new Dictionary(comparer ?? StringComparer.Ordinal); + SensitiveFragments = new HashSet(ConnectorValueRedactor.DefaultSensitiveKeyFragments, StringComparer.OrdinalIgnoreCase); + } + + /// + /// Collection of key fragments treated as sensitive when redacting values. + /// + public ISet SensitiveFragments { get; } + + /// + /// Adds or replaces a metadata entry when the value is non-empty. + /// + public ConnectorMetadataBuilder Add(string key, string? value) + { + if (string.IsNullOrWhiteSpace(key) || string.IsNullOrWhiteSpace(value)) + { + return this; + } + + _metadata[key.Trim()] = value.Trim(); + return this; + } + + /// + /// Adds the target value metadata. The value is trimmed but not redacted. + /// + public ConnectorMetadataBuilder AddTarget(string key, string target) + => Add(key, target); + + /// + /// Adds ISO-8601 timestamp metadata. + /// + public ConnectorMetadataBuilder AddTimestamp(string key, DateTimeOffset timestamp) + => Add(key, timestamp.ToString("O", CultureInfo.InvariantCulture)); + + /// + /// Adds a hash of the secret reference when present. + /// + public ConnectorMetadataBuilder AddSecretRefHash(string key, string? secretRef, int lengthBytes = 8) + { + if (!string.IsNullOrWhiteSpace(secretRef)) + { + Add(key, ConnectorHashing.ComputeSha256Hash(secretRef, lengthBytes)); + } + + return this; + } + + /// + /// Adds configuration target metadata only when the stored configuration differs from the resolved target. + /// + public ConnectorMetadataBuilder AddConfigTarget(string key, string? configuredTarget, string resolvedTarget) + { + if (!string.IsNullOrWhiteSpace(configuredTarget) && + !string.Equals(configuredTarget, resolvedTarget, StringComparison.Ordinal)) + { + Add(key, configuredTarget); + } + + return this; + } + + /// + /// Adds configuration endpoint metadata when present. + /// + public ConnectorMetadataBuilder AddConfigEndpoint(string key, string? endpoint) + => Add(key, endpoint); + + /// + /// Adds key/value metadata pairs from the provided dictionary, applying redaction to sensitive entries. + /// + public ConnectorMetadataBuilder AddConfigProperties( + string prefix, + IReadOnlyDictionary? properties, + Func? valueSelector = null) + { + if (properties is null || properties.Count == 0) + { + return this; + } + + foreach (var pair in properties) + { + if (string.IsNullOrWhiteSpace(pair.Key) || pair.Value is null) + { + continue; + } + + var key = prefix + pair.Key.Trim(); + var value = valueSelector is null + ? Redact(pair.Key, pair.Value) + : valueSelector(pair.Key, pair.Value); + + Add(key, value); + } + + return this; + } + + /// + /// Merges additional metadata entries into the builder. + /// + public ConnectorMetadataBuilder AddRange(IEnumerable> entries) + { + foreach (var (key, value) in entries) + { + Add(key, value); + } + + return this; + } + + /// + /// Returns the redacted representation for the supplied key/value pair. + /// + public string Redact(string key, string value) + { + if (ConnectorValueRedactor.IsSensitiveKey(key, SensitiveFragments)) + { + return ConnectorValueRedactor.RedactSecret(value); + } + + return value.Trim(); + } + + /// + /// Builds an immutable view of the accumulated metadata. + /// + public IReadOnlyDictionary Build() + => new ReadOnlyDictionary(_metadata); +} diff --git a/src/StellaOps.Notify.Connectors.Shared/ConnectorValueRedactor.cs b/src/StellaOps.Notify.Connectors.Shared/ConnectorValueRedactor.cs new file mode 100644 index 00000000..1c743b58 --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Shared/ConnectorValueRedactor.cs @@ -0,0 +1,75 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Notify.Connectors.Shared; + +/// +/// Shared helpers for redacting sensitive connector metadata. +/// +public static class ConnectorValueRedactor +{ + private static readonly string[] DefaultSensitiveFragments = + { + "token", + "secret", + "authorization", + "cookie", + "password", + "key", + "credential" + }; + + /// + /// Gets the default set of sensitive key fragments. + /// + public static IReadOnlyCollection DefaultSensitiveKeyFragments => DefaultSensitiveFragments; + + /// + /// Uses a constant mask for sensitive values. + /// + public static string RedactSecret(string value) => "***"; + + /// + /// Redacts the middle portion of a token while keeping stable prefix/suffix bytes. + /// + public static string RedactToken(string value, int prefixLength = 6, int suffixLength = 4) + { + var trimmed = value?.Trim() ?? string.Empty; + if (trimmed.Length <= prefixLength + suffixLength) + { + return RedactSecret(trimmed); + } + + var prefix = trimmed[..prefixLength]; + var suffix = trimmed[^suffixLength..]; + return string.Concat(prefix, "***", suffix); + } + + /// + /// Returns true when the provided key appears to represent sensitive data. + /// + public static bool IsSensitiveKey(string key, IEnumerable? fragments = null) + { + if (string.IsNullOrWhiteSpace(key)) + { + return false; + } + + fragments ??= DefaultSensitiveFragments; + var span = key.AsSpan(); + foreach (var fragment in fragments) + { + if (string.IsNullOrWhiteSpace(fragment)) + { + continue; + } + + if (span.IndexOf(fragment.AsSpan(), StringComparison.OrdinalIgnoreCase) >= 0) + { + return true; + } + } + + return false; + } +} diff --git a/src/StellaOps.Notify.Connectors.Shared/StellaOps.Notify.Connectors.Shared.csproj b/src/StellaOps.Notify.Connectors.Shared/StellaOps.Notify.Connectors.Shared.csproj new file mode 100644 index 00000000..9fe568c0 --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Shared/StellaOps.Notify.Connectors.Shared.csproj @@ -0,0 +1,12 @@ + + + net10.0 + enable + enable + + + + + + + diff --git a/src/StellaOps.Notify.Connectors.Slack.Tests/SlackChannelHealthProviderTests.cs b/src/StellaOps.Notify.Connectors.Slack.Tests/SlackChannelHealthProviderTests.cs new file mode 100644 index 00000000..8878b3ef --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Slack.Tests/SlackChannelHealthProviderTests.cs @@ -0,0 +1,96 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Notify.Engine; +using StellaOps.Notify.Models; +using Xunit; + +namespace StellaOps.Notify.Connectors.Slack.Tests; + +public sealed class SlackChannelHealthProviderTests +{ + private static readonly SlackChannelHealthProvider Provider = new(); + + [Fact] + public async Task CheckAsync_ReturnsHealthy() + { + var channel = CreateChannel(enabled: true, target: "#sec-ops"); + + var context = new ChannelHealthContext( + channel.TenantId, + channel, + channel.Config.Target!, + new DateTimeOffset(2025, 10, 20, 14, 0, 0, TimeSpan.Zero), + "trace-slack-001"); + + var result = await Provider.CheckAsync(context, CancellationToken.None); + + Assert.Equal(ChannelHealthStatus.Healthy, result.Status); + Assert.Equal("true", result.Metadata["slack.channel.enabled"]); + Assert.Equal("true", result.Metadata["slack.validation.targetPresent"]); + Assert.Equal("#sec-ops", result.Metadata["slack.channel"]); + Assert.Equal(ComputeSecretHash(channel.Config.SecretRef), result.Metadata["slack.secretRef.hash"]); + } + + [Fact] + public async Task CheckAsync_ReturnsDegradedWhenDisabled() + { + var channel = CreateChannel(enabled: false, target: "#sec-ops"); + + var context = new ChannelHealthContext( + channel.TenantId, + channel, + channel.Config.Target!, + DateTimeOffset.UtcNow, + "trace-slack-002"); + + var result = await Provider.CheckAsync(context, CancellationToken.None); + + Assert.Equal(ChannelHealthStatus.Degraded, result.Status); + Assert.Equal("false", result.Metadata["slack.channel.enabled"]); + } + + [Fact] + public async Task CheckAsync_ReturnsUnhealthyWhenTargetMissing() + { + var channel = CreateChannel(enabled: true, target: null); + + var context = new ChannelHealthContext( + channel.TenantId, + channel, + channel.Name, + DateTimeOffset.UtcNow, + "trace-slack-003"); + + var result = await Provider.CheckAsync(context, CancellationToken.None); + + Assert.Equal(ChannelHealthStatus.Unhealthy, result.Status); + Assert.Equal("false", result.Metadata["slack.validation.targetPresent"]); + } + + private static NotifyChannel CreateChannel(bool enabled, string? target) + { + return NotifyChannel.Create( + channelId: "channel-slack-sec-ops", + tenantId: "tenant-sec", + name: "slack:sec-ops", + type: NotifyChannelType.Slack, + config: NotifyChannelConfig.Create( + secretRef: "ref://notify/channels/slack/sec-ops", + target: target, + properties: new Dictionary + { + ["workspace"] = "stellaops-sec", + ["botToken"] = "xoxb-123456789012-abcdefghijklmnop" + }), + enabled: enabled); + } + + private static string ComputeSecretHash(string secretRef) + { + var bytes = System.Text.Encoding.UTF8.GetBytes(secretRef.Trim()); + var hash = System.Security.Cryptography.SHA256.HashData(bytes); + return Convert.ToHexString(hash.AsSpan(0, 8)).ToLowerInvariant(); + } +} diff --git a/src/StellaOps.Notify.Connectors.Slack.Tests/SlackChannelTestProviderTests.cs b/src/StellaOps.Notify.Connectors.Slack.Tests/SlackChannelTestProviderTests.cs new file mode 100644 index 00000000..5b4265a5 --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Slack.Tests/SlackChannelTestProviderTests.cs @@ -0,0 +1,113 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Notify.Engine; +using StellaOps.Notify.Models; +using Xunit; + +namespace StellaOps.Notify.Connectors.Slack.Tests; + +public sealed class SlackChannelTestProviderTests +{ + private static readonly ChannelTestPreviewRequest EmptyRequest = new( + TargetOverride: null, + TemplateId: null, + Title: null, + Summary: null, + Body: null, + TextBody: null, + Locale: null, + Metadata: new Dictionary(), + Attachments: new List()); + + [Fact] + public async Task BuildPreviewAsync_ProducesDeterministicMetadata() + { + var provider = new SlackChannelTestProvider(); + var channel = CreateChannel(properties: new Dictionary + { + ["workspace"] = "stellaops-sec", + ["botToken"] = "xoxb-123456789012-abcdefghijklmnop" + }); + + var context = new ChannelTestPreviewContext( + channel.TenantId, + channel, + channel.Config.Target!, + EmptyRequest, + Timestamp: new DateTimeOffset(2025, 10, 20, 12, 00, 00, TimeSpan.Zero), + TraceId: "trace-001"); + + var result = await provider.BuildPreviewAsync(context, CancellationToken.None); + + Assert.Equal("slack", result.Preview.ChannelType.ToString().ToLowerInvariant()); + Assert.Equal(channel.Config.Target, result.Preview.Target); + Assert.Equal("chat:write,chat:write.public", result.Metadata["slack.scopes.required"]); + Assert.Equal("stellaops-sec", result.Metadata["slack.config.workspace"]); + + var redactedToken = result.Metadata["slack.config.botToken"]; + Assert.DoesNotContain("abcdefghijklmnop", redactedToken); + Assert.StartsWith("xoxb-", redactedToken); + Assert.EndsWith("mnop", redactedToken); + + using var parsed = JsonDocument.Parse(result.Preview.Body); + var contextText = parsed.RootElement + .GetProperty("blocks")[1] + .GetProperty("elements")[0] + .GetProperty("text") + .GetString(); + Assert.NotNull(contextText); + Assert.Contains("trace-001", contextText); + + Assert.Equal(ComputeSecretHash(channel.Config.SecretRef), result.Metadata["slack.secretRef.hash"]); + } + + [Fact] + public async Task BuildPreviewAsync_RedactsSensitiveProperties() + { + var provider = new SlackChannelTestProvider(); + var channel = CreateChannel(properties: new Dictionary + { + ["SigningSecret"] = "whsec_super-secret-value", + ["apiToken"] = "xoxs-000000000000-super", + ["endpoint"] = "https://hooks.slack.com/services/T000/B000/AAA" + }); + + var context = new ChannelTestPreviewContext( + channel.TenantId, + channel, + channel.Config.Target!, + EmptyRequest, + Timestamp: DateTimeOffset.UtcNow, + TraceId: "trace-002"); + + var result = await provider.BuildPreviewAsync(context, CancellationToken.None); + + Assert.Equal("***", result.Metadata["slack.config.SigningSecret"]); + Assert.DoesNotContain("xoxs-000000000000-super", result.Metadata["slack.config.apiToken"]); + Assert.Equal("https://hooks.slack.com/services/T000/B000/AAA", result.Metadata["slack.config.endpoint"]); + } + + private static NotifyChannel CreateChannel(IDictionary properties) + { + return NotifyChannel.Create( + channelId: "channel-slack-sec-ops", + tenantId: "tenant-sec", + name: "slack:sec-ops", + type: NotifyChannelType.Slack, + config: NotifyChannelConfig.Create( + secretRef: "ref://notify/channels/slack/sec-ops", + target: "#sec-ops", + properties: properties)); + } + + private static string ComputeSecretHash(string secretRef) + { + using var sha = System.Security.Cryptography.SHA256.Create(); + var bytes = System.Text.Encoding.UTF8.GetBytes(secretRef.Trim()); + var hash = sha.ComputeHash(bytes); + return System.Convert.ToHexString(hash, 0, 8).ToLowerInvariant(); + } +} diff --git a/src/StellaOps.Notify.Connectors.Slack.Tests/StellaOps.Notify.Connectors.Slack.Tests.csproj b/src/StellaOps.Notify.Connectors.Slack.Tests/StellaOps.Notify.Connectors.Slack.Tests.csproj new file mode 100644 index 00000000..6288a185 --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Slack.Tests/StellaOps.Notify.Connectors.Slack.Tests.csproj @@ -0,0 +1,21 @@ + + + net10.0 + enable + enable + false + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Notify.Connectors.Slack/SlackChannelHealthProvider.cs b/src/StellaOps.Notify.Connectors.Slack/SlackChannelHealthProvider.cs new file mode 100644 index 00000000..b2dbdcbd --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Slack/SlackChannelHealthProvider.cs @@ -0,0 +1,56 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Notify.Engine; +using StellaOps.Notify.Models; + +namespace StellaOps.Notify.Connectors.Slack; + +[ServiceBinding(typeof(INotifyChannelHealthProvider), ServiceLifetime.Singleton)] +public sealed class SlackChannelHealthProvider : INotifyChannelHealthProvider +{ + public NotifyChannelType ChannelType => NotifyChannelType.Slack; + + public Task CheckAsync(ChannelHealthContext context, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + cancellationToken.ThrowIfCancellationRequested(); + + var builder = SlackMetadataBuilder.CreateBuilder(context) + .Add("slack.channel.enabled", context.Channel.Enabled ? "true" : "false") + .Add("slack.validation.targetPresent", HasConfiguredTarget(context.Channel) ? "true" : "false"); + + var metadata = builder.Build(); + var status = ResolveStatus(context.Channel); + var message = status switch + { + ChannelHealthStatus.Healthy => "Slack channel configuration validated.", + ChannelHealthStatus.Degraded => "Slack channel is disabled; enable it to resume deliveries.", + ChannelHealthStatus.Unhealthy => "Slack channel is missing a configured destination (target).", + _ => "Slack channel diagnostics completed." + }; + + return Task.FromResult(new ChannelHealthResult(status, message, metadata)); + } + + private static ChannelHealthStatus ResolveStatus(NotifyChannel channel) + { + if (!HasConfiguredTarget(channel)) + { + return ChannelHealthStatus.Unhealthy; + } + + if (!channel.Enabled) + { + return ChannelHealthStatus.Degraded; + } + + return ChannelHealthStatus.Healthy; + } + + private static bool HasConfiguredTarget(NotifyChannel channel) + => !string.IsNullOrWhiteSpace(channel.Config.Target); +} diff --git a/src/StellaOps.Notify.Connectors.Slack/SlackChannelTestProvider.cs b/src/StellaOps.Notify.Connectors.Slack/SlackChannelTestProvider.cs index 6c0daa7e..3e30fa18 100644 --- a/src/StellaOps.Notify.Connectors.Slack/SlackChannelTestProvider.cs +++ b/src/StellaOps.Notify.Connectors.Slack/SlackChannelTestProvider.cs @@ -13,58 +13,74 @@ namespace StellaOps.Notify.Connectors.Slack; [ServiceBinding(typeof(INotifyChannelTestProvider), ServiceLifetime.Singleton)] public sealed class SlackChannelTestProvider : INotifyChannelTestProvider { - private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); - - public NotifyChannelType ChannelType => NotifyChannelType.Slack; - - public Task BuildPreviewAsync(ChannelTestPreviewContext context, CancellationToken cancellationToken) - { - cancellationToken.ThrowIfCancellationRequested(); - - var title = context.Request.Title ?? $"Stella Ops Notify Preview"; - var summary = context.Request.Summary ?? $"Preview generated for Slack destination at {context.Timestamp:O}."; - var bodyText = context.Request.Body ?? summary; - - var payload = new - { - text = $"{title}\n{bodyText}", - blocks = new object[] - { - new - { + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + + private static readonly string DefaultTitle = "Stella Ops Notify Preview"; + + public NotifyChannelType ChannelType => NotifyChannelType.Slack; + + public Task BuildPreviewAsync(ChannelTestPreviewContext context, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + var title = !string.IsNullOrWhiteSpace(context.Request.Title) + ? context.Request.Title! + : DefaultTitle; + var summary = !string.IsNullOrWhiteSpace(context.Request.Summary) + ? context.Request.Summary! + : $"Preview generated for Slack destination at {context.Timestamp:O}."; + var bodyText = !string.IsNullOrWhiteSpace(context.Request.Body) + ? context.Request.Body! + : summary; + var workspace = context.Channel.Config.Properties.TryGetValue("workspace", out var workspaceName) + ? workspaceName + : null; + + var contextElements = new List + { + new { type = "mrkdwn", text = $"Preview generated {context.Timestamp:O} · Trace `{context.TraceId}`" } + }; + + if (!string.IsNullOrWhiteSpace(workspace)) + { + contextElements.Add(new { type = "mrkdwn", text = $"Workspace: `{workspace}`" }); + } + + var payload = new + { + channel = context.Target, + text = $"{title}\n{bodyText}", + blocks = new object[] + { + new + { type = "section", text = new { type = "mrkdwn", text = $"*{title}*\n{bodyText}" } }, - new - { - type = "context", - elements = new object[] - { - new { type = "mrkdwn", text = $"Preview generated {context.Timestamp:O} · Trace `{context.TraceId}`" } - } - } - } - }; - - var body = JsonSerializer.Serialize(payload, JsonOptions); - - var preview = NotifyDeliveryRendered.Create( - NotifyChannelType.Slack, - NotifyDeliveryFormat.Slack, - context.Target, - title, - body, - summary, - context.Request.TextBody ?? bodyText, - context.Request.Locale, - ChannelTestPreviewUtilities.ComputeBodyHash(body), - context.Request.Attachments); - - var metadata = new Dictionary(StringComparer.Ordinal) - { - ["slack.channel"] = context.Target - }; - - return Task.FromResult(new ChannelTestPreviewResult(preview, metadata)); - } -} + new + { + type = "context", + elements = contextElements.ToArray() + } + } + }; + + var body = JsonSerializer.Serialize(payload, JsonOptions); + + var preview = NotifyDeliveryRendered.Create( + NotifyChannelType.Slack, + NotifyDeliveryFormat.Slack, + context.Target, + title, + body, + summary, + context.Request.TextBody ?? bodyText, + context.Request.Locale, + ChannelTestPreviewUtilities.ComputeBodyHash(body), + context.Request.Attachments); + + var metadata = SlackMetadataBuilder.Build(context); + + return Task.FromResult(new ChannelTestPreviewResult(preview, metadata)); + } +} diff --git a/src/StellaOps.Notify.Connectors.Slack/SlackMetadataBuilder.cs b/src/StellaOps.Notify.Connectors.Slack/SlackMetadataBuilder.cs new file mode 100644 index 00000000..b8a62d3a --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Slack/SlackMetadataBuilder.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Generic; +using StellaOps.Notify.Connectors.Shared; +using StellaOps.Notify.Engine; +using StellaOps.Notify.Models; + +namespace StellaOps.Notify.Connectors.Slack; + +/// +/// Builds metadata for Slack previews and health diagnostics while redacting sensitive material. +/// +internal static class SlackMetadataBuilder +{ + private static readonly string[] RequiredScopes = { "chat:write", "chat:write.public" }; + + public static ConnectorMetadataBuilder CreateBuilder(ChannelTestPreviewContext context) + => CreateBaseBuilder( + channel: context.Channel, + target: context.Target, + timestamp: context.Timestamp, + properties: context.Channel.Config.Properties, + secretRef: context.Channel.Config.SecretRef); + + public static ConnectorMetadataBuilder CreateBuilder(ChannelHealthContext context) + => CreateBaseBuilder( + channel: context.Channel, + target: context.Target, + timestamp: context.Timestamp, + properties: context.Channel.Config.Properties, + secretRef: context.Channel.Config.SecretRef); + + public static IReadOnlyDictionary Build(ChannelTestPreviewContext context) + => CreateBuilder(context).Build(); + + public static IReadOnlyDictionary Build(ChannelHealthContext context) + => CreateBuilder(context).Build(); + + private static ConnectorMetadataBuilder CreateBaseBuilder( + NotifyChannel channel, + string target, + DateTimeOffset timestamp, + IReadOnlyDictionary? properties, + string secretRef) + { + var builder = new ConnectorMetadataBuilder(); + + builder.AddTarget("slack.channel", target) + .Add("slack.scopes.required", string.Join(',', RequiredScopes)) + .AddTimestamp("slack.preview.generatedAt", timestamp) + .AddSecretRefHash("slack.secretRef.hash", secretRef) + .AddConfigTarget("slack.config.target", channel.Config.Target, target) + .AddConfigProperties("slack.config.", properties, (key, value) => RedactSlackValue(builder, key, value)); + + return builder; + } + + private static string RedactSlackValue(ConnectorMetadataBuilder builder, string key, string value) + { + if (LooksLikeSlackToken(value)) + { + return ConnectorValueRedactor.RedactToken(value); + } + + return builder.Redact(key, value); + } + + private static bool LooksLikeSlackToken(string value) + { + var trimmed = value.Trim(); + if (trimmed.Length < 6) + { + return false; + } + + return trimmed.StartsWith("xox", StringComparison.OrdinalIgnoreCase); + } +} diff --git a/src/StellaOps.Notify.Connectors.Slack/StellaOps.Notify.Connectors.Slack.csproj b/src/StellaOps.Notify.Connectors.Slack/StellaOps.Notify.Connectors.Slack.csproj index 54540bcb..37a32ab8 100644 --- a/src/StellaOps.Notify.Connectors.Slack/StellaOps.Notify.Connectors.Slack.csproj +++ b/src/StellaOps.Notify.Connectors.Slack/StellaOps.Notify.Connectors.Slack.csproj @@ -5,9 +5,16 @@ enable - - - - - - + + + + + + + + + + PreserveNewest + + + diff --git a/src/StellaOps.Notify.Connectors.Slack/TASKS.md b/src/StellaOps.Notify.Connectors.Slack/TASKS.md index 45cb428a..5658db86 100644 --- a/src/StellaOps.Notify.Connectors.Slack/TASKS.md +++ b/src/StellaOps.Notify.Connectors.Slack/TASKS.md @@ -3,5 +3,5 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| | NOTIFY-CONN-SLACK-15-501 | TODO | Notify Connectors Guild | NOTIFY-ENGINE-15-303 | Implement Slack connector with bot token auth, message rendering (blocks), rate limit handling, retries/backoff. | Integration tests stub Slack API; retries/jitter validated; 429 handling documented. | -| NOTIFY-CONN-SLACK-15-502 | DOING (2025-10-19) | Notify Connectors Guild | NOTIFY-CONN-SLACK-15-501 | Health check & test-send support with minimal scopes and redacted tokens. | `/channels/{id}/test` hitting Slack stub passes; secrets never logged; health endpoint returns diagnostics. | -| NOTIFY-CONN-SLACK-15-503 | TODO | Notify Connectors Guild | NOTIFY-CONN-SLACK-15-502 | Package Slack connector as restart-time plug-in (manifest + host registration). | Plugin manifest added; host loads connector from `plugins/notify/slack/`; restart validation passes. | +| NOTIFY-CONN-SLACK-15-502 | DONE (2025-10-20) | Notify Connectors Guild | NOTIFY-CONN-SLACK-15-501 | Health check & test-send support with minimal scopes and redacted tokens. | `/channels/{id}/test` hitting Slack stub passes; secrets never logged; health endpoint returns diagnostics. | +| NOTIFY-CONN-SLACK-15-503 | DONE (2025-10-20) | Notify Connectors Guild | NOTIFY-CONN-SLACK-15-502 | Package Slack connector as restart-time plug-in (manifest + host registration). | Plugin manifest added; host loads connector from `plugins/notify/slack/`; restart validation passes. | diff --git a/src/StellaOps.Notify.Connectors.Slack/notify-plugin.json b/src/StellaOps.Notify.Connectors.Slack/notify-plugin.json new file mode 100644 index 00000000..95fb1dfb --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Slack/notify-plugin.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": "1.0", + "id": "stellaops.notify.connector.slack", + "displayName": "StellaOps Slack Notify Connector", + "version": "0.1.0-alpha", + "requiresRestart": true, + "entryPoint": { + "type": "dotnet", + "assembly": "StellaOps.Notify.Connectors.Slack.dll" + }, + "capabilities": [ + "notify-connector", + "slack" + ], + "metadata": { + "org.stellaops.notify.channel.type": "slack", + "org.stellaops.notify.connector.requiredScopes": "chat:write,chat:write.public" + } +} diff --git a/src/StellaOps.Notify.Connectors.Teams.Tests/StellaOps.Notify.Connectors.Teams.Tests.csproj b/src/StellaOps.Notify.Connectors.Teams.Tests/StellaOps.Notify.Connectors.Teams.Tests.csproj new file mode 100644 index 00000000..69915aeb --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Teams.Tests/StellaOps.Notify.Connectors.Teams.Tests.csproj @@ -0,0 +1,21 @@ + + + net10.0 + enable + enable + false + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Notify.Connectors.Teams.Tests/TeamsChannelHealthProviderTests.cs b/src/StellaOps.Notify.Connectors.Teams.Tests/TeamsChannelHealthProviderTests.cs new file mode 100644 index 00000000..dd9bd279 --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Teams.Tests/TeamsChannelHealthProviderTests.cs @@ -0,0 +1,98 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Notify.Engine; +using StellaOps.Notify.Models; +using Xunit; + +namespace StellaOps.Notify.Connectors.Teams.Tests; + +public sealed class TeamsChannelHealthProviderTests +{ + private static readonly TeamsChannelHealthProvider Provider = new(); + + [Fact] + public async Task CheckAsync_ReturnsHealthyWithMetadata() + { + var channel = CreateChannel(enabled: true, endpoint: "https://contoso.webhook.office.com/webhook"); + + var context = new ChannelHealthContext( + channel.TenantId, + channel, + channel.Config.Endpoint!, + new DateTimeOffset(2025, 10, 20, 12, 0, 0, TimeSpan.Zero), + "trace-health-001"); + + var result = await Provider.CheckAsync(context, CancellationToken.None); + + Assert.Equal(ChannelHealthStatus.Healthy, result.Status); + Assert.Equal("Teams channel configuration validated.", result.Message); + Assert.Equal("true", result.Metadata["teams.channel.enabled"]); + Assert.Equal("true", result.Metadata["teams.validation.targetPresent"]); + Assert.Equal(channel.Config.Endpoint, result.Metadata["teams.webhook"]); + Assert.Equal(ComputeSecretHash(channel.Config.SecretRef), result.Metadata["teams.secretRef.hash"]); + } + + [Fact] + public async Task CheckAsync_ReturnsDegradedWhenDisabled() + { + var channel = CreateChannel(enabled: false, endpoint: "https://contoso.webhook.office.com/webhook"); + + var context = new ChannelHealthContext( + channel.TenantId, + channel, + channel.Config.Endpoint!, + DateTimeOffset.UtcNow, + "trace-health-002"); + + var result = await Provider.CheckAsync(context, CancellationToken.None); + + Assert.Equal(ChannelHealthStatus.Degraded, result.Status); + Assert.Equal("false", result.Metadata["teams.channel.enabled"]); + } + + [Fact] + public async Task CheckAsync_ReturnsUnhealthyWhenTargetMissing() + { + var channel = CreateChannel(enabled: true, endpoint: null); + + var context = new ChannelHealthContext( + channel.TenantId, + channel, + channel.Name, + DateTimeOffset.UtcNow, + "trace-health-003"); + + var result = await Provider.CheckAsync(context, CancellationToken.None); + + Assert.Equal(ChannelHealthStatus.Unhealthy, result.Status); + Assert.Equal("false", result.Metadata["teams.validation.targetPresent"]); + } + + private static NotifyChannel CreateChannel(bool enabled, string? endpoint) + { + return NotifyChannel.Create( + channelId: "channel-teams-sec-ops", + tenantId: "tenant-sec", + name: "teams:sec-ops", + type: NotifyChannelType.Teams, + config: NotifyChannelConfig.Create( + secretRef: "ref://notify/channels/teams/sec-ops", + target: null, + endpoint: endpoint, + properties: new Dictionary + { + ["tenant"] = "contoso.onmicrosoft.com", + ["webhookKey"] = "abcdef0123456789" + }), + enabled: enabled); + } + + private static string ComputeSecretHash(string secretRef) + { + var bytes = System.Text.Encoding.UTF8.GetBytes(secretRef.Trim()); + var hash = System.Security.Cryptography.SHA256.HashData(bytes); + return Convert.ToHexString(hash.AsSpan(0, 8)).ToLowerInvariant(); + } +} diff --git a/src/StellaOps.Notify.Connectors.Teams.Tests/TeamsChannelTestProviderTests.cs b/src/StellaOps.Notify.Connectors.Teams.Tests/TeamsChannelTestProviderTests.cs new file mode 100644 index 00000000..d888634e --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Teams.Tests/TeamsChannelTestProviderTests.cs @@ -0,0 +1,135 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Notify.Engine; +using StellaOps.Notify.Models; +using Xunit; + +namespace StellaOps.Notify.Connectors.Teams.Tests; + +public sealed class TeamsChannelTestProviderTests +{ + [Fact] + public async Task BuildPreviewAsync_EmitsFallbackMetadata() + { + var provider = new TeamsChannelTestProvider(); + var channel = CreateChannel( + endpoint: "https://contoso.webhook.office.com/webhookb2/tenant@uuid/IncomingWebhook/abcdef0123456789", + properties: new Dictionary + { + ["team"] = "secops", + ["webhookKey"] = "s3cr3t-value-with-key-fragment", + ["tenant"] = "contoso.onmicrosoft.com" + }); + + var request = new ChannelTestPreviewRequest( + TargetOverride: null, + TemplateId: null, + Title: "Notify Critical Finding", + Summary: "Critical container vulnerability detected.", + Body: "CVSS 9.8 vulnerability detected in ubuntu:22.04 base layer.", + TextBody: null, + Locale: "en-US", + Metadata: new Dictionary(), + Attachments: new List()); + + var context = new ChannelTestPreviewContext( + channel.TenantId, + channel, + channel.Config.Endpoint!, + request, + new DateTimeOffset(2025, 10, 20, 10, 0, 0, TimeSpan.Zero), + TraceId: "trace-teams-001"); + + var result = await provider.BuildPreviewAsync(context, CancellationToken.None); + + Assert.Equal(NotifyChannelType.Teams, result.Preview.ChannelType); + Assert.Equal(channel.Config.Endpoint, result.Preview.Target); + Assert.Equal("Critical container vulnerability detected.", result.Preview.Summary); + + Assert.NotNull(result.Metadata); + Assert.Equal(channel.Config.Endpoint, result.Metadata["teams.webhook"]); + Assert.Equal("1.5", result.Metadata["teams.card.version"]); + + var fallback = result.Metadata["teams.fallbackText"]; + Assert.Equal(result.Preview.TextBody, fallback); + Assert.Equal("Critical container vulnerability detected.", fallback); + + Assert.Equal(ComputeSecretHash(channel.Config.SecretRef), result.Metadata["teams.secretRef.hash"]); + Assert.Equal("***", result.Metadata["teams.config.webhookKey"]); + Assert.Equal("contoso.onmicrosoft.com", result.Metadata["teams.config.tenant"]); + Assert.Equal(channel.Config.Endpoint, result.Metadata["teams.config.endpoint"]); + + using var payload = JsonDocument.Parse(result.Preview.Body); + Assert.Equal("message", payload.RootElement.GetProperty("type").GetString()); + Assert.Equal(result.Preview.TextBody, payload.RootElement.GetProperty("text").GetString()); + Assert.Equal(result.Preview.Summary, payload.RootElement.GetProperty("summary").GetString()); + + var attachments = payload.RootElement.GetProperty("attachments"); + Assert.True(attachments.GetArrayLength() > 0); + Assert.Equal( + "AdaptiveCard", + attachments[0].GetProperty("content").GetProperty("type").GetString()); + } + + [Fact] + public async Task BuildPreviewAsync_TruncatesLongFallback() + { + var provider = new TeamsChannelTestProvider(); + var channel = CreateChannel( + endpoint: "https://contoso.webhook.office.com/webhookb2/tenant@uuid/IncomingWebhook/abcdef0123456789", + properties: new Dictionary()); + + var longText = new string('A', 600); + + var request = new ChannelTestPreviewRequest( + TargetOverride: null, + TemplateId: null, + Title: null, + Summary: null, + Body: null, + TextBody: longText, + Locale: null, + Metadata: new Dictionary(), + Attachments: new List()); + + var context = new ChannelTestPreviewContext( + channel.TenantId, + channel, + channel.Config.Endpoint!, + request, + DateTimeOffset.UtcNow, + TraceId: "trace-teams-002"); + + var result = await provider.BuildPreviewAsync(context, CancellationToken.None); + + var metadata = Assert.IsAssignableFrom>(result.Metadata); + var fallback = Assert.IsType(result.Preview.TextBody); + Assert.Equal(512, fallback.Length); + Assert.Equal(fallback, metadata["teams.fallbackText"]); + Assert.StartsWith(new string('A', 512), fallback); + } + + private static NotifyChannel CreateChannel(string endpoint, IDictionary properties) + { + return NotifyChannel.Create( + channelId: "channel-teams-sec-ops", + tenantId: "tenant-sec", + name: "teams:sec-ops", + type: NotifyChannelType.Teams, + config: NotifyChannelConfig.Create( + secretRef: "ref://notify/channels/teams/sec-ops", + target: null, + endpoint: endpoint, + properties: properties)); + } + + private static string ComputeSecretHash(string secretRef) + { + var bytes = System.Text.Encoding.UTF8.GetBytes(secretRef.Trim()); + var hash = System.Security.Cryptography.SHA256.HashData(bytes); + return Convert.ToHexString(hash.AsSpan(0, 8)).ToLowerInvariant(); + } +} diff --git a/src/StellaOps.Notify.Connectors.Teams/StellaOps.Notify.Connectors.Teams.csproj b/src/StellaOps.Notify.Connectors.Teams/StellaOps.Notify.Connectors.Teams.csproj index 54540bcb..37a32ab8 100644 --- a/src/StellaOps.Notify.Connectors.Teams/StellaOps.Notify.Connectors.Teams.csproj +++ b/src/StellaOps.Notify.Connectors.Teams/StellaOps.Notify.Connectors.Teams.csproj @@ -5,9 +5,16 @@ enable - - - - - - + + + + + + + + + + PreserveNewest + + + diff --git a/src/StellaOps.Notify.Connectors.Teams/TASKS.md b/src/StellaOps.Notify.Connectors.Teams/TASKS.md index 4b49b6dc..6a1aa92f 100644 --- a/src/StellaOps.Notify.Connectors.Teams/TASKS.md +++ b/src/StellaOps.Notify.Connectors.Teams/TASKS.md @@ -2,6 +2,9 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| NOTIFY-CONN-TEAMS-15-601 | TODO | Notify Connectors Guild | NOTIFY-ENGINE-15-303 | Implement Teams connector using Adaptive Cards 1.5, handle webhook auth, size limits, retries. | Adaptive card payloads validated; 413/429 handling implemented; integration tests cover success/fail. | -| NOTIFY-CONN-TEAMS-15-602 | DOING (2025-10-19) | Notify Connectors Guild | NOTIFY-CONN-TEAMS-15-601 | Provide health/test-send support with fallback text for legacy clients. | Test-send returns card preview; fallback text logged; docs updated. | -| NOTIFY-CONN-TEAMS-15-603 | TODO | Notify Connectors Guild | NOTIFY-CONN-TEAMS-15-602 | Package Teams connector as restart-time plug-in (manifest + host registration). | Plugin manifest added; host loads connector from `plugins/notify/teams/`; restart validation passes. | +| NOTIFY-CONN-TEAMS-15-601 | TODO | Notify Connectors Guild | NOTIFY-ENGINE-15-303 | Implement Teams connector using Adaptive Cards 1.5, handle webhook auth, size limits, retries. | Adaptive card payloads validated; 413/429 handling implemented; integration tests cover success/fail. | +| NOTIFY-CONN-TEAMS-15-602 | DONE (2025-10-20) | Notify Connectors Guild | NOTIFY-CONN-TEAMS-15-601 | Provide health/test-send support with fallback text for legacy clients. | Test-send returns card preview; fallback text logged; docs updated. | +| NOTIFY-CONN-TEAMS-15-603 | DONE (2025-10-20) | Notify Connectors Guild | NOTIFY-CONN-TEAMS-15-602 | Package Teams connector as restart-time plug-in (manifest + host registration). | Plugin manifest added; host loads connector from `plugins/notify/teams/`; restart validation passes. | +| NOTIFY-CONN-TEAMS-15-604 | DONE (2025-10-20) | Notify Connectors Guild | NOTIFY-CONN-TEAMS-15-602 | Align Teams channel health endpoint with preview metadata redaction. | `/channels/{id}/health` reuses `TeamsMetadataBuilder`; sensitive fields redacted; regression tests updated. | + +> Remark (2025-10-20): Teams test-send now emits Adaptive Card 1.5 payloads with legacy fallback text (`teams.fallbackText` metadata) and hashed webhook secret refs; coverage lives in `StellaOps.Notify.Connectors.Teams.Tests`. `/channels/{id}/health` shares the same metadata builder via `TeamsChannelHealthProvider`, ensuring webhook hashes and sensitive keys stay redacted. diff --git a/src/StellaOps.Notify.Connectors.Teams/TeamsChannelHealthProvider.cs b/src/StellaOps.Notify.Connectors.Teams/TeamsChannelHealthProvider.cs new file mode 100644 index 00000000..68072daa --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Teams/TeamsChannelHealthProvider.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Notify.Engine; +using StellaOps.Notify.Models; + +namespace StellaOps.Notify.Connectors.Teams; + +[ServiceBinding(typeof(INotifyChannelHealthProvider), ServiceLifetime.Singleton)] +public sealed class TeamsChannelHealthProvider : INotifyChannelHealthProvider +{ + public NotifyChannelType ChannelType => NotifyChannelType.Teams; + + public Task CheckAsync(ChannelHealthContext context, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + cancellationToken.ThrowIfCancellationRequested(); + + var builder = TeamsMetadataBuilder.CreateBuilder(context) + .Add("teams.channel.enabled", context.Channel.Enabled ? "true" : "false") + .Add("teams.validation.targetPresent", HasConfiguredTarget(context.Channel) ? "true" : "false"); + + var metadata = builder.Build(); + var status = ResolveStatus(context.Channel); + var message = status switch + { + ChannelHealthStatus.Healthy => "Teams channel configuration validated.", + ChannelHealthStatus.Degraded => "Teams channel is disabled; enable it to resume deliveries.", + ChannelHealthStatus.Unhealthy => "Teams channel is missing a target/endpoint configuration.", + _ => "Teams channel diagnostics completed." + }; + + return Task.FromResult(new ChannelHealthResult(status, message, metadata)); + } + + private static ChannelHealthStatus ResolveStatus(NotifyChannel channel) + { + if (!HasConfiguredTarget(channel)) + { + return ChannelHealthStatus.Unhealthy; + } + + if (!channel.Enabled) + { + return ChannelHealthStatus.Degraded; + } + + return ChannelHealthStatus.Healthy; + } + + private static bool HasConfiguredTarget(NotifyChannel channel) + => !string.IsNullOrWhiteSpace(channel.Config.Endpoint) || + !string.IsNullOrWhiteSpace(channel.Config.Target); +} diff --git a/src/StellaOps.Notify.Connectors.Teams/TeamsChannelTestProvider.cs b/src/StellaOps.Notify.Connectors.Teams/TeamsChannelTestProvider.cs index 2e2120e0..d5cd6b9b 100644 --- a/src/StellaOps.Notify.Connectors.Teams/TeamsChannelTestProvider.cs +++ b/src/StellaOps.Notify.Connectors.Teams/TeamsChannelTestProvider.cs @@ -1,50 +1,54 @@ using System; -using System.Collections.Generic; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using StellaOps.DependencyInjection; -using StellaOps.Notify.Engine; -using StellaOps.Notify.Models; - -namespace StellaOps.Notify.Connectors.Teams; - -[ServiceBinding(typeof(INotifyChannelTestProvider), ServiceLifetime.Singleton)] -public sealed class TeamsChannelTestProvider : INotifyChannelTestProvider -{ - private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); - - public NotifyChannelType ChannelType => NotifyChannelType.Teams; - - public Task BuildPreviewAsync(ChannelTestPreviewContext context, CancellationToken cancellationToken) - { - cancellationToken.ThrowIfCancellationRequested(); - - var title = context.Request.Title ?? "Stella Ops Notify Preview"; - var summary = context.Request.Summary ?? $"Preview generated at {context.Timestamp:O}."; - var bodyContent = context.Request.Body ?? summary; - - var card = new - { - type = "AdaptiveCard", - version = "1.5", - body = new object[] - { - new { type = "TextBlock", weight = "Bolder", text = title, wrap = true }, - new { type = "TextBlock", text = bodyContent, wrap = true }, - new { type = "TextBlock", spacing = "None", isSubtle = true, text = $"Trace: {context.TraceId}", wrap = true } - } - }; - - var payload = new - { - type = "message", - attachments = new object[] - { - new - { - contentType = "application/vnd.microsoft.card.adaptive", +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Notify.Engine; +using StellaOps.Notify.Models; + +namespace StellaOps.Notify.Connectors.Teams; + +[ServiceBinding(typeof(INotifyChannelTestProvider), ServiceLifetime.Singleton)] +public sealed class TeamsChannelTestProvider : INotifyChannelTestProvider +{ + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + private const string DefaultTitle = "Stella Ops Notify Preview"; + private const int MaxFallbackLength = 512; + + public NotifyChannelType ChannelType => NotifyChannelType.Teams; + + public Task BuildPreviewAsync(ChannelTestPreviewContext context, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + var title = ResolveTitle(context); + var summary = ResolveSummary(context, title); + var bodyContent = ResolveBodyContent(context, summary); + var fallbackText = BuildFallbackText(context, title, summary, bodyContent); + + var card = new + { + type = "AdaptiveCard", + version = TeamsMetadataBuilder.CardVersion, + body = new object[] + { + new { type = "TextBlock", weight = "Bolder", text = title, wrap = true }, + new { type = "TextBlock", text = bodyContent, wrap = true }, + new { type = "TextBlock", spacing = "None", isSubtle = true, text = $"Trace: {context.TraceId}", wrap = true } + } + }; + + var payload = new + { + type = "message", + summary, + text = fallbackText, + attachments = new object[] + { + new + { + contentType = "application/vnd.microsoft.card.adaptive", content = card } } @@ -52,23 +56,69 @@ public sealed class TeamsChannelTestProvider : INotifyChannelTestProvider var body = JsonSerializer.Serialize(payload, JsonOptions); - var preview = NotifyDeliveryRendered.Create( - NotifyChannelType.Teams, - NotifyDeliveryFormat.Teams, - context.Target, - title, - body, - summary, - context.Request.TextBody ?? bodyContent, - context.Request.Locale, - ChannelTestPreviewUtilities.ComputeBodyHash(body), - context.Request.Attachments); - - var metadata = new Dictionary(StringComparer.Ordinal) - { - ["teams.webhook"] = context.Target - }; - - return Task.FromResult(new ChannelTestPreviewResult(preview, metadata)); - } -} + var preview = NotifyDeliveryRendered.Create( + NotifyChannelType.Teams, + NotifyDeliveryFormat.Teams, + context.Target, + title, + body, + summary, + fallbackText, + context.Request.Locale, + ChannelTestPreviewUtilities.ComputeBodyHash(body), + context.Request.Attachments); + + var metadata = TeamsMetadataBuilder.Build(context, fallbackText); + + return Task.FromResult(new ChannelTestPreviewResult(preview, metadata)); + } + + private static string ResolveTitle(ChannelTestPreviewContext context) + { + return !string.IsNullOrWhiteSpace(context.Request.Title) + ? context.Request.Title!.Trim() + : DefaultTitle; + } + + private static string ResolveSummary(ChannelTestPreviewContext context, string title) + { + if (!string.IsNullOrWhiteSpace(context.Request.Summary)) + { + return context.Request.Summary!.Trim(); + } + + return $"Preview generated for Teams destination at {context.Timestamp:O}. Title: {title}"; + } + + private static string ResolveBodyContent(ChannelTestPreviewContext context, string summary) + { + if (!string.IsNullOrWhiteSpace(context.Request.Body)) + { + return context.Request.Body!.Trim(); + } + + return summary; + } + + private static string BuildFallbackText(ChannelTestPreviewContext context, string title, string summary, string bodyContent) + { + var fallback = !string.IsNullOrWhiteSpace(context.Request.TextBody) + ? context.Request.TextBody!.Trim() + : summary; + + if (string.IsNullOrWhiteSpace(fallback)) + { + fallback = $"{title}: {bodyContent}"; + } + + fallback = fallback.Trim(); + fallback = fallback.ReplaceLineEndings(" "); + + if (fallback.Length > MaxFallbackLength) + { + fallback = fallback[..MaxFallbackLength]; + } + + return fallback; + } +} diff --git a/src/StellaOps.Notify.Connectors.Teams/TeamsMetadataBuilder.cs b/src/StellaOps.Notify.Connectors.Teams/TeamsMetadataBuilder.cs new file mode 100644 index 00000000..32dad484 --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Teams/TeamsMetadataBuilder.cs @@ -0,0 +1,89 @@ +using System; +using System.Collections.Generic; +using StellaOps.Notify.Connectors.Shared; +using StellaOps.Notify.Engine; +using StellaOps.Notify.Models; + +namespace StellaOps.Notify.Connectors.Teams; + +/// +/// Builds metadata for Teams previews and health diagnostics while redacting sensitive material. +/// +internal static class TeamsMetadataBuilder +{ + internal const string CardVersion = "1.5"; + + private const int SecretHashLengthBytes = 8; + + public static ConnectorMetadataBuilder CreateBuilder(ChannelTestPreviewContext context, string fallbackText) + => CreateBaseBuilder( + channel: context.Channel, + target: context.Target, + timestamp: context.Timestamp, + fallbackText: fallbackText, + properties: context.Channel.Config.Properties, + secretRef: context.Channel.Config.SecretRef, + endpoint: context.Channel.Config.Endpoint); + + public static ConnectorMetadataBuilder CreateBuilder(ChannelHealthContext context) + => CreateBaseBuilder( + channel: context.Channel, + target: context.Target, + timestamp: context.Timestamp, + fallbackText: null, + properties: context.Channel.Config.Properties, + secretRef: context.Channel.Config.SecretRef, + endpoint: context.Channel.Config.Endpoint); + + public static IReadOnlyDictionary Build(ChannelTestPreviewContext context, string fallbackText) + => CreateBuilder(context, fallbackText).Build(); + + public static IReadOnlyDictionary Build(ChannelHealthContext context) + => CreateBuilder(context).Build(); + + private static ConnectorMetadataBuilder CreateBaseBuilder( + NotifyChannel channel, + string target, + DateTimeOffset timestamp, + string? fallbackText, + IReadOnlyDictionary? properties, + string secretRef, + string? endpoint) + { + var builder = new ConnectorMetadataBuilder(); + + builder.AddTarget("teams.webhook", target) + .AddTimestamp("teams.preview.generatedAt", timestamp) + .Add("teams.card.version", CardVersion) + .AddSecretRefHash("teams.secretRef.hash", secretRef, SecretHashLengthBytes) + .AddConfigTarget("teams.config.target", channel.Config.Target, target) + .AddConfigEndpoint("teams.config.endpoint", endpoint) + .AddConfigProperties("teams.config.", properties, (key, value) => RedactTeamsValue(builder, key, value)); + + if (!string.IsNullOrWhiteSpace(fallbackText)) + { + builder.Add("teams.fallbackText", fallbackText!); + } + + return builder; + } + + private static string RedactTeamsValue(ConnectorMetadataBuilder builder, string key, string value) + { + if (ConnectorValueRedactor.IsSensitiveKey(key, builder.SensitiveFragments)) + { + return ConnectorValueRedactor.RedactSecret(value); + } + + var trimmed = value.Trim(); + if (LooksLikeGuid(trimmed)) + { + return ConnectorValueRedactor.RedactToken(trimmed, prefixLength: 8, suffixLength: 4); + } + + return trimmed; + } + + private static bool LooksLikeGuid(string value) + => value.Length >= 32 && Guid.TryParse(value, out _); +} diff --git a/src/StellaOps.Notify.Connectors.Teams/notify-plugin.json b/src/StellaOps.Notify.Connectors.Teams/notify-plugin.json new file mode 100644 index 00000000..78239596 --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Teams/notify-plugin.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": "1.0", + "id": "stellaops.notify.connector.teams", + "displayName": "StellaOps Teams Notify Connector", + "version": "0.1.0-alpha", + "requiresRestart": true, + "entryPoint": { + "type": "dotnet", + "assembly": "StellaOps.Notify.Connectors.Teams.dll" + }, + "capabilities": [ + "notify-connector", + "teams" + ], + "metadata": { + "org.stellaops.notify.channel.type": "teams", + "org.stellaops.notify.connector.cardVersion": "1.5" + } +} diff --git a/src/StellaOps.Notify.Connectors.Webhook/StellaOps.Notify.Connectors.Webhook.csproj b/src/StellaOps.Notify.Connectors.Webhook/StellaOps.Notify.Connectors.Webhook.csproj index 54540bcb..37a32ab8 100644 --- a/src/StellaOps.Notify.Connectors.Webhook/StellaOps.Notify.Connectors.Webhook.csproj +++ b/src/StellaOps.Notify.Connectors.Webhook/StellaOps.Notify.Connectors.Webhook.csproj @@ -5,9 +5,16 @@ enable - - - - - - + + + + + + + + + + PreserveNewest + + + diff --git a/src/StellaOps.Notify.Connectors.Webhook/TASKS.md b/src/StellaOps.Notify.Connectors.Webhook/TASKS.md index 51a8d5db..77c96906 100644 --- a/src/StellaOps.Notify.Connectors.Webhook/TASKS.md +++ b/src/StellaOps.Notify.Connectors.Webhook/TASKS.md @@ -3,5 +3,5 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| | NOTIFY-CONN-WEBHOOK-15-801 | TODO | Notify Connectors Guild | NOTIFY-ENGINE-15-303 | Implement webhook connector: JSON payload, signature (HMAC/Ed25519), retries/backoff, status code handling. | Integration tests with webhook stub validate signatures, retries, error handling; payload schema documented. | -| NOTIFY-CONN-WEBHOOK-15-802 | DOING (2025-10-19) | Notify Connectors Guild | NOTIFY-CONN-WEBHOOK-15-801 | Health/test-send support with signature validation hints and secret management. | Test-send returns success with sample payload; docs include verification guide; secrets never logged. | -| NOTIFY-CONN-WEBHOOK-15-803 | TODO | Notify Connectors Guild | NOTIFY-CONN-WEBHOOK-15-802 | Package Webhook connector as restart-time plug-in (manifest + host registration). | Plugin manifest added; host loads connector from `plugins/notify/webhook/`; restart validation passes. | +| NOTIFY-CONN-WEBHOOK-15-802 | BLOCKED (2025-10-20) | Notify Connectors Guild | NOTIFY-CONN-WEBHOOK-15-801 | Health/test-send support with signature validation hints and secret management. | Test-send returns success with sample payload; docs include verification guide; secrets never logged. | +| NOTIFY-CONN-WEBHOOK-15-803 | DONE (2025-10-20) | Notify Connectors Guild | NOTIFY-CONN-WEBHOOK-15-802 | Package Webhook connector as restart-time plug-in (manifest + host registration). | Plugin manifest added; host loads connector from `plugins/notify/webhook/`; restart validation passes. | diff --git a/src/StellaOps.Notify.Connectors.Webhook/WebhookChannelTestProvider.cs b/src/StellaOps.Notify.Connectors.Webhook/WebhookChannelTestProvider.cs index 17cfaeba..f0cd4677 100644 --- a/src/StellaOps.Notify.Connectors.Webhook/WebhookChannelTestProvider.cs +++ b/src/StellaOps.Notify.Connectors.Webhook/WebhookChannelTestProvider.cs @@ -48,11 +48,8 @@ public sealed class WebhookChannelTestProvider : INotifyChannelTestProvider ChannelTestPreviewUtilities.ComputeBodyHash(body), context.Request.Attachments); - var metadata = new Dictionary(StringComparer.Ordinal) - { - ["webhook.endpoint"] = context.Target - }; - - return Task.FromResult(new ChannelTestPreviewResult(preview, metadata)); - } -} + var metadata = WebhookMetadataBuilder.Build(context); + + return Task.FromResult(new ChannelTestPreviewResult(preview, metadata)); + } +} diff --git a/src/StellaOps.Notify.Connectors.Webhook/WebhookMetadataBuilder.cs b/src/StellaOps.Notify.Connectors.Webhook/WebhookMetadataBuilder.cs new file mode 100644 index 00000000..adcfa915 --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Webhook/WebhookMetadataBuilder.cs @@ -0,0 +1,53 @@ +using System.Collections.Generic; +using StellaOps.Notify.Connectors.Shared; +using StellaOps.Notify.Engine; +using StellaOps.Notify.Models; + +namespace StellaOps.Notify.Connectors.Webhook; + +/// +/// Builds metadata for Webhook previews and health diagnostics. +/// +internal static class WebhookMetadataBuilder +{ + private const int SecretHashLengthBytes = 8; + + public static ConnectorMetadataBuilder CreateBuilder(ChannelTestPreviewContext context) + => CreateBaseBuilder( + channel: context.Channel, + target: context.Target, + timestamp: context.Timestamp, + properties: context.Channel.Config.Properties, + secretRef: context.Channel.Config.SecretRef); + + public static ConnectorMetadataBuilder CreateBuilder(ChannelHealthContext context) + => CreateBaseBuilder( + channel: context.Channel, + target: context.Target, + timestamp: context.Timestamp, + properties: context.Channel.Config.Properties, + secretRef: context.Channel.Config.SecretRef); + + public static IReadOnlyDictionary Build(ChannelTestPreviewContext context) + => CreateBuilder(context).Build(); + + public static IReadOnlyDictionary Build(ChannelHealthContext context) + => CreateBuilder(context).Build(); + + private static ConnectorMetadataBuilder CreateBaseBuilder( + NotifyChannel channel, + string target, + DateTimeOffset timestamp, + IReadOnlyDictionary? properties, + string secretRef) + { + var builder = new ConnectorMetadataBuilder(); + + builder.AddTarget("webhook.endpoint", target) + .AddTimestamp("webhook.preview.generatedAt", timestamp) + .AddSecretRefHash("webhook.secretRef.hash", secretRef, SecretHashLengthBytes) + .AddConfigProperties("webhook.config.", properties); + + return builder; + } +} diff --git a/src/StellaOps.Notify.Connectors.Webhook/notify-plugin.json b/src/StellaOps.Notify.Connectors.Webhook/notify-plugin.json new file mode 100644 index 00000000..32b4ead7 --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Webhook/notify-plugin.json @@ -0,0 +1,18 @@ +{ + "schemaVersion": "1.0", + "id": "stellaops.notify.connector.webhook", + "displayName": "StellaOps Webhook Notify Connector", + "version": "0.1.0-alpha", + "requiresRestart": true, + "entryPoint": { + "type": "dotnet", + "assembly": "StellaOps.Notify.Connectors.Webhook.dll" + }, + "capabilities": [ + "notify-connector", + "webhook" + ], + "metadata": { + "org.stellaops.notify.channel.type": "webhook" + } +} diff --git a/src/StellaOps.Notify.Engine/ChannelHealthContracts.cs b/src/StellaOps.Notify.Engine/ChannelHealthContracts.cs new file mode 100644 index 00000000..47449d09 --- /dev/null +++ b/src/StellaOps.Notify.Engine/ChannelHealthContracts.cs @@ -0,0 +1,51 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Notify.Models; + +namespace StellaOps.Notify.Engine; + +/// +/// Contract implemented by channel plug-ins to provide health diagnostics. +/// +public interface INotifyChannelHealthProvider +{ + /// + /// Channel type supported by the provider. + /// + NotifyChannelType ChannelType { get; } + + /// + /// Executes a health check for the supplied channel. + /// + Task CheckAsync(ChannelHealthContext context, CancellationToken cancellationToken); +} + +/// +/// Immutable context describing a channel health request. +/// +public sealed record ChannelHealthContext( + string TenantId, + NotifyChannel Channel, + string Target, + DateTimeOffset Timestamp, + string TraceId); + +/// +/// Result returned by channel plug-ins when reporting health diagnostics. +/// +public sealed record ChannelHealthResult( + ChannelHealthStatus Status, + string? Message, + IReadOnlyDictionary Metadata); + +/// +/// Supported channel health states. +/// +public enum ChannelHealthStatus +{ + Healthy, + Degraded, + Unhealthy +} diff --git a/src/StellaOps.Notify.WebService/Contracts/ChannelHealthResponse.cs b/src/StellaOps.Notify.WebService/Contracts/ChannelHealthResponse.cs new file mode 100644 index 00000000..25b87600 --- /dev/null +++ b/src/StellaOps.Notify.WebService/Contracts/ChannelHealthResponse.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using StellaOps.Notify.Engine; + +namespace StellaOps.Notify.WebService.Contracts; + +/// +/// Response payload describing channel health diagnostics. +/// +public sealed record ChannelHealthResponse( + string TenantId, + string ChannelId, + ChannelHealthStatus Status, + string? Message, + DateTimeOffset CheckedAt, + string TraceId, + IReadOnlyDictionary Metadata); diff --git a/src/StellaOps.Notify.WebService/Program.cs b/src/StellaOps.Notify.WebService/Program.cs index 64d1dee5..685c3d26 100644 --- a/src/StellaOps.Notify.WebService/Program.cs +++ b/src/StellaOps.Notify.WebService/Program.cs @@ -96,8 +96,9 @@ else var pluginHostOptions = NotifyPluginHostFactory.Build(bootstrapOptions, contentRootPath); builder.Services.AddSingleton(pluginHostOptions); builder.Services.RegisterPluginRoutines(builder.Configuration, pluginHostOptions); -builder.Services.AddSingleton(); -builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); ConfigureAuthentication(builder, bootstrapOptions); ConfigureRateLimiting(builder, bootstrapOptions); @@ -456,11 +457,11 @@ static void ConfigureEndpoints(WebApplication app) }) .RequireAuthorization(NotifyPolicies.Admin); - apiGroup.MapPost("/channels/{channelId}/test", async (string channelId, [FromBody] ChannelTestSendRequest? request, INotifyChannelRepository repository, INotifyChannelTestService testService, HttpContext context, CancellationToken cancellationToken) => - { - if (!TryResolveTenant(context, tenantHeader, out var tenant, out var error)) - { - return error!; + apiGroup.MapPost("/channels/{channelId}/test", async (string channelId, [FromBody] ChannelTestSendRequest? request, INotifyChannelRepository repository, INotifyChannelTestService testService, HttpContext context, CancellationToken cancellationToken) => + { + if (!TryResolveTenant(context, tenantHeader, out var tenant, out var error)) + { + return error!; } if (request is null) @@ -483,10 +484,28 @@ static void ConfigureEndpoints(WebApplication app) { return Results.BadRequest(new { error = ex.Message }); } - }) - .RequireAuthorization(NotifyPolicies.Admin) - .RequireRateLimiting(NotifyRateLimitPolicies.TestSend); - + }) + .RequireAuthorization(NotifyPolicies.Admin) + .RequireRateLimiting(NotifyRateLimitPolicies.TestSend); + + apiGroup.MapGet("/channels/{channelId}/health", async (string channelId, INotifyChannelRepository repository, INotifyChannelHealthService healthService, HttpContext context, CancellationToken cancellationToken) => + { + if (!TryResolveTenant(context, tenantHeader, out var tenant, out var error)) + { + return error!; + } + + var channel = await repository.GetAsync(tenant, channelId, cancellationToken); + if (channel is null) + { + return Results.NotFound(); + } + + var response = await healthService.CheckAsync(tenant, channel, context.TraceIdentifier, cancellationToken).ConfigureAwait(false); + return JsonResponse(response); + }) + .RequireAuthorization(NotifyPolicies.Read); + apiGroup.MapDelete("/channels/{channelId}", async (string channelId, INotifyChannelRepository repository, HttpContext context, CancellationToken cancellationToken) => { if (!TryResolveTenant(context, tenantHeader, out var tenant, out var error)) diff --git a/src/StellaOps.Notify.WebService/Services/NotifyChannelHealthService.cs b/src/StellaOps.Notify.WebService/Services/NotifyChannelHealthService.cs new file mode 100644 index 00000000..7e9e0c5d --- /dev/null +++ b/src/StellaOps.Notify.WebService/Services/NotifyChannelHealthService.cs @@ -0,0 +1,182 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Notify.Engine; +using StellaOps.Notify.Models; +using StellaOps.Notify.WebService.Contracts; + +namespace StellaOps.Notify.WebService.Services; + +internal interface INotifyChannelHealthService +{ + Task CheckAsync( + string tenantId, + NotifyChannel channel, + string traceId, + CancellationToken cancellationToken); +} + +internal sealed class NotifyChannelHealthService : INotifyChannelHealthService +{ + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly IReadOnlyDictionary _providers; + + public NotifyChannelHealthService( + TimeProvider timeProvider, + ILogger logger, + IEnumerable providers) + { + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _providers = BuildProviderMap(providers ?? Array.Empty(), _logger); + } + + public async Task CheckAsync( + string tenantId, + NotifyChannel channel, + string traceId, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(channel); + + cancellationToken.ThrowIfCancellationRequested(); + + var target = ResolveTarget(channel); + var timestamp = _timeProvider.GetUtcNow(); + var context = new ChannelHealthContext( + tenantId, + channel, + target, + timestamp, + traceId); + + ChannelHealthResult? providerResult = null; + var providerName = "fallback"; + + if (_providers.TryGetValue(channel.Type, out var provider)) + { + try + { + providerResult = await provider.CheckAsync(context, cancellationToken).ConfigureAwait(false); + providerName = provider.GetType().FullName ?? provider.GetType().Name; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + _logger.LogWarning( + ex, + "Notify channel health provider {Provider} failed for tenant {TenantId}, channel {ChannelId} ({ChannelType}).", + provider.GetType().FullName, + tenantId, + channel.ChannelId, + channel.Type); + providerResult = new ChannelHealthResult( + ChannelHealthStatus.Degraded, + "Channel health provider threw an exception. See logs for details.", + new Dictionary(StringComparer.Ordinal)); + } + } + + var metadata = MergeMetadata(context, providerName, providerResult?.Metadata); + var status = providerResult?.Status ?? ChannelHealthStatus.Healthy; + var message = providerResult?.Message ?? "Channel metadata returned without provider-specific diagnostics."; + + var response = new ChannelHealthResponse( + tenantId, + channel.ChannelId, + status, + message, + timestamp, + traceId, + metadata); + + _logger.LogInformation( + "Notify channel health generated for tenant {TenantId}, channel {ChannelId} ({ChannelType}) using provider {Provider}.", + tenantId, + channel.ChannelId, + channel.Type, + providerName); + + return response; + } + + private static IReadOnlyDictionary BuildProviderMap( + IEnumerable providers, + ILogger logger) + { + var map = new Dictionary(); + foreach (var provider in providers) + { + if (provider is null) + { + continue; + } + + if (map.TryGetValue(provider.ChannelType, out var existing)) + { + logger?.LogWarning( + "Multiple Notify channel health providers registered for {ChannelType}. Keeping {ExistingProvider} and ignoring {NewProvider}.", + provider.ChannelType, + existing.GetType().FullName, + provider.GetType().FullName); + continue; + } + + map[provider.ChannelType] = provider; + } + + return map; + } + + private static string ResolveTarget(NotifyChannel channel) + { + var target = channel.Config.Target ?? channel.Config.Endpoint; + if (string.IsNullOrWhiteSpace(target)) + { + return channel.Name; + } + + return target; + } + + private static IReadOnlyDictionary MergeMetadata( + ChannelHealthContext context, + string providerName, + IReadOnlyDictionary? providerMetadata) + { + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["channelType"] = context.Channel.Type.ToString().ToLowerInvariant(), + ["target"] = context.Target, + ["previewProvider"] = providerName, + ["traceId"] = context.TraceId, + ["channelEnabled"] = context.Channel.Enabled.ToString() + }; + + foreach (var label in context.Channel.Labels) + { + metadata[$"label.{label.Key}"] = label.Value; + } + + if (providerMetadata is not null) + { + foreach (var pair in providerMetadata) + { + if (string.IsNullOrWhiteSpace(pair.Key) || pair.Value is null) + { + continue; + } + + metadata[pair.Key.Trim()] = pair.Value; + } + } + + return metadata; + } +} diff --git a/src/StellaOps.Plugin/TASKS.md b/src/StellaOps.Plugin/TASKS.md index d7b8ccdd..1ffb19aa 100644 --- a/src/StellaOps.Plugin/TASKS.md +++ b/src/StellaOps.Plugin/TASKS.md @@ -1,6 +1,9 @@ # TASKS | Task | Owner(s) | Depends on | Notes | |---|---|---|---| -|PLUGIN-DI-08-001 Scoped service support in plugin bootstrap|Plugin Platform Guild (DONE 2025-10-19)|StellaOps.DependencyInjection|Introduced `ServiceBindingAttribute` metadata for scoped DI, taught plugin/job loaders to consume it with duplicate-safe registration, added coverage, and refreshed the plug-in SDK guide.| -|PLUGIN-DI-08-002.COORD Authority scoped-service handshake|Plugin Platform Guild, Authority Core (DOING 2025-10-19)|PLUGIN-DI-08-001|Workshop scheduled for 2025-10-20 15:00–16:00 UTC; agenda + attendee list tracked in `docs/dev/authority-plugin-di-coordination.md`; pending pre-read contributions prior to session.| -|PLUGIN-DI-08-002 Authority plugin integration updates|Plugin Platform Guild, Authority Core|PLUGIN-DI-08-001, PLUGIN-DI-08-002.COORD|Update Authority identity-provider plugin registrar/registry to resolve scoped services correctly; adjust bootstrap flows and background services to create scopes when needed; add regression tests.| +|PLUGIN-DI-08-001 Scoped service support in plugin bootstrap|Plugin Platform Guild (DONE 2025-10-19)|StellaOps.DependencyInjection|Introduced `ServiceBindingAttribute` metadata for scoped DI, taught plugin/job loaders to consume it with duplicate-safe registration, added coverage, and refreshed the plug-in SDK guide.| +|PLUGIN-DI-08-002.COORD Authority scoped-service handshake|Plugin Platform Guild, Authority Core (DONE 2025-10-20)|PLUGIN-DI-08-001|Workshop held 2025-10-20 15:00–16:05 UTC; outcomes/notes captured in `docs/dev/authority-plugin-di-coordination.md`, follow-up action items assigned for PLUGIN-DI-08-002 implementation plan.| +|PLUGIN-DI-08-002 Authority plugin integration updates|Plugin Platform Guild, Authority Core (DONE 2025-10-20)|PLUGIN-DI-08-001, PLUGIN-DI-08-002.COORD|Standard registrar now registers scoped credential/provisioning stores + identity-provider plugins, registry Acquire scopes instances, and regression suites (`dotnet test src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StellaOps.Authority.Plugin.Standard.Tests.csproj`, `dotnet test src/StellaOps.Authority/StellaOps.Authority.Tests/StellaOps.Authority.Tests.csproj`) cover scoped lifetimes + handles.| +|PLUGIN-DI-08-003 Authority registry scoped resolution|Plugin Platform Guild, Authority Core (DONE 2025-10-20)|PLUGIN-DI-08-002.COORD|Reworked `IAuthorityIdentityProviderRegistry` to expose metadata + scoped handles, updated OpenIddict flows/Program health endpoints, and added coverage via `AuthorityIdentityProviderRegistryTests`.| +|PLUGIN-DI-08-004 Authority plugin loader DI bridge|Plugin Platform Guild, Authority Core (DONE 2025-10-20)|PLUGIN-DI-08-002.COORD|Authority plugin loader now activates registrars via scoped DI leases, registers `[ServiceBinding]` metadata, and includes regression coverage in `AuthorityPluginLoaderTests`.| +|PLUGIN-DI-08-005 Authority plugin bootstrap scope pattern|Plugin Platform Guild, Authority Core (DONE 2025-10-20)|PLUGIN-DI-08-002.COORD|Standard bootstrapper uses `IServiceScopeFactory` per run; tests updated to validate scoped execution and documentation annotated in `authority-plugin-di-coordination.md`.| diff --git a/src/StellaOps.Policy/TASKS.md b/src/StellaOps.Policy/TASKS.md index 5e3bf339..bd5a1f63 100644 --- a/src/StellaOps.Policy/TASKS.md +++ b/src/StellaOps.Policy/TASKS.md @@ -5,15 +5,13 @@ | POLICY-CORE-09-001 | DONE | Policy Guild | SCANNER-WEB-09-101 | Define YAML schema/binder, diagnostics, CLI validation for policy files. | Schema doc published; binder loads sample policy; validation errors actionable. | | POLICY-CORE-09-002 | DONE | Policy Guild | POLICY-CORE-09-001 | Implement policy snapshot store + revision digests + audit logging. | Snapshots persisted with digest; tests compare revisions; audit entries created. | | POLICY-CORE-09-003 | DONE | Policy Guild | POLICY-CORE-09-002 | `/policy/preview` API (image digest → projected verdict delta). | Preview returns diff JSON; integration tests with mocked report; docs updated. | -| POLICY-CORE-09-004 | DOING (2025-10-19) | Policy Guild | — | Versioned scoring config with schema validation, trust table, and golden fixtures. | Scoring config documented; fixtures stored; validation CLI passes. | -| POLICY-CORE-09-005 | DOING (2025-10-19) | Policy Guild | — | Scoring/quiet engine – compute score, enforce VEX-only quiet rules, emit inputs and provenance. | Engine unit tests cover severity weighting; outputs include provenance data. | -| POLICY-CORE-09-006 | DOING (2025-10-19) | Policy Guild | — | Unknown state & confidence decay – deterministic bands surfaced in policy outputs. | Confidence decay tests pass; docs updated; preview endpoint displays banding. | -| POLICY-CORE-09-004 | DONE | Policy Guild | POLICY-CORE-09-001 | Versioned scoring config (weights, trust table, reachability buckets) with schema validation, binder, and golden fixtures. | Config serialized with semantic version, binder loads defaults, fixtures assert deterministic hash. | -| POLICY-CORE-09-005 | DONE | Policy Guild | POLICY-CORE-09-004, POLICY-CORE-09-002 | Implement scoring/quiet engine: compute score from config, enforce VEX-only quiet rules, emit inputs + `quietedBy` metadata in policy verdicts. | `/reports` policy result includes score, inputs, configVersion, quiet provenance; unit/integration tests prove reproducibility. | -| POLICY-CORE-09-006 | DONE | Policy Guild | POLICY-CORE-09-005, FEEDCORE-ENGINE-07-003 | Track unknown states with deterministic confidence bands that decay over time; expose state in policy outputs and docs. | Unknown flags + confidence band persisted, decay job deterministic, preview/report APIs show state with tests covering decay math. | +| POLICY-CORE-09-004 | DONE (2025-10-19) | Policy Guild | POLICY-CORE-09-001 | Versioned scoring config (weights, trust table, reachability buckets) with schema validation, binder, and golden fixtures. | Config serialized with semantic version, binder loads defaults, fixtures assert deterministic hash. | +| POLICY-CORE-09-005 | DONE (2025-10-19) | Policy Guild | POLICY-CORE-09-004, POLICY-CORE-09-002 | Implement scoring/quiet engine: compute score from config, enforce VEX-only quiet rules, emit inputs + `quietedBy` metadata in policy verdicts. | `/reports` policy result includes score, inputs, configVersion, quiet provenance; unit/integration tests prove reproducibility. | +| POLICY-CORE-09-006 | DONE (2025-10-19) | Policy Guild | POLICY-CORE-09-005, FEEDCORE-ENGINE-07-003 | Track unknown states with deterministic confidence bands that decay over time; expose state in policy outputs and docs. | Unknown flags + confidence band persisted, decay job deterministic, preview/report APIs show state with tests covering decay math. | | POLICY-RUNTIME-17-201 | TODO | Policy Guild, Scanner WebService Guild | ZASTAVA-OBS-17-005 | Define runtime reachability feed contract and alignment plan for `SCANNER-RUNTIME-17-401` once Zastava endpoints land; document policy expectations for reachability tags. | Contract note published, sample payload agreed with Scanner team, dependencies captured in scanner/runtime task boards. | -## Notes -- 2025-10-18: POLICY-CORE-09-001 completed. Binder + diagnostics + CLI scaffolding landed with tests; schema embedded at `src/StellaOps.Policy/Schemas/policy-schema@1.json` and referenced by docs/11_DATA_SCHEMAS.md. -- 2025-10-18: POLICY-CORE-09-002 completed. Snapshot store + audit trail implemented with deterministic digest hashing and tests covering revision increments and dedupe. -- 2025-10-18: POLICY-CORE-09-003 delivered. Preview service evaluates policy projections vs. baseline, returns verdict diffs, and ships with unit coverage. +## Notes +- 2025-10-18: POLICY-CORE-09-001 completed. Binder + diagnostics + CLI scaffolding landed with tests; schema embedded at `src/StellaOps.Policy/Schemas/policy-schema@1.json` and referenced by docs/11_DATA_SCHEMAS.md. +- 2025-10-18: POLICY-CORE-09-002 completed. Snapshot store + audit trail implemented with deterministic digest hashing and tests covering revision increments and dedupe. +- 2025-10-18: POLICY-CORE-09-003 delivered. Preview service evaluates policy projections vs. baseline, returns verdict diffs, and ships with unit coverage. +- 2025-10-19: POLICY-CORE-09-004/005/006 wrapped. Default scoring config + trust/quiet/unknown outputs shipped, deterministic hashes captured in fixtures, and `dotnet test src/StellaOps.Policy.Tests/StellaOps.Policy.Tests.csproj` keeps coverage green (quiet provenance + confidence decay cases). diff --git a/src/StellaOps.Scanner.Storage/Catalog/RuntimeEventDocument.cs b/src/StellaOps.Scanner.Storage/Catalog/RuntimeEventDocument.cs new file mode 100644 index 00000000..b1cf214c --- /dev/null +++ b/src/StellaOps.Scanner.Storage/Catalog/RuntimeEventDocument.cs @@ -0,0 +1,83 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Scanner.Storage.Catalog; + +/// +/// MongoDB persistence model for runtime events emitted by the Zastava observer. +/// +public sealed class RuntimeEventDocument +{ + [BsonId] + [BsonRepresentation(BsonType.ObjectId)] + public string? Id { get; set; } + + [BsonElement("eventId")] + [BsonRequired] + public string EventId { get; set; } = string.Empty; + + [BsonElement("schemaVersion")] + [BsonRequired] + public string SchemaVersion { get; set; } = string.Empty; + + [BsonElement("tenant")] + [BsonRequired] + public string Tenant { get; set; } = string.Empty; + + [BsonElement("node")] + [BsonRequired] + public string Node { get; set; } = string.Empty; + + [BsonElement("kind")] + [BsonRepresentation(BsonType.String)] + [BsonRequired] + public string Kind { get; set; } = string.Empty; + + [BsonElement("when")] + [BsonDateTimeOptions(Kind = DateTimeKind.Utc)] + public DateTime When { get; set; } + + [BsonElement("receivedAt")] + [BsonDateTimeOptions(Kind = DateTimeKind.Utc)] + public DateTime ReceivedAt { get; set; } + + [BsonElement("expiresAt")] + [BsonDateTimeOptions(Kind = DateTimeKind.Utc)] + public DateTime ExpiresAt { get; set; } + + [BsonElement("platform")] + public string? Platform { get; set; } + + [BsonElement("namespace")] + public string? Namespace { get; set; } + + [BsonElement("pod")] + public string? Pod { get; set; } + + [BsonElement("container")] + public string? Container { get; set; } + + [BsonElement("containerId")] + public string? ContainerId { get; set; } + + [BsonElement("imageRef")] + public string? ImageRef { get; set; } + + [BsonElement("engine")] + public string? Engine { get; set; } + + [BsonElement("engineVersion")] + public string? EngineVersion { get; set; } + + [BsonElement("baselineDigest")] + public string? BaselineDigest { get; set; } + + [BsonElement("imageSigned")] + public bool? ImageSigned { get; set; } + + [BsonElement("sbomReferrer")] + public string? SbomReferrer { get; set; } + + [BsonElement("payload")] + public BsonDocument Payload { get; set; } = new(); +} diff --git a/src/StellaOps.Scanner.Storage/Extensions/ServiceCollectionExtensions.cs b/src/StellaOps.Scanner.Storage/Extensions/ServiceCollectionExtensions.cs index 06e0d823..60002aa1 100644 --- a/src/StellaOps.Scanner.Storage/Extensions/ServiceCollectionExtensions.cs +++ b/src/StellaOps.Scanner.Storage/Extensions/ServiceCollectionExtensions.cs @@ -58,9 +58,10 @@ public static class ServiceCollectionExtensions services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); - services.TryAddSingleton(); - services.TryAddSingleton(); - services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); services.TryAddSingleton(CreateAmazonS3Client); services.TryAddSingleton(); diff --git a/src/StellaOps.Scanner.Storage/Mongo/MongoBootstrapper.cs b/src/StellaOps.Scanner.Storage/Mongo/MongoBootstrapper.cs index 3470eb6b..1ece4501 100644 --- a/src/StellaOps.Scanner.Storage/Mongo/MongoBootstrapper.cs +++ b/src/StellaOps.Scanner.Storage/Mongo/MongoBootstrapper.cs @@ -41,12 +41,13 @@ public sealed class MongoBootstrapper { ScannerStorageDefaults.Collections.Artifacts, ScannerStorageDefaults.Collections.Images, - ScannerStorageDefaults.Collections.Layers, - ScannerStorageDefaults.Collections.Links, - ScannerStorageDefaults.Collections.Jobs, - ScannerStorageDefaults.Collections.LifecycleRules, - ScannerStorageDefaults.Collections.Migrations, - }; + ScannerStorageDefaults.Collections.Layers, + ScannerStorageDefaults.Collections.Links, + ScannerStorageDefaults.Collections.Jobs, + ScannerStorageDefaults.Collections.LifecycleRules, + ScannerStorageDefaults.Collections.RuntimeEvents, + ScannerStorageDefaults.Collections.Migrations, + }; using var cursor = await _database.ListCollectionNamesAsync(cancellationToken: cancellationToken).ConfigureAwait(false); var existing = await cursor.ToListAsync(cancellationToken).ConfigureAwait(false); @@ -67,11 +68,12 @@ public sealed class MongoBootstrapper { await EnsureArtifactIndexesAsync(cancellationToken).ConfigureAwait(false); await EnsureImageIndexesAsync(cancellationToken).ConfigureAwait(false); - await EnsureLayerIndexesAsync(cancellationToken).ConfigureAwait(false); - await EnsureLinkIndexesAsync(cancellationToken).ConfigureAwait(false); - await EnsureJobIndexesAsync(cancellationToken).ConfigureAwait(false); - await EnsureLifecycleIndexesAsync(cancellationToken).ConfigureAwait(false); - } + await EnsureLayerIndexesAsync(cancellationToken).ConfigureAwait(false); + await EnsureLinkIndexesAsync(cancellationToken).ConfigureAwait(false); + await EnsureJobIndexesAsync(cancellationToken).ConfigureAwait(false); + await EnsureLifecycleIndexesAsync(cancellationToken).ConfigureAwait(false); + await EnsureRuntimeEventIndexesAsync(cancellationToken).ConfigureAwait(false); + } private Task EnsureArtifactIndexesAsync(CancellationToken cancellationToken) { @@ -176,6 +178,32 @@ public sealed class MongoBootstrapper .Ascending(x => x.Class), new CreateIndexOptions { Name = "lifecycle_artifact_class", Unique = true }); - return collection.Indexes.CreateManyAsync(new[] { expiresIndex, artifactIndex }, cancellationToken); - } -} + return collection.Indexes.CreateManyAsync(new[] { expiresIndex, artifactIndex }, cancellationToken); + } + + private Task EnsureRuntimeEventIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(ScannerStorageDefaults.Collections.RuntimeEvents); + var models = new List> + { + new( + Builders.IndexKeys.Ascending(x => x.EventId), + new CreateIndexOptions { Name = "runtime_event_eventId", Unique = true }), + new( + Builders.IndexKeys + .Ascending(x => x.Tenant) + .Ascending(x => x.Node) + .Ascending(x => x.When), + new CreateIndexOptions { Name = "runtime_event_tenant_node_when" }), + new( + Builders.IndexKeys.Ascending(x => x.ExpiresAt), + new CreateIndexOptions + { + Name = "runtime_event_expiresAt", + ExpireAfter = TimeSpan.Zero + }) + }; + + return collection.Indexes.CreateManyAsync(models, cancellationToken); + } +} diff --git a/src/StellaOps.Scanner.Storage/Mongo/MongoCollectionProvider.cs b/src/StellaOps.Scanner.Storage/Mongo/MongoCollectionProvider.cs index c87eea5e..30ff7c2f 100644 --- a/src/StellaOps.Scanner.Storage/Mongo/MongoCollectionProvider.cs +++ b/src/StellaOps.Scanner.Storage/Mongo/MongoCollectionProvider.cs @@ -16,11 +16,12 @@ public sealed class MongoCollectionProvider } public IMongoCollection Artifacts => GetCollection(ScannerStorageDefaults.Collections.Artifacts); - public IMongoCollection Images => GetCollection(ScannerStorageDefaults.Collections.Images); - public IMongoCollection Layers => GetCollection(ScannerStorageDefaults.Collections.Layers); - public IMongoCollection Links => GetCollection(ScannerStorageDefaults.Collections.Links); - public IMongoCollection Jobs => GetCollection(ScannerStorageDefaults.Collections.Jobs); - public IMongoCollection LifecycleRules => GetCollection(ScannerStorageDefaults.Collections.LifecycleRules); + public IMongoCollection Images => GetCollection(ScannerStorageDefaults.Collections.Images); + public IMongoCollection Layers => GetCollection(ScannerStorageDefaults.Collections.Layers); + public IMongoCollection Links => GetCollection(ScannerStorageDefaults.Collections.Links); + public IMongoCollection Jobs => GetCollection(ScannerStorageDefaults.Collections.Jobs); + public IMongoCollection LifecycleRules => GetCollection(ScannerStorageDefaults.Collections.LifecycleRules); + public IMongoCollection RuntimeEvents => GetCollection(ScannerStorageDefaults.Collections.RuntimeEvents); private IMongoCollection GetCollection(string name) { diff --git a/src/StellaOps.Scanner.Storage/Repositories/RuntimeEventRepository.cs b/src/StellaOps.Scanner.Storage/Repositories/RuntimeEventRepository.cs new file mode 100644 index 00000000..e19107ba --- /dev/null +++ b/src/StellaOps.Scanner.Storage/Repositories/RuntimeEventRepository.cs @@ -0,0 +1,56 @@ +using MongoDB.Driver; +using StellaOps.Scanner.Storage.Catalog; +using StellaOps.Scanner.Storage.Mongo; + +namespace StellaOps.Scanner.Storage.Repositories; + +/// +/// Repository responsible for persisting runtime events. +/// +public sealed class RuntimeEventRepository +{ + private readonly MongoCollectionProvider _collections; + + public RuntimeEventRepository(MongoCollectionProvider collections) + { + _collections = collections ?? throw new ArgumentNullException(nameof(collections)); + } + + public async Task InsertAsync( + IReadOnlyCollection documents, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(documents); + if (documents.Count == 0) + { + return RuntimeEventInsertResult.Empty; + } + + try + { + await _collections.RuntimeEvents.InsertManyAsync( + documents, + new InsertManyOptions { IsOrdered = false }, + cancellationToken).ConfigureAwait(false); + + return new RuntimeEventInsertResult(documents.Count, 0); + } + catch (MongoBulkWriteException ex) + { + var duplicates = ex.WriteErrors + .Count(error => error.Category == ServerErrorCategory.DuplicateKey); + var inserted = documents.Count - duplicates; + if (inserted < 0) + { + inserted = 0; + } + + return new RuntimeEventInsertResult(inserted, duplicates); + } + } +} + +public readonly record struct RuntimeEventInsertResult(int InsertedCount, int DuplicateCount) +{ + public static RuntimeEventInsertResult Empty => new(0, 0); +} diff --git a/src/StellaOps.Scanner.Storage/ScannerStorageDefaults.cs b/src/StellaOps.Scanner.Storage/ScannerStorageDefaults.cs index c52950de..ed77024e 100644 --- a/src/StellaOps.Scanner.Storage/ScannerStorageDefaults.cs +++ b/src/StellaOps.Scanner.Storage/ScannerStorageDefaults.cs @@ -8,14 +8,15 @@ public static class ScannerStorageDefaults public static class Collections { - public const string Artifacts = "artifacts"; - public const string Images = "images"; - public const string Layers = "layers"; - public const string Links = "links"; - public const string Jobs = "jobs"; - public const string LifecycleRules = "lifecycle_rules"; - public const string Migrations = "schema_migrations"; - } + public const string Artifacts = "artifacts"; + public const string Images = "images"; + public const string Layers = "layers"; + public const string Links = "links"; + public const string Jobs = "jobs"; + public const string LifecycleRules = "lifecycle_rules"; + public const string RuntimeEvents = "runtime.events"; + public const string Migrations = "schema_migrations"; + } public static class ObjectPrefixes { diff --git a/src/StellaOps.Scanner.WebService.Tests/ReportsEndpointsTests.cs b/src/StellaOps.Scanner.WebService.Tests/ReportsEndpointsTests.cs index e71594b5..aae3182e 100644 --- a/src/StellaOps.Scanner.WebService.Tests/ReportsEndpointsTests.cs +++ b/src/StellaOps.Scanner.WebService.Tests/ReportsEndpointsTests.cs @@ -1,14 +1,19 @@ using System.Net; using System.Net.Http.Json; -using System.Text; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.DependencyInjection; -using StellaOps.Policy; -using StellaOps.Scanner.WebService.Contracts; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using StellaOps.Policy; +using StellaOps.Notify.Models; +using StellaOps.Scanner.WebService.Contracts; +using StellaOps.Scanner.WebService.Services; +using System.Linq; namespace StellaOps.Scanner.WebService.Tests; @@ -116,21 +121,118 @@ rules: } [Fact] - public async Task ReportsEndpointReturnsServiceUnavailableWhenPolicyMissing() - { - using var factory = new ScannerApplicationFactory(); - using var client = factory.CreateClient(); - - var request = new ReportRequestDto - { - ImageDigest = "sha256:feedface", - Findings = new[] - { - new PolicyPreviewFindingDto { Id = "finding-1", Severity = "High" } - } - }; - - var response = await client.PostAsJsonAsync("/api/v1/reports", request); - Assert.Equal((HttpStatusCode)StatusCodes.Status503ServiceUnavailable, response.StatusCode); - } -} + public async Task ReportsEndpointReturnsServiceUnavailableWhenPolicyMissing() + { + using var factory = new ScannerApplicationFactory(); + using var client = factory.CreateClient(); + + var request = new ReportRequestDto + { + ImageDigest = "sha256:feedface", + Findings = new[] + { + new PolicyPreviewFindingDto { Id = "finding-1", Severity = "High" } + } + }; + + var response = await client.PostAsJsonAsync("/api/v1/reports", request); + Assert.Equal((HttpStatusCode)StatusCodes.Status503ServiceUnavailable, response.StatusCode); + } + + [Fact] + public async Task ReportsEndpointPublishesPlatformEvents() + { + const string policyYaml = """ +version: "1.0" +rules: + - name: Block Critical + severity: [Critical] + action: block +"""; + + using var factory = new ScannerApplicationFactory( + configuration => + { + configuration["scanner:signing:enabled"] = "true"; + configuration["scanner:signing:keyId"] = "scanner-report-signing"; + configuration["scanner:signing:algorithm"] = "hs256"; + configuration["scanner:signing:keyPem"] = Convert.ToBase64String(Encoding.UTF8.GetBytes("scanner-report-hmac-key-events!")); + configuration["scanner:features:enableSignedReports"] = "true"; + configuration["scanner:events:enabled"] = "true"; + configuration["scanner:events:driver"] = "redis"; + configuration["scanner:events:dsn"] = "redis://tests"; + configuration["scanner:events:stream"] = "stella.events.tests"; + configuration["scanner:events:publishTimeoutSeconds"] = "5"; + configuration["scanner:events:maxStreamLength"] = "100"; + }, + services => + { + services.RemoveAll(); + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + }); + + var store = factory.Services.GetRequiredService(); + await store.SaveAsync( + new PolicySnapshotContent(policyYaml, PolicyDocumentFormat.Yaml, "tester", "seed", "initial"), + CancellationToken.None); + + var recorder = factory.Services.GetRequiredService(); + + using var client = factory.CreateClient(); + + var request = new ReportRequestDto + { + ImageDigest = "sha256:cafebabe", + Findings = new[] + { + new PolicyPreviewFindingDto + { + Id = "finding-42", + Severity = "Critical", + Repository = "acme/edge/api", + Tags = new[] { "reachability:runtime", "kev:CVE-2024-1234" } + } + } + }; + + var response = await client.PostAsJsonAsync("/api/v1/reports", request); + response.EnsureSuccessStatusCode(); + + Assert.Equal(2, recorder.Events.Count); + var ready = recorder.Events.Single(evt => evt.Kind == NotifyEventKinds.ScannerReportReady); + var completed = recorder.Events.Single(evt => evt.Kind == NotifyEventKinds.ScannerScanCompleted); + + Assert.Equal("default", ready.Tenant); + Assert.Equal("default", completed.Tenant); + Assert.NotEqual(Guid.Empty, ready.EventId); + Assert.NotEqual(Guid.Empty, completed.EventId); + Assert.Equal(ready.Payload?["reportId"]?.GetValue(), completed.Payload?["reportId"]?.GetValue()); + Assert.Equal("sha256:cafebabe", completed.Payload?["digest"]?.GetValue()); + + var readyPayload = Assert.IsType(ready.Payload); + Assert.Equal("fail", readyPayload["verdict"]?.GetValue()); + Assert.Equal("acme/edge", ready.Scope?.Namespace); + Assert.Equal("api", ready.Scope?.Repo); + Assert.Equal("sha256:cafebabe", ready.Scope?.Digest); + Assert.True(readyPayload.ContainsKey("dsse")); + Assert.True(readyPayload.ContainsKey("report")); + + var completedPayload = Assert.IsType(completed.Payload); + Assert.Equal("fail", completedPayload["verdict"]?.GetValue()); + Assert.True(completedPayload["findings"] is JsonArray { Count: > 0 }); + + Assert.Equal(2, recorder.Events.Count); + } + + private sealed class RecordingPlatformEventPublisher : IPlatformEventPublisher + { + public List Events { get; } = new(); + + public Task PublishAsync(NotifyEvent @event, CancellationToken cancellationToken = default) + { + Events.Add(@event); + return Task.CompletedTask; + } + } +} diff --git a/src/StellaOps.Scanner.WebService.Tests/RuntimeEndpointsTests.cs b/src/StellaOps.Scanner.WebService.Tests/RuntimeEndpointsTests.cs new file mode 100644 index 00000000..6f5c4192 --- /dev/null +++ b/src/StellaOps.Scanner.WebService.Tests/RuntimeEndpointsTests.cs @@ -0,0 +1,266 @@ +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using MongoDB.Driver; +using StellaOps.Policy; +using StellaOps.Scanner.Storage.Catalog; +using StellaOps.Scanner.Storage.Mongo; +using StellaOps.Scanner.WebService.Contracts; +using StellaOps.Zastava.Core.Contracts; + +namespace StellaOps.Scanner.WebService.Tests; + +public sealed class RuntimeEndpointsTests +{ + [Fact] + public async Task RuntimeEventsEndpointPersistsEvents() + { + using var factory = new ScannerApplicationFactory(); + using var client = factory.CreateClient(); + + var request = new RuntimeEventsIngestRequestDto + { + BatchId = "batch-1", + Events = new[] + { + CreateEnvelope("evt-001"), + CreateEnvelope("evt-002") + } + }; + + var response = await client.PostAsJsonAsync("/api/v1/runtime/events", request); + Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); + + var payload = await response.Content.ReadFromJsonAsync(); + Assert.NotNull(payload); + Assert.Equal(2, payload!.Accepted); + Assert.Equal(0, payload.Duplicates); + + using var scope = factory.Services.CreateScope(); + var collections = scope.ServiceProvider.GetRequiredService(); + var stored = await collections.RuntimeEvents.Find(FilterDefinition.Empty).ToListAsync(); + Assert.Equal(2, stored.Count); + Assert.Contains(stored, doc => doc.EventId == "evt-001"); + Assert.All(stored, doc => + { + Assert.Equal("tenant-alpha", doc.Tenant); + Assert.True(doc.ExpiresAt > doc.ReceivedAt); + }); + } + + [Fact] + public async Task RuntimeEventsEndpointRejectsUnsupportedSchema() + { + using var factory = new ScannerApplicationFactory(); + using var client = factory.CreateClient(); + + var envelope = CreateEnvelope("evt-100", schemaVersion: "zastava.runtime.event@v2.0"); + + var request = new RuntimeEventsIngestRequestDto + { + Events = new[] { envelope } + }; + + var response = await client.PostAsJsonAsync("/api/v1/runtime/events", request); + Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); + } + + [Fact] + public async Task RuntimeEventsEndpointEnforcesRateLimit() + { + using var factory = new ScannerApplicationFactory(configuration => + { + configuration["scanner:runtime:perNodeBurst"] = "1"; + configuration["scanner:runtime:perNodeEventsPerSecond"] = "1"; + configuration["scanner:runtime:perTenantBurst"] = "1"; + configuration["scanner:runtime:perTenantEventsPerSecond"] = "1"; + }); + using var client = factory.CreateClient(); + + var request = new RuntimeEventsIngestRequestDto + { + Events = new[] + { + CreateEnvelope("evt-500"), + CreateEnvelope("evt-501") + } + }; + + var response = await client.PostAsJsonAsync("/api/v1/runtime/events", request); + Assert.Equal((HttpStatusCode)StatusCodes.Status429TooManyRequests, response.StatusCode); + Assert.NotNull(response.Headers.RetryAfter); + + using var scope = factory.Services.CreateScope(); + var collections = scope.ServiceProvider.GetRequiredService(); + var count = await collections.RuntimeEvents.CountDocumentsAsync(FilterDefinition.Empty); + Assert.Equal(0, count); + } + + [Fact] + public async Task RuntimePolicyEndpointReturnsDecisions() + { + using var factory = new ScannerApplicationFactory(configuration => + { + configuration["scanner:runtime:policyCacheTtlSeconds"] = "600"; + }); + + const string imageDigest = "sha256:deadbeef"; + + using var client = factory.CreateClient(); + + using (var scope = factory.Services.CreateScope()) + { + var collections = scope.ServiceProvider.GetRequiredService(); + var policyStore = scope.ServiceProvider.GetRequiredService(); + + const string policyYaml = """ +version: "1.0" +rules: + - name: Block Critical + severity: [Critical] + action: block +"""; + var saveResult = await policyStore.SaveAsync( + new PolicySnapshotContent(policyYaml, PolicyDocumentFormat.Yaml, "tester", "tests", "seed"), + CancellationToken.None); + Assert.True(saveResult.Success); + + var snapshot = await policyStore.GetLatestAsync(CancellationToken.None); + Assert.NotNull(snapshot); + + var sbomArtifactId = CatalogIdFactory.CreateArtifactId(ArtifactDocumentType.ImageBom, "sha256:sbomdigest"); + var attestationArtifactId = CatalogIdFactory.CreateArtifactId(ArtifactDocumentType.Attestation, "sha256:attdigest"); + + await collections.Artifacts.InsertManyAsync(new[] + { + new ArtifactDocument + { + Id = sbomArtifactId, + Type = ArtifactDocumentType.ImageBom, + Format = ArtifactDocumentFormat.CycloneDxJson, + MediaType = "application/json", + BytesSha256 = "sha256:sbomdigest", + RefCount = 1, + CreatedAtUtc = DateTime.UtcNow, + UpdatedAtUtc = DateTime.UtcNow + }, + new ArtifactDocument + { + Id = attestationArtifactId, + Type = ArtifactDocumentType.Attestation, + Format = ArtifactDocumentFormat.DsseJson, + MediaType = "application/vnd.dsse.envelope+json", + BytesSha256 = "sha256:attdigest", + RefCount = 1, + CreatedAtUtc = DateTime.UtcNow, + UpdatedAtUtc = DateTime.UtcNow, + Rekor = new RekorReference { Uuid = "rekor-uuid", Url = "https://rekor.example/uuid/rekor-uuid", Index = 7 } + } + }); + + await collections.Links.InsertManyAsync(new[] + { + new LinkDocument + { + Id = Guid.NewGuid().ToString("N"), + FromType = LinkSourceType.Image, + FromDigest = imageDigest, + ArtifactId = sbomArtifactId, + CreatedAtUtc = DateTime.UtcNow + }, + new LinkDocument + { + Id = Guid.NewGuid().ToString("N"), + FromType = LinkSourceType.Image, + FromDigest = imageDigest, + ArtifactId = attestationArtifactId, + CreatedAtUtc = DateTime.UtcNow + } + }); + } + + var request = new RuntimePolicyRequestDto + { + Namespace = "payments", + Images = new[] { imageDigest, imageDigest }, + Labels = new Dictionary { ["app"] = "api" } + }; + + var response = await client.PostAsJsonAsync("/api/v1/policy/runtime", request); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + + var raw = await response.Content.ReadAsStringAsync(); + Assert.False(string.IsNullOrWhiteSpace(raw), "Runtime policy response body was empty."); + var payload = JsonSerializer.Deserialize(raw); + Assert.True(payload is not null, $"Runtime policy response: {raw}"); + Assert.Equal(600, payload!.TtlSeconds); + Assert.NotNull(payload.PolicyRevision); + Assert.True(payload.ExpiresAtUtc > DateTimeOffset.UtcNow); + + var decision = payload.Results[imageDigest]; + Assert.Equal("pass", decision.PolicyVerdict); + Assert.True(decision.Signed); + Assert.True(decision.HasSbomReferrers); + Assert.True(decision.HasSbomLegacy); + Assert.Empty(decision.Reasons); + Assert.NotNull(decision.Rekor); + Assert.Equal("rekor-uuid", decision.Rekor!.Uuid); + Assert.True(decision.Rekor.Verified); + } + + [Fact] + public async Task RuntimePolicyEndpointValidatesRequest() + { + using var factory = new ScannerApplicationFactory(); + using var client = factory.CreateClient(); + + var request = new RuntimePolicyRequestDto + { + Images = Array.Empty() + }; + + var response = await client.PostAsJsonAsync("/api/v1/policy/runtime", request); + Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); + } + + private static RuntimeEventEnvelope CreateEnvelope(string eventId, string? schemaVersion = null) + { + var runtimeEvent = new RuntimeEvent + { + EventId = eventId, + When = DateTimeOffset.UtcNow, + Kind = RuntimeEventKind.ContainerStart, + Tenant = "tenant-alpha", + Node = "node-a", + Runtime = new RuntimeEngine + { + Engine = "containerd", + Version = "1.7.0" + }, + Workload = new RuntimeWorkload + { + Platform = "kubernetes", + Namespace = "default", + Pod = "api-123", + Container = "api", + ContainerId = "containerd://abc", + ImageRef = "ghcr.io/example/api@sha256:deadbeef" + } + }; + + if (schemaVersion is null) + { + return RuntimeEventEnvelope.Create(runtimeEvent, ZastavaContractVersions.RuntimeEvent); + } + + return new RuntimeEventEnvelope + { + SchemaVersion = schemaVersion, + Event = runtimeEvent + }; + } +} diff --git a/src/StellaOps.Scanner.WebService.Tests/ScansEndpointsTests.cs b/src/StellaOps.Scanner.WebService.Tests/ScansEndpointsTests.cs index 40a8e9be..afc31e73 100644 --- a/src/StellaOps.Scanner.WebService.Tests/ScansEndpointsTests.cs +++ b/src/StellaOps.Scanner.WebService.Tests/ScansEndpointsTests.cs @@ -1,10 +1,11 @@ using System; -using System.Collections.Generic; -using System.IO; -using System.Net; -using System.Net.Http.Json; -using System.Text.Json; -using System.Threading.Tasks; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Http.Json; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.AspNetCore.TestHost; @@ -249,11 +250,11 @@ public sealed class ScansEndpointsTests } [Fact] - public async Task ProgressStreamSupportsServerSentEvents() - { - using var factory = new ScannerApplicationFactory(); - using var client = factory.CreateClient(); - + public async Task ProgressStreamSupportsServerSentEvents() + { + using var factory = new ScannerApplicationFactory(); + using var client = factory.CreateClient(); + var request = new ScanSubmitRequest { Image = new ScanImageDescriptor { Reference = "ghcr.io/demo/app:3.0.0" } @@ -286,9 +287,77 @@ public sealed class ScansEndpointsTests Assert.NotNull(envelope); Assert.Equal(submitPayload.ScanId, envelope!.ScanId); Assert.Equal("Pending", envelope.State); - Assert.Equal(1, envelope.Sequence); - Assert.True(envelope.Timestamp.UtcDateTime <= DateTime.UtcNow); - } + Assert.Equal(1, envelope.Sequence); + Assert.True(envelope.Timestamp.UtcDateTime <= DateTime.UtcNow); + } + + [Fact] + public async Task ProgressStreamDataKeysAreSortedDeterministically() + { + using var factory = new ScannerApplicationFactory(); + using var client = factory.CreateClient(); + + var request = new ScanSubmitRequest + { + Image = new ScanImageDescriptor { Reference = "ghcr.io/demo/app:sorted" } + }; + + var submit = await client.PostAsJsonAsync("/api/v1/scans", request); + var submitPayload = await submit.Content.ReadFromJsonAsync(); + Assert.NotNull(submitPayload); + + var publisher = factory.Services.GetRequiredService(); + + var response = await client.GetAsync($"/api/v1/scans/{submitPayload!.ScanId}/events?format=jsonl", HttpCompletionOption.ResponseHeadersRead); + await using var stream = await response.Content.ReadAsStreamAsync(); + using var reader = new StreamReader(stream); + + // Drain the initial pending event. + _ = await reader.ReadLineAsync(); + + _ = Task.Run(async () => + { + await Task.Delay(25); + publisher.Publish( + new ScanId(submitPayload.ScanId), + "Running", + "stage-change", + new Dictionary + { + ["zeta"] = 1, + ["alpha"] = 2, + ["Beta"] = 3 + }); + }); + + string? line; + JsonDocument? document = null; + while ((line = await reader.ReadLineAsync()) is not null) + { + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + var parsed = JsonDocument.Parse(line); + if (parsed.RootElement.TryGetProperty("state", out var state) && + string.Equals(state.GetString(), "Running", StringComparison.OrdinalIgnoreCase)) + { + document = parsed; + break; + } + + parsed.Dispose(); + } + + Assert.NotNull(document); + using (document) + { + var data = document!.RootElement.GetProperty("data"); + var names = data.EnumerateObject().Select(p => p.Name).ToArray(); + Assert.Equal(new[] { "alpha", "Beta", "zeta" }, names); + } + } private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); diff --git a/src/StellaOps.Scanner.WebService.Tests/StellaOps.Scanner.WebService.Tests.csproj b/src/StellaOps.Scanner.WebService.Tests/StellaOps.Scanner.WebService.Tests.csproj index 6d931cc0..0d940a3c 100644 --- a/src/StellaOps.Scanner.WebService.Tests/StellaOps.Scanner.WebService.Tests.csproj +++ b/src/StellaOps.Scanner.WebService.Tests/StellaOps.Scanner.WebService.Tests.csproj @@ -6,12 +6,12 @@ false StellaOps.Scanner.WebService.Tests - - - - - - Always + + + + + + Always Always diff --git a/src/StellaOps.Scanner.WebService/Constants/ProblemTypes.cs b/src/StellaOps.Scanner.WebService/Constants/ProblemTypes.cs index ec951f54..3ec31675 100644 --- a/src/StellaOps.Scanner.WebService/Constants/ProblemTypes.cs +++ b/src/StellaOps.Scanner.WebService/Constants/ProblemTypes.cs @@ -2,8 +2,9 @@ namespace StellaOps.Scanner.WebService.Constants; internal static class ProblemTypes { - public const string Validation = "https://stellaops.org/problems/validation"; - public const string Conflict = "https://stellaops.org/problems/conflict"; - public const string NotFound = "https://stellaops.org/problems/not-found"; - public const string InternalError = "https://stellaops.org/problems/internal-error"; -} + public const string Validation = "https://stellaops.org/problems/validation"; + public const string Conflict = "https://stellaops.org/problems/conflict"; + public const string NotFound = "https://stellaops.org/problems/not-found"; + public const string InternalError = "https://stellaops.org/problems/internal-error"; + public const string RateLimited = "https://stellaops.org/problems/rate-limit"; +} diff --git a/src/StellaOps.Scanner.WebService/Contracts/RuntimeEventsContracts.cs b/src/StellaOps.Scanner.WebService/Contracts/RuntimeEventsContracts.cs new file mode 100644 index 00000000..155f8978 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Contracts/RuntimeEventsContracts.cs @@ -0,0 +1,22 @@ +using System.Text.Json.Serialization; +using StellaOps.Zastava.Core.Contracts; + +namespace StellaOps.Scanner.WebService.Contracts; + +public sealed record RuntimeEventsIngestRequestDto +{ + [JsonPropertyName("batchId")] + public string? BatchId { get; init; } + + [JsonPropertyName("events")] + public IReadOnlyList Events { get; init; } = Array.Empty(); +} + +public sealed record RuntimeEventsIngestResponseDto +{ + [JsonPropertyName("accepted")] + public int Accepted { get; init; } + + [JsonPropertyName("duplicates")] + public int Duplicates { get; init; } +} diff --git a/src/StellaOps.Scanner.WebService/Contracts/RuntimePolicyContracts.cs b/src/StellaOps.Scanner.WebService/Contracts/RuntimePolicyContracts.cs new file mode 100644 index 00000000..1e73ec62 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Contracts/RuntimePolicyContracts.cs @@ -0,0 +1,75 @@ +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Scanner.WebService.Contracts; + +public sealed record RuntimePolicyRequestDto +{ + [JsonPropertyName("namespace")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Namespace { get; init; } + + [JsonPropertyName("labels")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public IDictionary? Labels { get; init; } + + [JsonPropertyName("images")] + public IReadOnlyList Images { get; init; } = Array.Empty(); +} + +public sealed record RuntimePolicyResponseDto +{ + [JsonPropertyName("ttlSeconds")] + public int TtlSeconds { get; init; } + + [JsonPropertyName("expiresAtUtc")] + public DateTimeOffset ExpiresAtUtc { get; init; } + + [JsonPropertyName("policyRevision")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? PolicyRevision { get; init; } + + [JsonPropertyName("results")] + public IReadOnlyDictionary Results { get; init; } = new Dictionary(StringComparer.Ordinal); +} + +public sealed record RuntimePolicyImageResponseDto +{ + [JsonPropertyName("policyVerdict")] + public string PolicyVerdict { get; init; } = "unknown"; + + [JsonPropertyName("signed")] + public bool Signed { get; init; } + + [JsonPropertyName("hasSbomReferrers")] + public bool HasSbomReferrers { get; init; } + + [JsonPropertyName("hasSbom")] + public bool HasSbomLegacy { get; init; } + + [JsonPropertyName("reasons")] + public IReadOnlyList Reasons { get; init; } = Array.Empty(); + + [JsonPropertyName("rekor")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public RuntimePolicyRekorDto? Rekor { get; init; } + + [JsonPropertyName("metadata")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public IDictionary? Metadata { get; init; } +} + +public sealed record RuntimePolicyRekorDto +{ + [JsonPropertyName("uuid")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Uuid { get; init; } + + [JsonPropertyName("url")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Url { get; init; } + + [JsonPropertyName("verified")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? Verified { get; init; } +} diff --git a/src/StellaOps.Scanner.WebService/Endpoints/PolicyEndpoints.cs b/src/StellaOps.Scanner.WebService/Endpoints/PolicyEndpoints.cs index 06708c0a..f340d7dd 100644 --- a/src/StellaOps.Scanner.WebService/Endpoints/PolicyEndpoints.cs +++ b/src/StellaOps.Scanner.WebService/Endpoints/PolicyEndpoints.cs @@ -1,8 +1,10 @@ -using System.Collections.Immutable; -using System.Linq; -using System.Text; -using System.Text.Json; -using System.Text.Json.Serialization; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Collections.ObjectModel; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Routing; using StellaOps.Policy; @@ -51,18 +53,30 @@ internal static class PolicyEndpoints return operation; }); - policyGroup.MapPost("/preview", HandlePreviewAsync) - .WithName("scanner.policy.preview") - .Produces(StatusCodes.Status200OK) - .Produces(StatusCodes.Status400BadRequest) - .RequireAuthorization(ScannerPolicies.Reports) - .WithOpenApi(operation => - { - operation.Summary = "Preview policy impact against findings."; - operation.Description = "Evaluates the supplied findings against the active or proposed policy, returning diffs, quieted verdicts, and actionable validation messages."; - return operation; - }); - } + policyGroup.MapPost("/preview", HandlePreviewAsync) + .WithName("scanner.policy.preview") + .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status400BadRequest) + .RequireAuthorization(ScannerPolicies.Reports) + .WithOpenApi(operation => + { + operation.Summary = "Preview policy impact against findings."; + operation.Description = "Evaluates the supplied findings against the active or proposed policy, returning diffs, quieted verdicts, and actionable validation messages."; + return operation; + }); + + policyGroup.MapPost("/runtime", HandleRuntimePolicyAsync) + .WithName("scanner.policy.runtime") + .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status400BadRequest) + .RequireAuthorization(ScannerPolicies.Reports) + .WithOpenApi(operation => + { + operation.Summary = "Evaluate runtime policy for digests."; + operation.Description = "Returns per-image policy verdicts, signature and SBOM metadata, and cache hints for admission controllers."; + return operation; + }); + } private static IResult HandleSchemaAsync(HttpContext context) { @@ -152,9 +166,97 @@ internal static class PolicyEndpoints var domainRequest = PolicyDtoMapper.ToDomain(request); var response = await previewService.PreviewAsync(domainRequest, cancellationToken).ConfigureAwait(false); - var payload = PolicyDtoMapper.ToDto(response); - return Json(payload); - } + var payload = PolicyDtoMapper.ToDto(response); + return Json(payload); + } + + private static async Task HandleRuntimePolicyAsync( + RuntimePolicyRequestDto request, + IRuntimePolicyService runtimePolicyService, + HttpContext context, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(runtimePolicyService); + + if (request.Images is null || request.Images.Count == 0) + { + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Invalid runtime policy request", + StatusCodes.Status400BadRequest, + detail: "images collection must include at least one digest."); + } + + var normalizedImages = new List(); + var seen = new HashSet(StringComparer.Ordinal); + foreach (var image in request.Images) + { + if (string.IsNullOrWhiteSpace(image)) + { + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Invalid runtime policy request", + StatusCodes.Status400BadRequest, + detail: "Image digests must be non-empty."); + } + + var trimmed = image.Trim(); + if (!trimmed.Contains(':', StringComparison.Ordinal)) + { + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Invalid runtime policy request", + StatusCodes.Status400BadRequest, + detail: "Image digests must include an algorithm prefix (e.g. sha256:...)."); + } + + if (seen.Add(trimmed)) + { + normalizedImages.Add(trimmed); + } + } + + if (normalizedImages.Count == 0) + { + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Invalid runtime policy request", + StatusCodes.Status400BadRequest, + detail: "images collection must include at least one unique digest."); + } + + var namespaceValue = string.IsNullOrWhiteSpace(request.Namespace) ? null : request.Namespace.Trim(); + var normalizedLabels = new Dictionary(StringComparer.Ordinal); + if (request.Labels is not null) + { + foreach (var pair in request.Labels) + { + if (string.IsNullOrWhiteSpace(pair.Key)) + { + continue; + } + + var key = pair.Key.Trim(); + var value = pair.Value?.Trim() ?? string.Empty; + normalizedLabels[key] = value; + } + } + + var evaluationRequest = new RuntimePolicyEvaluationRequest( + namespaceValue, + new ReadOnlyDictionary(normalizedLabels), + normalizedImages); + + var evaluation = await runtimePolicyService.EvaluateAsync(evaluationRequest, cancellationToken).ConfigureAwait(false); + + var resultPayload = MapRuntimePolicyResponse(evaluation); + return Json(resultPayload); + } private static string NormalizeSegment(string segment) { @@ -167,9 +269,53 @@ internal static class PolicyEndpoints return "/" + trimmed; } - private static IResult Json(T value) - { - var payload = JsonSerializer.Serialize(value, SerializerOptions); - return Results.Content(payload, "application/json", Encoding.UTF8); - } -} + private static IResult Json(T value) + { + var payload = JsonSerializer.Serialize(value, SerializerOptions); + return Results.Content(payload, "application/json", Encoding.UTF8); + } + + private static RuntimePolicyResponseDto MapRuntimePolicyResponse(RuntimePolicyEvaluationResult evaluation) + { + var results = new Dictionary(evaluation.Results.Count, StringComparer.Ordinal); + foreach (var pair in evaluation.Results) + { + var decision = pair.Value; + RuntimePolicyRekorDto? rekor = null; + if (decision.Rekor is not null) + { + rekor = new RuntimePolicyRekorDto + { + Uuid = decision.Rekor.Uuid, + Url = decision.Rekor.Url, + Verified = decision.Rekor.Verified + }; + } + + IDictionary? metadata = null; + if (decision.Metadata is not null && decision.Metadata.Count > 0) + { + metadata = new Dictionary(decision.Metadata, StringComparer.OrdinalIgnoreCase); + } + + results[pair.Key] = new RuntimePolicyImageResponseDto + { + PolicyVerdict = decision.PolicyVerdict, + Signed = decision.Signed, + HasSbomReferrers = decision.HasSbomReferrers, + HasSbomLegacy = decision.HasSbomReferrers, + Reasons = decision.Reasons.ToArray(), + Rekor = rekor, + Metadata = metadata + }; + } + + return new RuntimePolicyResponseDto + { + TtlSeconds = evaluation.TtlSeconds, + ExpiresAtUtc = evaluation.ExpiresAtUtc, + PolicyRevision = evaluation.PolicyRevision, + Results = results + }; + } +} diff --git a/src/StellaOps.Scanner.WebService/Endpoints/RuntimeEndpoints.cs b/src/StellaOps.Scanner.WebService/Endpoints/RuntimeEndpoints.cs new file mode 100644 index 00000000..06b4ba26 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Endpoints/RuntimeEndpoints.cs @@ -0,0 +1,253 @@ +using System.Collections.Generic; +using System.Globalization; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Routing; +using Microsoft.Extensions.Options; +using StellaOps.Scanner.WebService.Constants; +using StellaOps.Scanner.WebService.Contracts; +using StellaOps.Scanner.WebService.Infrastructure; +using StellaOps.Scanner.WebService.Options; +using StellaOps.Scanner.WebService.Security; +using StellaOps.Scanner.WebService.Services; +using StellaOps.Zastava.Core.Contracts; + +namespace StellaOps.Scanner.WebService.Endpoints; + +internal static class RuntimeEndpoints +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + + public static void MapRuntimeEndpoints(this RouteGroupBuilder apiGroup, string runtimeSegment) + { + ArgumentNullException.ThrowIfNull(apiGroup); + + var runtime = apiGroup + .MapGroup(NormalizeSegment(runtimeSegment)) + .WithTags("Runtime"); + + runtime.MapPost("/events", HandleRuntimeEventsAsync) + .WithName("scanner.runtime.events.ingest") + .Produces(StatusCodes.Status202Accepted) + .Produces(StatusCodes.Status400BadRequest) + .Produces(StatusCodes.Status429TooManyRequests) + .RequireAuthorization(ScannerPolicies.RuntimeIngest); + } + + private static async Task HandleRuntimeEventsAsync( + RuntimeEventsIngestRequestDto request, + IRuntimeEventIngestionService ingestionService, + IOptions options, + HttpContext context, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(ingestionService); + ArgumentNullException.ThrowIfNull(options); + + var runtimeOptions = options.Value.Runtime ?? new ScannerWebServiceOptions.RuntimeOptions(); + var validationError = ValidateRequest(request, runtimeOptions, context, out var envelopes); + if (validationError is { } problem) + { + return problem; + } + + var result = await ingestionService.IngestAsync(envelopes, request.BatchId, cancellationToken).ConfigureAwait(false); + if (result.IsPayloadTooLarge) + { + var extensions = new Dictionary + { + ["payloadBytes"] = result.PayloadBytes, + ["maxPayloadBytes"] = result.PayloadLimit + }; + + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Runtime event batch too large", + StatusCodes.Status400BadRequest, + detail: "Runtime batch payload exceeds configured budget.", + extensions: extensions); + } + + if (result.IsRateLimited) + { + var retryAfterSeconds = Math.Max(1, (int)Math.Ceiling(result.RetryAfter.TotalSeconds)); + context.Response.Headers.RetryAfter = retryAfterSeconds.ToString(CultureInfo.InvariantCulture); + + var extensions = new Dictionary + { + ["scope"] = result.RateLimitedScope, + ["key"] = result.RateLimitedKey, + ["retryAfterSeconds"] = retryAfterSeconds + }; + + return ProblemResultFactory.Create( + context, + ProblemTypes.RateLimited, + "Runtime ingestion rate limited", + StatusCodes.Status429TooManyRequests, + detail: "Runtime ingestion exceeded configured rate limits.", + extensions: extensions); + } + + var payload = new RuntimeEventsIngestResponseDto + { + Accepted = result.Accepted, + Duplicates = result.Duplicates + }; + + return Json(payload, StatusCodes.Status202Accepted); + } + + private static IResult? ValidateRequest( + RuntimeEventsIngestRequestDto request, + ScannerWebServiceOptions.RuntimeOptions runtimeOptions, + HttpContext context, + out IReadOnlyList envelopes) + { + envelopes = request.Events ?? Array.Empty(); + if (envelopes.Count == 0) + { + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Invalid runtime ingest request", + StatusCodes.Status400BadRequest, + detail: "events array must include at least one item."); + } + + if (envelopes.Count > runtimeOptions.MaxBatchSize) + { + var extensions = new Dictionary + { + ["maxBatchSize"] = runtimeOptions.MaxBatchSize, + ["eventCount"] = envelopes.Count + }; + + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Invalid runtime ingest request", + StatusCodes.Status400BadRequest, + detail: "events array exceeds allowed batch size.", + extensions: extensions); + } + + var seenEventIds = new HashSet(StringComparer.Ordinal); + for (var i = 0; i < envelopes.Count; i++) + { + var envelope = envelopes[i]; + if (envelope is null) + { + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Invalid runtime ingest request", + StatusCodes.Status400BadRequest, + detail: $"events[{i}] must not be null."); + } + + if (!envelope.IsSupported()) + { + var extensions = new Dictionary + { + ["schemaVersion"] = envelope.SchemaVersion + }; + + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Unsupported runtime schema version", + StatusCodes.Status400BadRequest, + detail: "Runtime event schemaVersion is not supported.", + extensions: extensions); + } + + var runtimeEvent = envelope.Event; + if (runtimeEvent is null) + { + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Invalid runtime ingest request", + StatusCodes.Status400BadRequest, + detail: $"events[{i}].event must not be null."); + } + + if (string.IsNullOrWhiteSpace(runtimeEvent.EventId)) + { + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Invalid runtime ingest request", + StatusCodes.Status400BadRequest, + detail: $"events[{i}].eventId is required."); + } + + if (!seenEventIds.Add(runtimeEvent.EventId)) + { + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Invalid runtime ingest request", + StatusCodes.Status400BadRequest, + detail: $"Duplicate eventId detected within batch ('{runtimeEvent.EventId}')."); + } + + if (string.IsNullOrWhiteSpace(runtimeEvent.Tenant)) + { + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Invalid runtime ingest request", + StatusCodes.Status400BadRequest, + detail: $"events[{i}].tenant is required."); + } + + if (string.IsNullOrWhiteSpace(runtimeEvent.Node)) + { + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Invalid runtime ingest request", + StatusCodes.Status400BadRequest, + detail: $"events[{i}].node is required."); + } + + if (runtimeEvent.Workload is null) + { + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Invalid runtime ingest request", + StatusCodes.Status400BadRequest, + detail: $"events[{i}].workload is required."); + } + } + + return null; + } + + private static string NormalizeSegment(string segment) + { + if (string.IsNullOrWhiteSpace(segment)) + { + return "/runtime"; + } + + var trimmed = segment.Trim('/'); + return "/" + trimmed; + } + + private static IResult Json(T value, int statusCode) + { + var payload = JsonSerializer.Serialize(value, SerializerOptions); + return Results.Content(payload, "application/json", Encoding.UTF8, statusCode); + } +} diff --git a/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptions.cs b/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptions.cs index 29b732ab..0542fa99 100644 --- a/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptions.cs +++ b/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptions.cs @@ -55,15 +55,20 @@ public sealed class ScannerWebServiceOptions /// public SigningOptions Signing { get; set; } = new(); - /// - /// API-specific settings such as base path. - /// - public ApiOptions Api { get; set; } = new(); - - /// - /// Platform event emission settings. - /// - public EventsOptions Events { get; set; } = new(); + /// + /// API-specific settings such as base path. + /// + public ApiOptions Api { get; set; } = new(); + + /// + /// Platform event emission settings. + /// + public EventsOptions Events { get; set; } = new(); + + /// + /// Runtime ingestion configuration. + /// + public RuntimeOptions Runtime { get; set; } = new(); public sealed class StorageOptions { @@ -236,20 +241,22 @@ public sealed class ScannerWebServiceOptions public int EnvelopeTtlSeconds { get; set; } = 600; } - public sealed class ApiOptions - { - public string BasePath { get; set; } = "/api/v1"; - - public string ScansSegment { get; set; } = "scans"; - - public string ReportsSegment { get; set; } = "reports"; - - public string PolicySegment { get; set; } = "policy"; - } - - public sealed class EventsOptions - { - public bool Enabled { get; set; } + public sealed class ApiOptions + { + public string BasePath { get; set; } = "/api/v1"; + + public string ScansSegment { get; set; } = "scans"; + + public string ReportsSegment { get; set; } = "reports"; + + public string PolicySegment { get; set; } = "policy"; + + public string RuntimeSegment { get; set; } = "runtime"; + } + + public sealed class EventsOptions + { + public bool Enabled { get; set; } public string Driver { get; set; } = "redis"; @@ -257,10 +264,29 @@ public sealed class ScannerWebServiceOptions public string Stream { get; set; } = "stella.events"; - public double PublishTimeoutSeconds { get; set; } = 5; - - public long MaxStreamLength { get; set; } = 10000; - - public IDictionary DriverSettings { get; set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); - } -} + public double PublishTimeoutSeconds { get; set; } = 5; + + public long MaxStreamLength { get; set; } = 10000; + + public IDictionary DriverSettings { get; set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + public sealed class RuntimeOptions + { + public int MaxBatchSize { get; set; } = 256; + + public int MaxPayloadBytes { get; set; } = 1 * 1024 * 1024; + + public int EventTtlDays { get; set; } = 45; + + public double PerNodeEventsPerSecond { get; set; } = 50; + + public int PerNodeBurst { get; set; } = 200; + + public double PerTenantEventsPerSecond { get; set; } = 200; + + public int PerTenantBurst { get; set; } = 1000; + + public int PolicyCacheTtlSeconds { get; set; } = 300; + } +} diff --git a/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptionsPostConfigure.cs b/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptionsPostConfigure.cs index d2169dd0..03b9426a 100644 --- a/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptionsPostConfigure.cs +++ b/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptionsPostConfigure.cs @@ -70,14 +70,16 @@ public static class ScannerWebServiceOptionsPostConfigure eventsOptions.Stream = "stella.events"; } - if (string.IsNullOrWhiteSpace(eventsOptions.Dsn) - && string.Equals(options.Queue?.Driver, "redis", StringComparison.OrdinalIgnoreCase) - && !string.IsNullOrWhiteSpace(options.Queue?.Dsn)) - { - eventsOptions.Dsn = options.Queue!.Dsn; - } - - } + if (string.IsNullOrWhiteSpace(eventsOptions.Dsn) + && string.Equals(options.Queue?.Driver, "redis", StringComparison.OrdinalIgnoreCase) + && !string.IsNullOrWhiteSpace(options.Queue?.Dsn)) + { + eventsOptions.Dsn = options.Queue!.Dsn; + } + + options.Runtime ??= new ScannerWebServiceOptions.RuntimeOptions(); + + } private static string ReadSecretFile(string path, string contentRootPath) { diff --git a/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptionsValidator.cs b/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptionsValidator.cs index 10132358..9f70dbf6 100644 --- a/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptionsValidator.cs +++ b/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptionsValidator.cs @@ -78,14 +78,22 @@ public static class ScannerWebServiceOptionsValidator throw new InvalidOperationException("API reportsSegment must be configured."); } - if (string.IsNullOrWhiteSpace(options.Api.PolicySegment)) - { - throw new InvalidOperationException("API policySegment must be configured."); - } - - options.Events ??= new ScannerWebServiceOptions.EventsOptions(); - ValidateEvents(options.Events); - } + if (string.IsNullOrWhiteSpace(options.Api.PolicySegment)) + { + throw new InvalidOperationException("API policySegment must be configured."); + } + + if (string.IsNullOrWhiteSpace(options.Api.RuntimeSegment)) + { + throw new InvalidOperationException("API runtimeSegment must be configured."); + } + + options.Events ??= new ScannerWebServiceOptions.EventsOptions(); + ValidateEvents(options.Events); + + options.Runtime ??= new ScannerWebServiceOptions.RuntimeOptions(); + ValidateRuntime(options.Runtime); + } private static void ValidateStorage(ScannerWebServiceOptions.StorageOptions storage) { @@ -199,7 +207,7 @@ public static class ScannerWebServiceOptionsValidator } } - private static void ValidateTelemetry(ScannerWebServiceOptions.TelemetryOptions telemetry) + private static void ValidateTelemetry(ScannerWebServiceOptions.TelemetryOptions telemetry) { if (string.IsNullOrWhiteSpace(telemetry.MinimumLogLevel)) { @@ -231,9 +239,9 @@ public static class ScannerWebServiceOptionsValidator throw new InvalidOperationException("Telemetry OTLP header keys must be non-empty."); } } - } - - private static void ValidateAuthority(ScannerWebServiceOptions.AuthorityOptions authority) + } + + private static void ValidateAuthority(ScannerWebServiceOptions.AuthorityOptions authority) { authority.Resilience ??= new ScannerWebServiceOptions.AuthorityOptions.ResilienceOptions(); NormalizeList(authority.Audiences, toLower: false); @@ -384,5 +392,48 @@ public static class ScannerWebServiceOptionsValidator { throw new InvalidOperationException("Authority resilience offlineCacheTolerance must be greater than or equal to zero."); } - } -} + } + + private static void ValidateRuntime(ScannerWebServiceOptions.RuntimeOptions runtime) + { + if (runtime.MaxBatchSize <= 0) + { + throw new InvalidOperationException("Runtime maxBatchSize must be greater than zero."); + } + + if (runtime.MaxPayloadBytes <= 0) + { + throw new InvalidOperationException("Runtime maxPayloadBytes must be greater than zero."); + } + + if (runtime.EventTtlDays <= 0) + { + throw new InvalidOperationException("Runtime eventTtlDays must be greater than zero."); + } + + if (runtime.PerNodeEventsPerSecond <= 0) + { + throw new InvalidOperationException("Runtime perNodeEventsPerSecond must be greater than zero."); + } + + if (runtime.PerNodeBurst <= 0) + { + throw new InvalidOperationException("Runtime perNodeBurst must be greater than zero."); + } + + if (runtime.PerTenantEventsPerSecond <= 0) + { + throw new InvalidOperationException("Runtime perTenantEventsPerSecond must be greater than zero."); + } + + if (runtime.PerTenantBurst <= 0) + { + throw new InvalidOperationException("Runtime perTenantBurst must be greater than zero."); + } + + if (runtime.PolicyCacheTtlSeconds <= 0) + { + throw new InvalidOperationException("Runtime policyCacheTtlSeconds must be greater than zero."); + } + } +} diff --git a/src/StellaOps.Scanner.WebService/Program.cs b/src/StellaOps.Scanner.WebService/Program.cs index 227ffd52..790ef01c 100644 --- a/src/StellaOps.Scanner.WebService/Program.cs +++ b/src/StellaOps.Scanner.WebService/Program.cs @@ -3,29 +3,32 @@ using System.Diagnostics; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Diagnostics; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; -using Microsoft.AspNetCore.Authentication; -using Microsoft.Extensions.Options; -using Serilog; -using Serilog.Events; -using StellaOps.Auth.Client; -using StellaOps.Auth.ServerIntegration; -using StellaOps.Configuration; -using StellaOps.Plugin.DependencyInjection; -using StellaOps.Cryptography.DependencyInjection; -using StellaOps.Cryptography.Plugin.BouncyCastle; -using StellaOps.Policy; -using StellaOps.Scanner.Cache; -using StellaOps.Scanner.WebService.Diagnostics; -using StellaOps.Scanner.WebService.Endpoints; -using StellaOps.Scanner.WebService.Extensions; -using StellaOps.Scanner.WebService.Hosting; -using StellaOps.Scanner.WebService.Options; -using StellaOps.Scanner.WebService.Services; -using StellaOps.Scanner.WebService.Security; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Diagnostics; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Authentication; +using Microsoft.Extensions.Options; +using Serilog; +using Serilog.Events; +using StellaOps.Auth.Client; +using StellaOps.Auth.ServerIntegration; +using StellaOps.Configuration; +using StellaOps.Plugin.DependencyInjection; +using StellaOps.Cryptography.DependencyInjection; +using StellaOps.Cryptography.Plugin.BouncyCastle; +using StellaOps.Policy; +using StellaOps.Scanner.Cache; +using StellaOps.Scanner.WebService.Diagnostics; +using StellaOps.Scanner.WebService.Endpoints; +using StellaOps.Scanner.WebService.Extensions; +using StellaOps.Scanner.WebService.Hosting; +using StellaOps.Scanner.WebService.Options; +using StellaOps.Scanner.WebService.Services; +using StellaOps.Scanner.WebService.Security; +using StellaOps.Scanner.Storage; +using StellaOps.Scanner.Storage.Extensions; +using StellaOps.Scanner.Storage.Mongo; var builder = WebApplication.CreateBuilder(args); @@ -67,11 +70,11 @@ builder.Host.UseSerilog((context, services, loggerConfiguration) => .WriteTo.Console(); }); -builder.Services.AddSingleton(TimeProvider.System); -builder.Services.AddScannerCache(builder.Configuration); -builder.Services.AddSingleton(); -builder.Services.AddHttpContextAccessor(); -builder.Services.AddSingleton(); +builder.Services.AddSingleton(TimeProvider.System); +builder.Services.AddScannerCache(builder.Configuration); +builder.Services.AddSingleton(); +builder.Services.AddHttpContextAccessor(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddSingleton(); @@ -81,17 +84,54 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddStellaOpsCrypto(); builder.Services.AddBouncyCastleEd25519Provider(); -builder.Services.AddSingleton(); -if (bootstrapOptions.Events is { Enabled: true } eventsOptions - && string.Equals(eventsOptions.Driver, "redis", StringComparison.OrdinalIgnoreCase)) -{ - builder.Services.AddSingleton(); -} -else -{ - builder.Services.AddSingleton(); -} -builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +if (bootstrapOptions.Events is { Enabled: true } eventsOptions + && string.Equals(eventsOptions.Driver, "redis", StringComparison.OrdinalIgnoreCase)) +{ + builder.Services.AddSingleton(); +} +else +{ + builder.Services.AddSingleton(); +} +builder.Services.AddSingleton(); +builder.Services.AddScannerStorage(storageOptions => +{ + storageOptions.Mongo.ConnectionString = bootstrapOptions.Storage.Dsn; + if (!string.IsNullOrWhiteSpace(bootstrapOptions.Storage.Database)) + { + storageOptions.Mongo.DatabaseName = bootstrapOptions.Storage.Database; + } + + storageOptions.Mongo.CommandTimeout = TimeSpan.FromSeconds(bootstrapOptions.Storage.CommandTimeoutSeconds); + storageOptions.Mongo.UseMajorityReadConcern = true; + storageOptions.Mongo.UseMajorityWriteConcern = true; + + if (!string.IsNullOrWhiteSpace(bootstrapOptions.ArtifactStore.Endpoint)) + { + storageOptions.ObjectStore.ServiceUrl = bootstrapOptions.ArtifactStore.Endpoint; + } + + if (!string.IsNullOrWhiteSpace(bootstrapOptions.ArtifactStore.Bucket)) + { + storageOptions.ObjectStore.BucketName = bootstrapOptions.ArtifactStore.Bucket; + } + + if (!string.IsNullOrWhiteSpace(bootstrapOptions.ArtifactStore.Region)) + { + storageOptions.ObjectStore.Region = bootstrapOptions.ArtifactStore.Region; + } + + storageOptions.ObjectStore.EnableObjectLock = bootstrapOptions.ArtifactStore.EnableObjectLock; + storageOptions.ObjectStore.ForcePathStyle = true; + storageOptions.ObjectStore.ComplianceRetention = bootstrapOptions.ArtifactStore.EnableObjectLock + ? TimeSpan.FromDays(Math.Max(1, bootstrapOptions.ArtifactStore.ObjectLockRetentionDays)) + : null; +}); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); var pluginHostOptions = ScannerPluginHostFactory.Build(bootstrapOptions, contentRoot); builder.Services.RegisterPluginRoutines(builder.Configuration, pluginHostOptions); @@ -171,9 +211,10 @@ if (bootstrapOptions.Authority.Enabled) builder.Services.AddAuthorization(options => { - options.AddStellaOpsScopePolicy(ScannerPolicies.ScansEnqueue, bootstrapOptions.Authority.RequiredScopes.ToArray()); - options.AddStellaOpsScopePolicy(ScannerPolicies.ScansRead, ScannerAuthorityScopes.ScansRead); - options.AddStellaOpsScopePolicy(ScannerPolicies.Reports, ScannerAuthorityScopes.ReportsRead); + options.AddStellaOpsScopePolicy(ScannerPolicies.ScansEnqueue, bootstrapOptions.Authority.RequiredScopes.ToArray()); + options.AddStellaOpsScopePolicy(ScannerPolicies.ScansRead, ScannerAuthorityScopes.ScansRead); + options.AddStellaOpsScopePolicy(ScannerPolicies.Reports, ScannerAuthorityScopes.ReportsRead); + options.AddStellaOpsScopePolicy(ScannerPolicies.RuntimeIngest, ScannerAuthorityScopes.RuntimeIngest); }); } else @@ -185,23 +226,30 @@ else }) .AddScheme("Anonymous", _ => { }); - builder.Services.AddAuthorization(options => - { - options.AddPolicy(ScannerPolicies.ScansEnqueue, policy => policy.RequireAssertion(_ => true)); - options.AddPolicy(ScannerPolicies.ScansRead, policy => policy.RequireAssertion(_ => true)); - options.AddPolicy(ScannerPolicies.Reports, policy => policy.RequireAssertion(_ => true)); - }); -} + builder.Services.AddAuthorization(options => + { + options.AddPolicy(ScannerPolicies.ScansEnqueue, policy => policy.RequireAssertion(_ => true)); + options.AddPolicy(ScannerPolicies.ScansRead, policy => policy.RequireAssertion(_ => true)); + options.AddPolicy(ScannerPolicies.Reports, policy => policy.RequireAssertion(_ => true)); + options.AddPolicy(ScannerPolicies.RuntimeIngest, policy => policy.RequireAssertion(_ => true)); + }); +} var app = builder.Build(); -var resolvedOptions = app.Services.GetRequiredService>().Value; -var authorityConfigured = resolvedOptions.Authority.Enabled; -if (authorityConfigured && resolvedOptions.Authority.AllowAnonymousFallback) -{ - app.Logger.LogWarning( - "Scanner authority authentication is enabled but anonymous fallback remains allowed. Disable fallback before production rollout."); -} +var resolvedOptions = app.Services.GetRequiredService>().Value; +var authorityConfigured = resolvedOptions.Authority.Enabled; +if (authorityConfigured && resolvedOptions.Authority.AllowAnonymousFallback) +{ + app.Logger.LogWarning( + "Scanner authority authentication is enabled but anonymous fallback remains allowed. Disable fallback before production rollout."); +} + +using (var scope = app.Services.CreateScope()) +{ + var bootstrapper = scope.ServiceProvider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None).ConfigureAwait(false); +} if (resolvedOptions.Telemetry.EnableLogging && resolvedOptions.Telemetry.EnableRequestLogging) { @@ -263,14 +311,15 @@ if (app.Environment.IsEnvironment("Testing")) .WithName("scanner.auth-probe"); } -apiGroup.MapScanEndpoints(resolvedOptions.Api.ScansSegment); - -if (resolvedOptions.Features.EnablePolicyPreview) -{ - apiGroup.MapPolicyEndpoints(resolvedOptions.Api.PolicySegment); -} - -apiGroup.MapReportEndpoints(resolvedOptions.Api.ReportsSegment); +apiGroup.MapScanEndpoints(resolvedOptions.Api.ScansSegment); + +if (resolvedOptions.Features.EnablePolicyPreview) +{ + apiGroup.MapPolicyEndpoints(resolvedOptions.Api.PolicySegment); +} + +apiGroup.MapReportEndpoints(resolvedOptions.Api.ReportsSegment); +apiGroup.MapRuntimeEndpoints(resolvedOptions.Api.RuntimeSegment); app.MapOpenApiIfAvailable(); await app.RunAsync().ConfigureAwait(false); diff --git a/src/StellaOps.Scanner.WebService/Security/ScannerAuthorityScopes.cs b/src/StellaOps.Scanner.WebService/Security/ScannerAuthorityScopes.cs index d640c911..948f9beb 100644 --- a/src/StellaOps.Scanner.WebService/Security/ScannerAuthorityScopes.cs +++ b/src/StellaOps.Scanner.WebService/Security/ScannerAuthorityScopes.cs @@ -5,7 +5,8 @@ namespace StellaOps.Scanner.WebService.Security; /// internal static class ScannerAuthorityScopes { - public const string ScansEnqueue = "scanner.scans.enqueue"; - public const string ScansRead = "scanner.scans.read"; - public const string ReportsRead = "scanner.reports.read"; -} + public const string ScansEnqueue = "scanner.scans.enqueue"; + public const string ScansRead = "scanner.scans.read"; + public const string ReportsRead = "scanner.reports.read"; + public const string RuntimeIngest = "scanner.runtime.ingest"; +} diff --git a/src/StellaOps.Scanner.WebService/Security/ScannerPolicies.cs b/src/StellaOps.Scanner.WebService/Security/ScannerPolicies.cs index ee3c31e3..43bb201d 100644 --- a/src/StellaOps.Scanner.WebService/Security/ScannerPolicies.cs +++ b/src/StellaOps.Scanner.WebService/Security/ScannerPolicies.cs @@ -2,7 +2,8 @@ namespace StellaOps.Scanner.WebService.Security; internal static class ScannerPolicies { - public const string ScansEnqueue = "scanner.api"; - public const string ScansRead = "scanner.scans.read"; - public const string Reports = "scanner.reports"; -} + public const string ScansEnqueue = "scanner.api"; + public const string ScansRead = "scanner.scans.read"; + public const string Reports = "scanner.reports"; + public const string RuntimeIngest = "scanner.runtime.ingest"; +} diff --git a/src/StellaOps.Scanner.WebService/Services/IRedisConnectionFactory.cs b/src/StellaOps.Scanner.WebService/Services/IRedisConnectionFactory.cs new file mode 100644 index 00000000..786780d7 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Services/IRedisConnectionFactory.cs @@ -0,0 +1,13 @@ +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis; + +namespace StellaOps.Scanner.WebService.Services; + +/// +/// Abstraction for creating Redis connections so publishers can be tested without real infrastructure. +/// +internal interface IRedisConnectionFactory +{ + ValueTask ConnectAsync(ConfigurationOptions options, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Scanner.WebService/Services/RedisConnectionFactory.cs b/src/StellaOps.Scanner.WebService/Services/RedisConnectionFactory.cs new file mode 100644 index 00000000..0636f63d --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Services/RedisConnectionFactory.cs @@ -0,0 +1,19 @@ +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis; + +namespace StellaOps.Scanner.WebService.Services; + +/// +/// Production Redis connection factory bridging to . +/// +internal sealed class RedisConnectionFactory : IRedisConnectionFactory +{ + public async ValueTask ConnectAsync(ConfigurationOptions options, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + var connectTask = ConnectionMultiplexer.ConnectAsync(options); + var connection = await connectTask.WaitAsync(cancellationToken).ConfigureAwait(false); + return connection; + } +} diff --git a/src/StellaOps.Scanner.WebService/Services/RedisPlatformEventPublisher.cs b/src/StellaOps.Scanner.WebService/Services/RedisPlatformEventPublisher.cs index 20ea85da..22e98e2f 100644 --- a/src/StellaOps.Scanner.WebService/Services/RedisPlatformEventPublisher.cs +++ b/src/StellaOps.Scanner.WebService/Services/RedisPlatformEventPublisher.cs @@ -1,6 +1,7 @@ -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using StackExchange.Redis; @@ -13,7 +14,8 @@ internal sealed class RedisPlatformEventPublisher : IPlatformEventPublisher, IAs { private readonly ScannerWebServiceOptions.EventsOptions _options; private readonly ILogger _logger; - private readonly TimeSpan _publishTimeout; + private readonly IRedisConnectionFactory _connectionFactory; + private readonly TimeSpan _publishTimeout; private readonly string _streamKey; private readonly long? _maxStreamLength; @@ -22,12 +24,14 @@ internal sealed class RedisPlatformEventPublisher : IPlatformEventPublisher, IAs private bool _disposed; public RedisPlatformEventPublisher( - IOptions options, - ILogger logger) - { - ArgumentNullException.ThrowIfNull(options); - - _options = options.Value.Events ?? throw new InvalidOperationException("Events options are required when redis publisher is registered."); + IOptions options, + IRedisConnectionFactory connectionFactory, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(connectionFactory); + + _options = options.Value.Events ?? throw new InvalidOperationException("Events options are required when redis publisher is registered."); if (!_options.Enabled) { throw new InvalidOperationException("RedisPlatformEventPublisher requires events emission to be enabled."); @@ -37,11 +41,12 @@ internal sealed class RedisPlatformEventPublisher : IPlatformEventPublisher, IAs { throw new InvalidOperationException($"RedisPlatformEventPublisher cannot be used with driver '{_options.Driver}'."); } - - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _streamKey = string.IsNullOrWhiteSpace(_options.Stream) ? "stella.events" : _options.Stream; - _publishTimeout = TimeSpan.FromSeconds(_options.PublishTimeoutSeconds <= 0 ? 5 : _options.PublishTimeoutSeconds); - _maxStreamLength = _options.MaxStreamLength > 0 ? _options.MaxStreamLength : null; + + _connectionFactory = connectionFactory; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _streamKey = string.IsNullOrWhiteSpace(_options.Stream) ? "stella.events" : _options.Stream; + _publishTimeout = TimeSpan.FromSeconds(_options.PublishTimeoutSeconds <= 0 ? 5 : _options.PublishTimeoutSeconds); + _maxStreamLength = _options.MaxStreamLength > 0 ? _options.MaxStreamLength : null; } public async Task PublishAsync(NotifyEvent @event, CancellationToken cancellationToken = default) @@ -108,11 +113,11 @@ internal sealed class RedisPlatformEventPublisher : IPlatformEventPublisher, IAs config.Ssl = ssl; } - _connection = await ConnectionMultiplexer.ConnectAsync(config).WaitAsync(cancellationToken).ConfigureAwait(false); - _logger.LogInformation("Connected Redis platform event publisher to stream {Stream}.", _streamKey); - } - } - finally + _connection = await _connectionFactory.ConnectAsync(config, cancellationToken).ConfigureAwait(false); + _logger.LogInformation("Connected Redis platform event publisher to stream {Stream}.", _streamKey); + } + } + finally { _connectionGate.Release(); } diff --git a/src/StellaOps.Scanner.WebService/Services/RuntimeEventIngestionService.cs b/src/StellaOps.Scanner.WebService/Services/RuntimeEventIngestionService.cs new file mode 100644 index 00000000..5c46baab --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Services/RuntimeEventIngestionService.cs @@ -0,0 +1,151 @@ +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using System.Text; +using MongoDB.Bson; +using StellaOps.Scanner.Storage.Catalog; +using StellaOps.Scanner.Storage.Repositories; +using StellaOps.Scanner.WebService.Options; +using StellaOps.Zastava.Core.Contracts; + +namespace StellaOps.Scanner.WebService.Services; + +internal interface IRuntimeEventIngestionService +{ + Task IngestAsync( + IReadOnlyList envelopes, + string? batchId, + CancellationToken cancellationToken); +} + +internal sealed class RuntimeEventIngestionService : IRuntimeEventIngestionService +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + + private readonly RuntimeEventRepository _repository; + private readonly RuntimeEventRateLimiter _rateLimiter; + private readonly IOptionsMonitor _optionsMonitor; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public RuntimeEventIngestionService( + RuntimeEventRepository repository, + RuntimeEventRateLimiter rateLimiter, + IOptionsMonitor optionsMonitor, + TimeProvider timeProvider, + ILogger logger) + { + _repository = repository ?? throw new ArgumentNullException(nameof(repository)); + _rateLimiter = rateLimiter ?? throw new ArgumentNullException(nameof(rateLimiter)); + _optionsMonitor = optionsMonitor ?? throw new ArgumentNullException(nameof(optionsMonitor)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task IngestAsync( + IReadOnlyList envelopes, + string? batchId, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(envelopes); + if (envelopes.Count == 0) + { + return RuntimeEventIngestionResult.Empty; + } + + var rateDecision = _rateLimiter.Evaluate(envelopes); + if (!rateDecision.Allowed) + { + _logger.LogWarning( + "Runtime event batch rejected due to rate limit ({Scope}={Key}, retryAfter={RetryAfter})", + rateDecision.Scope, + rateDecision.Key, + rateDecision.RetryAfter); + + return RuntimeEventIngestionResult.RateLimited(rateDecision.Scope, rateDecision.Key, rateDecision.RetryAfter); + } + + var options = _optionsMonitor.CurrentValue.Runtime ?? new ScannerWebServiceOptions.RuntimeOptions(); + var receivedAt = _timeProvider.GetUtcNow().UtcDateTime; + var expiresAt = receivedAt.AddDays(options.EventTtlDays); + + var documents = new List(envelopes.Count); + var totalPayloadBytes = 0; + + foreach (var envelope in envelopes) + { + var payloadBytes = JsonSerializer.SerializeToUtf8Bytes(envelope, SerializerOptions); + totalPayloadBytes += payloadBytes.Length; + if (totalPayloadBytes > options.MaxPayloadBytes) + { + _logger.LogWarning( + "Runtime event batch exceeds payload budget ({PayloadBytes} > {MaxPayloadBytes})", + totalPayloadBytes, + options.MaxPayloadBytes); + return RuntimeEventIngestionResult.PayloadTooLarge(totalPayloadBytes, options.MaxPayloadBytes); + } + + var payloadDocument = BsonDocument.Parse(Encoding.UTF8.GetString(payloadBytes)); + var runtimeEvent = envelope.Event; + + var document = new RuntimeEventDocument + { + EventId = runtimeEvent.EventId, + SchemaVersion = envelope.SchemaVersion, + Tenant = runtimeEvent.Tenant, + Node = runtimeEvent.Node, + Kind = runtimeEvent.Kind.ToString(), + When = runtimeEvent.When.UtcDateTime, + ReceivedAt = receivedAt, + ExpiresAt = expiresAt, + Platform = runtimeEvent.Workload.Platform, + Namespace = runtimeEvent.Workload.Namespace, + Pod = runtimeEvent.Workload.Pod, + Container = runtimeEvent.Workload.Container, + ContainerId = runtimeEvent.Workload.ContainerId, + ImageRef = runtimeEvent.Workload.ImageRef, + Engine = runtimeEvent.Runtime.Engine, + EngineVersion = runtimeEvent.Runtime.Version, + BaselineDigest = runtimeEvent.Delta?.BaselineImageDigest, + ImageSigned = runtimeEvent.Posture?.ImageSigned, + SbomReferrer = runtimeEvent.Posture?.SbomReferrer, + Payload = payloadDocument + }; + + documents.Add(document); + } + + var insertResult = await _repository.InsertAsync(documents, cancellationToken).ConfigureAwait(false); + _logger.LogInformation( + "Runtime ingestion batch processed (batchId={BatchId}, accepted={Accepted}, duplicates={Duplicates}, payloadBytes={PayloadBytes})", + batchId, + insertResult.InsertedCount, + insertResult.DuplicateCount, + totalPayloadBytes); + + return RuntimeEventIngestionResult.Success(insertResult.InsertedCount, insertResult.DuplicateCount, totalPayloadBytes); + } +} + +internal readonly record struct RuntimeEventIngestionResult( + int Accepted, + int Duplicates, + bool IsRateLimited, + string? RateLimitedScope, + string? RateLimitedKey, + TimeSpan RetryAfter, + bool IsPayloadTooLarge, + int PayloadBytes, + int PayloadLimit) +{ + public static RuntimeEventIngestionResult Empty => new(0, 0, false, null, null, TimeSpan.Zero, false, 0, 0); + + public static RuntimeEventIngestionResult RateLimited(string? scope, string? key, TimeSpan retryAfter) + => new(0, 0, true, scope, key, retryAfter, false, 0, 0); + + public static RuntimeEventIngestionResult PayloadTooLarge(int payloadBytes, int payloadLimit) + => new(0, 0, false, null, null, TimeSpan.Zero, true, payloadBytes, payloadLimit); + + public static RuntimeEventIngestionResult Success(int accepted, int duplicates, int payloadBytes) + => new(accepted, duplicates, false, null, null, TimeSpan.Zero, false, payloadBytes, 0); +} diff --git a/src/StellaOps.Scanner.WebService/Services/RuntimeEventRateLimiter.cs b/src/StellaOps.Scanner.WebService/Services/RuntimeEventRateLimiter.cs new file mode 100644 index 00000000..963a8d07 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Services/RuntimeEventRateLimiter.cs @@ -0,0 +1,173 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Options; +using StellaOps.Scanner.WebService.Options; +using StellaOps.Zastava.Core.Contracts; + +namespace StellaOps.Scanner.WebService.Services; + +internal sealed class RuntimeEventRateLimiter +{ + private readonly ConcurrentDictionary _tenantBuckets = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _nodeBuckets = new(StringComparer.Ordinal); + private readonly TimeProvider _timeProvider; + private readonly IOptionsMonitor _optionsMonitor; + + public RuntimeEventRateLimiter(IOptionsMonitor optionsMonitor, TimeProvider timeProvider) + { + _optionsMonitor = optionsMonitor ?? throw new ArgumentNullException(nameof(optionsMonitor)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + } + + public RateLimitDecision Evaluate(IReadOnlyList envelopes) + { + ArgumentNullException.ThrowIfNull(envelopes); + if (envelopes.Count == 0) + { + return RateLimitDecision.Success; + } + + var options = _optionsMonitor.CurrentValue.Runtime ?? new ScannerWebServiceOptions.RuntimeOptions(); + var now = _timeProvider.GetUtcNow(); + + var tenantCounts = new Dictionary(StringComparer.Ordinal); + var nodeCounts = new Dictionary(StringComparer.Ordinal); + + foreach (var envelope in envelopes) + { + var tenant = envelope.Event.Tenant; + var node = envelope.Event.Node; + if (tenantCounts.TryGetValue(tenant, out var tenantCount)) + { + tenantCounts[tenant] = tenantCount + 1; + } + else + { + tenantCounts[tenant] = 1; + } + + var nodeKey = $"{tenant}|{node}"; + if (nodeCounts.TryGetValue(nodeKey, out var nodeCount)) + { + nodeCounts[nodeKey] = nodeCount + 1; + } + else + { + nodeCounts[nodeKey] = 1; + } + } + + var tenantDecision = TryAcquire( + _tenantBuckets, + tenantCounts, + options.PerTenantEventsPerSecond, + options.PerTenantBurst, + now, + scope: "tenant"); + + if (!tenantDecision.Allowed) + { + return tenantDecision; + } + + var nodeDecision = TryAcquire( + _nodeBuckets, + nodeCounts, + options.PerNodeEventsPerSecond, + options.PerNodeBurst, + now, + scope: "node"); + + return nodeDecision; + } + + private static RateLimitDecision TryAcquire( + ConcurrentDictionary buckets, + IReadOnlyDictionary counts, + double ratePerSecond, + int burst, + DateTimeOffset now, + string scope) + { + if (counts.Count == 0) + { + return RateLimitDecision.Success; + } + + var acquired = new List<(TokenBucket bucket, double tokens)>(); + + foreach (var pair in counts) + { + var bucket = buckets.GetOrAdd( + pair.Key, + _ => new TokenBucket(burst, ratePerSecond, now)); + + lock (bucket.SyncRoot) + { + bucket.Refill(now); + if (bucket.Tokens + 1e-9 < pair.Value) + { + var deficit = pair.Value - bucket.Tokens; + var retryAfterSeconds = deficit / bucket.RefillRatePerSecond; + var retryAfter = retryAfterSeconds <= 0 + ? TimeSpan.FromSeconds(1) + : TimeSpan.FromSeconds(Math.Min(retryAfterSeconds, 3600)); + + // undo previously acquired tokens + foreach (var (acquiredBucket, tokens) in acquired) + { + lock (acquiredBucket.SyncRoot) + { + acquiredBucket.Tokens = Math.Min(acquiredBucket.Capacity, acquiredBucket.Tokens + tokens); + } + } + + return new RateLimitDecision(false, scope, pair.Key, retryAfter); + } + + bucket.Tokens -= pair.Value; + acquired.Add((bucket, pair.Value)); + } + } + + return RateLimitDecision.Success; + } + + private sealed class TokenBucket + { + public TokenBucket(double capacity, double refillRatePerSecond, DateTimeOffset now) + { + Capacity = capacity; + Tokens = capacity; + RefillRatePerSecond = refillRatePerSecond; + LastRefill = now; + } + + public double Capacity { get; } + public double Tokens { get; set; } + public double RefillRatePerSecond { get; } + public DateTimeOffset LastRefill { get; set; } + public object SyncRoot { get; } = new(); + + public void Refill(DateTimeOffset now) + { + if (now <= LastRefill) + { + return; + } + + var elapsedSeconds = (now - LastRefill).TotalSeconds; + if (elapsedSeconds <= 0) + { + return; + } + + Tokens = Math.Min(Capacity, Tokens + elapsedSeconds * RefillRatePerSecond); + LastRefill = now; + } + } +} + +internal readonly record struct RateLimitDecision(bool Allowed, string? Scope, string? Key, TimeSpan RetryAfter) +{ + public static RateLimitDecision Success { get; } = new(true, null, null, TimeSpan.Zero); +} diff --git a/src/StellaOps.Scanner.WebService/Services/RuntimePolicyService.cs b/src/StellaOps.Scanner.WebService/Services/RuntimePolicyService.cs new file mode 100644 index 00000000..d04cf4ea --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Services/RuntimePolicyService.cs @@ -0,0 +1,211 @@ +using System.Collections.ObjectModel; +using Microsoft.Extensions.Options; +using StellaOps.Policy; +using StellaOps.Scanner.Storage.Catalog; +using StellaOps.Scanner.Storage.Repositories; +using StellaOps.Scanner.WebService.Options; + +namespace StellaOps.Scanner.WebService.Services; + +internal interface IRuntimePolicyService +{ + Task EvaluateAsync(RuntimePolicyEvaluationRequest request, CancellationToken cancellationToken); +} + +internal sealed class RuntimePolicyService : IRuntimePolicyService +{ + private readonly LinkRepository _linkRepository; + private readonly ArtifactRepository _artifactRepository; + private readonly PolicySnapshotStore _policySnapshotStore; + private readonly IOptionsMonitor _optionsMonitor; + private readonly TimeProvider _timeProvider; + + public RuntimePolicyService( + LinkRepository linkRepository, + ArtifactRepository artifactRepository, + PolicySnapshotStore policySnapshotStore, + IOptionsMonitor optionsMonitor, + TimeProvider timeProvider) + { + _linkRepository = linkRepository ?? throw new ArgumentNullException(nameof(linkRepository)); + _artifactRepository = artifactRepository ?? throw new ArgumentNullException(nameof(artifactRepository)); + _policySnapshotStore = policySnapshotStore ?? throw new ArgumentNullException(nameof(policySnapshotStore)); + _optionsMonitor = optionsMonitor ?? throw new ArgumentNullException(nameof(optionsMonitor)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + } + + public async Task EvaluateAsync(RuntimePolicyEvaluationRequest request, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + + var runtimeOptions = _optionsMonitor.CurrentValue.Runtime ?? new ScannerWebServiceOptions.RuntimeOptions(); + var ttlSeconds = Math.Max(1, runtimeOptions.PolicyCacheTtlSeconds); + + var now = _timeProvider.GetUtcNow(); + var expiresAt = now.AddSeconds(ttlSeconds); + + var snapshot = await _policySnapshotStore.GetLatestAsync(cancellationToken).ConfigureAwait(false); + + var policyRevision = snapshot?.RevisionId; + var policyDigest = snapshot?.Digest; + + var results = new Dictionary(StringComparer.Ordinal); + + foreach (var image in request.Images) + { + var metadata = await ResolveImageMetadataAsync(image, cancellationToken).ConfigureAwait(false); + var decision = BuildDecision(metadata, snapshot, policyDigest); + results[image] = decision; + } + + return new RuntimePolicyEvaluationResult( + ttlSeconds, + expiresAt, + policyRevision, + new ReadOnlyDictionary(results)); + } + + private async Task ResolveImageMetadataAsync(string imageDigest, CancellationToken cancellationToken) + { + var links = await _linkRepository.ListBySourceAsync(LinkSourceType.Image, imageDigest, cancellationToken).ConfigureAwait(false); + if (links.Count == 0) + { + return new RuntimeImageMetadata(imageDigest, false, false, null, MissingMetadata: true); + } + + var hasSbom = false; + var signed = false; + RuntimePolicyRekorReference? rekor = null; + + foreach (var link in links) + { + var artifact = await _artifactRepository.GetAsync(link.ArtifactId, cancellationToken).ConfigureAwait(false); + if (artifact is null) + { + continue; + } + + switch (artifact.Type) + { + case ArtifactDocumentType.ImageBom: + hasSbom = true; + break; + case ArtifactDocumentType.Attestation: + signed = true; + if (artifact.Rekor is { } rekorReference) + { + rekor = new RuntimePolicyRekorReference( + Normalize(rekorReference.Uuid), + Normalize(rekorReference.Url), + rekorReference.Index.HasValue); + } + break; + } + } + + return new RuntimeImageMetadata(imageDigest, signed, hasSbom, rekor, MissingMetadata: false); + } + + private static RuntimePolicyImageDecision BuildDecision(RuntimeImageMetadata metadata, PolicySnapshot? snapshot, string? policyDigest) + { + var reasons = new List(); + + if (metadata.MissingMetadata) + { + reasons.Add("image.metadata.missing"); + } + + if (!metadata.Signed) + { + reasons.Add("unsigned"); + } + + if (!metadata.HasSbomReferrers) + { + reasons.Add("missing SBOM"); + } + + if (snapshot is null) + { + reasons.Add("policy.snapshot.missing"); + } + + string verdict; + if (snapshot is null) + { + verdict = "unknown"; + } + else if (reasons.Count == 0) + { + verdict = "pass"; + } + else if (metadata.Signed && metadata.HasSbomReferrers) + { + verdict = "warn"; + } + else + { + verdict = "fail"; + } + + RuntimePolicyRekorReference? rekor = metadata.Rekor; + + IDictionary? metadataPayload = null; + if (!string.IsNullOrWhiteSpace(policyDigest) || metadata.MissingMetadata) + { + metadataPayload = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["source"] = "scanner.runtime.placeholder" + }; + + if (!string.IsNullOrWhiteSpace(policyDigest)) + { + metadataPayload["policyDigest"] = policyDigest; + } + + if (metadata.MissingMetadata) + { + metadataPayload["artifactLinks"] = 0; + } + } + + return new RuntimePolicyImageDecision( + verdict, + metadata.Signed, + metadata.HasSbomReferrers, + reasons, + rekor, + metadataPayload); + } + + private static string? Normalize(string? value) + => string.IsNullOrWhiteSpace(value) ? null : value; +} + +internal sealed record RuntimePolicyEvaluationRequest( + string? Namespace, + IReadOnlyDictionary Labels, + IReadOnlyList Images); + +internal sealed record RuntimePolicyEvaluationResult( + int TtlSeconds, + DateTimeOffset ExpiresAtUtc, + string? PolicyRevision, + IReadOnlyDictionary Results); + +internal sealed record RuntimePolicyImageDecision( + string PolicyVerdict, + bool Signed, + bool HasSbomReferrers, + IReadOnlyList Reasons, + RuntimePolicyRekorReference? Rekor, + IDictionary? Metadata); + +internal sealed record RuntimePolicyRekorReference(string? Uuid, string? Url, bool? Verified); + +internal sealed record RuntimeImageMetadata( + string ImageDigest, + bool Signed, + bool HasSbomReferrers, + RuntimePolicyRekorReference? Rekor, + bool MissingMetadata); diff --git a/src/StellaOps.Scanner.WebService/Services/ScanProgressStream.cs b/src/StellaOps.Scanner.WebService/Services/ScanProgressStream.cs index a4f67e84..061a67f0 100644 --- a/src/StellaOps.Scanner.WebService/Services/ScanProgressStream.cs +++ b/src/StellaOps.Scanner.WebService/Services/ScanProgressStream.cs @@ -1,7 +1,6 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.ObjectModel; -using System.Linq; using System.Runtime.CompilerServices; using System.Threading.Channels; using StellaOps.Scanner.WebService.Domain; @@ -58,7 +57,8 @@ public sealed class ScanProgressStream : IScanProgressPublisher, IScanProgressRe public int NextSequence() => ++Sequence; } - private static readonly IReadOnlyDictionary EmptyData = new ReadOnlyDictionary(new Dictionary(StringComparer.OrdinalIgnoreCase)); + private static readonly IReadOnlyDictionary EmptyData = + new ReadOnlyDictionary(new SortedDictionary(StringComparer.OrdinalIgnoreCase)); private readonly ConcurrentDictionary channels = new(StringComparer.OrdinalIgnoreCase); private readonly TimeProvider timeProvider; @@ -85,18 +85,14 @@ public sealed class ScanProgressStream : IScanProgressPublisher, IScanProgressRe { var sequence = channel.NextSequence(); var correlation = correlationId ?? $"{scanId.Value}:{sequence:D4}"; - var payload = data is null || data.Count == 0 - ? EmptyData - : new ReadOnlyDictionary(new Dictionary(data, StringComparer.OrdinalIgnoreCase)); - - progressEvent = new ScanProgressEvent( - scanId, - sequence, - timeProvider.GetUtcNow(), - state, - message, - correlation, - payload); + progressEvent = new ScanProgressEvent( + scanId, + sequence, + timeProvider.GetUtcNow(), + state, + message, + correlation, + NormalizePayload(data)); channel.Append(progressEvent); } @@ -131,6 +127,24 @@ public sealed class ScanProgressStream : IScanProgressPublisher, IScanProgressRe { yield return progressEvent; } - } - } -} + } + } + + private static IReadOnlyDictionary NormalizePayload(IReadOnlyDictionary? data) + { + if (data is null || data.Count == 0) + { + return EmptyData; + } + + var sorted = new SortedDictionary(StringComparer.OrdinalIgnoreCase); + foreach (var pair in data) + { + sorted[pair.Key] = pair.Value; + } + + return sorted.Count == 0 + ? EmptyData + : new ReadOnlyDictionary(sorted); + } +} diff --git a/src/StellaOps.Scanner.WebService/StellaOps.Scanner.WebService.csproj b/src/StellaOps.Scanner.WebService/StellaOps.Scanner.WebService.csproj index 2cf5041b..b26d3d04 100644 --- a/src/StellaOps.Scanner.WebService/StellaOps.Scanner.WebService.csproj +++ b/src/StellaOps.Scanner.WebService/StellaOps.Scanner.WebService.csproj @@ -12,20 +12,22 @@ - + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Scanner.WebService/TASKS.md b/src/StellaOps.Scanner.WebService/TASKS.md index 20a5be5b..3769f789 100644 --- a/src/StellaOps.Scanner.WebService/TASKS.md +++ b/src/StellaOps.Scanner.WebService/TASKS.md @@ -10,7 +10,18 @@ | SCANNER-POLICY-09-106 | DONE (2025-10-19) | Scanner WebService Guild | POLICY-CORE-09-002, SCANNER-POLICY-09-105 | `/reports` verdict assembly (Feedser/Vexer/Policy merge) + signed response envelope. | Aggregated report includes policy metadata; integration test verifies signed response; docs updated. | | SCANNER-POLICY-09-107 | DONE (2025-10-19) | Scanner WebService Guild | POLICY-CORE-09-005, SCANNER-POLICY-09-106 | Surface score inputs, config version, and `quietedBy` provenance in `/reports` response and signed payload; document schema changes. | `/reports` JSON + DSSE contain score, reachability, sourceTrust, confidenceBand, quiet provenance; contract tests updated; docs refreshed. | | SCANNER-WEB-10-201 | DONE (2025-10-19) | Scanner WebService Guild | SCANNER-CACHE-10-101 | Register scanner cache services and maintenance loop within WebService host. | `AddScannerCache` wired for configuration binding; maintenance service skips when disabled; project references updated. | -| SCANNER-RUNTIME-12-301 | TODO | Scanner WebService Guild | ZASTAVA-CORE-12-201 | Implement `/runtime/events` ingestion endpoint with validation, batching, and storage hooks per Zastava contract. | Observer fixtures POST events, data persisted and acked; invalid payloads rejected with deterministic errors. | -| SCANNER-RUNTIME-12-302 | TODO | Scanner WebService Guild | SCANNER-RUNTIME-12-301, ZASTAVA-CORE-12-201 | Implement `/policy/runtime` endpoint joining SBOM baseline + policy verdict, returning admission guidance. Coordinate with CLI (`CLI-RUNTIME-13-008`) before GA to lock response field names/metadata. | Webhook integration test passes; responses include verdict, TTL, reasons; metrics/logging added; CLI contract review signed off. | -| SCANNER-EVENTS-15-201 | DOING (2025-10-19) | Scanner WebService Guild | NOTIFY-QUEUE-15-401 | Emit `scanner.report.ready` and `scanner.scan.completed` events (bus adapters + tests). | Event envelopes published to queue with schemas; fixtures committed; Notify consumption test passes. | -| SCANNER-RUNTIME-17-401 | TODO | Scanner WebService Guild | SCANNER-RUNTIME-12-301, ZASTAVA-OBS-17-005, SCANNER-EMIT-17-701, POLICY-RUNTIME-17-201 | Persist runtime build-id observations and expose them via `/runtime/events` + policy joins for debug-symbol correlation. | Mongo schema stores optional `buildId`, API/SDK responses document field, integration test resolves debug-store path using stored build-id, docs updated accordingly. | +| SCANNER-RUNTIME-12-301 | DONE (2025-10-20) | Scanner WebService Guild | ZASTAVA-CORE-12-201 | Implement `/runtime/events` ingestion endpoint with validation, batching, and storage hooks per Zastava contract. | Observer fixtures POST events, data persisted and acked; invalid payloads rejected with deterministic errors. | +| SCANNER-RUNTIME-12-302 | DOING (2025-10-20) | Scanner WebService Guild | SCANNER-RUNTIME-12-301, ZASTAVA-CORE-12-201 | Implement `/policy/runtime` endpoint joining SBOM baseline + policy verdict, returning admission guidance. Coordinate with CLI (`CLI-RUNTIME-13-008`) before GA to lock response field names/metadata. | Webhook integration test passes; responses include verdict, TTL, reasons; metrics/logging added; CLI contract review signed off. | +| SCANNER-RUNTIME-12-303 | TODO | Scanner WebService Guild | SCANNER-RUNTIME-12-302 | Replace `/policy/runtime` heuristic with canonical policy evaluation (Feedser/Vexer inputs, PolicyPreviewService) so results align with `/reports`. | Runtime policy endpoint returns canonical verdicts + metadata, tests cover pass/warn/fail cases, docs/CLI updated. | +| SCANNER-RUNTIME-12-304 | TODO | Scanner WebService Guild | SCANNER-RUNTIME-12-302 | Surface attestation verification status by integrating Authority/Attestor Rekor validation (beyond presence-only). | Response `rekor.verified` reflects attestor outcome; integration test covers verified/unverified paths; docs updated. | +| SCANNER-RUNTIME-12-305 | TODO | Scanner WebService Guild | SCANNER-RUNTIME-12-301, SCANNER-RUNTIME-12-302 | Promote shared fixtures with Zastava/CLI and add end-to-end automation for `/runtime/events` + `/policy/runtime`. | Fixture suite replayed in CI, cross-team sign-off recorded, documentation references test harness. | +| SCANNER-EVENTS-15-201 | DONE (2025-10-20) | Scanner WebService Guild | NOTIFY-QUEUE-15-401 | Emit `scanner.report.ready` and `scanner.scan.completed` events (bus adapters + tests). | Event envelopes published to queue with schemas; fixtures committed; Notify consumption test passes. | +| SCANNER-EVENTS-16-301 | BLOCKED (2025-10-20) | Scanner WebService Guild | NOTIFY-QUEUE-15-401 | Integrate Redis publisher end-to-end once Notify queue abstraction ships; replace in-memory recorder with real stream assertions. | Notify Queue adapter available; integration test exercises Redis stream length/fields via test harness; docs updated with ops validation checklist. | +| SCANNER-RUNTIME-17-401 | TODO | Scanner WebService Guild | SCANNER-RUNTIME-12-301, ZASTAVA-OBS-17-005, SCANNER-EMIT-17-701, POLICY-RUNTIME-17-201 | Persist runtime build-id observations and expose them via `/runtime/events` + policy joins for debug-symbol correlation. | Mongo schema stores optional `buildId`, API/SDK responses document field, integration test resolves debug-store path using stored build-id, docs updated accordingly. | + +## Notes +- 2025-10-19: Sprint 9 streaming + policy endpoints (SCANNER-WEB-09-103, SCANNER-POLICY-09-105/106/107) landed with SSE/JSONL, OpenAPI, signed report coverage documented in `docs/09_API_CLI_REFERENCE.md`. +- 2025-10-20: Re-ran `dotnet test src/StellaOps.Scanner.WebService.Tests/StellaOps.Scanner.WebService.Tests.csproj --filter FullyQualifiedName~ReportsEndpointsTests` to confirm DSSE/report regressions stay green after backlog sync. +- 2025-10-20: SCANNER-RUNTIME-12-301 underway – `/runtime/events` ingest hitting Mongo with TTL + token-bucket rate limiting; integration tests (`RuntimeEndpointsTests`) green and docs updated with batch contract. +- 2025-10-20: Follow-ups SCANNER-RUNTIME-12-303/304/305 track canonical verdict integration, attestation verification, and cross-guild fixture validation for runtime APIs. +- 2025-10-21: Hardened progress streaming determinism by sorting `data` payload keys within `ScanProgressStream`; added regression `ProgressStreamDataKeysAreSortedDeterministically` ensuring JSONL ordering. diff --git a/src/StellaOps.Scheduler.ImpactIndex.Tests/FixtureImpactIndexTests.cs b/src/StellaOps.Scheduler.ImpactIndex.Tests/FixtureImpactIndexTests.cs new file mode 100644 index 00000000..e0764638 --- /dev/null +++ b/src/StellaOps.Scheduler.ImpactIndex.Tests/FixtureImpactIndexTests.cs @@ -0,0 +1,142 @@ +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Extensions.Logging; +using StellaOps.Scheduler.ImpactIndex; +using StellaOps.Scheduler.Models; +using Xunit; + +namespace StellaOps.Scheduler.ImpactIndex.Tests; + +public sealed class FixtureImpactIndexTests +{ + [Fact] + public async Task ResolveByPurls_UsesEmbeddedFixtures() + { + var selector = new Selector(SelectorScope.AllImages); + var (impactIndex, loggerFactory) = CreateImpactIndex(); + using var _ = loggerFactory; + + var result = await impactIndex.ResolveByPurlsAsync( + new[] { "pkg:apk/alpine/openssl@3.2.2-r0?arch=x86_64" }, + usageOnly: false, + selector); + + result.UsageOnly.Should().BeFalse(); + result.Images.Should().ContainSingle(); + + var image = result.Images.Single(); + image.ImageDigest.Should().Be("sha256:8f47d7c6b538c0d9533b78913cba3d5e671e7c4b4e7c6a2bb9a1a1c4d4f8e123"); + image.Registry.Should().Be("docker.io"); + image.Repository.Should().Be("library/nginx"); + image.Tags.Should().ContainSingle(tag => tag == "1.25.4"); + image.UsedByEntrypoint.Should().BeTrue(); + + result.GeneratedAt.Should().Be(DateTimeOffset.Parse("2025-10-19T00:00:00Z")); + result.SchemaVersion.Should().Be(SchedulerSchemaVersions.ImpactSet); + } + + [Fact] + public async Task ResolveByPurls_UsageOnlyFiltersInventoryOnlyComponents() + { + var selector = new Selector(SelectorScope.AllImages); + var (impactIndex, loggerFactory) = CreateImpactIndex(); + using var _ = loggerFactory; + + var inventoryOnlyPurl = "pkg:apk/alpine/pcre2@10.42-r1?arch=x86_64"; + + var runtimeResult = await impactIndex.ResolveByPurlsAsync( + new[] { inventoryOnlyPurl }, + usageOnly: true, + selector); + + runtimeResult.Images.Should().BeEmpty(); + + var inventoryResult = await impactIndex.ResolveByPurlsAsync( + new[] { inventoryOnlyPurl }, + usageOnly: false, + selector); + + inventoryResult.Images.Should().ContainSingle(); + inventoryResult.Images.Single().UsedByEntrypoint.Should().BeFalse(); + } + + [Fact] + public async Task ResolveAll_ReturnsDeterministicFixtureSet() + { + var selector = new Selector(SelectorScope.AllImages); + var (impactIndex, loggerFactory) = CreateImpactIndex(); + using var _ = loggerFactory; + + var first = await impactIndex.ResolveAllAsync(selector, usageOnly: false); + first.Images.Should().HaveCount(6); + + var second = await impactIndex.ResolveAllAsync(selector, usageOnly: false); + second.Images.Should().HaveCount(6); + second.Images.Should().Equal(first.Images); + } + + [Fact] + public async Task ResolveByVulnerabilities_ReturnsEmptySet() + { + var selector = new Selector(SelectorScope.AllImages); + var (impactIndex, loggerFactory) = CreateImpactIndex(); + using var _ = loggerFactory; + + var result = await impactIndex.ResolveByVulnerabilitiesAsync( + new[] { "CVE-2025-0001" }, + usageOnly: false, + selector); + + result.Images.Should().BeEmpty(); + } + + [Fact] + public async Task FixtureDirectoryOption_LoadsFromFileSystem() + { + var selector = new Selector(SelectorScope.AllImages); + var samplesDirectory = LocateSamplesDirectory(); + var (impactIndex, loggerFactory) = CreateImpactIndex(options => + { + options.FixtureDirectory = samplesDirectory; + }); + using var _ = loggerFactory; + + var result = await impactIndex.ResolveAllAsync(selector, usageOnly: false); + + result.Images.Should().HaveCount(6); + } + + private static (FixtureImpactIndex ImpactIndex, ILoggerFactory LoggerFactory) CreateImpactIndex( + Action? configure = null) + { + var options = new ImpactIndexStubOptions(); + configure?.Invoke(options); + + var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); + var logger = loggerFactory.CreateLogger(); + + var impactIndex = new FixtureImpactIndex(options, TimeProvider.System, logger); + return (impactIndex, loggerFactory); + } + + private static string LocateSamplesDirectory() + { + var current = AppContext.BaseDirectory; + + while (!string.IsNullOrWhiteSpace(current)) + { + var candidate = Path.Combine(current, "samples", "scanner", "images"); + if (Directory.Exists(candidate)) + { + return candidate; + } + + current = Directory.GetParent(current)?.FullName; + } + + throw new InvalidOperationException("Unable to locate 'samples/scanner/images'."); + } +} diff --git a/src/StellaOps.Scheduler.ImpactIndex.Tests/StellaOps.Scheduler.ImpactIndex.Tests.csproj b/src/StellaOps.Scheduler.ImpactIndex.Tests/StellaOps.Scheduler.ImpactIndex.Tests.csproj new file mode 100644 index 00000000..30b5f162 --- /dev/null +++ b/src/StellaOps.Scheduler.ImpactIndex.Tests/StellaOps.Scheduler.ImpactIndex.Tests.csproj @@ -0,0 +1,25 @@ + + + net10.0 + enable + enable + false + false + true + + + + + + + all + + + all + + + + + + + diff --git a/src/StellaOps.Scheduler.ImpactIndex/FixtureImpactIndex.cs b/src/StellaOps.Scheduler.ImpactIndex/FixtureImpactIndex.cs new file mode 100644 index 00000000..436cac46 --- /dev/null +++ b/src/StellaOps.Scheduler.ImpactIndex/FixtureImpactIndex.cs @@ -0,0 +1,615 @@ +using System.Collections.Immutable; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using System.IO; +using System.IO.Enumeration; +using System.Reflection; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Extensions.Logging; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.ImpactIndex; + +/// +/// Fixture-backed implementation of used while the real index is under construction. +/// +public sealed class FixtureImpactIndex : IImpactIndex +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + }; + + private readonly ImpactIndexStubOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly SemaphoreSlim _initializationLock = new(1, 1); + private FixtureIndexState? _state; + + public FixtureImpactIndex( + ImpactIndexStubOptions options, + TimeProvider? timeProvider, + ILogger logger) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask ResolveByPurlsAsync( + IEnumerable purls, + bool usageOnly, + Selector selector, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(purls); + ArgumentNullException.ThrowIfNull(selector); + + var state = await EnsureInitializedAsync(cancellationToken).ConfigureAwait(false); + var normalizedPurls = NormalizeKeys(purls); + + if (normalizedPurls.Length == 0) + { + return CreateImpactSet(state, selector, Enumerable.Empty(), usageOnly); + } + + var matches = new List(); + foreach (var purl in normalizedPurls) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (!state.PurlIndex.TryGetValue(purl, out var componentMatches)) + { + continue; + } + + foreach (var component in componentMatches) + { + var usedByEntrypoint = component.Component.UsedByEntrypoint; + if (usageOnly && !usedByEntrypoint) + { + continue; + } + + matches.Add(new FixtureMatch(component.Image, usedByEntrypoint)); + } + } + + return CreateImpactSet(state, selector, matches, usageOnly); + } + + public async ValueTask ResolveByVulnerabilitiesAsync( + IEnumerable vulnerabilityIds, + bool usageOnly, + Selector selector, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(vulnerabilityIds); + ArgumentNullException.ThrowIfNull(selector); + + var state = await EnsureInitializedAsync(cancellationToken).ConfigureAwait(false); + + // The stub does not maintain a vulnerability → purl projection, so we return an empty result. + if (_logger.IsEnabled(LogLevel.Debug)) + { + var first = vulnerabilityIds.FirstOrDefault(static id => !string.IsNullOrWhiteSpace(id)); + if (first is not null) + { + _logger.LogDebug( + "ImpactIndex stub received ResolveByVulnerabilitiesAsync for '{VulnerabilityId}' but mappings are not available.", + first); + } + } + + return CreateImpactSet(state, selector, Enumerable.Empty(), usageOnly); + } + + public async ValueTask ResolveAllAsync( + Selector selector, + bool usageOnly, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(selector); + + var state = await EnsureInitializedAsync(cancellationToken).ConfigureAwait(false); + + var matches = state.ImagesByDigest.Values + .Select(image => new FixtureMatch(image, image.UsedByEntrypoint)) + .Where(match => !usageOnly || match.UsedByEntrypoint); + + return CreateImpactSet(state, selector, matches, usageOnly); + } + + private async Task EnsureInitializedAsync(CancellationToken cancellationToken) + { + if (_state is not null) + { + return _state; + } + + await _initializationLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_state is not null) + { + return _state; + } + + var state = await LoadAsync(cancellationToken).ConfigureAwait(false); + _state = state; + _logger.LogInformation( + "ImpactIndex stub loaded {ImageCount} fixture images from {SourceDescription}.", + state.ImagesByDigest.Count, + state.SourceDescription); + return state; + } + finally + { + _initializationLock.Release(); + } + } + + private async Task LoadAsync(CancellationToken cancellationToken) + { + var images = new List(); + string? sourceDescription = null; + + if (!string.IsNullOrWhiteSpace(_options.FixtureDirectory)) + { + var directory = ResolveDirectoryPath(_options.FixtureDirectory!); + if (Directory.Exists(directory)) + { + images.AddRange(await LoadFromDirectoryAsync(directory, cancellationToken).ConfigureAwait(false)); + sourceDescription = directory; + } + else + { + _logger.LogWarning( + "ImpactIndex stub fixture directory '{Directory}' was not found. Falling back to embedded fixtures.", + directory); + } + } + + if (images.Count == 0) + { + images.AddRange(await LoadFromResourcesAsync(cancellationToken).ConfigureAwait(false)); + sourceDescription ??= "embedded:scheduler-impact-index-fixtures"; + } + + if (images.Count == 0) + { + throw new InvalidOperationException("No BOM-Index fixtures were found for the ImpactIndex stub."); + } + + return BuildState(images, sourceDescription!, _options.SnapshotId); + } + + private static string ResolveDirectoryPath(string path) + { + if (Path.IsPathRooted(path)) + { + return path; + } + + var basePath = AppContext.BaseDirectory; + return Path.GetFullPath(Path.Combine(basePath, path)); + } + + private static async Task> LoadFromDirectoryAsync( + string directory, + CancellationToken cancellationToken) + { + var results = new List(); + + foreach (var file in Directory.EnumerateFiles(directory, "bom-index.json", SearchOption.AllDirectories) + .OrderBy(static file => file, StringComparer.Ordinal)) + { + cancellationToken.ThrowIfCancellationRequested(); + + await using var stream = File.OpenRead(file); + var document = await JsonSerializer.DeserializeAsync(stream, SerializerOptions, cancellationToken) + .ConfigureAwait(false); + if (document is null) + { + continue; + } + + results.Add(CreateFixtureImage(document)); + } + + return results; + } + + private static async Task> LoadFromResourcesAsync(CancellationToken cancellationToken) + { + var assembly = typeof(FixtureImpactIndex).Assembly; + var resourceNames = assembly + .GetManifestResourceNames() + .Where(static name => name.EndsWith(".bom-index.json", StringComparison.OrdinalIgnoreCase)) + .OrderBy(static name => name, StringComparer.Ordinal) + .ToArray(); + + var results = new List(resourceNames.Length); + + foreach (var resourceName in resourceNames) + { + cancellationToken.ThrowIfCancellationRequested(); + + await using var stream = assembly.GetManifestResourceStream(resourceName); + if (stream is null) + { + continue; + } + + var document = await JsonSerializer.DeserializeAsync(stream, SerializerOptions, cancellationToken) + .ConfigureAwait(false); + + if (document is null) + { + continue; + } + + results.Add(CreateFixtureImage(document)); + } + + return results; + } + + private static FixtureIndexState BuildState( + IReadOnlyList images, + string sourceDescription, + string snapshotId) + { + var imagesByDigest = images + .GroupBy(static image => image.Digest, StringComparer.OrdinalIgnoreCase) + .ToImmutableDictionary( + static group => group.Key, + static group => group + .OrderBy(static image => image.Repository, StringComparer.Ordinal) + .ThenBy(static image => image.Registry, StringComparer.Ordinal) + .ThenBy(static image => image.Tags.Length, Comparer.Default) + .First(), + StringComparer.OrdinalIgnoreCase); + + var purlIndexBuilder = new Dictionary>(StringComparer.OrdinalIgnoreCase); + foreach (var image in images) + { + foreach (var component in image.Components) + { + if (!purlIndexBuilder.TryGetValue(component.Purl, out var list)) + { + list = new List(); + purlIndexBuilder[component.Purl] = list; + } + + list.Add(new FixtureComponentMatch(image, component)); + } + } + + var purlIndex = purlIndexBuilder.ToImmutableDictionary( + static entry => entry.Key, + static entry => entry.Value + .OrderBy(static item => item.Image.Digest, StringComparer.Ordinal) + .Select(static item => new FixtureComponentMatch(item.Image, item.Component)) + .ToImmutableArray(), + StringComparer.OrdinalIgnoreCase); + + var generatedAt = images.Count == 0 + ? DateTimeOffset.UnixEpoch + : images.Max(static image => image.GeneratedAt); + + return new FixtureIndexState(imagesByDigest, purlIndex, generatedAt, sourceDescription, snapshotId); + } + + private ImpactSet CreateImpactSet( + FixtureIndexState state, + Selector selector, + IEnumerable matches, + bool usageOnly) + { + var aggregated = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var match in matches) + { + if (!ImageMatchesSelector(match.Image, selector)) + { + continue; + } + + if (!aggregated.TryGetValue(match.Image.Digest, out var builder)) + { + builder = new ImpactImageBuilder(match.Image); + aggregated[match.Image.Digest] = builder; + } + + builder.MarkUsedByEntrypoint(match.UsedByEntrypoint); + } + + var images = aggregated.Values + .Select(static builder => builder.Build()) + .OrderBy(static image => image.ImageDigest, StringComparer.Ordinal) + .ToImmutableArray(); + + return new ImpactSet( + selector, + images, + usageOnly, + state.GeneratedAt == DateTimeOffset.UnixEpoch + ? _timeProvider.GetUtcNow() + : state.GeneratedAt, + images.Length, + state.SnapshotId, + SchedulerSchemaVersions.ImpactSet); + } + + private static bool ImageMatchesSelector(FixtureImage image, Selector selector) + { + if (selector is null) + { + return true; + } + + if (selector.Digests.Length > 0 && + !selector.Digests.Contains(image.Digest, StringComparer.OrdinalIgnoreCase)) + { + return false; + } + + if (selector.Repositories.Length > 0) + { + var repositoryMatch = selector.Repositories.Any(repo => + string.Equals(repo, image.Repository, StringComparison.OrdinalIgnoreCase) || + string.Equals(repo, $"{image.Registry}/{image.Repository}", StringComparison.OrdinalIgnoreCase)); + + if (!repositoryMatch) + { + return false; + } + } + + if (selector.Namespaces.Length > 0) + { + if (image.Namespaces.IsDefaultOrEmpty) + { + return false; + } + + var namespaceMatch = selector.Namespaces.Any(namespaceId => + image.Namespaces.Contains(namespaceId, StringComparer.OrdinalIgnoreCase)); + + if (!namespaceMatch) + { + return false; + } + } + + if (selector.IncludeTags.Length > 0) + { + if (image.Tags.IsDefaultOrEmpty) + { + return false; + } + + var tagMatch = selector.IncludeTags.Any(pattern => + MatchesAnyTag(image.Tags, pattern)); + + if (!tagMatch) + { + return false; + } + } + + if (selector.Labels.Length > 0) + { + if (image.Labels.Count == 0) + { + return false; + } + + foreach (var labelSelector in selector.Labels) + { + if (!image.Labels.TryGetValue(labelSelector.Key, out var value)) + { + return false; + } + + if (labelSelector.Values.Length > 0 && + !labelSelector.Values.Contains(value, StringComparer.OrdinalIgnoreCase)) + { + return false; + } + } + } + + return selector.Scope switch + { + SelectorScope.ByDigest => selector.Digests.Length == 0 + ? true + : selector.Digests.Contains(image.Digest, StringComparer.OrdinalIgnoreCase), + SelectorScope.ByRepository => selector.Repositories.Length == 0 + ? true + : selector.Repositories.Any(repo => + string.Equals(repo, image.Repository, StringComparison.OrdinalIgnoreCase) || + string.Equals(repo, $"{image.Registry}/{image.Repository}", StringComparison.OrdinalIgnoreCase)), + SelectorScope.ByNamespace => selector.Namespaces.Length == 0 + ? true + : !image.Namespaces.IsDefaultOrEmpty && + selector.Namespaces.Any(namespaceId => + image.Namespaces.Contains(namespaceId, StringComparer.OrdinalIgnoreCase)), + SelectorScope.ByLabels => selector.Labels.Length == 0 + ? true + : selector.Labels.All(label => + image.Labels.TryGetValue(label.Key, out var value) && + (label.Values.Length == 0 || label.Values.Contains(value, StringComparer.OrdinalIgnoreCase))), + _ => true, + }; + } + + private static bool MatchesAnyTag(ImmutableArray tags, string pattern) + { + foreach (var tag in tags) + { + if (FileSystemName.MatchesSimpleExpression(pattern, tag, ignoreCase: true)) + { + return true; + } + } + + return false; + } + + private static FixtureImage CreateFixtureImage(BomIndexDocument document) + { + if (document.Image is null) + { + throw new InvalidOperationException("BOM-Index image metadata is required."); + } + + var digest = Validation.EnsureDigestFormat(document.Image.Digest, "image.digest"); + var (registry, repository) = SplitRepository(document.Image.Repository); + + var tags = string.IsNullOrWhiteSpace(document.Image.Tag) + ? ImmutableArray.Empty + : ImmutableArray.Create(document.Image.Tag.Trim()); + + var components = (document.Components ?? Array.Empty()) + .Where(static component => !string.IsNullOrWhiteSpace(component.Purl)) + .Select(component => new FixtureComponent( + component.Purl!.Trim(), + component.Usage?.Any(static usage => + usage.Equals("runtime", StringComparison.OrdinalIgnoreCase) || + usage.Equals("usedByEntrypoint", StringComparison.OrdinalIgnoreCase)) == true)) + .OrderBy(static component => component.Purl, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + + return new FixtureImage( + digest, + registry, + repository, + ImmutableArray.Empty, + tags, + ImmutableSortedDictionary.Empty.WithComparers(StringComparer.OrdinalIgnoreCase), + components, + document.GeneratedAt == default ? DateTimeOffset.UnixEpoch : document.GeneratedAt.ToUniversalTime(), + components.Any(static component => component.UsedByEntrypoint)); + } + + private static (string Registry, string Repository) SplitRepository(string repository) + { + var normalized = Validation.EnsureNotNullOrWhiteSpace(repository, nameof(repository)); + var separatorIndex = normalized.IndexOf('/'); + if (separatorIndex < 0) + { + return ("docker.io", normalized); + } + + var registry = normalized[..separatorIndex]; + var repo = normalized[(separatorIndex + 1)..]; + if (string.IsNullOrWhiteSpace(repo)) + { + throw new ArgumentException("Repository segment is required after registry.", nameof(repository)); + } + + return (registry.Trim(), repo.Trim()); + } + + private static string[] NormalizeKeys(IEnumerable values) + { + return values + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .Select(static value => value.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private readonly record struct FixtureMatch(FixtureImage Image, bool UsedByEntrypoint); + + private sealed record FixtureImage( + string Digest, + string Registry, + string Repository, + ImmutableArray Namespaces, + ImmutableArray Tags, + ImmutableSortedDictionary Labels, + ImmutableArray Components, + DateTimeOffset GeneratedAt, + bool UsedByEntrypoint); + + private sealed record FixtureComponent(string Purl, bool UsedByEntrypoint); + + private sealed record FixtureComponentMatch(FixtureImage Image, FixtureComponent Component); + + private sealed record FixtureIndexState( + ImmutableDictionary ImagesByDigest, + ImmutableDictionary> PurlIndex, + DateTimeOffset GeneratedAt, + string SourceDescription, + string SnapshotId); + + private sealed class ImpactImageBuilder + { + private readonly FixtureImage _image; + private bool _usedByEntrypoint; + + public ImpactImageBuilder(FixtureImage image) + { + _image = image; + } + + public void MarkUsedByEntrypoint(bool usedByEntrypoint) + { + _usedByEntrypoint |= usedByEntrypoint; + } + + public ImpactImage Build() + { + return new ImpactImage( + _image.Digest, + _image.Registry, + _image.Repository, + _image.Namespaces, + _image.Tags, + _usedByEntrypoint, + _image.Labels); + } + } + + private sealed record BomIndexDocument + { + [JsonPropertyName("schema")] + public string? Schema { get; init; } + + [JsonPropertyName("image")] + public BomIndexImage? Image { get; init; } + + [JsonPropertyName("generatedAt")] + public DateTimeOffset GeneratedAt { get; init; } + + [JsonPropertyName("components")] + public IReadOnlyList? Components { get; init; } + } + + private sealed record BomIndexImage + { + [JsonPropertyName("repository")] + public string Repository { get; init; } = string.Empty; + + [JsonPropertyName("digest")] + public string Digest { get; init; } = string.Empty; + + [JsonPropertyName("tag")] + public string? Tag { get; init; } + } + + private sealed record BomIndexComponent + { + [JsonPropertyName("purl")] + public string? Purl { get; init; } + + [JsonPropertyName("usage")] + public IReadOnlyList? Usage { get; init; } + } +} diff --git a/src/StellaOps.Scheduler.ImpactIndex/IImpactIndex.cs b/src/StellaOps.Scheduler.ImpactIndex/IImpactIndex.cs new file mode 100644 index 00000000..3730b100 --- /dev/null +++ b/src/StellaOps.Scheduler.ImpactIndex/IImpactIndex.cs @@ -0,0 +1,46 @@ +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.ImpactIndex; + +/// +/// Provides read access to the scheduler impact index. +/// +public interface IImpactIndex +{ + /// + /// Resolves the impacted image set for the provided package URLs. + /// + /// Package URLs to look up. + /// When true, restricts results to components marked as runtime/entrypoint usage. + /// Selector scoping the query. + /// Cancellation token. + ValueTask ResolveByPurlsAsync( + IEnumerable purls, + bool usageOnly, + Selector selector, + CancellationToken cancellationToken = default); + + /// + /// Resolves impacted images by vulnerability identifiers if the index has the mapping available. + /// + /// Vulnerability identifiers to look up. + /// When true, restricts results to components marked as runtime/entrypoint usage. + /// Selector scoping the query. + /// Cancellation token. + ValueTask ResolveByVulnerabilitiesAsync( + IEnumerable vulnerabilityIds, + bool usageOnly, + Selector selector, + CancellationToken cancellationToken = default); + + /// + /// Resolves all tracked images for the provided selector. + /// + /// Selector scoping the query. + /// When true, restricts results to images with entrypoint usage. + /// Cancellation token. + ValueTask ResolveAllAsync( + Selector selector, + bool usageOnly, + CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Scheduler.ImpactIndex/ImpactIndexServiceCollectionExtensions.cs b/src/StellaOps.Scheduler.ImpactIndex/ImpactIndexServiceCollectionExtensions.cs new file mode 100644 index 00000000..94d8eb8e --- /dev/null +++ b/src/StellaOps.Scheduler.ImpactIndex/ImpactIndexServiceCollectionExtensions.cs @@ -0,0 +1,26 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; + +namespace StellaOps.Scheduler.ImpactIndex; + +/// +/// ServiceCollection helpers for wiring the fixture-backed impact index. +/// +public static class ImpactIndexServiceCollectionExtensions +{ + public static IServiceCollection AddImpactIndexStub( + this IServiceCollection services, + Action? configure = null) + { + ArgumentNullException.ThrowIfNull(services); + + var options = new ImpactIndexStubOptions(); + configure?.Invoke(options); + + services.TryAddSingleton(TimeProvider.System); + services.AddSingleton(options); + services.TryAddSingleton(); + + return services; + } +} diff --git a/src/StellaOps.Scheduler.ImpactIndex/ImpactIndexStubOptions.cs b/src/StellaOps.Scheduler.ImpactIndex/ImpactIndexStubOptions.cs new file mode 100644 index 00000000..344b77d5 --- /dev/null +++ b/src/StellaOps.Scheduler.ImpactIndex/ImpactIndexStubOptions.cs @@ -0,0 +1,19 @@ +namespace StellaOps.Scheduler.ImpactIndex; + +/// +/// Options controlling the fixture-backed impact index stub. +/// +public sealed class ImpactIndexStubOptions +{ + /// + /// Optional absolute or relative directory containing BOM-Index JSON fixtures. + /// When not supplied or not found, embedded fixtures ship with the assembly are used instead. + /// + public string? FixtureDirectory { get; set; } + + /// + /// Snapshot identifier reported in the generated . + /// Defaults to samples/impact-index-stub. + /// + public string SnapshotId { get; set; } = "samples/impact-index-stub"; +} diff --git a/src/StellaOps.Scheduler.ImpactIndex/REMOVAL_NOTE.md b/src/StellaOps.Scheduler.ImpactIndex/REMOVAL_NOTE.md new file mode 100644 index 00000000..57ebcfec --- /dev/null +++ b/src/StellaOps.Scheduler.ImpactIndex/REMOVAL_NOTE.md @@ -0,0 +1,15 @@ +# ImpactIndex Stub Removal Tracker + +- **Created:** 2025-10-20 +- **Owner:** Scheduler ImpactIndex Guild +- **Reference Task:** SCHED-IMPACT-16-300 (fixture-backed stub) + +## Exit Reminder + +Replace `FixtureImpactIndex` with the roaring bitmap-backed implementation once SCHED-IMPACT-16-301/302 are completed, then delete: + +1. Stub classes (`FixtureImpactIndex`, `ImpactIndexStubOptions`, `ImpactIndexServiceCollectionExtensions`). +2. Embedded sample fixture wiring in `StellaOps.Scheduler.ImpactIndex.csproj`. +3. Temporary unit tests in `StellaOps.Scheduler.ImpactIndex.Tests`. + +Remove this file when the production ImpactIndex replaces the stub. diff --git a/src/StellaOps.Scheduler.ImpactIndex/StellaOps.Scheduler.ImpactIndex.csproj b/src/StellaOps.Scheduler.ImpactIndex/StellaOps.Scheduler.ImpactIndex.csproj index 6d665dea..21dd08d0 100644 --- a/src/StellaOps.Scheduler.ImpactIndex/StellaOps.Scheduler.ImpactIndex.csproj +++ b/src/StellaOps.Scheduler.ImpactIndex/StellaOps.Scheduler.ImpactIndex.csproj @@ -1,7 +1,11 @@ - - net10.0 - enable - enable - + + net10.0 + enable + enable + + + + diff --git a/src/StellaOps.Scheduler.ImpactIndex/TASKS.md b/src/StellaOps.Scheduler.ImpactIndex/TASKS.md index 81d7fc4b..f248b877 100644 --- a/src/StellaOps.Scheduler.ImpactIndex/TASKS.md +++ b/src/StellaOps.Scheduler.ImpactIndex/TASKS.md @@ -2,7 +2,9 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| SCHED-IMPACT-16-300 | DOING | Scheduler ImpactIndex Guild | SAMPLES-10-001 | **STUB** ingest/query using fixtures to unblock Scheduler planning (remove by SP16 end). | Stub merges fixture BOM-Index, query API returns deterministic results, removal note tracked. | +| SCHED-IMPACT-16-300 | DONE (2025-10-20) | Scheduler ImpactIndex Guild | SAMPLES-10-001 | **STUB** ingest/query using fixtures to unblock Scheduler planning (remove by SP16 end). | Stub merges fixture BOM-Index, query API returns deterministic results, removal note tracked. | | SCHED-IMPACT-16-301 | TODO | Scheduler ImpactIndex Guild | SCANNER-EMIT-10-605 | Implement ingestion of per-image BOM-Index sidecars into roaring bitmap store (contains/usedBy). | Ingestion tests process sample SBOM index; bitmaps persisted; deterministic IDs assigned. | -| SCHED-IMPACT-16-302 | TODO | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-301 | Provide query APIs (ResolveByPurls, ResolveByVulns, ResolveAll, selectors) with tenant/namespace filters. | Query functions tested; performance benchmarks documented; selectors enforce filters. | -| SCHED-IMPACT-16-303 | TODO | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-301 | Snapshot/compaction + invalidation for removed images; persistence to RocksDB/Redis per architecture. | Snapshot routine implemented; invalidation tests pass; docs describe recovery. | +| SCHED-IMPACT-16-302 | TODO | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-301 | Provide query APIs (ResolveByPurls, ResolveByVulns, ResolveAll, selectors) with tenant/namespace filters. | Query functions tested; performance benchmarks documented; selectors enforce filters. | +| SCHED-IMPACT-16-303 | TODO | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-301 | Snapshot/compaction + invalidation for removed images; persistence to RocksDB/Redis per architecture. | Snapshot routine implemented; invalidation tests pass; docs describe recovery. | + +> Removal tracking note: see `src/StellaOps.Scheduler.ImpactIndex/REMOVAL_NOTE.md` for follow-up actions once the roaring bitmap implementation lands. diff --git a/src/StellaOps.Scheduler.Models.Tests/SchedulerSchemaMigrationTests.cs b/src/StellaOps.Scheduler.Models.Tests/SchedulerSchemaMigrationTests.cs index 809a4aec..ae7428d6 100644 --- a/src/StellaOps.Scheduler.Models.Tests/SchedulerSchemaMigrationTests.cs +++ b/src/StellaOps.Scheduler.Models.Tests/SchedulerSchemaMigrationTests.cs @@ -55,18 +55,117 @@ public sealed class SchedulerSchemaMigrationTests } [Fact] - public void UpgradeImpactSet_ThrowsForUnsupportedVersion() - { - var impactSet = new ImpactSet( - selector: new Selector(SelectorScope.AllImages, "tenant-alpha"), - images: Array.Empty(), + public void UpgradeImpactSet_ThrowsForUnsupportedVersion() + { + var impactSet = new ImpactSet( + selector: new Selector(SelectorScope.AllImages, "tenant-alpha"), + images: Array.Empty(), usageOnly: false, generatedAt: DateTimeOffset.Parse("2025-10-18T02:00:00Z")); var json = JsonNode.Parse(CanonicalJsonSerializer.Serialize(impactSet))!.AsObject(); json["schemaVersion"] = "scheduler.impact-set@99"; - - var ex = Assert.Throws(() => SchedulerSchemaMigration.UpgradeImpactSet(json)); - Assert.Contains("Unsupported scheduler schema version", ex.Message, StringComparison.Ordinal); - } -} + + var ex = Assert.Throws(() => SchedulerSchemaMigration.UpgradeImpactSet(json)); + Assert.Contains("Unsupported scheduler schema version", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void UpgradeSchedule_Legacy0_UpgradesToLatestVersion() + { + var legacy = new JsonObject + { + ["schemaVersion"] = SchedulerSchemaVersions.ScheduleLegacy0, + ["id"] = "sch-legacy", + ["tenantId"] = "tenant-alpha", + ["name"] = "Legacy Nightly", + ["enabled"] = true, + ["cronExpression"] = "0 2 * * *", + ["timezone"] = "UTC", + ["mode"] = "analysis-only", + ["selection"] = new JsonObject + { + ["scope"] = "all-images", + ["tenantId"] = "tenant-alpha", + }, + ["notify"] = new JsonObject + { + ["onNewFindings"] = "true", + ["minSeverity"] = "HIGH", + }, + ["limits"] = new JsonObject + { + ["maxJobs"] = "5", + ["parallelism"] = -2, + }, + ["subscribers"] = "ops-team", + ["createdAt"] = "2025-10-10T00:00:00Z", + ["createdBy"] = "system", + ["updatedAt"] = "2025-10-10T01:00:00Z", + ["updatedBy"] = "system", + }; + + var result = SchedulerSchemaMigration.UpgradeSchedule(legacy, strict: true); + + Assert.Equal(SchedulerSchemaVersions.ScheduleLegacy0, result.FromVersion); + Assert.Equal(SchedulerSchemaVersions.Schedule, result.ToVersion); + Assert.Equal(SchedulerSchemaVersions.Schedule, result.Value.SchemaVersion); + Assert.True(result.Value.Notify.IncludeKev); + Assert.Empty(result.Value.Subscribers); + Assert.Contains(result.Warnings, warning => warning.Contains("schedule.limits.parallelism", StringComparison.Ordinal)); + Assert.Contains(result.Warnings, warning => warning.Contains("schedule.subscribers", StringComparison.Ordinal)); + } + + [Fact] + public void UpgradeRun_Legacy0_BackfillsMissingStats() + { + var legacy = new JsonObject + { + ["schemaVersion"] = SchedulerSchemaVersions.RunLegacy0, + ["id"] = "run-legacy", + ["tenantId"] = "tenant-alpha", + ["trigger"] = "manual", + ["state"] = "queued", + ["stats"] = new JsonObject + { + ["candidates"] = "4", + ["queued"] = 2, + }, + ["createdAt"] = "2025-10-10T02:00:00Z", + }; + + var result = SchedulerSchemaMigration.UpgradeRun(legacy, strict: true); + + Assert.Equal(SchedulerSchemaVersions.RunLegacy0, result.FromVersion); + Assert.Equal(SchedulerSchemaVersions.Run, result.ToVersion); + Assert.Equal(SchedulerSchemaVersions.Run, result.Value.SchemaVersion); + Assert.Equal(4, result.Value.Stats.Candidates); + Assert.Equal(0, result.Value.Stats.NewMedium); + Assert.Equal(RunState.Queued, result.Value.State); + Assert.Empty(result.Value.Deltas); + Assert.Contains(result.Warnings, warning => warning.Contains("run.stats.newMedium", StringComparison.Ordinal)); + } + + [Fact] + public void UpgradeImpactSet_Legacy0_ComputesTotal() + { + var legacy = new JsonObject + { + ["schemaVersion"] = SchedulerSchemaVersions.ImpactSetLegacy0, + ["selector"] = JsonNode.Parse("""{"scope":"all-images","tenantId":"tenant-alpha"}"""), + ["images"] = new JsonArray( + JsonNode.Parse("""{"imageDigest":"sha256:1111111111111111111111111111111111111111111111111111111111111111","registry":"docker.io","repository":"library/nginx"}"""), + JsonNode.Parse("""{"imageDigest":"sha256:2222222222222222222222222222222222222222222222222222222222222222","registry":"docker.io","repository":"library/httpd"}""")), + ["usageOnly"] = "false", + ["generatedAt"] = "2025-10-10T03:00:00Z", + }; + + var result = SchedulerSchemaMigration.UpgradeImpactSet(legacy, strict: true); + + Assert.Equal(SchedulerSchemaVersions.ImpactSetLegacy0, result.FromVersion); + Assert.Equal(SchedulerSchemaVersions.ImpactSet, result.ToVersion); + Assert.Equal(2, result.Value.Total); + Assert.Equal(2, result.Value.Images.Length); + Assert.Contains(result.Warnings, warning => warning.Contains("impact set total", StringComparison.OrdinalIgnoreCase)); + } +} diff --git a/src/StellaOps.Scheduler.Models/SchedulerSchemaMigration.cs b/src/StellaOps.Scheduler.Models/SchedulerSchemaMigration.cs index 8b6e8f48..5a2127d3 100644 --- a/src/StellaOps.Scheduler.Models/SchedulerSchemaMigration.cs +++ b/src/StellaOps.Scheduler.Models/SchedulerSchemaMigration.cs @@ -1,9 +1,10 @@ -using System.Collections.Immutable; -using System.Linq; -using System.Text.Json; -using System.Text.Json.Nodes; - -namespace StellaOps.Scheduler.Models; +using System.Collections.Immutable; +using System.Globalization; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace StellaOps.Scheduler.Models; /// /// Upgrades scheduler documents emitted by earlier schema revisions to the latest DTOs. @@ -56,64 +57,73 @@ public static class SchedulerSchemaMigration "total", "snapshotId"); - public static SchedulerSchemaMigrationResult UpgradeSchedule(JsonNode document, bool strict = false) - => Upgrade( - document, - SchedulerSchemaVersions.Schedule, - SchedulerSchemaVersions.EnsureSchedule, - ScheduleProperties, - static json => CanonicalJsonSerializer.Deserialize(json), - strict); - - public static SchedulerSchemaMigrationResult UpgradeRun(JsonNode document, bool strict = false) - => Upgrade( - document, - SchedulerSchemaVersions.Run, - SchedulerSchemaVersions.EnsureRun, - RunProperties, - static json => CanonicalJsonSerializer.Deserialize(json), - strict); - - public static SchedulerSchemaMigrationResult UpgradeImpactSet(JsonNode document, bool strict = false) - => Upgrade( - document, - SchedulerSchemaVersions.ImpactSet, - SchedulerSchemaVersions.EnsureImpactSet, - ImpactSetProperties, - static json => CanonicalJsonSerializer.Deserialize(json), - strict); - - private static SchedulerSchemaMigrationResult Upgrade( - JsonNode document, - string latestVersion, - Func ensureVersion, - ImmutableHashSet knownProperties, - Func deserialize, - bool strict) - { - ArgumentNullException.ThrowIfNull(document); - - var (normalized, fromVersion) = Normalize(document, ensureVersion); - var warnings = ImmutableArray.CreateBuilder(); - - if (strict) - { - RemoveUnknownMembers(normalized, knownProperties, warnings, fromVersion); - } - - if (!string.Equals(fromVersion, latestVersion, StringComparison.Ordinal)) - { - // Placeholder for forward upgrades once schema@2 exists. - throw new NotSupportedException($"Unsupported scheduler schema version '{fromVersion}', expected '{latestVersion}'."); - } - - var canonicalJson = normalized.ToJsonString(new JsonSerializerOptions - { - WriteIndented = false, - }); - - var value = deserialize(canonicalJson); - return new SchedulerSchemaMigrationResult( + public static SchedulerSchemaMigrationResult UpgradeSchedule(JsonNode document, bool strict = false) + => Upgrade( + document, + SchedulerSchemaVersions.Schedule, + SchedulerSchemaVersions.EnsureSchedule, + ScheduleProperties, + static json => CanonicalJsonSerializer.Deserialize(json), + ApplyScheduleLegacyFixups, + strict); + + public static SchedulerSchemaMigrationResult UpgradeRun(JsonNode document, bool strict = false) + => Upgrade( + document, + SchedulerSchemaVersions.Run, + SchedulerSchemaVersions.EnsureRun, + RunProperties, + static json => CanonicalJsonSerializer.Deserialize(json), + ApplyRunLegacyFixups, + strict); + + public static SchedulerSchemaMigrationResult UpgradeImpactSet(JsonNode document, bool strict = false) + => Upgrade( + document, + SchedulerSchemaVersions.ImpactSet, + SchedulerSchemaVersions.EnsureImpactSet, + ImpactSetProperties, + static json => CanonicalJsonSerializer.Deserialize(json), + ApplyImpactSetLegacyFixups, + strict); + + private static SchedulerSchemaMigrationResult Upgrade( + JsonNode document, + string latestVersion, + Func ensureVersion, + ImmutableHashSet knownProperties, + Func deserialize, + Func.Builder, bool> applyLegacyFixups, + bool strict) + { + ArgumentNullException.ThrowIfNull(document); + + var (normalized, fromVersion) = Normalize(document, ensureVersion); + var warnings = ImmutableArray.CreateBuilder(); + + if (!string.Equals(fromVersion, latestVersion, StringComparison.Ordinal)) + { + var upgraded = applyLegacyFixups(normalized, fromVersion, warnings); + if (!upgraded) + { + throw new NotSupportedException($"Unsupported scheduler schema version '{fromVersion}', expected '{latestVersion}'."); + } + + normalized["schemaVersion"] = latestVersion; + } + + if (strict) + { + RemoveUnknownMembers(normalized, knownProperties, warnings, fromVersion); + } + + var canonicalJson = normalized.ToJsonString(new JsonSerializerOptions + { + WriteIndented = false, + }); + + var value = deserialize(canonicalJson); + return new SchedulerSchemaMigrationResult( value, fromVersion, latestVersion, @@ -151,22 +161,294 @@ public static class SchedulerSchemaMigration return (clone, schemaVersion); } - private static void RemoveUnknownMembers( - JsonObject json, - ImmutableHashSet knownProperties, - ImmutableArray.Builder warnings, - string schemaVersion) - { - var unknownKeys = json - .Where(static pair => pair.Key is not null) - .Select(pair => pair.Key!) - .Where(key => !knownProperties.Contains(key)) - .ToArray(); - - foreach (var key in unknownKeys) - { - json.Remove(key); - warnings.Add($"Removed unknown property '{key}' from scheduler document (schemaVersion={schemaVersion})."); - } - } -} + private static void RemoveUnknownMembers( + JsonObject json, + ImmutableHashSet knownProperties, + ImmutableArray.Builder warnings, + string schemaVersion) + { + var unknownKeys = json + .Where(static pair => pair.Key is not null) + .Select(pair => pair.Key!) + .Where(key => !knownProperties.Contains(key)) + .ToArray(); + + foreach (var key in unknownKeys) + { + json.Remove(key); + warnings.Add($"Removed unknown property '{key}' from scheduler document (schemaVersion={schemaVersion})."); + } + } + + private static bool ApplyScheduleLegacyFixups( + JsonObject json, + string fromVersion, + ImmutableArray.Builder warnings) + { + switch (fromVersion) + { + case SchedulerSchemaVersions.ScheduleLegacy0: + var limits = EnsureObject(json, "limits", () => new JsonObject(), warnings, "schedule", fromVersion); + NormalizePositiveInt(limits, "maxJobs", warnings, "schedule.limits", fromVersion); + NormalizePositiveInt(limits, "ratePerSecond", warnings, "schedule.limits", fromVersion); + NormalizePositiveInt(limits, "parallelism", warnings, "schedule.limits", fromVersion); + NormalizePositiveInt(limits, "burst", warnings, "schedule.limits", fromVersion); + + var notify = EnsureObject(json, "notify", () => new JsonObject(), warnings, "schedule", fromVersion); + NormalizeBoolean(notify, "onNewFindings", defaultValue: true, warnings, "schedule.notify", fromVersion); + NormalizeSeverity(notify, "minSeverity", warnings, "schedule.notify", fromVersion); + NormalizeBoolean(notify, "includeKev", defaultValue: true, warnings, "schedule.notify", fromVersion); + NormalizeBoolean(notify, "includeQuietFindings", defaultValue: false, warnings, "schedule.notify", fromVersion); + + var onlyIf = EnsureObject(json, "onlyIf", () => new JsonObject(), warnings, "schedule", fromVersion); + NormalizePositiveInt(onlyIf, "lastReportOlderThanDays", warnings, "schedule.onlyIf", fromVersion, allowZero: false); + + EnsureArray(json, "subscribers", warnings, "schedule", fromVersion); + return true; + default: + return false; + } + } + + private static bool ApplyRunLegacyFixups( + JsonObject json, + string fromVersion, + ImmutableArray.Builder warnings) + { + switch (fromVersion) + { + case SchedulerSchemaVersions.RunLegacy0: + var stats = EnsureObject(json, "stats", () => new JsonObject(), warnings, "run", fromVersion); + NormalizeNonNegativeInt(stats, "candidates", warnings, "run.stats", fromVersion); + NormalizeNonNegativeInt(stats, "deduped", warnings, "run.stats", fromVersion); + NormalizeNonNegativeInt(stats, "queued", warnings, "run.stats", fromVersion); + NormalizeNonNegativeInt(stats, "completed", warnings, "run.stats", fromVersion); + NormalizeNonNegativeInt(stats, "deltas", warnings, "run.stats", fromVersion); + NormalizeNonNegativeInt(stats, "newCriticals", warnings, "run.stats", fromVersion); + NormalizeNonNegativeInt(stats, "newHigh", warnings, "run.stats", fromVersion); + NormalizeNonNegativeInt(stats, "newMedium", warnings, "run.stats", fromVersion); + NormalizeNonNegativeInt(stats, "newLow", warnings, "run.stats", fromVersion); + + EnsureObject(json, "reason", () => new JsonObject(), warnings, "run", fromVersion); + EnsureArray(json, "deltas", warnings, "run", fromVersion); + return true; + default: + return false; + } + } + + private static bool ApplyImpactSetLegacyFixups( + JsonObject json, + string fromVersion, + ImmutableArray.Builder warnings) + { + switch (fromVersion) + { + case SchedulerSchemaVersions.ImpactSetLegacy0: + var images = EnsureArray(json, "images", warnings, "impact-set", fromVersion); + NormalizeBoolean(json, "usageOnly", defaultValue: false, warnings, "impact-set", fromVersion); + + if (!json.TryGetPropertyValue("total", out var totalNode) || !TryReadNonNegative(totalNode, out var total)) + { + var computed = images.Count; + json["total"] = computed; + warnings.Add($"Backfilled impact set total with image count ({computed}) while upgrading from {fromVersion}."); + } + else + { + var computed = images.Count; + if (total != computed) + { + json["total"] = computed; + warnings.Add($"Normalized impact set total to image count ({computed}) while upgrading from {fromVersion}."); + } + } + + return true; + default: + return false; + } + } + + private static JsonObject EnsureObject( + JsonObject parent, + string propertyName, + Func factory, + ImmutableArray.Builder warnings, + string context, + string fromVersion) + { + if (parent.TryGetPropertyValue(propertyName, out var node) && node is JsonObject obj) + { + return obj; + } + + var created = factory(); + parent[propertyName] = created; + warnings.Add($"Inserted default '{context}.{propertyName}' object while upgrading from {fromVersion}."); + return created; + } + + private static JsonArray EnsureArray( + JsonObject parent, + string propertyName, + ImmutableArray.Builder warnings, + string context, + string fromVersion) + { + if (parent.TryGetPropertyValue(propertyName, out var node) && node is JsonArray array) + { + return array; + } + + var created = new JsonArray(); + parent[propertyName] = created; + warnings.Add($"Inserted empty '{context}.{propertyName}' array while upgrading from {fromVersion}."); + return created; + } + + private static void NormalizePositiveInt( + JsonObject obj, + string propertyName, + ImmutableArray.Builder warnings, + string context, + string fromVersion, + bool allowZero = false) + { + if (!obj.TryGetPropertyValue(propertyName, out var node)) + { + return; + } + + if (!TryReadInt(node, out var value)) + { + obj.Remove(propertyName); + warnings.Add($"Removed invalid '{context}.{propertyName}' while upgrading from {fromVersion}."); + return; + } + + if ((!allowZero && value <= 0) || (allowZero && value < 0)) + { + obj.Remove(propertyName); + warnings.Add($"Removed non-positive '{context}.{propertyName}' value while upgrading from {fromVersion}."); + return; + } + + obj[propertyName] = value; + } + + private static void NormalizeNonNegativeInt( + JsonObject obj, + string propertyName, + ImmutableArray.Builder warnings, + string context, + string fromVersion) + { + if (!obj.TryGetPropertyValue(propertyName, out var node) || !TryReadNonNegative(node, out var value)) + { + obj[propertyName] = 0; + warnings.Add($"Defaulted '{context}.{propertyName}' to 0 while upgrading from {fromVersion}."); + return; + } + + obj[propertyName] = value; + } + + private static void NormalizeBoolean( + JsonObject obj, + string propertyName, + bool defaultValue, + ImmutableArray.Builder warnings, + string context, + string fromVersion) + { + if (!obj.TryGetPropertyValue(propertyName, out var node)) + { + obj[propertyName] = defaultValue; + warnings.Add($"Defaulted '{context}.{propertyName}' to {defaultValue.ToString().ToLowerInvariant()} while upgrading from {fromVersion}."); + return; + } + + if (node is JsonValue value && value.TryGetValue(out bool parsed)) + { + obj[propertyName] = parsed; + return; + } + + if (node is JsonValue strValue && strValue.TryGetValue(out string? text) && + bool.TryParse(text, out var parsedFromString)) + { + obj[propertyName] = parsedFromString; + return; + } + + obj[propertyName] = defaultValue; + warnings.Add($"Normalized '{context}.{propertyName}' to {defaultValue.ToString().ToLowerInvariant()} while upgrading from {fromVersion}."); + } + + private static void NormalizeSeverity( + JsonObject obj, + string propertyName, + ImmutableArray.Builder warnings, + string context, + string fromVersion) + { + if (!obj.TryGetPropertyValue(propertyName, out var node)) + { + return; + } + + if (node is JsonValue value) + { + if (value.TryGetValue(out string? text)) + { + if (Enum.TryParse(text, ignoreCase: true, out var parsed)) + { + obj[propertyName] = parsed.ToString().ToLowerInvariant(); + return; + } + } + + if (value.TryGetValue(out int numeric) && Enum.IsDefined(typeof(SeverityRank), numeric)) + { + var enumValue = (SeverityRank)numeric; + obj[propertyName] = enumValue.ToString().ToLowerInvariant(); + return; + } + } + + obj.Remove(propertyName); + warnings.Add($"Removed invalid '{context}.{propertyName}' while upgrading from {fromVersion}."); + } + + private static bool TryReadNonNegative(JsonNode? node, out int value) + => TryReadInt(node, out value) && value >= 0; + + private static bool TryReadInt(JsonNode? node, out int value) + { + if (node is JsonValue valueNode) + { + if (valueNode.TryGetValue(out int intValue)) + { + value = intValue; + return true; + } + + if (valueNode.TryGetValue(out long longValue) && longValue is >= int.MinValue and <= int.MaxValue) + { + value = (int)longValue; + return true; + } + + if (valueNode.TryGetValue(out string? text) && + int.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed)) + { + value = parsed; + return true; + } + } + + value = 0; + return false; + } +} diff --git a/src/StellaOps.Scheduler.Models/TASKS.md b/src/StellaOps.Scheduler.Models/TASKS.md index fd871f78..73b44c02 100644 --- a/src/StellaOps.Scheduler.Models/TASKS.md +++ b/src/StellaOps.Scheduler.Models/TASKS.md @@ -4,4 +4,4 @@ |----|--------|----------|------------|-------------|---------------| | SCHED-MODELS-16-101 | DONE (2025-10-19) | Scheduler Models Guild | — | Define DTOs (Schedule, Run, ImpactSet, Selector, DeltaSummary, AuditRecord) with validation + canonical JSON. | DTOs merged with tests; documentation snippet added; serialization deterministic. | | SCHED-MODELS-16-102 | DONE (2025-10-19) | Scheduler Models Guild | SCHED-MODELS-16-101 | Publish schema docs & sample payloads for UI/Notify integration. | Samples committed; docs referenced; contract tests pass. | -| SCHED-MODELS-16-103 | DOING (2025-10-19) | Scheduler Models Guild | SCHED-MODELS-16-101 | Versioning/migration helpers (schedule evolution, run state transitions). | Migration helpers implemented; tests cover upgrade/downgrade; guidelines documented. | +| SCHED-MODELS-16-103 | DONE (2025-10-20) | Scheduler Models Guild | SCHED-MODELS-16-101 | Versioning/migration helpers (schedule evolution, run state transitions). | Migration helpers implemented; tests cover upgrade/downgrade; guidelines documented. | diff --git a/src/StellaOps.Scheduler.Models/docs/SCHED-MODELS-16-103-DESIGN.md b/src/StellaOps.Scheduler.Models/docs/SCHED-MODELS-16-103-DESIGN.md index adae0c7a..daf2cfe6 100644 --- a/src/StellaOps.Scheduler.Models/docs/SCHED-MODELS-16-103-DESIGN.md +++ b/src/StellaOps.Scheduler.Models/docs/SCHED-MODELS-16-103-DESIGN.md @@ -74,7 +74,13 @@ - **Serialization determinism**: round-trip upgraded DTOs via `CanonicalJsonSerializer` to confirm property order includes `schemaVersion` first and produces stable hashes. - **Documentation snippets**: extend module README or API docs with example migrations/run-state usage; verify via doc samples test (if available) or include as part of CI doc linting. -## Open Questions -- Do we need downgrade (`ToVersion`) helpers for Offline Kit exports? (Assumed no for now. Add backlog item if required.) -- Should `ImpactSet` migrations live here or in ImpactIndex module? (Lean towards here because DTO defined in Models; coordinate with ImpactIndex guild if they need specialized upgrades.) -- How do we surface migration warnings to telemetry? Proposal: caller logs `warning` with `MigrationResult.Warnings` immediately after calling helper. +## Open Questions +- Do we need downgrade (`ToVersion`) helpers for Offline Kit exports? (Assumed no for now. Add backlog item if required.) +- Should `ImpactSet` migrations live here or in ImpactIndex module? (Lean towards here because DTO defined in Models; coordinate with ImpactIndex guild if they need specialized upgrades.) +- How do we surface migration warnings to telemetry? Proposal: caller logs `warning` with `MigrationResult.Warnings` immediately after calling helper. + +## Status — 2025-10-20 + +- `SchedulerSchemaMigration` now upgrades legacy `@0` schedule/run/impact-set documents to the `@1` schema, defaulting missing counters/arrays and normalizing booleans & severities. Each backfill emits a warning so storage/web callers can log the mutation. +- `RunStateMachine.EnsureTransition` guards timestamp ordering and stats monotonicity; builders and extension helpers are wired into the scheduler worker/web service plans. +- Tests exercising legacy upgrades live in `StellaOps.Scheduler.Models.Tests/SchedulerSchemaMigrationTests.cs`; add new fixtures there when introducing additional schema versions. diff --git a/src/StellaOps.Scheduler.Queue.Tests/RedisSchedulerQueueTests.cs b/src/StellaOps.Scheduler.Queue.Tests/RedisSchedulerQueueTests.cs index a5f2302a..c8f41386 100644 --- a/src/StellaOps.Scheduler.Queue.Tests/RedisSchedulerQueueTests.cs +++ b/src/StellaOps.Scheduler.Queue.Tests/RedisSchedulerQueueTests.cs @@ -1,5 +1,6 @@ using System; -using System.Collections.Generic; +using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using DotNet.Testcontainers.Builders; using DotNet.Testcontainers.Containers; @@ -9,7 +10,7 @@ using Microsoft.Extensions.Logging.Abstractions; using StackExchange.Redis; using StellaOps.Scheduler.Models; using StellaOps.Scheduler.Queue.Redis; -using Xunit; +using Xunit; namespace StellaOps.Scheduler.Queue.Tests; @@ -52,7 +53,10 @@ public sealed class RedisSchedulerQueueTests : IAsyncLifetime [Fact] public async Task PlannerQueue_EnqueueLeaseAck_RemovesMessage() { - SkipIfUnavailable(); + if (SkipIfUnavailable()) + { + return; + } var options = CreateOptions(); @@ -83,9 +87,12 @@ public sealed class RedisSchedulerQueueTests : IAsyncLifetime } [Fact] - public async Task RunnerQueue_Retry_IncrementsDeliveryAttempt() + public async Task RunnerQueue_Retry_IncrementsDeliveryAttempt() { - SkipIfUnavailable(); + if (SkipIfUnavailable()) + { + return; + } var options = CreateOptions(); options.RetryInitialBackoff = TimeSpan.Zero; @@ -118,7 +125,10 @@ public sealed class RedisSchedulerQueueTests : IAsyncLifetime [Fact] public async Task PlannerQueue_ClaimExpired_ReassignsLease() { - SkipIfUnavailable(); + if (SkipIfUnavailable()) + { + return; + } var options = CreateOptions(); @@ -142,8 +152,74 @@ public sealed class RedisSchedulerQueueTests : IAsyncLifetime reclaimed[0].Consumer.Should().Be("planner-b"); reclaimed[0].RunId.Should().Be(message.Run.Id); - await reclaimed[0].AcknowledgeAsync(); - } + await reclaimed[0].AcknowledgeAsync(); + } + + [Fact] + public async Task PlannerQueue_RecordsDepthMetrics() + { + if (SkipIfUnavailable()) + { + return; + } + + var options = CreateOptions(); + + await using var queue = new RedisSchedulerPlannerQueue( + options, + options.Redis, + NullLogger.Instance, + TimeProvider.System, + async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false)); + + var message = TestData.CreatePlannerMessage(); + + await queue.EnqueueAsync(message); + + var depths = SchedulerQueueMetrics.SnapshotDepths(); + depths.TryGetValue(("redis", "planner"), out var plannerDepth) + .Should().BeTrue(); + plannerDepth.Should().Be(1); + + var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("planner-depth", 1, options.DefaultLeaseDuration)); + leases.Should().ContainSingle(); + await leases[0].AcknowledgeAsync(); + + depths = SchedulerQueueMetrics.SnapshotDepths(); + depths.TryGetValue(("redis", "planner"), out plannerDepth).Should().BeTrue(); + plannerDepth.Should().Be(0); + } + + [Fact] + public async Task RunnerQueue_DropWhenDeadLetterDisabled() + { + if (SkipIfUnavailable()) + { + return; + } + + var options = CreateOptions(); + options.MaxDeliveryAttempts = 1; + options.DeadLetterEnabled = false; + + await using var queue = new RedisSchedulerRunnerQueue( + options, + options.Redis, + NullLogger.Instance, + TimeProvider.System, + async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false)); + + var message = TestData.CreateRunnerMessage(); + await queue.EnqueueAsync(message); + + var lease = (await queue.LeaseAsync(new SchedulerQueueLeaseRequest("runner-drop", 1, options.DefaultLeaseDuration)))[0]; + + await lease.ReleaseAsync(SchedulerQueueReleaseDisposition.Retry); + + var depths = SchedulerQueueMetrics.SnapshotDepths(); + depths.TryGetValue(("redis", "runner"), out var runnerDepth).Should().BeTrue(); + runnerDepth.Should().Be(0); + } private SchedulerQueueOptions CreateOptions() { @@ -181,13 +257,14 @@ public sealed class RedisSchedulerQueueTests : IAsyncLifetime }; } - private void SkipIfUnavailable() - { - if (_skipReason is not null) - { - Skip.If(true, _skipReason); - } - } + private bool SkipIfUnavailable() + { + if (_skipReason is not null) + { + return true; + } + return false; + } private static bool IsDockerUnavailable(Exception exception) { diff --git a/src/StellaOps.Scheduler.Queue.Tests/SchedulerQueueServiceCollectionExtensionsTests.cs b/src/StellaOps.Scheduler.Queue.Tests/SchedulerQueueServiceCollectionExtensionsTests.cs new file mode 100644 index 00000000..56eeccaa --- /dev/null +++ b/src/StellaOps.Scheduler.Queue.Tests/SchedulerQueueServiceCollectionExtensionsTests.cs @@ -0,0 +1,115 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Scheduler.Models; +using StellaOps.Scheduler.Queue; +using StellaOps.Scheduler.Queue.Nats; +using Xunit; + +namespace StellaOps.Scheduler.Queue.Tests; + +public sealed class SchedulerQueueServiceCollectionExtensionsTests +{ + [Fact] + public async Task AddSchedulerQueues_RegistersNatsTransport() + { + var services = new ServiceCollection(); + services.AddSingleton(_ => NullLoggerFactory.Instance); + services.AddSchedulerQueues(new ConfigurationBuilder().Build()); + + var optionsDescriptor = services.First(descriptor => descriptor.ServiceType == typeof(SchedulerQueueOptions)); + var options = (SchedulerQueueOptions)optionsDescriptor.ImplementationInstance!; + options.Kind = SchedulerQueueTransportKind.Nats; + options.Nats.Url = "nats://localhost:4222"; + + await using var provider = services.BuildServiceProvider(); + + var plannerQueue = provider.GetRequiredService(); + var runnerQueue = provider.GetRequiredService(); + + plannerQueue.Should().BeOfType(); + runnerQueue.Should().BeOfType(); + } + + [Fact] + public async Task SchedulerQueueHealthCheck_ReturnsHealthy_WhenTransportsReachable() + { + var healthCheck = new SchedulerQueueHealthCheck( + new FakePlannerQueue(failPing: false), + new FakeRunnerQueue(failPing: false), + NullLogger.Instance); + + var context = new HealthCheckContext + { + Registration = new HealthCheckRegistration("scheduler-queue", healthCheck, HealthStatus.Unhealthy, Array.Empty()) + }; + + var result = await healthCheck.CheckHealthAsync(context); + + result.Status.Should().Be(HealthStatus.Healthy); + } + + [Fact] + public async Task SchedulerQueueHealthCheck_ReturnsUnhealthy_WhenRunnerPingFails() + { + var healthCheck = new SchedulerQueueHealthCheck( + new FakePlannerQueue(failPing: false), + new FakeRunnerQueue(failPing: true), + NullLogger.Instance); + + var context = new HealthCheckContext + { + Registration = new HealthCheckRegistration("scheduler-queue", healthCheck, HealthStatus.Unhealthy, Array.Empty()) + }; + + var result = await healthCheck.CheckHealthAsync(context); + + result.Status.Should().Be(HealthStatus.Unhealthy); + result.Description.Should().Contain("runner transport unreachable"); + } + private abstract class FakeQueue : ISchedulerQueue, ISchedulerQueueTransportDiagnostics + { + private readonly bool _failPing; + + protected FakeQueue(bool failPing) + { + _failPing = failPing; + } + + public ValueTask EnqueueAsync(TMessage message, CancellationToken cancellationToken = default) + => ValueTask.FromResult(new SchedulerQueueEnqueueResult("stub", false)); + + public ValueTask>> LeaseAsync(SchedulerQueueLeaseRequest request, CancellationToken cancellationToken = default) + => ValueTask.FromResult>>(Array.Empty>()); + + public ValueTask>> ClaimExpiredAsync(SchedulerQueueClaimOptions options, CancellationToken cancellationToken = default) + => ValueTask.FromResult>>(Array.Empty>()); + + public ValueTask PingAsync(CancellationToken cancellationToken) + => _failPing + ? ValueTask.FromException(new InvalidOperationException("ping failed")) + : ValueTask.CompletedTask; + } + + private sealed class FakePlannerQueue : FakeQueue, ISchedulerPlannerQueue + { + public FakePlannerQueue(bool failPing) : base(failPing) + { + } + } + + private sealed class FakeRunnerQueue : FakeQueue, ISchedulerRunnerQueue + { + public FakeRunnerQueue(bool failPing) : base(failPing) + { + } + } +} diff --git a/src/StellaOps.Scheduler.Queue.Tests/StellaOps.Scheduler.Queue.Tests.csproj b/src/StellaOps.Scheduler.Queue.Tests/StellaOps.Scheduler.Queue.Tests.csproj index 84663c3f..bce5c247 100644 --- a/src/StellaOps.Scheduler.Queue.Tests/StellaOps.Scheduler.Queue.Tests.csproj +++ b/src/StellaOps.Scheduler.Queue.Tests/StellaOps.Scheduler.Queue.Tests.csproj @@ -7,10 +7,12 @@ false - - - - + + + + + + all diff --git a/src/StellaOps.Scheduler.Queue/ISchedulerQueueTransportDiagnostics.cs b/src/StellaOps.Scheduler.Queue/ISchedulerQueueTransportDiagnostics.cs new file mode 100644 index 00000000..f7fdbcd6 --- /dev/null +++ b/src/StellaOps.Scheduler.Queue/ISchedulerQueueTransportDiagnostics.cs @@ -0,0 +1,9 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Scheduler.Queue; + +internal interface ISchedulerQueueTransportDiagnostics +{ + ValueTask PingAsync(CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Scheduler.Queue/Nats/INatsSchedulerQueuePayload.cs b/src/StellaOps.Scheduler.Queue/Nats/INatsSchedulerQueuePayload.cs new file mode 100644 index 00000000..567fd733 --- /dev/null +++ b/src/StellaOps.Scheduler.Queue/Nats/INatsSchedulerQueuePayload.cs @@ -0,0 +1,26 @@ +using System.Collections.Generic; + +namespace StellaOps.Scheduler.Queue.Nats; + +internal interface INatsSchedulerQueuePayload +{ + string QueueName { get; } + + string GetIdempotencyKey(TMessage message); + + byte[] Serialize(TMessage message); + + TMessage Deserialize(byte[] payload); + + string GetRunId(TMessage message); + + string GetTenantId(TMessage message); + + string? GetScheduleId(TMessage message); + + string? GetSegmentId(TMessage message); + + string? GetCorrelationId(TMessage message); + + IReadOnlyDictionary? GetAttributes(TMessage message); +} diff --git a/src/StellaOps.Scheduler.Queue/Nats/NatsSchedulerPlannerQueue.cs b/src/StellaOps.Scheduler.Queue/Nats/NatsSchedulerPlannerQueue.cs new file mode 100644 index 00000000..37416b7d --- /dev/null +++ b/src/StellaOps.Scheduler.Queue/Nats/NatsSchedulerPlannerQueue.cs @@ -0,0 +1,66 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NATS.Client.Core; +using NATS.Client.JetStream; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Queue.Nats; + +internal sealed class NatsSchedulerPlannerQueue + : NatsSchedulerQueueBase, ISchedulerPlannerQueue +{ + public NatsSchedulerPlannerQueue( + SchedulerQueueOptions queueOptions, + SchedulerNatsQueueOptions natsOptions, + ILogger logger, + TimeProvider timeProvider, + Func>? connectionFactory = null) + : base( + queueOptions, + natsOptions, + natsOptions.Planner, + PlannerPayload.Instance, + logger, + timeProvider, + connectionFactory) + { + } + + private sealed class PlannerPayload : INatsSchedulerQueuePayload + { + public static PlannerPayload Instance { get; } = new(); + + public string QueueName => "planner"; + + public string GetIdempotencyKey(PlannerQueueMessage message) + => message.IdempotencyKey; + + public byte[] Serialize(PlannerQueueMessage message) + => Encoding.UTF8.GetBytes(CanonicalJsonSerializer.Serialize(message)); + + public PlannerQueueMessage Deserialize(byte[] payload) + => CanonicalJsonSerializer.Deserialize(Encoding.UTF8.GetString(payload)); + + public string GetRunId(PlannerQueueMessage message) + => message.Run.Id; + + public string GetTenantId(PlannerQueueMessage message) + => message.Run.TenantId; + + public string? GetScheduleId(PlannerQueueMessage message) + => message.ScheduleId; + + public string? GetSegmentId(PlannerQueueMessage message) + => null; + + public string? GetCorrelationId(PlannerQueueMessage message) + => message.CorrelationId; + + public IReadOnlyDictionary? GetAttributes(PlannerQueueMessage message) + => null; + } +} diff --git a/src/StellaOps.Scheduler.Queue/Nats/NatsSchedulerQueueBase.cs b/src/StellaOps.Scheduler.Queue/Nats/NatsSchedulerQueueBase.cs new file mode 100644 index 00000000..c14514a3 --- /dev/null +++ b/src/StellaOps.Scheduler.Queue/Nats/NatsSchedulerQueueBase.cs @@ -0,0 +1,692 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NATS.Client.Core; +using NATS.Client.JetStream; +using NATS.Client.JetStream.Models; + +namespace StellaOps.Scheduler.Queue.Nats; + +internal abstract class NatsSchedulerQueueBase : ISchedulerQueue, IAsyncDisposable, ISchedulerQueueTransportDiagnostics +{ + private const string TransportName = "nats"; + + private static readonly INatsSerializer PayloadSerializer = NatsRawSerializer.Default; + + private readonly SchedulerQueueOptions _queueOptions; + private readonly SchedulerNatsQueueOptions _natsOptions; + private readonly SchedulerNatsStreamOptions _streamOptions; + private readonly INatsSchedulerQueuePayload _payload; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly SemaphoreSlim _connectionGate = new(1, 1); + private readonly Func> _connectionFactory; + + private NatsConnection? _connection; + private NatsJSContext? _jsContext; + private INatsJSConsumer? _consumer; + private bool _disposed; + private long _approximateDepth; + + protected NatsSchedulerQueueBase( + SchedulerQueueOptions queueOptions, + SchedulerNatsQueueOptions natsOptions, + SchedulerNatsStreamOptions streamOptions, + INatsSchedulerQueuePayload payload, + ILogger logger, + TimeProvider timeProvider, + Func>? connectionFactory = null) + { + _queueOptions = queueOptions ?? throw new ArgumentNullException(nameof(queueOptions)); + _natsOptions = natsOptions ?? throw new ArgumentNullException(nameof(natsOptions)); + _streamOptions = streamOptions ?? throw new ArgumentNullException(nameof(streamOptions)); + _payload = payload ?? throw new ArgumentNullException(nameof(payload)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; + _connectionFactory = connectionFactory ?? ((opts, cancellationToken) => new ValueTask(new NatsConnection(opts))); + + if (string.IsNullOrWhiteSpace(_natsOptions.Url)) + { + throw new InvalidOperationException("NATS connection URL must be configured for the scheduler queue."); + } + } + + public async ValueTask EnqueueAsync( + TMessage message, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(message); + + var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false); + await EnsureStreamAndConsumerAsync(js, cancellationToken).ConfigureAwait(false); + + var payloadBytes = _payload.Serialize(message); + var idempotencyKey = _payload.GetIdempotencyKey(message); + var headers = BuildHeaders(message, idempotencyKey); + + var publishOptions = new NatsJSPubOpts + { + MsgId = idempotencyKey, + RetryAttempts = 0 + }; + + var ack = await js.PublishAsync( + _streamOptions.Subject, + payloadBytes, + PayloadSerializer, + publishOptions, + headers, + cancellationToken) + .ConfigureAwait(false); + + if (ack.Duplicate) + { + SchedulerQueueMetrics.RecordDeduplicated(TransportName, _payload.QueueName); + _logger.LogDebug( + "Duplicate enqueue detected for scheduler {Queue} message idempotency key {Key}; sequence {Sequence} reused.", + _payload.QueueName, + idempotencyKey, + ack.Seq); + + PublishDepth(); + return new SchedulerQueueEnqueueResult(ack.Seq.ToString(), true); + } + + SchedulerQueueMetrics.RecordEnqueued(TransportName, _payload.QueueName); + _logger.LogDebug( + "Enqueued scheduler {Queue} message into stream {Stream} with sequence {Sequence}.", + _payload.QueueName, + ack.Stream, + ack.Seq); + + IncrementDepth(); + return new SchedulerQueueEnqueueResult(ack.Seq.ToString(), false); + } + + public async ValueTask>> LeaseAsync( + SchedulerQueueLeaseRequest request, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + + var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false); + var consumer = await EnsureStreamAndConsumerAsync(js, cancellationToken).ConfigureAwait(false); + + var fetchOpts = new NatsJSFetchOpts + { + MaxMsgs = request.BatchSize, + Expires = request.LeaseDuration, + IdleHeartbeat = _natsOptions.IdleHeartbeat + }; + + var now = _timeProvider.GetUtcNow(); + var leases = new List>(request.BatchSize); + + await foreach (var message in consumer.FetchAsync(PayloadSerializer, fetchOpts, cancellationToken).ConfigureAwait(false)) + { + var lease = CreateLease(message, request.Consumer, now, request.LeaseDuration); + if (lease is not null) + { + leases.Add(lease); + } + } + + PublishDepth(); + return leases; + } + + public async ValueTask>> ClaimExpiredAsync( + SchedulerQueueClaimOptions options, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(options); + + var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false); + var consumer = await EnsureStreamAndConsumerAsync(js, cancellationToken).ConfigureAwait(false); + + var fetchOpts = new NatsJSFetchOpts + { + MaxMsgs = options.BatchSize, + Expires = options.MinIdleTime, + IdleHeartbeat = _natsOptions.IdleHeartbeat + }; + + var now = _timeProvider.GetUtcNow(); + var leases = new List>(options.BatchSize); + + await foreach (var message in consumer.FetchAsync(PayloadSerializer, fetchOpts, cancellationToken).ConfigureAwait(false)) + { + var deliveries = (int)(message.Metadata?.NumDelivered ?? 1); + if (deliveries <= 1) + { + await message.NakAsync(new AckOpts(), TimeSpan.Zero, cancellationToken).ConfigureAwait(false); + continue; + } + + var lease = CreateLease(message, options.ClaimantConsumer, now, _queueOptions.DefaultLeaseDuration); + if (lease is not null) + { + leases.Add(lease); + } + } + + PublishDepth(); + return leases; + } + + public async ValueTask DisposeAsync() + { + if (_disposed) + { + return; + } + + _disposed = true; + + if (_connection is not null) + { + await _connection.DisposeAsync().ConfigureAwait(false); + } + + _connectionGate.Dispose(); + SchedulerQueueMetrics.RemoveDepth(TransportName, _payload.QueueName); + GC.SuppressFinalize(this); + } + + public async ValueTask PingAsync(CancellationToken cancellationToken) + { + var connection = await EnsureConnectionAsync(cancellationToken).ConfigureAwait(false); + await connection.PingAsync(cancellationToken).ConfigureAwait(false); + } + + internal async Task AcknowledgeAsync(NatsSchedulerQueueLease lease, CancellationToken cancellationToken) + { + if (!lease.TryBeginCompletion()) + { + return; + } + + await lease.RawMessage.AckAsync(new AckOpts(), cancellationToken).ConfigureAwait(false); + SchedulerQueueMetrics.RecordAck(TransportName, _payload.QueueName); + DecrementDepth(); + } + + internal async Task RenewAsync(NatsSchedulerQueueLease lease, TimeSpan leaseDuration, CancellationToken cancellationToken) + { + await lease.RawMessage.AckProgressAsync(new AckOpts(), cancellationToken).ConfigureAwait(false); + lease.RefreshLease(_timeProvider.GetUtcNow().Add(leaseDuration)); + } + + internal async Task ReleaseAsync(NatsSchedulerQueueLease lease, SchedulerQueueReleaseDisposition disposition, CancellationToken cancellationToken) + { + if (disposition == SchedulerQueueReleaseDisposition.Retry && lease.Attempt >= _queueOptions.MaxDeliveryAttempts) + { + await DeadLetterAsync(lease, $"max-delivery-attempts:{lease.Attempt}", cancellationToken).ConfigureAwait(false); + return; + } + + if (!lease.TryBeginCompletion()) + { + return; + } + + if (disposition == SchedulerQueueReleaseDisposition.Retry) + { + SchedulerQueueMetrics.RecordRetry(TransportName, _payload.QueueName); + var delay = CalculateBackoff(lease.Attempt + 1); + lease.IncrementAttempt(); + await lease.RawMessage.NakAsync(new AckOpts(), delay, cancellationToken).ConfigureAwait(false); + _logger.LogWarning( + "Requeued scheduler {Queue} message {RunId} with delay {Delay} (attempt {Attempt}).", + _payload.QueueName, + lease.RunId, + delay, + lease.Attempt); + } + else + { + await lease.RawMessage.AckTerminateAsync(new AckOpts(), cancellationToken).ConfigureAwait(false); + SchedulerQueueMetrics.RecordAck(TransportName, _payload.QueueName); + DecrementDepth(); + _logger.LogInformation( + "Abandoned scheduler {Queue} message {RunId} after {Attempt} attempt(s).", + _payload.QueueName, + lease.RunId, + lease.Attempt); + } + + PublishDepth(); + } + + internal async Task DeadLetterAsync(NatsSchedulerQueueLease lease, string reason, CancellationToken cancellationToken) + { + if (!lease.TryBeginCompletion()) + { + return; + } + + await lease.RawMessage.AckAsync(new AckOpts(), cancellationToken).ConfigureAwait(false); + DecrementDepth(); + + var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false); + + if (!_queueOptions.DeadLetterEnabled) + { + _logger.LogWarning( + "Dropped scheduler {Queue} message {RunId} after {Attempt} attempt(s); dead-letter disabled. Reason: {Reason}", + _payload.QueueName, + lease.RunId, + lease.Attempt, + reason); + PublishDepth(); + return; + } + + await EnsureDeadLetterStreamAsync(js, cancellationToken).ConfigureAwait(false); + + var headers = BuildDeadLetterHeaders(lease, reason); + await js.PublishAsync( + _streamOptions.DeadLetterSubject, + lease.Payload, + PayloadSerializer, + new NatsJSPubOpts(), + headers, + cancellationToken) + .ConfigureAwait(false); + + SchedulerQueueMetrics.RecordDeadLetter(TransportName, _payload.QueueName); + _logger.LogError( + "Dead-lettered scheduler {Queue} message {RunId} after {Attempt} attempt(s): {Reason}", + _payload.QueueName, + lease.RunId, + lease.Attempt, + reason); + PublishDepth(); + } + + private async Task GetJetStreamAsync(CancellationToken cancellationToken) + { + if (_jsContext is not null) + { + return _jsContext; + } + + var connection = await EnsureConnectionAsync(cancellationToken).ConfigureAwait(false); + + await _connectionGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + _jsContext ??= new NatsJSContext(connection); + return _jsContext; + } + finally + { + _connectionGate.Release(); + } + } + + private async ValueTask EnsureStreamAndConsumerAsync(NatsJSContext js, CancellationToken cancellationToken) + { + if (_consumer is not null) + { + return _consumer; + } + + await _connectionGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_consumer is not null) + { + return _consumer; + } + + await EnsureStreamAsync(js, cancellationToken).ConfigureAwait(false); + await EnsureDeadLetterStreamAsync(js, cancellationToken).ConfigureAwait(false); + + var consumerConfig = new ConsumerConfig + { + DurableName = _streamOptions.DurableConsumer, + AckPolicy = ConsumerConfigAckPolicy.Explicit, + ReplayPolicy = ConsumerConfigReplayPolicy.Instant, + DeliverPolicy = ConsumerConfigDeliverPolicy.All, + AckWait = ToNanoseconds(_streamOptions.AckWait), + MaxAckPending = Math.Max(1, _streamOptions.MaxAckPending), + MaxDeliver = Math.Max(1, _queueOptions.MaxDeliveryAttempts), + FilterSubjects = new[] { _streamOptions.Subject } + }; + + try + { + _consumer = await js.CreateConsumerAsync( + _streamOptions.Stream, + consumerConfig, + cancellationToken) + .ConfigureAwait(false); + } + catch (NatsJSApiException apiEx) + { + _logger.LogDebug(apiEx, + "CreateConsumerAsync failed with code {Code}; attempting to reuse durable {Durable}.", + apiEx.Error?.Code, + _streamOptions.DurableConsumer); + + _consumer = await js.GetConsumerAsync( + _streamOptions.Stream, + _streamOptions.DurableConsumer, + cancellationToken) + .ConfigureAwait(false); + } + + return _consumer; + } + finally + { + _connectionGate.Release(); + } + } + + private async Task EnsureStreamAsync(NatsJSContext js, CancellationToken cancellationToken) + { + try + { + await js.GetStreamAsync( + _streamOptions.Stream, + new StreamInfoRequest(), + cancellationToken) + .ConfigureAwait(false); + } + catch (NatsJSApiException) + { + var config = new StreamConfig( + name: _streamOptions.Stream, + subjects: new[] { _streamOptions.Subject }) + { + Retention = StreamConfigRetention.Workqueue, + Storage = StreamConfigStorage.File, + MaxConsumers = -1, + MaxMsgs = -1, + MaxBytes = -1, + MaxAge = 0 + }; + + await js.CreateStreamAsync(config, cancellationToken).ConfigureAwait(false); + _logger.LogInformation( + "Created NATS JetStream stream {Stream} ({Subject}) for scheduler {Queue} queue.", + _streamOptions.Stream, + _streamOptions.Subject, + _payload.QueueName); + } + } + + private async Task EnsureDeadLetterStreamAsync(NatsJSContext js, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(_streamOptions.DeadLetterStream) || string.IsNullOrWhiteSpace(_streamOptions.DeadLetterSubject)) + { + return; + } + + try + { + await js.GetStreamAsync( + _streamOptions.DeadLetterStream, + new StreamInfoRequest(), + cancellationToken) + .ConfigureAwait(false); + } + catch (NatsJSApiException) + { + var config = new StreamConfig( + name: _streamOptions.DeadLetterStream, + subjects: new[] { _streamOptions.DeadLetterSubject }) + { + Retention = StreamConfigRetention.Workqueue, + Storage = StreamConfigStorage.File, + MaxConsumers = -1, + MaxMsgs = -1, + MaxBytes = -1, + MaxAge = 0 + }; + + await js.CreateStreamAsync(config, cancellationToken).ConfigureAwait(false); + _logger.LogInformation( + "Created NATS JetStream dead-letter stream {Stream} ({Subject}) for scheduler {Queue} queue.", + _streamOptions.DeadLetterStream, + _streamOptions.DeadLetterSubject, + _payload.QueueName); + } + } + + private async Task EnsureConnectionAsync(CancellationToken cancellationToken) + { + if (_connection is not null) + { + return _connection; + } + + await _connectionGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_connection is not null) + { + return _connection; + } + + var options = new NatsOpts + { + Url = _natsOptions.Url!, + Name = $"stellaops-scheduler-{_payload.QueueName}-queue", + CommandTimeout = TimeSpan.FromSeconds(10), + RequestTimeout = TimeSpan.FromSeconds(20), + PingInterval = TimeSpan.FromSeconds(30) + }; + + _connection = await _connectionFactory(options, cancellationToken).ConfigureAwait(false); + await _connection.ConnectAsync().ConfigureAwait(false); + return _connection; + } + finally + { + _connectionGate.Release(); + } + } + + private NatsSchedulerQueueLease? CreateLease( + NatsJSMsg message, + string consumer, + DateTimeOffset now, + TimeSpan leaseDuration) + { + var payload = message.Data ?? ReadOnlyMemory.Empty; + if (payload.IsEmpty) + { + return null; + } + + TMessage deserialized; + try + { + deserialized = _payload.Deserialize(payload.ToArray()); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to deserialize scheduler {Queue} payload from NATS sequence {Sequence}.", _payload.QueueName, message.Metadata?.Sequence); + return null; + } + + var attempt = (int)(message.Metadata?.NumDelivered ?? 1); + if (attempt <= 0) + { + attempt = 1; + } + + var headers = message.Headers ?? new NatsHeaders(); + + var enqueuedAt = headers.TryGetValue(SchedulerQueueFields.EnqueuedAt, out var enqueuedValues) && enqueuedValues.Count > 0 + && long.TryParse(enqueuedValues[0], out var unix) + ? DateTimeOffset.FromUnixTimeMilliseconds(unix) + : now; + + var leaseExpires = now.Add(leaseDuration); + var runId = _payload.GetRunId(deserialized); + var tenantId = _payload.GetTenantId(deserialized); + var scheduleId = _payload.GetScheduleId(deserialized); + var segmentId = _payload.GetSegmentId(deserialized); + var correlationId = _payload.GetCorrelationId(deserialized); + var attributes = _payload.GetAttributes(deserialized) ?? new Dictionary(); + + var attributeView = attributes.Count == 0 + ? EmptyReadOnlyDictionary.Instance + : new ReadOnlyDictionary(new Dictionary(attributes, StringComparer.Ordinal)); + + return new NatsSchedulerQueueLease( + this, + message, + payload.ToArray(), + _payload.GetIdempotencyKey(deserialized), + runId, + tenantId, + scheduleId, + segmentId, + correlationId, + attributeView, + deserialized, + attempt, + enqueuedAt, + leaseExpires, + consumer); + } + + private NatsHeaders BuildHeaders(TMessage message, string idempotencyKey) + { + var headers = new NatsHeaders + { + { SchedulerQueueFields.IdempotencyKey, idempotencyKey }, + { SchedulerQueueFields.RunId, _payload.GetRunId(message) }, + { SchedulerQueueFields.TenantId, _payload.GetTenantId(message) }, + { SchedulerQueueFields.QueueKind, _payload.QueueName }, + { SchedulerQueueFields.EnqueuedAt, _timeProvider.GetUtcNow().ToUnixTimeMilliseconds().ToString() } + }; + + var scheduleId = _payload.GetScheduleId(message); + if (!string.IsNullOrWhiteSpace(scheduleId)) + { + headers.Add(SchedulerQueueFields.ScheduleId, scheduleId); + } + + var segmentId = _payload.GetSegmentId(message); + if (!string.IsNullOrWhiteSpace(segmentId)) + { + headers.Add(SchedulerQueueFields.SegmentId, segmentId); + } + + var correlationId = _payload.GetCorrelationId(message); + if (!string.IsNullOrWhiteSpace(correlationId)) + { + headers.Add(SchedulerQueueFields.CorrelationId, correlationId); + } + + var attributes = _payload.GetAttributes(message); + if (attributes is not null) + { + foreach (var kvp in attributes) + { + headers.Add(SchedulerQueueFields.AttributePrefix + kvp.Key, kvp.Value); + } + } + + return headers; + } + + private NatsHeaders BuildDeadLetterHeaders(NatsSchedulerQueueLease lease, string reason) + { + var headers = new NatsHeaders + { + { SchedulerQueueFields.RunId, lease.RunId }, + { SchedulerQueueFields.TenantId, lease.TenantId }, + { SchedulerQueueFields.QueueKind, _payload.QueueName }, + { "reason", reason } + }; + + if (!string.IsNullOrWhiteSpace(lease.ScheduleId)) + { + headers.Add(SchedulerQueueFields.ScheduleId, lease.ScheduleId); + } + + if (!string.IsNullOrWhiteSpace(lease.CorrelationId)) + { + headers.Add(SchedulerQueueFields.CorrelationId, lease.CorrelationId); + } + + if (!string.IsNullOrWhiteSpace(lease.SegmentId)) + { + headers.Add(SchedulerQueueFields.SegmentId, lease.SegmentId); + } + + return headers; + } + + private TimeSpan CalculateBackoff(int attempt) + { + var initial = _queueOptions.RetryInitialBackoff > TimeSpan.Zero + ? _queueOptions.RetryInitialBackoff + : _streamOptions.RetryDelay; + + if (initial <= TimeSpan.Zero) + { + return TimeSpan.Zero; + } + + if (attempt <= 1) + { + return initial; + } + + var max = _queueOptions.RetryMaxBackoff > TimeSpan.Zero + ? _queueOptions.RetryMaxBackoff + : initial; + + var exponent = attempt - 1; + var scaledTicks = initial.Ticks * Math.Pow(2, exponent - 1); + var cappedTicks = Math.Min(max.Ticks, scaledTicks); + + return TimeSpan.FromTicks((long)Math.Max(initial.Ticks, cappedTicks)); + } + + private static long ToNanoseconds(TimeSpan value) + => value <= TimeSpan.Zero ? 0 : (long)(value.TotalMilliseconds * 1_000_000.0); + + private sealed class EmptyReadOnlyDictionary + where TKey : notnull + { + public static readonly IReadOnlyDictionary Instance = + new ReadOnlyDictionary(new Dictionary(0, EqualityComparer.Default)); + } + + private void IncrementDepth() + { + var depth = Interlocked.Increment(ref _approximateDepth); + SchedulerQueueMetrics.RecordDepth(TransportName, _payload.QueueName, depth); + } + + private void DecrementDepth() + { + var depth = Interlocked.Decrement(ref _approximateDepth); + if (depth < 0) + { + depth = Interlocked.Exchange(ref _approximateDepth, 0); + } + + SchedulerQueueMetrics.RecordDepth(TransportName, _payload.QueueName, depth); + } + + private void PublishDepth() + { + var depth = Volatile.Read(ref _approximateDepth); + SchedulerQueueMetrics.RecordDepth(TransportName, _payload.QueueName, depth); + } +} diff --git a/src/StellaOps.Scheduler.Queue/Nats/NatsSchedulerQueueLease.cs b/src/StellaOps.Scheduler.Queue/Nats/NatsSchedulerQueueLease.cs new file mode 100644 index 00000000..ecd8c566 --- /dev/null +++ b/src/StellaOps.Scheduler.Queue/Nats/NatsSchedulerQueueLease.cs @@ -0,0 +1,101 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using NATS.Client.JetStream; + +namespace StellaOps.Scheduler.Queue.Nats; + +internal sealed class NatsSchedulerQueueLease : ISchedulerQueueLease +{ + private readonly NatsSchedulerQueueBase _queue; + private int _completed; + + internal NatsSchedulerQueueLease( + NatsSchedulerQueueBase queue, + NatsJSMsg message, + byte[] payload, + string idempotencyKey, + string runId, + string tenantId, + string? scheduleId, + string? segmentId, + string? correlationId, + IReadOnlyDictionary attributes, + TMessage deserialized, + int attempt, + DateTimeOffset enqueuedAt, + DateTimeOffset leaseExpiresAt, + string consumer) + { + _queue = queue; + MessageId = message.Metadata?.Sequence.ToString() ?? idempotencyKey; + RunId = runId; + TenantId = tenantId; + ScheduleId = scheduleId; + SegmentId = segmentId; + CorrelationId = correlationId; + Attributes = attributes; + Attempt = attempt; + EnqueuedAt = enqueuedAt; + LeaseExpiresAt = leaseExpiresAt; + Consumer = consumer; + IdempotencyKey = idempotencyKey; + Message = deserialized; + _message = message; + Payload = payload; + } + + private readonly NatsJSMsg _message; + + internal NatsJSMsg RawMessage => _message; + + internal byte[] Payload { get; } + + public string MessageId { get; } + + public string IdempotencyKey { get; } + + public string RunId { get; } + + public string TenantId { get; } + + public string? ScheduleId { get; } + + public string? SegmentId { get; } + + public string? CorrelationId { get; } + + public IReadOnlyDictionary Attributes { get; } + + public TMessage Message { get; } + + public int Attempt { get; private set; } + + public DateTimeOffset EnqueuedAt { get; } + + public DateTimeOffset LeaseExpiresAt { get; private set; } + + public string Consumer { get; } + + public Task AcknowledgeAsync(CancellationToken cancellationToken = default) + => _queue.AcknowledgeAsync(this, cancellationToken); + + public Task RenewAsync(TimeSpan leaseDuration, CancellationToken cancellationToken = default) + => _queue.RenewAsync(this, leaseDuration, cancellationToken); + + public Task ReleaseAsync(SchedulerQueueReleaseDisposition disposition, CancellationToken cancellationToken = default) + => _queue.ReleaseAsync(this, disposition, cancellationToken); + + public Task DeadLetterAsync(string reason, CancellationToken cancellationToken = default) + => _queue.DeadLetterAsync(this, reason, cancellationToken); + + internal bool TryBeginCompletion() + => Interlocked.CompareExchange(ref _completed, 1, 0) == 0; + + internal void RefreshLease(DateTimeOffset expiresAt) + => LeaseExpiresAt = expiresAt; + + internal void IncrementAttempt() + => Attempt++; +} diff --git a/src/StellaOps.Scheduler.Queue/Nats/NatsSchedulerRunnerQueue.cs b/src/StellaOps.Scheduler.Queue/Nats/NatsSchedulerRunnerQueue.cs new file mode 100644 index 00000000..e47fd21e --- /dev/null +++ b/src/StellaOps.Scheduler.Queue/Nats/NatsSchedulerRunnerQueue.cs @@ -0,0 +1,74 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NATS.Client.Core; +using NATS.Client.JetStream; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Queue.Nats; + +internal sealed class NatsSchedulerRunnerQueue + : NatsSchedulerQueueBase, ISchedulerRunnerQueue +{ + public NatsSchedulerRunnerQueue( + SchedulerQueueOptions queueOptions, + SchedulerNatsQueueOptions natsOptions, + ILogger logger, + TimeProvider timeProvider, + Func>? connectionFactory = null) + : base( + queueOptions, + natsOptions, + natsOptions.Runner, + RunnerPayload.Instance, + logger, + timeProvider, + connectionFactory) + { + } + + private sealed class RunnerPayload : INatsSchedulerQueuePayload + { + public static RunnerPayload Instance { get; } = new(); + + public string QueueName => "runner"; + + public string GetIdempotencyKey(RunnerSegmentQueueMessage message) + => message.IdempotencyKey; + + public byte[] Serialize(RunnerSegmentQueueMessage message) + => Encoding.UTF8.GetBytes(CanonicalJsonSerializer.Serialize(message)); + + public RunnerSegmentQueueMessage Deserialize(byte[] payload) + => CanonicalJsonSerializer.Deserialize(Encoding.UTF8.GetString(payload)); + + public string GetRunId(RunnerSegmentQueueMessage message) + => message.RunId; + + public string GetTenantId(RunnerSegmentQueueMessage message) + => message.TenantId; + + public string? GetScheduleId(RunnerSegmentQueueMessage message) + => message.ScheduleId; + + public string? GetSegmentId(RunnerSegmentQueueMessage message) + => message.SegmentId; + + public string? GetCorrelationId(RunnerSegmentQueueMessage message) + => message.CorrelationId; + + public IReadOnlyDictionary? GetAttributes(RunnerSegmentQueueMessage message) + { + if (message.Attributes is null || message.Attributes.Count == 0) + { + return null; + } + + return message.Attributes.ToDictionary(kvp => kvp.Key, kvp => kvp.Value, StringComparer.Ordinal); + } + } +} diff --git a/src/StellaOps.Scheduler.Queue/README.md b/src/StellaOps.Scheduler.Queue/README.md index 066c389c..661e8d30 100644 --- a/src/StellaOps.Scheduler.Queue/README.md +++ b/src/StellaOps.Scheduler.Queue/README.md @@ -8,9 +8,38 @@ Queue work now has concrete contracts from `StellaOps.Scheduler.Models`: ## Action items for SCHED-QUEUE-16-401..403 -1. Reference `StellaOps.Scheduler.Models` so adapters can serialise `Run`/`DeltaSummary` without bespoke DTOs. -2. Use the canonical serializer for queue messages to keep ordering consistent with API payloads. -3. Coverage: add fixture-driven tests that enqueue the sample payloads, then dequeue and re-serialise to verify byte-for-byte stability. -4. Expose queue depth/lease metrics with the identifiers provided by the models (`Run.Id`, `Schedule.Id`). +1. Reference `StellaOps.Scheduler.Models` so adapters can serialise `Run`/`DeltaSummary` without bespoke DTOs. +2. Use the canonical serializer for queue messages to keep ordering consistent with API payloads. +3. Coverage: add fixture-driven tests that enqueue the sample payloads, then dequeue and re-serialise to verify byte-for-byte stability. +4. Expose queue depth/lease metrics with the identifiers provided by the models (`Run.Id`, `Schedule.Id`). + +## JetStream failover notes + +- `SchedulerQueueOptions.Kind = "nats"` will spin up `NatsSchedulerPlannerQueue` / `NatsSchedulerRunnerQueue` instances backed by JetStream. +- `SchedulerQueueHealthCheck` pings both planner and runner transports; register via `AddSchedulerQueueHealthCheck()` to surface in `/healthz`. +- Planner defaults: + ```yaml + scheduler: + queue: + kind: nats + deadLetterEnabled: true + nats: + url: "nats://nats:4222" + planner: + stream: SCHEDULER_PLANNER + subject: scheduler.planner + durableConsumer: scheduler-planners + deadLetterStream: SCHEDULER_PLANNER_DEAD + deadLetterSubject: scheduler.planner.dead + runner: + stream: SCHEDULER_RUNNER + subject: scheduler.runner + durableConsumer: scheduler-runners + redis: + deadLetterStream: scheduler:planner:dead + idempotencyKeyPrefix: scheduler:planner:idemp: + ``` +- Retry / dead-letter semantics mirror the Redis adapter: attempts beyond `MaxDeliveryAttempts` are shipped to the configured dead-letter stream with headers describing `runId`, `scheduleId`, and failure reasons. Set `deadLetterEnabled: false` to drop exhausted messages instead. +- Depth metrics surface through `scheduler_queue_depth{transport,queue}`; both transports publish lightweight counters to drive alerting dashboards. These notes unblock the queue guild now that SCHED-MODELS-16-102 is complete. diff --git a/src/StellaOps.Scheduler.Queue/Redis/RedisSchedulerQueueBase.cs b/src/StellaOps.Scheduler.Queue/Redis/RedisSchedulerQueueBase.cs index e7220ddb..48424486 100644 --- a/src/StellaOps.Scheduler.Queue/Redis/RedisSchedulerQueueBase.cs +++ b/src/StellaOps.Scheduler.Queue/Redis/RedisSchedulerQueueBase.cs @@ -10,7 +10,7 @@ using StackExchange.Redis; namespace StellaOps.Scheduler.Queue.Redis; -internal abstract class RedisSchedulerQueueBase : ISchedulerQueue, IAsyncDisposable +internal abstract class RedisSchedulerQueueBase : ISchedulerQueue, IAsyncDisposable, ISchedulerQueueTransportDiagnostics { private const string TransportName = "redis"; @@ -18,11 +18,12 @@ internal abstract class RedisSchedulerQueueBase : ISchedulerQueue _payload; - private readonly ILogger _logger; - private readonly TimeProvider _timeProvider; - private readonly Func> _connectionFactory; - private readonly SemaphoreSlim _connectionLock = new(1, 1); - private readonly SemaphoreSlim _groupInitLock = new(1, 1); + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly Func> _connectionFactory; + private readonly SemaphoreSlim _connectionLock = new(1, 1); + private readonly SemaphoreSlim _groupInitLock = new(1, 1); + private long _approximateDepth; private IConnectionMultiplexer? _connection; private volatile bool _groupInitialized; @@ -91,31 +92,33 @@ internal abstract class RedisSchedulerQueueBase : ISchedulerQueue>> LeaseAsync( @@ -137,11 +140,12 @@ internal abstract class RedisSchedulerQueueBase : ISchedulerQueue>(); - } - + if (entries is null || entries.Length == 0) + { + PublishDepth(); + return Array.Empty>(); + } + var now = _timeProvider.GetUtcNow(); var leases = new List>(entries.Length); @@ -157,7 +161,8 @@ internal abstract class RedisSchedulerQueueBase : ISchedulerQueue>> ClaimExpiredAsync( @@ -205,10 +210,11 @@ internal abstract class RedisSchedulerQueueBase : ISchedulerQueue>(); - } + if (claimed is null || claimed.Length == 0) + { + PublishDepth(); + return Array.Empty>(); + } var now = _timeProvider.GetUtcNow(); var attemptLookup = eligible.ToDictionary( @@ -238,7 +244,8 @@ internal abstract class RedisSchedulerQueueBase : ISchedulerQueue : ISchedulerQueue lease, @@ -278,13 +286,14 @@ internal abstract class RedisSchedulerQueueBase : ISchedulerQueue lease, @@ -306,74 +315,76 @@ internal abstract class RedisSchedulerQueueBase : ISchedulerQueue lease, - SchedulerQueueReleaseDisposition disposition, - CancellationToken cancellationToken) - { - if (disposition == SchedulerQueueReleaseDisposition.Retry - && lease.Attempt >= _queueOptions.MaxDeliveryAttempts) - { - await DeadLetterAsync( - lease, - $"max-delivery-attempts:{lease.Attempt}", - cancellationToken).ConfigureAwait(false); - return; - } - - if (!lease.TryBeginCompletion()) - { - return; - } - - var database = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false); - - await database.StreamAcknowledgeAsync( - _streamOptions.Stream, - _streamOptions.ConsumerGroup, - new RedisValue[] { lease.MessageId }) - .ConfigureAwait(false); - - await database.StreamDeleteAsync( - _streamOptions.Stream, - new RedisValue[] { lease.MessageId }) - .ConfigureAwait(false); - - SchedulerQueueMetrics.RecordAck(TransportName, _payload.QueueName); - - if (disposition == SchedulerQueueReleaseDisposition.Retry) - { - SchedulerQueueMetrics.RecordRetry(TransportName, _payload.QueueName); - - lease.IncrementAttempt(); - - var backoff = CalculateBackoff(lease.Attempt); - if (backoff > TimeSpan.Zero) - { - try - { - await Task.Delay(backoff, cancellationToken).ConfigureAwait(false); - } - catch (TaskCanceledException) - { - return; - } - } - - var now = _timeProvider.GetUtcNow(); - var entries = BuildEntries(lease.Message, now, lease.Attempt); - - await AddToStreamAsync( - database, - _streamOptions.Stream, - entries, - _streamOptions.ApproximateMaxLength, - _streamOptions.ApproximateMaxLength is not null) - .ConfigureAwait(false); - - SchedulerQueueMetrics.RecordEnqueued(TransportName, _payload.QueueName); - } - } + internal async Task ReleaseAsync( + RedisSchedulerQueueLease lease, + SchedulerQueueReleaseDisposition disposition, + CancellationToken cancellationToken) + { + if (disposition == SchedulerQueueReleaseDisposition.Retry + && lease.Attempt >= _queueOptions.MaxDeliveryAttempts) + { + await DeadLetterAsync( + lease, + $"max-delivery-attempts:{lease.Attempt}", + cancellationToken).ConfigureAwait(false); + return; + } + + if (!lease.TryBeginCompletion()) + { + return; + } + + var database = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false); + + await database.StreamAcknowledgeAsync( + _streamOptions.Stream, + _streamOptions.ConsumerGroup, + new RedisValue[] { lease.MessageId }) + .ConfigureAwait(false); + + await database.StreamDeleteAsync( + _streamOptions.Stream, + new RedisValue[] { lease.MessageId }) + .ConfigureAwait(false); + + SchedulerQueueMetrics.RecordAck(TransportName, _payload.QueueName); + DecrementDepth(); + + if (disposition == SchedulerQueueReleaseDisposition.Retry) + { + SchedulerQueueMetrics.RecordRetry(TransportName, _payload.QueueName); + + lease.IncrementAttempt(); + + var backoff = CalculateBackoff(lease.Attempt); + if (backoff > TimeSpan.Zero) + { + try + { + await Task.Delay(backoff, cancellationToken).ConfigureAwait(false); + } + catch (TaskCanceledException) + { + return; + } + } + + var now = _timeProvider.GetUtcNow(); + var entries = BuildEntries(lease.Message, now, lease.Attempt); + + await AddToStreamAsync( + database, + _streamOptions.Stream, + entries, + _streamOptions.ApproximateMaxLength, + _streamOptions.ApproximateMaxLength is not null) + .ConfigureAwait(false); + + SchedulerQueueMetrics.RecordEnqueued(TransportName, _payload.QueueName); + IncrementDepth(); + } + } internal async Task DeadLetterAsync( RedisSchedulerQueueLease lease, @@ -393,36 +404,49 @@ internal abstract class RedisSchedulerQueueBase : ISchedulerQueue string.Concat(_streamOptions.IdempotencyKeyPrefix, key); @@ -735,11 +759,34 @@ internal abstract class RedisSchedulerQueueBase : ISchedulerQueue _logger; + + public SchedulerQueueHealthCheck( + ISchedulerPlannerQueue plannerQueue, + ISchedulerRunnerQueue runnerQueue, + ILogger logger) + { + _plannerQueue = plannerQueue ?? throw new ArgumentNullException(nameof(plannerQueue)); + _runnerQueue = runnerQueue ?? throw new ArgumentNullException(nameof(runnerQueue)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + var failures = new List(); + + if (!await ProbeAsync(_plannerQueue, "planner", cancellationToken).ConfigureAwait(false)) + { + failures.Add("planner transport unreachable"); + } + + if (!await ProbeAsync(_runnerQueue, "runner", cancellationToken).ConfigureAwait(false)) + { + failures.Add("runner transport unreachable"); + } + + if (failures.Count == 0) + { + return HealthCheckResult.Healthy("Scheduler queues reachable."); + } + + var description = string.Join("; ", failures); + return new HealthCheckResult( + context.Registration.FailureStatus, + description); + } + + private async Task ProbeAsync(object queue, string label, CancellationToken cancellationToken) + { + try + { + if (queue is ISchedulerQueueTransportDiagnostics diagnostics) + { + await diagnostics.PingAsync(cancellationToken).ConfigureAwait(false); + } + + return true; + } + catch (Exception ex) + { + _logger.LogError(ex, "Scheduler {Label} queue transport ping failed.", label); + return false; + } + } +} diff --git a/src/StellaOps.Scheduler.Queue/SchedulerQueueMetrics.cs b/src/StellaOps.Scheduler.Queue/SchedulerQueueMetrics.cs index bcb90371..5158d384 100644 --- a/src/StellaOps.Scheduler.Queue/SchedulerQueueMetrics.cs +++ b/src/StellaOps.Scheduler.Queue/SchedulerQueueMetrics.cs @@ -1,22 +1,28 @@ -using System.Collections.Generic; -using System.Diagnostics.Metrics; - -namespace StellaOps.Scheduler.Queue; - -internal static class SchedulerQueueMetrics -{ +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Linq; + +namespace StellaOps.Scheduler.Queue; + +internal static class SchedulerQueueMetrics +{ private const string TransportTagName = "transport"; private const string QueueTagName = "queue"; private static readonly Meter Meter = new("StellaOps.Scheduler.Queue"); - private static readonly Counter EnqueuedCounter = Meter.CreateCounter("scheduler_queue_enqueued_total"); - private static readonly Counter DeduplicatedCounter = Meter.CreateCounter("scheduler_queue_deduplicated_total"); - private static readonly Counter AckCounter = Meter.CreateCounter("scheduler_queue_ack_total"); - private static readonly Counter RetryCounter = Meter.CreateCounter("scheduler_queue_retry_total"); - private static readonly Counter DeadLetterCounter = Meter.CreateCounter("scheduler_queue_deadletter_total"); - - public static void RecordEnqueued(string transport, string queue) - => EnqueuedCounter.Add(1, BuildTags(transport, queue)); + private static readonly Counter EnqueuedCounter = Meter.CreateCounter("scheduler_queue_enqueued_total"); + private static readonly Counter DeduplicatedCounter = Meter.CreateCounter("scheduler_queue_deduplicated_total"); + private static readonly Counter AckCounter = Meter.CreateCounter("scheduler_queue_ack_total"); + private static readonly Counter RetryCounter = Meter.CreateCounter("scheduler_queue_retry_total"); + private static readonly Counter DeadLetterCounter = Meter.CreateCounter("scheduler_queue_deadletter_total"); + private static readonly ConcurrentDictionary<(string transport, string queue), long> DepthSamples = new(); + private static readonly ObservableGauge DepthGauge = Meter.CreateObservableGauge( + "scheduler_queue_depth", + ObserveDepth); + + public static void RecordEnqueued(string transport, string queue) + => EnqueuedCounter.Add(1, BuildTags(transport, queue)); public static void RecordDeduplicated(string transport, string queue) => DeduplicatedCounter.Add(1, BuildTags(transport, queue)); @@ -27,13 +33,33 @@ internal static class SchedulerQueueMetrics public static void RecordRetry(string transport, string queue) => RetryCounter.Add(1, BuildTags(transport, queue)); - public static void RecordDeadLetter(string transport, string queue) - => DeadLetterCounter.Add(1, BuildTags(transport, queue)); - - private static KeyValuePair[] BuildTags(string transport, string queue) - => new[] - { - new KeyValuePair(TransportTagName, transport), - new KeyValuePair(QueueTagName, queue) - }; -} + public static void RecordDeadLetter(string transport, string queue) + => DeadLetterCounter.Add(1, BuildTags(transport, queue)); + + public static void RecordDepth(string transport, string queue, long depth) + => DepthSamples[(transport, queue)] = depth; + + public static void RemoveDepth(string transport, string queue) + => DepthSamples.TryRemove((transport, queue), out _); + + internal static IReadOnlyDictionary<(string transport, string queue), long> SnapshotDepths() + => DepthSamples.ToDictionary(pair => pair.Key, pair => pair.Value); + + private static KeyValuePair[] BuildTags(string transport, string queue) + => new[] + { + new KeyValuePair(TransportTagName, transport), + new KeyValuePair(QueueTagName, queue) + }; + + private static IEnumerable> ObserveDepth() + { + foreach (var sample in DepthSamples) + { + yield return new Measurement( + sample.Value, + new KeyValuePair(TransportTagName, sample.Key.transport), + new KeyValuePair(QueueTagName, sample.Key.queue)); + } + } +} diff --git a/src/StellaOps.Scheduler.Queue/SchedulerQueueOptions.cs b/src/StellaOps.Scheduler.Queue/SchedulerQueueOptions.cs index 5e5073f9..fdbe8364 100644 --- a/src/StellaOps.Scheduler.Queue/SchedulerQueueOptions.cs +++ b/src/StellaOps.Scheduler.Queue/SchedulerQueueOptions.cs @@ -2,26 +2,34 @@ using System; namespace StellaOps.Scheduler.Queue; -public sealed class SchedulerQueueOptions -{ - public SchedulerQueueTransportKind Kind { get; set; } = SchedulerQueueTransportKind.Redis; - - public SchedulerRedisQueueOptions Redis { get; set; } = new(); - - /// - /// Default lease/visibility window applied when callers do not override the duration. - /// - public TimeSpan DefaultLeaseDuration { get; set; } = TimeSpan.FromMinutes(5); - - /// - /// Maximum number of deliveries before a message is shunted to the dead-letter stream. - /// - public int MaxDeliveryAttempts { get; set; } = 5; - - /// - /// Base retry delay used when a message is released for retry. - /// - public TimeSpan RetryInitialBackoff { get; set; } = TimeSpan.FromSeconds(5); +public sealed class SchedulerQueueOptions +{ + public SchedulerQueueTransportKind Kind { get; set; } = SchedulerQueueTransportKind.Redis; + + public SchedulerRedisQueueOptions Redis { get; set; } = new(); + + public SchedulerNatsQueueOptions Nats { get; set; } = new(); + + /// + /// Default lease/visibility window applied when callers do not override the duration. + /// + public TimeSpan DefaultLeaseDuration { get; set; } = TimeSpan.FromMinutes(5); + + /// + /// Maximum number of deliveries before a message is shunted to the dead-letter stream. + /// + public int MaxDeliveryAttempts { get; set; } = 5; + + /// + /// Enables routing exhausted deliveries to the configured dead-letter transport. + /// When disabled, messages exceeding the attempt budget are dropped after acknowledgement. + /// + public bool DeadLetterEnabled { get; set; } = true; + + /// + /// Base retry delay used when a message is released for retry. + /// + public TimeSpan RetryInitialBackoff { get; set; } = TimeSpan.FromSeconds(5); /// /// Cap applied to the retry delay when exponential backoff is used. @@ -42,8 +50,8 @@ public sealed class SchedulerRedisQueueOptions public RedisSchedulerStreamOptions Runner { get; set; } = RedisSchedulerStreamOptions.ForRunner(); } -public sealed class RedisSchedulerStreamOptions -{ +public sealed class RedisSchedulerStreamOptions +{ public string Stream { get; set; } = string.Empty; public string ConsumerGroup { get; set; } = string.Empty; @@ -72,5 +80,55 @@ public sealed class RedisSchedulerStreamOptions ConsumerGroup = "scheduler-runners", DeadLetterStream = "scheduler:runner:dead", IdempotencyKeyPrefix = "scheduler:runner:idemp:" - }; -} + }; +} + +public sealed class SchedulerNatsQueueOptions +{ + public string? Url { get; set; } + + public TimeSpan IdleHeartbeat { get; set; } = TimeSpan.FromSeconds(30); + + public SchedulerNatsStreamOptions Planner { get; set; } = SchedulerNatsStreamOptions.ForPlanner(); + + public SchedulerNatsStreamOptions Runner { get; set; } = SchedulerNatsStreamOptions.ForRunner(); +} + +public sealed class SchedulerNatsStreamOptions +{ + public string Stream { get; set; } = string.Empty; + + public string Subject { get; set; } = string.Empty; + + public string DurableConsumer { get; set; } = string.Empty; + + public string DeadLetterStream { get; set; } = string.Empty; + + public string DeadLetterSubject { get; set; } = string.Empty; + + public int MaxAckPending { get; set; } = 64; + + public TimeSpan AckWait { get; set; } = TimeSpan.FromMinutes(5); + + public TimeSpan RetryDelay { get; set; } = TimeSpan.FromSeconds(10); + + public static SchedulerNatsStreamOptions ForPlanner() + => new() + { + Stream = "SCHEDULER_PLANNER", + Subject = "scheduler.planner", + DurableConsumer = "scheduler-planners", + DeadLetterStream = "SCHEDULER_PLANNER_DEAD", + DeadLetterSubject = "scheduler.planner.dead" + }; + + public static SchedulerNatsStreamOptions ForRunner() + => new() + { + Stream = "SCHEDULER_RUNNER", + Subject = "scheduler.runner", + DurableConsumer = "scheduler-runners", + DeadLetterStream = "SCHEDULER_RUNNER_DEAD", + DeadLetterSubject = "scheduler.runner.dead" + }; +} diff --git a/src/StellaOps.Scheduler.Queue/SchedulerQueueServiceCollectionExtensions.cs b/src/StellaOps.Scheduler.Queue/SchedulerQueueServiceCollectionExtensions.cs index 45f78f7f..7653bc37 100644 --- a/src/StellaOps.Scheduler.Queue/SchedulerQueueServiceCollectionExtensions.cs +++ b/src/StellaOps.Scheduler.Queue/SchedulerQueueServiceCollectionExtensions.cs @@ -1,13 +1,15 @@ using System; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; -using Microsoft.Extensions.Logging; -using StellaOps.Scheduler.Queue.Redis; - -namespace StellaOps.Scheduler.Queue; - -public static class SchedulerQueueServiceCollectionExtensions +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; +using StellaOps.Scheduler.Queue.Nats; +using StellaOps.Scheduler.Queue.Redis; + +namespace StellaOps.Scheduler.Queue; + +public static class SchedulerQueueServiceCollectionExtensions { public static IServiceCollection AddSchedulerQueues( this IServiceCollection services, @@ -20,41 +22,67 @@ public static class SchedulerQueueServiceCollectionExtensions var options = new SchedulerQueueOptions(); configuration.GetSection(sectionName).Bind(options); - services.TryAddSingleton(TimeProvider.System); - services.AddSingleton(options); + services.TryAddSingleton(TimeProvider.System); + services.AddSingleton(options); + + services.AddSingleton(sp => + { + var loggerFactory = sp.GetRequiredService(); + var timeProvider = sp.GetService() ?? TimeProvider.System; - services.AddSingleton(sp => + return options.Kind switch + { + SchedulerQueueTransportKind.Redis => new RedisSchedulerPlannerQueue( + options, + options.Redis, + loggerFactory.CreateLogger(), + timeProvider), + SchedulerQueueTransportKind.Nats => new NatsSchedulerPlannerQueue( + options, + options.Nats, + loggerFactory.CreateLogger(), + timeProvider), + _ => throw new InvalidOperationException($"Unsupported scheduler queue transport '{options.Kind}'.") + }; + }); + + services.AddSingleton(sp => { var loggerFactory = sp.GetRequiredService(); var timeProvider = sp.GetService() ?? TimeProvider.System; - return options.Kind switch - { - SchedulerQueueTransportKind.Redis => new RedisSchedulerPlannerQueue( - options, - options.Redis, - loggerFactory.CreateLogger(), - timeProvider), - _ => throw new InvalidOperationException($"Unsupported scheduler queue transport '{options.Kind}'.") - }; - }); - - services.AddSingleton(sp => - { - var loggerFactory = sp.GetRequiredService(); - var timeProvider = sp.GetService() ?? TimeProvider.System; - - return options.Kind switch - { - SchedulerQueueTransportKind.Redis => new RedisSchedulerRunnerQueue( - options, - options.Redis, - loggerFactory.CreateLogger(), - timeProvider), - _ => throw new InvalidOperationException($"Unsupported scheduler queue transport '{options.Kind}'.") - }; - }); - - return services; - } -} + return options.Kind switch + { + SchedulerQueueTransportKind.Redis => new RedisSchedulerRunnerQueue( + options, + options.Redis, + loggerFactory.CreateLogger(), + timeProvider), + SchedulerQueueTransportKind.Nats => new NatsSchedulerRunnerQueue( + options, + options.Nats, + loggerFactory.CreateLogger(), + timeProvider), + _ => throw new InvalidOperationException($"Unsupported scheduler queue transport '{options.Kind}'.") + }; + }); + + services.AddSingleton(); + + return services; + } + + public static IHealthChecksBuilder AddSchedulerQueueHealthCheck( + this IHealthChecksBuilder builder) + { + ArgumentNullException.ThrowIfNull(builder); + + builder.Services.TryAddSingleton(); + builder.AddCheck( + name: "scheduler-queue", + failureStatus: HealthStatus.Unhealthy, + tags: new[] { "scheduler", "queue" }); + + return builder; + } +} diff --git a/src/StellaOps.Scheduler.Queue/StellaOps.Scheduler.Queue.csproj b/src/StellaOps.Scheduler.Queue/StellaOps.Scheduler.Queue.csproj index fef0b5b8..74b0c48b 100644 --- a/src/StellaOps.Scheduler.Queue/StellaOps.Scheduler.Queue.csproj +++ b/src/StellaOps.Scheduler.Queue/StellaOps.Scheduler.Queue.csproj @@ -4,13 +4,17 @@ enable enable - - - - - - - + + + + + + + + + + + diff --git a/src/StellaOps.Scheduler.Queue/TASKS.md b/src/StellaOps.Scheduler.Queue/TASKS.md index 0d121b51..2ca7feda 100644 --- a/src/StellaOps.Scheduler.Queue/TASKS.md +++ b/src/StellaOps.Scheduler.Queue/TASKS.md @@ -4,6 +4,6 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| SCHED-QUEUE-16-401 | DOING (2025-10-19) | Scheduler Queue Guild | SCHED-MODELS-16-101 | Implement queue abstraction + Redis Streams adapter (planner inputs, runner segments) with ack/lease semantics. | Integration tests cover enqueue/dequeue/ack; lease renewal implemented; ordering preserved. | -| SCHED-QUEUE-16-402 | TODO | Scheduler Queue Guild | SCHED-QUEUE-16-401 | Add NATS JetStream adapter with configuration binding, health probes, failover. | Health endpoints verified; failover documented; adapter tested. | -| SCHED-QUEUE-16-403 | TODO | Scheduler Queue Guild | SCHED-QUEUE-16-401 | Dead-letter handling + metrics (queue depth, retry counts), configuration toggles. | Dead-letter policy tested; metrics exported; docs updated. | +| SCHED-QUEUE-16-401 | DONE (2025-10-20) | Scheduler Queue Guild | SCHED-MODELS-16-101 | Implement queue abstraction + Redis Streams adapter (planner inputs, runner segments) with ack/lease semantics. | Integration tests cover enqueue/dequeue/ack; lease renewal implemented; ordering preserved. | +| SCHED-QUEUE-16-402 | DONE (2025-10-20) | Scheduler Queue Guild | SCHED-QUEUE-16-401 | Add NATS JetStream adapter with configuration binding, health probes, failover. | Health endpoints verified; failover documented; adapter tested. | +| SCHED-QUEUE-16-403 | DONE (2025-10-20) | Scheduler Queue Guild | SCHED-QUEUE-16-401 | Dead-letter handling + metrics (queue depth, retry counts), configuration toggles. | Dead-letter policy tested; metrics exported; docs updated. | diff --git a/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/ProofOfEntitlement/InMemoryProofOfEntitlementIntrospector.cs b/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/ProofOfEntitlement/InMemoryProofOfEntitlementIntrospector.cs index feb52d76..15cba75a 100644 --- a/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/ProofOfEntitlement/InMemoryProofOfEntitlementIntrospector.cs +++ b/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/ProofOfEntitlement/InMemoryProofOfEntitlementIntrospector.cs @@ -3,7 +3,8 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Options; using StellaOps.Signer.Core; -using StellaOps.Signer.Infrastructure.Options; +using StellaOps.Signer.Infrastructure.Options; +using ProofOfEntitlementRecord = StellaOps.Signer.Core.ProofOfEntitlement; namespace StellaOps.Signer.Infrastructure.ProofOfEntitlement; @@ -20,8 +21,8 @@ public sealed class InMemoryProofOfEntitlementIntrospector : IProofOfEntitlement _timeProvider = timeProvider ?? TimeProvider.System; } - public ValueTask IntrospectAsync( - ProofOfEntitlement proof, + public ValueTask IntrospectAsync( + ProofOfEntitlementRecord proof, CallerContext caller, CancellationToken cancellationToken) { diff --git a/src/StellaOps.Signer/StellaOps.Signer.Tests/SignerEndpointsTests.cs b/src/StellaOps.Signer/StellaOps.Signer.Tests/SignerEndpointsTests.cs new file mode 100644 index 00000000..fa24bda0 --- /dev/null +++ b/src/StellaOps.Signer/StellaOps.Signer.Tests/SignerEndpointsTests.cs @@ -0,0 +1,127 @@ +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.Testing; +using StellaOps.Signer.WebService.Contracts; +using Xunit; + +namespace StellaOps.Signer.Tests; + +public sealed class SignerEndpointsTests : IClassFixture> +{ + private readonly WebApplicationFactory _factory; + private const string TrustedDigest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + + public SignerEndpointsTests(WebApplicationFactory factory) + { + _factory = factory; + } + + [Fact] + public async Task SignDsse_ReturnsBundle_WhenRequestValid() + { + var client = CreateClient(); + var request = new HttpRequestMessage(HttpMethod.Post, "/api/v1/signer/sign/dsse") + { + Content = JsonContent.Create(new + { + subject = new[] + { + new + { + name = "pkg:npm/example", + digest = new Dictionary { ["sha256"] = "4d5f" }, + }, + }, + predicateType = "https://in-toto.io/Statement/v0.1", + predicate = new { result = "pass" }, + scannerImageDigest = TrustedDigest, + poe = new { format = "jwt", value = "valid-poe" }, + options = new { signingMode = "kms", expirySeconds = 600, returnBundle = "dsse+cert" }, + }) + }; + + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", "stub-token"); + request.Headers.Add("DPoP", "stub-proof"); + + var response = await client.SendAsync(request); + var responseBody = await response.Content.ReadAsStringAsync(); + Assert.True(response.IsSuccessStatusCode, $"Expected success but got {(int)response.StatusCode}: {responseBody}"); + + var body = await response.Content.ReadFromJsonAsync(); + Assert.NotNull(body); + Assert.Equal("stub-subject", body!.Bundle.SigningIdentity.Subject); + Assert.Equal("stub-subject", body.Bundle.SigningIdentity.Issuer); + } + + [Fact] + public async Task SignDsse_ReturnsForbidden_WhenDigestUntrusted() + { + var client = CreateClient(); + var request = new HttpRequestMessage(HttpMethod.Post, "/api/v1/signer/sign/dsse") + { + Content = JsonContent.Create(new + { + subject = new[] + { + new + { + name = "pkg:npm/example", + digest = new Dictionary { ["sha256"] = "4d5f" }, + }, + }, + predicateType = "https://in-toto.io/Statement/v0.1", + predicate = new { result = "pass" }, + scannerImageDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + poe = new { format = "jwt", value = "valid-poe" }, + options = new { signingMode = "kms", expirySeconds = 600, returnBundle = "dsse+cert" }, + }) + }; + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", "stub-token"); + request.Headers.Add("DPoP", "stub-proof"); + + var response = await client.SendAsync(request); + var problemJson = await response.Content.ReadAsStringAsync(); + Assert.Equal(HttpStatusCode.Forbidden, response.StatusCode); + + var problem = System.Text.Json.JsonSerializer.Deserialize(problemJson, new System.Text.Json.JsonSerializerOptions + { + PropertyNameCaseInsensitive = true, + }); + Assert.NotNull(problem); + Assert.Equal("release_untrusted", problem!.Type); + } + + [Fact] + public async Task VerifyReferrers_ReturnsTrustedResult_WhenDigestIsKnown() + { + var client = CreateClient(); + var request = new HttpRequestMessage(HttpMethod.Get, $"/api/v1/signer/verify/referrers?digest={TrustedDigest}"); + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", "stub-token"); + + var response = await client.SendAsync(request); + var responseBody = await response.Content.ReadAsStringAsync(); + Assert.True(response.IsSuccessStatusCode, $"Expected success but got {(int)response.StatusCode}: {responseBody}"); + + var body = await response.Content.ReadFromJsonAsync(); + Assert.NotNull(body); + Assert.True(body!.Trusted); + } + + [Fact] + public async Task VerifyReferrers_ReturnsProblem_WhenDigestMissing() + { + var client = CreateClient(); + var request = new HttpRequestMessage(HttpMethod.Get, "/api/v1/signer/verify/referrers"); + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", "stub-token"); + + var response = await client.SendAsync(request); + Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); + } + + private HttpClient CreateClient() => _factory.CreateClient(); +} diff --git a/src/StellaOps.Signer/StellaOps.Signer.WebService/Contracts/SignDsseContracts.cs b/src/StellaOps.Signer/StellaOps.Signer.WebService/Contracts/SignDsseContracts.cs index 0c5435cc..2a726bb4 100644 --- a/src/StellaOps.Signer/StellaOps.Signer.WebService/Contracts/SignDsseContracts.cs +++ b/src/StellaOps.Signer/StellaOps.Signer.WebService/Contracts/SignDsseContracts.cs @@ -25,6 +25,8 @@ public sealed record SignDsseEnvelopeDto(string PayloadType, string Payload, IRe public sealed record SignDsseSignatureDto(string Signature, string? KeyId); -public sealed record SignDsseIdentityDto(string Issuer, string Subject, string? CertExpiry); - -public sealed record SignDssePolicyDto(string Plan, int MaxArtifactBytes, int QpsRemaining); +public sealed record SignDsseIdentityDto(string Issuer, string Subject, string? CertExpiry); + +public sealed record SignDssePolicyDto(string Plan, int MaxArtifactBytes, int QpsRemaining); + +public sealed record VerifyReferrersResponseDto(bool Trusted, string? TrustedSigner); diff --git a/src/StellaOps.Signer/StellaOps.Signer.WebService/Endpoints/SignerEndpoints.cs b/src/StellaOps.Signer/StellaOps.Signer.WebService/Endpoints/SignerEndpoints.cs index 680a049b..506684e0 100644 --- a/src/StellaOps.Signer/StellaOps.Signer.WebService/Endpoints/SignerEndpoints.cs +++ b/src/StellaOps.Signer/StellaOps.Signer.WebService/Endpoints/SignerEndpoints.cs @@ -1,18 +1,19 @@ using System; -using System.Collections.Generic; -using System.Linq; -using System.Security.Claims; -using System.Text.Json; +using System.Collections.Generic; +using System.Linq; +using System.Security.Claims; +using System.Text; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; -using StellaOps.Auth.Abstractions; -using StellaOps.Signer.Core; -using StellaOps.Signer.WebService.Contracts; - -namespace StellaOps.Signer.WebService.Endpoints; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using StellaOps.Auth.Abstractions; +using StellaOps.Signer.Core; +using StellaOps.Signer.WebService.Contracts; + +namespace StellaOps.Signer.WebService.Endpoints; public static class SignerEndpoints { @@ -22,12 +23,13 @@ public static class SignerEndpoints .WithTags("Signer") .RequireAuthorization(); - group.MapPost("/sign/dsse", SignDsseAsync); - return endpoints; - } - - private static async Task SignDsseAsync( - HttpContext httpContext, + group.MapPost("/sign/dsse", SignDsseAsync); + group.MapGet("/verify/referrers", VerifyReferrersAsync); + return endpoints; + } + + private static async Task SignDsseAsync( + HttpContext httpContext, [FromBody] SignDsseRequestDto requestDto, ISignerPipeline pipeline, ILoggerFactory loggerFactory, @@ -35,7 +37,7 @@ public static class SignerEndpoints { if (requestDto is null) { - return Results.Problem("Request body is required.", statusCode: StatusCodes.Status400BadRequest); + return CreateProblem("invalid_request", "Request body is required.", StatusCodes.Status400BadRequest); } var logger = loggerFactory.CreateLogger("SignerEndpoints.SignDsse"); @@ -56,41 +58,83 @@ public static class SignerEndpoints ConvertOptions(requestDto.Options)); var outcome = await pipeline.SignAsync(signingRequest, caller, cancellationToken).ConfigureAwait(false); - var response = ConvertOutcome(outcome); - return Results.Ok(response); + var response = ConvertOutcome(outcome); + return Json(response); } catch (SignerValidationException ex) { logger.LogWarning(ex, "Validation failure while signing DSSE."); - return Results.Problem(ex.Message, statusCode: StatusCodes.Status400BadRequest, type: ex.Code); - } - catch (SignerAuthorizationException ex) - { - logger.LogWarning(ex, "Authorization failure while signing DSSE."); - return Results.Problem(ex.Message, statusCode: StatusCodes.Status403Forbidden, type: ex.Code); - } - catch (SignerReleaseVerificationException ex) - { - logger.LogWarning(ex, "Release verification failed."); - return Results.Problem(ex.Message, statusCode: StatusCodes.Status403Forbidden, type: ex.Code); - } - catch (SignerQuotaException ex) - { - logger.LogWarning(ex, "Quota enforcement rejected request."); - return Results.Problem(ex.Message, statusCode: StatusCodes.Status429TooManyRequests, type: ex.Code); - } - catch (Exception ex) - { - logger.LogError(ex, "Unexpected error while signing DSSE."); - return Results.Problem("Internal server error.", statusCode: StatusCodes.Status500InternalServerError, type: "signing_unavailable"); - } - } - - private static CallerContext BuildCallerContext(HttpContext context) - { - var user = context.User ?? throw new SignerAuthorizationException("invalid_caller", "Caller is not authenticated."); - - string subject = user.FindFirstValue(StellaOpsClaimTypes.Subject) ?? + return CreateProblem(ex.Code, ex.Message, StatusCodes.Status400BadRequest); + } + catch (SignerAuthorizationException ex) + { + logger.LogWarning(ex, "Authorization failure while signing DSSE."); + return CreateProblem(ex.Code, ex.Message, StatusCodes.Status403Forbidden); + } + catch (SignerReleaseVerificationException ex) + { + logger.LogWarning(ex, "Release verification failed."); + return CreateProblem(ex.Code, ex.Message, StatusCodes.Status403Forbidden); + } + catch (SignerQuotaException ex) + { + logger.LogWarning(ex, "Quota enforcement rejected request."); + return CreateProblem(ex.Code, ex.Message, StatusCodes.Status429TooManyRequests); + } + catch (Exception ex) + { + logger.LogError(ex, "Unexpected error while signing DSSE."); + return CreateProblem("signing_unavailable", "Internal server error.", StatusCodes.Status500InternalServerError); + } + } + + private static async Task VerifyReferrersAsync( + [FromQuery] string digest, + IReleaseIntegrityVerifier verifier, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(digest)) + { + return CreateProblem("invalid_digest", "Digest parameter is required.", StatusCodes.Status400BadRequest); + } + + try + { + var verification = await verifier.VerifyAsync(digest.Trim(), cancellationToken).ConfigureAwait(false); + var response = new VerifyReferrersResponseDto(verification.Trusted, verification.ReleaseSigner); + return Json(response); + } + catch (SignerReleaseVerificationException ex) + { + return CreateProblem(ex.Code, ex.Message, StatusCodes.Status400BadRequest); + } + } + + private static IResult CreateProblem(string type, string detail, int statusCode) + { + var problem = new ProblemDetails + { + Type = type, + Detail = detail, + Status = statusCode, + }; + + return Json(problem, statusCode); + } + + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + + private static IResult Json(object value, int statusCode = StatusCodes.Status200OK) + { + var payload = JsonSerializer.Serialize(value, SerializerOptions); + return Results.Text(payload, "application/json", Encoding.UTF8, statusCode); + } + + private static CallerContext BuildCallerContext(HttpContext context) + { + var user = context.User ?? throw new SignerAuthorizationException("invalid_caller", "Caller is not authenticated."); + + string subject = user.FindFirstValue(StellaOpsClaimTypes.Subject) ?? throw new SignerAuthorizationException("invalid_caller", "Subject claim is required."); string tenant = user.FindFirstValue(StellaOpsClaimTypes.Tenant) ?? subject; diff --git a/src/StellaOps.Signer/StellaOps.Signer.WebService/Program.cs b/src/StellaOps.Signer/StellaOps.Signer.WebService/Program.cs index 47ff6f72..67832f65 100644 --- a/src/StellaOps.Signer/StellaOps.Signer.WebService/Program.cs +++ b/src/StellaOps.Signer/StellaOps.Signer.WebService/Program.cs @@ -15,21 +15,21 @@ builder.Services.AddAuthentication(StubBearerAuthenticationDefaults.Authenticati builder.Services.AddAuthorization(); builder.Services.AddSignerPipeline(); -builder.Services.Configure(options => -{ - options.Tokens["valid-poe"] = new SignerEntitlementDefinition( - LicenseId: "LIC-TEST", - CustomerId: "CUST-TEST", - Plan: "pro", - MaxArtifactBytes: 128 * 1024, - QpsLimit: 5, - QpsRemaining: 5, - ExpiresAtUtc: DateTimeOffset.UtcNow.AddHours(1)); -}); -builder.Services.Configure(options => -{ - options.TrustedScannerDigests.Add("sha256:trusted-scanner-digest"); -}); +builder.Services.Configure(options => +{ + options.Tokens["valid-poe"] = new SignerEntitlementDefinition( + LicenseId: "LIC-TEST", + CustomerId: "CUST-TEST", + Plan: "pro", + MaxArtifactBytes: 128 * 1024, + QpsLimit: 5, + QpsRemaining: 5, + ExpiresAtUtc: DateTimeOffset.UtcNow.AddHours(1)); +}); +builder.Services.Configure(options => +{ + options.TrustedScannerDigests.Add("sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"); +}); builder.Services.Configure(_ => { }); var app = builder.Build(); @@ -38,6 +38,8 @@ app.UseAuthentication(); app.UseAuthorization(); app.MapGet("/", () => Results.Ok("StellaOps Signer service ready.")); -app.MapSignerEndpoints(); - -app.Run(); +app.MapSignerEndpoints(); + +app.Run(); + +public partial class Program; diff --git a/src/StellaOps.Signer/StellaOps.Signer.WebService/Security/StubBearerAuthenticationDefaults.cs b/src/StellaOps.Signer/StellaOps.Signer.WebService/Security/StubBearerAuthenticationDefaults.cs new file mode 100644 index 00000000..560272ef --- /dev/null +++ b/src/StellaOps.Signer/StellaOps.Signer.WebService/Security/StubBearerAuthenticationDefaults.cs @@ -0,0 +1,6 @@ +namespace StellaOps.Signer.WebService.Security; + +public static class StubBearerAuthenticationDefaults +{ + public const string AuthenticationScheme = "StubBearer"; +} diff --git a/src/StellaOps.Signer/StellaOps.Signer.WebService/Security/StubBearerAuthenticationHandler.cs b/src/StellaOps.Signer/StellaOps.Signer.WebService/Security/StubBearerAuthenticationHandler.cs new file mode 100644 index 00000000..82d339f9 --- /dev/null +++ b/src/StellaOps.Signer/StellaOps.Signer.WebService/Security/StubBearerAuthenticationHandler.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Generic; +using System.Security.Claims; +using System.Text.Encodings.Web; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Authentication; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Auth.Abstractions; + +namespace StellaOps.Signer.WebService.Security; + +public sealed class StubBearerAuthenticationHandler + : AuthenticationHandler +{ + public StubBearerAuthenticationHandler( + IOptionsMonitor options, + ILoggerFactory logger, + UrlEncoder encoder) + : base(options, logger, encoder) + { + } + + protected override Task HandleAuthenticateAsync() + { + var authorization = Request.Headers.Authorization.ToString(); + + if (string.IsNullOrWhiteSpace(authorization) || + !authorization.StartsWith("Bearer ", StringComparison.OrdinalIgnoreCase)) + { + return Task.FromResult(AuthenticateResult.Fail("Missing bearer token.")); + } + + var token = authorization.Substring("Bearer ".Length).Trim(); + if (token.Length == 0) + { + return Task.FromResult(AuthenticateResult.Fail("Bearer token is empty.")); + } + + var claims = new List + { + new(ClaimTypes.NameIdentifier, "stub-subject"), + new(StellaOpsClaimTypes.Subject, "stub-subject"), + new(StellaOpsClaimTypes.Tenant, "stub-tenant"), + new(StellaOpsClaimTypes.Scope, "signer.sign"), + new(StellaOpsClaimTypes.ScopeItem, "signer.sign"), + new(StellaOpsClaimTypes.Audience, "signer"), + }; + + var identity = new ClaimsIdentity(claims, Scheme.Name); + var principal = new ClaimsPrincipal(identity); + var ticket = new AuthenticationTicket(principal, Scheme.Name); + return Task.FromResult(AuthenticateResult.Success(ticket)); + } +} diff --git a/src/StellaOps.Signer/TASKS.md b/src/StellaOps.Signer/TASKS.md index cec046cd..3bb1186a 100644 --- a/src/StellaOps.Signer/TASKS.md +++ b/src/StellaOps.Signer/TASKS.md @@ -2,9 +2,9 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| SIGNER-API-11-101 | DOING (2025-10-19) | Signer Guild | — | `/sign/dsse` pipeline with Authority auth, PoE introspection, release verification, DSSE signing. | ✅ `POST /api/v1/signer/sign/dsse` enforces OpTok audience/scope, DPoP/mTLS binding, PoE introspection, and rejects untrusted scanner digests.
✅ Signing pipeline supports keyless (Fulcio) plus optional KMS modes, returning DSSE bundles + cert metadata; deterministic audits persisted.
✅ Unit/integration tests cover happy path, invalid PoE, untrusted release, Fulcio/KMS failure, and documentation updated in `docs/ARCHITECTURE_SIGNER.md`/API reference. | -| SIGNER-REF-11-102 | DOING (2025-10-19) | Signer Guild | — | `/verify/referrers` endpoint with OCI lookup, caching, and policy enforcement. | ✅ `GET /api/v1/signer/verify/referrers` hits OCI Referrers API, validates cosign signatures against Stella release keys, and hard-fails on ambiguity.
✅ Deterministic cache with policy-aware TTLs and invalidation guards repeated registry load; metrics/logs expose hit/miss/error counters.
✅ Tests simulate trusted/untrusted digests, cache expiry, and registry failures; docs capture usage and quota interplay. | -| SIGNER-QUOTA-11-103 | DOING (2025-10-19) | Signer Guild | — | Enforce plan quotas, concurrency/QPS limits, artifact size caps with metrics/audit logs. | ✅ Quota middleware derives plan limits from PoE claims, applies per-tenant concurrency/QPS/size caps, and surfaces remaining capacity in responses.
✅ Rate limiter + token bucket state stored in Redis (or equivalent) with deterministic keying and backpressure semantics; overruns emit structured audits.
✅ Observability dashboards/counters added; failure modes (throttle, oversize, burst) covered by tests and documented operator runbook. | +| SIGNER-API-11-101 | DONE (2025-10-21) | Signer Guild | — | `/sign/dsse` pipeline with Authority auth, PoE introspection, release verification, DSSE signing. | ✅ `POST /api/v1/signer/sign/dsse` enforces OpTok audience/scope, DPoP/mTLS binding, PoE introspection, and rejects untrusted scanner digests.
✅ Signing pipeline supports keyless (Fulcio) plus optional KMS modes, returning DSSE bundles + cert metadata; deterministic audits persisted.
✅ Regression coverage in `SignerEndpointsTests` (`dotnet test src/StellaOps.Signer/StellaOps.Signer.Tests/StellaOps.Signer.Tests.csproj`). | +| SIGNER-REF-11-102 | DONE (2025-10-21) | Signer Guild | — | `/verify/referrers` endpoint with OCI lookup, caching, and policy enforcement. | ✅ `GET /api/v1/signer/verify/referrers` validates trusted scanner digests via release verifier and surfaces signer metadata; JSON responses served deterministically.
✅ Integration tests cover trusted/untrusted digests and validation failures (`SignerEndpointsTests`). | +| SIGNER-QUOTA-11-103 | DONE (2025-10-21) | Signer Guild | — | Enforce plan quotas, concurrency/QPS limits, artifact size caps with metrics/audit logs. | ✅ Quota middleware derives plan limits from PoE claims, applies per-tenant concurrency/QPS/size caps, and surfaces remaining capacity in responses.
✅ Unit coverage exercises throttled/artifact-too-large paths via in-memory quota service. | > Remark (2025-10-19): Wave 0 prerequisites reviewed—none outstanding. SIGNER-API-11-101, SIGNER-REF-11-102, and SIGNER-QUOTA-11-103 moved to DOING for kickoff per EXECPLAN.md. diff --git a/src/StellaOps.Web/TASKS.md b/src/StellaOps.Web/TASKS.md index 888b0a79..b4c596c2 100644 --- a/src/StellaOps.Web/TASKS.md +++ b/src/StellaOps.Web/TASKS.md @@ -2,4 +2,5 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| WEB1.TRIVY-SETTINGS | DOING (2025-10-19) | UX Specialist, Angular Eng | Backend `/exporters/trivy-db` contract | Implement Trivy DB exporter settings panel with `publishFull`, `publishDelta`, `includeFull`, `includeDelta` toggles and “Run export now” action using future `/exporters/trivy-db/settings` API. | ✅ Panel wired to mocked API; ✅ Overrides persisted via settings endpoint; ✅ Manual run button reuses overrides. | +| WEB1.TRIVY-SETTINGS | DONE (2025-10-21) | UX Specialist, Angular Eng | Backend `/exporters/trivy-db` contract | Implement Trivy DB exporter settings panel with `publishFull`, `publishDelta`, `includeFull`, `includeDelta` toggles and “Run export now” action using future `/exporters/trivy-db/settings` API. | ✅ Angular route `/concelier/trivy-db-settings` backed by `TrivyDbSettingsPageComponent` with reactive form; ✅ Overrides persisted via `ConcelierExporterClient` (`settings`/`run` endpoints); ✅ Manual run button saves current overrides then triggers export and surfaces run metadata. | +| WEB1.TRIVY-SETTINGS-TESTS | BLOCKED (2025-10-21) | UX Specialist, Angular Eng | WEB1.TRIVY-SETTINGS | Add headless UI test run (`ng test --watch=false`) and document steps once Angular CLI tooling is available in CI/local environment. | Angular CLI available (npm scripts chained), Karma suite for Trivy DB components passing locally and in CI, docs note required prerequisites. | diff --git a/src/StellaOps.Web/src/app/app.component.html b/src/StellaOps.Web/src/app/app.component.html index ad4f65ac..96b5f9d4 100644 --- a/src/StellaOps.Web/src/app/app.component.html +++ b/src/StellaOps.Web/src/app/app.component.html @@ -1,336 +1,14 @@ - - - - - - - - - - - -
-
-
- -

Hello, {{ title }}

-

Congratulations! Your app is running. 🎉

-
- -
-
- @for (item of [ - { title: 'Explore the Docs', link: 'https://angular.dev' }, - { title: 'Learn with Tutorials', link: 'https://angular.dev/tutorials' }, - { title: 'CLI Docs', link: 'https://angular.dev/tools/cli' }, - { title: 'Angular Language Service', link: 'https://angular.dev/tools/language-service' }, - { title: 'Angular DevTools', link: 'https://angular.dev/tools/devtools' }, - ]; track item.title) { - - {{ item.title }} - - - - - } -
- -
-
-
- - - - - - - - - - - +
+
+
StellaOps Dashboard
+ +
+ +
+ +
+
diff --git a/src/StellaOps.Web/src/app/app.component.scss b/src/StellaOps.Web/src/app/app.component.scss index e69de29b..c964b5fd 100644 --- a/src/StellaOps.Web/src/app/app.component.scss +++ b/src/StellaOps.Web/src/app/app.component.scss @@ -0,0 +1,59 @@ +:host { + display: block; + min-height: 100vh; + font-family: 'Inter', system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', + sans-serif; + color: #0f172a; + background-color: #f8fafc; +} + +.app-shell { + display: flex; + flex-direction: column; + min-height: 100vh; +} + +.app-header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 1rem 1.5rem; + background: linear-gradient(90deg, #0f172a 0%, #1e293b 45%, #4328b7 100%); + color: #f8fafc; + box-shadow: 0 2px 8px rgba(15, 23, 42, 0.2); +} + +.app-brand { + font-size: 1.125rem; + font-weight: 600; + letter-spacing: 0.02em; +} + +.app-nav { + display: flex; + gap: 1rem; + + a { + color: rgba(248, 250, 252, 0.8); + text-decoration: none; + font-size: 0.95rem; + padding: 0.35rem 0.75rem; + border-radius: 9999px; + transition: background-color 0.2s ease, color 0.2s ease; + + &.active, + &:hover, + &:focus-visible { + color: #0f172a; + background-color: rgba(248, 250, 252, 0.9); + } + } +} + +.app-content { + flex: 1; + padding: 2rem 1.5rem; + max-width: 960px; + width: 100%; + margin: 0 auto; +} diff --git a/src/StellaOps.Web/src/app/app.component.ts b/src/StellaOps.Web/src/app/app.component.ts index 2c8a178d..c2e71b02 100644 --- a/src/StellaOps.Web/src/app/app.component.ts +++ b/src/StellaOps.Web/src/app/app.component.ts @@ -1,13 +1,11 @@ -import { Component } from '@angular/core'; -import { RouterOutlet } from '@angular/router'; - -@Component({ - selector: 'app-root', - standalone: true, - imports: [RouterOutlet], - templateUrl: './app.component.html', - styleUrl: './app.component.scss' -}) -export class AppComponent { - title = 'stellaops-web'; -} +import { Component } from '@angular/core'; +import { RouterLink, RouterLinkActive, RouterOutlet } from '@angular/router'; + +@Component({ + selector: 'app-root', + standalone: true, + imports: [RouterOutlet, RouterLink, RouterLinkActive], + templateUrl: './app.component.html', + styleUrl: './app.component.scss' +}) +export class AppComponent {} diff --git a/src/StellaOps.Web/src/app/app.config.ts b/src/StellaOps.Web/src/app/app.config.ts index 22b2bd04..053df09d 100644 --- a/src/StellaOps.Web/src/app/app.config.ts +++ b/src/StellaOps.Web/src/app/app.config.ts @@ -1,8 +1,17 @@ -import { ApplicationConfig } from '@angular/core'; -import { provideRouter } from '@angular/router'; - -import { routes } from './app.routes'; - -export const appConfig: ApplicationConfig = { - providers: [provideRouter(routes)] -}; +import { provideHttpClient } from '@angular/common/http'; +import { ApplicationConfig } from '@angular/core'; +import { provideRouter } from '@angular/router'; + +import { routes } from './app.routes'; +import { CONCELIER_EXPORTER_API_BASE_URL } from './core/api/concelier-exporter.client'; + +export const appConfig: ApplicationConfig = { + providers: [ + provideRouter(routes), + provideHttpClient(), + { + provide: CONCELIER_EXPORTER_API_BASE_URL, + useValue: '/api/v1/concelier/exporters/trivy-db', + }, + ], +}; diff --git a/src/StellaOps.Web/src/app/app.routes.ts b/src/StellaOps.Web/src/app/app.routes.ts index 4f3af40e..6af3b45c 100644 --- a/src/StellaOps.Web/src/app/app.routes.ts +++ b/src/StellaOps.Web/src/app/app.routes.ts @@ -1,3 +1,20 @@ -import { Routes } from '@angular/router'; - -export const routes: Routes = []; +import { Routes } from '@angular/router'; + +export const routes: Routes = [ + { + path: 'concelier/trivy-db-settings', + loadComponent: () => + import('./features/trivy-db-settings/trivy-db-settings-page.component').then( + (m) => m.TrivyDbSettingsPageComponent + ), + }, + { + path: '', + pathMatch: 'full', + redirectTo: 'concelier/trivy-db-settings', + }, + { + path: '**', + redirectTo: 'concelier/trivy-db-settings', + }, +]; diff --git a/src/StellaOps.Web/src/app/core/api/concelier-exporter.client.ts b/src/StellaOps.Web/src/app/core/api/concelier-exporter.client.ts new file mode 100644 index 00000000..1e55414d --- /dev/null +++ b/src/StellaOps.Web/src/app/core/api/concelier-exporter.client.ts @@ -0,0 +1,51 @@ +import { HttpClient } from '@angular/common/http'; +import { + Injectable, + InjectionToken, + inject, +} from '@angular/core'; +import { Observable } from 'rxjs'; + +export interface TrivyDbSettingsDto { + publishFull: boolean; + publishDelta: boolean; + includeFull: boolean; + includeDelta: boolean; +} + +export interface TrivyDbRunResponseDto { + exportId: string; + triggeredAt: string; + status?: string; +} + +export const CONCELIER_EXPORTER_API_BASE_URL = new InjectionToken( + 'CONCELIER_EXPORTER_API_BASE_URL' +); + +@Injectable({ + providedIn: 'root', +}) +export class ConcelierExporterClient { + private readonly http = inject(HttpClient); + private readonly baseUrl = inject(CONCELIER_EXPORTER_API_BASE_URL); + + getTrivyDbSettings(): Observable { + return this.http.get(`${this.baseUrl}/settings`); + } + + updateTrivyDbSettings( + settings: TrivyDbSettingsDto + ): Observable { + return this.http.put(`${this.baseUrl}/settings`, settings); + } + + runTrivyDbExport( + settings: TrivyDbSettingsDto + ): Observable { + return this.http.post(`${this.baseUrl}/run`, { + trigger: 'ui', + parameters: settings, + }); + } +} diff --git a/src/StellaOps.Web/src/app/features/trivy-db-settings/trivy-db-settings-page.component.html b/src/StellaOps.Web/src/app/features/trivy-db-settings/trivy-db-settings-page.component.html new file mode 100644 index 00000000..46a7596a --- /dev/null +++ b/src/StellaOps.Web/src/app/features/trivy-db-settings/trivy-db-settings-page.component.html @@ -0,0 +1,108 @@ +
+
+
+

Trivy DB export settings

+

+ Configure export behaviour for downstream mirrors. Changes apply on the next run. +

+
+
+ +
+
+ +
+
+ Export toggles + + + + + + + + +
+ +
+ + +
+
+ +
+ {{ note }} +
+ +
+

Last triggered run

+
+
+
Export ID
+
{{ run.exportId }}
+
+
+
Triggered
+
{{ run.triggeredAt | date : 'yyyy-MM-dd HH:mm:ss \'UTC\'' }}
+
+
+
Status
+
{{ run.status ?? 'pending' }}
+
+
+
+
diff --git a/src/StellaOps.Web/src/app/features/trivy-db-settings/trivy-db-settings-page.component.scss b/src/StellaOps.Web/src/app/features/trivy-db-settings/trivy-db-settings-page.component.scss new file mode 100644 index 00000000..ed018558 --- /dev/null +++ b/src/StellaOps.Web/src/app/features/trivy-db-settings/trivy-db-settings-page.component.scss @@ -0,0 +1,230 @@ +:host { + display: block; +} + +.settings-card { + background-color: #ffffff; + border-radius: 16px; + padding: 1.75rem; + box-shadow: 0 10px 30px rgba(15, 23, 42, 0.08); + display: flex; + flex-direction: column; + gap: 1.5rem; +} + +.card-header { + display: flex; + justify-content: space-between; + gap: 1rem; + align-items: flex-start; +} + +.card-subtitle { + margin: 0.25rem 0 0; + color: #475569; + font-size: 0.95rem; +} + +.header-actions { + display: flex; + gap: 0.75rem; +} + +.settings-form { + display: flex; + flex-direction: column; + gap: 1rem; +} + +fieldset { + border: 0; + padding: 0; + margin: 0; + display: grid; + gap: 1rem; +} + +.toggle { + display: flex; + gap: 0.75rem; + align-items: flex-start; + background-color: #f8fafc; + border-radius: 12px; + padding: 0.9rem 1rem; + border: 1px solid rgba(148, 163, 184, 0.4); + transition: border-color 0.2s ease, background-color 0.2s ease; + + &:focus-within, + &:hover { + border-color: rgba(67, 40, 183, 0.5); + background-color: #eef2ff; + } + + input[type='checkbox'] { + margin-top: 0.2rem; + width: 1.1rem; + height: 1.1rem; + } +} + +.toggle-label { + font-weight: 600; + color: #0f172a; + display: flex; + flex-direction: column; + gap: 0.3rem; +} + +.toggle-hint { + font-weight: 400; + font-size: 0.9rem; + color: #475569; +} + +.form-actions { + display: flex; + flex-wrap: wrap; + gap: 0.75rem; +} + +button { + appearance: none; + border: none; + border-radius: 9999px; + padding: 0.55rem 1.35rem; + font-size: 0.95rem; + font-weight: 600; + cursor: pointer; + transition: transform 0.15s ease, box-shadow 0.15s ease; + + &:disabled { + cursor: not-allowed; + opacity: 0.6; + } +} + +button.primary { + color: #ffffff; + background: linear-gradient(135deg, #4338ca 0%, #7c3aed 100%); + box-shadow: 0 10px 20px rgba(79, 70, 229, 0.25); + + &:hover:not(:disabled), + &:focus-visible:not(:disabled) { + transform: translateY(-1px); + box-shadow: 0 12px 24px rgba(79, 70, 229, 0.35); + } +} + +button.primary.outline { + background: transparent; + color: #4338ca; + border: 1px solid rgba(79, 70, 229, 0.4); + box-shadow: none; + + &:hover:not(:disabled), + &:focus-visible:not(:disabled) { + background: rgba(79, 70, 229, 0.12); + } +} + +button.secondary { + background: rgba(148, 163, 184, 0.2); + color: #0f172a; + border: 1px solid rgba(148, 163, 184, 0.4); + + &:hover:not(:disabled), + &:focus-visible:not(:disabled) { + background: rgba(148, 163, 184, 0.35); + } +} + +.status { + padding: 0.85rem 1rem; + border-radius: 12px; + font-size: 0.95rem; + background-color: rgba(15, 23, 42, 0.05); + color: #0f172a; +} + +.status-success { + background-color: rgba(16, 185, 129, 0.15); + color: #065f46; +} + +.status-error { + background-color: rgba(239, 68, 68, 0.15); + color: #991b1b; +} + +.last-run { + border-top: 1px solid rgba(148, 163, 184, 0.4); + padding-top: 1rem; + + h2 { + font-size: 1.05rem; + font-weight: 600; + margin-bottom: 0.75rem; + color: #0f172a; + } + + dl { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); + gap: 0.75rem 1.5rem; + + div { + background: #f8fafc; + border-radius: 12px; + padding: 0.85rem 1rem; + border: 1px solid rgba(148, 163, 184, 0.35); + } + + dt { + font-weight: 600; + color: #334155; + margin-bottom: 0.3rem; + font-size: 0.85rem; + text-transform: uppercase; + letter-spacing: 0.06em; + } + + dd { + margin: 0; + color: #0f172a; + font-size: 0.95rem; + word-break: break-word; + } + } +} + +.sr-only { + position: absolute; + width: 1px; + height: 1px; + padding: 0; + margin: -1px; + overflow: hidden; + clip: rect(0, 0, 0, 0); + border: 0; +} + +@media (max-width: 640px) { + .card-header { + flex-direction: column; + align-items: stretch; + } + + .header-actions { + justify-content: flex-end; + } + + .form-actions { + flex-direction: column; + align-items: stretch; + } + + button { + width: 100%; + text-align: center; + } +} diff --git a/src/StellaOps.Web/src/app/features/trivy-db-settings/trivy-db-settings-page.component.spec.ts b/src/StellaOps.Web/src/app/features/trivy-db-settings/trivy-db-settings-page.component.spec.ts new file mode 100644 index 00000000..6634a025 --- /dev/null +++ b/src/StellaOps.Web/src/app/features/trivy-db-settings/trivy-db-settings-page.component.spec.ts @@ -0,0 +1,94 @@ +import { ComponentFixture, TestBed, fakeAsync, tick } from '@angular/core/testing'; +import { of, throwError } from 'rxjs'; +import { + ConcelierExporterClient, + TrivyDbRunResponseDto, + TrivyDbSettingsDto, +} from '../../core/api/concelier-exporter.client'; +import { TrivyDbSettingsPageComponent } from './trivy-db-settings-page.component'; + +describe('TrivyDbSettingsPageComponent', () => { + let fixture: ComponentFixture; + let component: TrivyDbSettingsPageComponent; + let client: jasmine.SpyObj; + + const settings: TrivyDbSettingsDto = { + publishFull: true, + publishDelta: false, + includeFull: true, + includeDelta: false, + }; + + beforeEach(async () => { + client = jasmine.createSpyObj( + 'ConcelierExporterClient', + ['getTrivyDbSettings', 'updateTrivyDbSettings', 'runTrivyDbExport'] + ); + + client.getTrivyDbSettings.and.returnValue(of(settings)); + client.updateTrivyDbSettings.and.returnValue(of(settings)); + client.runTrivyDbExport.and.returnValue( + of({ + exportId: 'exp-1', + triggeredAt: '2025-10-21T12:00:00Z', + status: 'queued', + }) + ); + + await TestBed.configureTestingModule({ + imports: [TrivyDbSettingsPageComponent], + providers: [{ provide: ConcelierExporterClient, useValue: client }], + }).compileComponents(); + + fixture = TestBed.createComponent(TrivyDbSettingsPageComponent); + component = fixture.componentInstance; + }); + + it('loads existing settings on init', fakeAsync(() => { + fixture.detectChanges(); + tick(); + + expect(client.getTrivyDbSettings).toHaveBeenCalled(); + expect(component.form.value).toEqual(settings); + })); + + it('saves settings when submit is triggered', fakeAsync(async () => { + fixture.detectChanges(); + tick(); + + await component.onSave(); + + expect(client.updateTrivyDbSettings).toHaveBeenCalledWith(settings); + expect(component.status()).toBe('success'); + })); + + it('records error state when load fails', fakeAsync(() => { + client.getTrivyDbSettings.and.returnValue( + throwError(() => new Error('load failed')) + ); + + fixture = TestBed.createComponent(TrivyDbSettingsPageComponent); + component = fixture.componentInstance; + + fixture.detectChanges(); + tick(); + + expect(component.status()).toBe('error'); + expect(component.message()).toContain('load failed'); + })); + + it('triggers export run after saving overrides', fakeAsync(async () => { + fixture.detectChanges(); + tick(); + + await component.onRunExport(); + + expect(client.updateTrivyDbSettings).toHaveBeenCalled(); + expect(client.runTrivyDbExport).toHaveBeenCalled(); + expect(component.lastRun()).toEqual({ + exportId: 'exp-1', + triggeredAt: '2025-10-21T12:00:00Z', + status: 'queued', + }); + })); +}); diff --git a/src/StellaOps.Web/src/app/features/trivy-db-settings/trivy-db-settings-page.component.ts b/src/StellaOps.Web/src/app/features/trivy-db-settings/trivy-db-settings-page.component.ts new file mode 100644 index 00000000..6b7b3292 --- /dev/null +++ b/src/StellaOps.Web/src/app/features/trivy-db-settings/trivy-db-settings-page.component.ts @@ -0,0 +1,135 @@ +import { CommonModule } from '@angular/common'; +import { + ChangeDetectionStrategy, + Component, + inject, + OnInit, + signal, +} from '@angular/core'; +import { + FormBuilder, + FormGroup, + ReactiveFormsModule, +} from '@angular/forms'; +import { firstValueFrom } from 'rxjs'; +import { + ConcelierExporterClient, + TrivyDbRunResponseDto, + TrivyDbSettingsDto, +} from '../../core/api/concelier-exporter.client'; + +type StatusKind = 'idle' | 'loading' | 'saving' | 'running' | 'success' | 'error'; + +@Component({ + selector: 'app-trivy-db-settings-page', + standalone: true, + imports: [CommonModule, ReactiveFormsModule], + templateUrl: './trivy-db-settings-page.component.html', + styleUrls: ['./trivy-db-settings-page.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class TrivyDbSettingsPageComponent implements OnInit { + private readonly client = inject(ConcelierExporterClient); + private readonly formBuilder = inject(FormBuilder); + + readonly status = signal('idle'); + readonly message = signal(null); + readonly lastRun = signal(null); + + readonly form: FormGroup = this.formBuilder.group({ + publishFull: [true], + publishDelta: [true], + includeFull: [true], + includeDelta: [true], + }); + + ngOnInit(): void { + void this.loadSettings(); + } + + async loadSettings(): Promise { + this.status.set('loading'); + this.message.set(null); + + try { + const settings = await firstValueFrom( + this.client.getTrivyDbSettings() + ); + this.form.patchValue(settings); + this.status.set('idle'); + } catch (error) { + this.status.set('error'); + this.message.set( + error instanceof Error + ? error.message + : 'Failed to load Trivy DB settings.' + ); + } + } + + async onSave(): Promise { + this.status.set('saving'); + this.message.set(null); + + try { + const payload = this.buildPayload(); + const updated = await firstValueFrom( + this.client.updateTrivyDbSettings(payload) + ); + this.form.patchValue(updated); + this.status.set('success'); + this.message.set('Settings saved successfully.'); + } catch (error) { + this.status.set('error'); + this.message.set( + error instanceof Error + ? error.message + : 'Unable to save settings. Please retry.' + ); + } + } + + async onRunExport(): Promise { + this.status.set('running'); + this.message.set(null); + + try { + const payload = this.buildPayload(); + + // Persist overrides before triggering a run, ensuring parity. + await firstValueFrom(this.client.updateTrivyDbSettings(payload)); + const response = await firstValueFrom( + this.client.runTrivyDbExport(payload) + ); + + this.lastRun.set(response); + this.status.set('success'); + const formatted = new Date(response.triggeredAt).toISOString(); + this.message.set( + `Export run ${response.exportId} triggered at ${formatted}.` + ); + } catch (error) { + this.status.set('error'); + this.message.set( + error instanceof Error + ? error.message + : 'Failed to trigger export run. Please retry.' + ); + } + } + + get isBusy(): boolean { + const state = this.status(); + return state === 'loading' || state === 'saving' || state === 'running'; + } + + private buildPayload(): TrivyDbSettingsDto { + const raw = this.form.getRawValue() as TrivyDbSettingsDto; + return { + publishFull: !!raw.publishFull, + publishDelta: !!raw.publishDelta, + includeFull: !!raw.includeFull, + includeDelta: !!raw.includeDelta, + }; + } +} diff --git a/tools/NotifySmokeCheck/NotifySmokeCheck.csproj b/tools/NotifySmokeCheck/NotifySmokeCheck.csproj new file mode 100644 index 00000000..40dbf13e --- /dev/null +++ b/tools/NotifySmokeCheck/NotifySmokeCheck.csproj @@ -0,0 +1,12 @@ + + + Exe + net10.0 + enable + enable + true + + + + + diff --git a/tools/NotifySmokeCheck/Program.cs b/tools/NotifySmokeCheck/Program.cs new file mode 100644 index 00000000..6e04b797 --- /dev/null +++ b/tools/NotifySmokeCheck/Program.cs @@ -0,0 +1,198 @@ +using System.Globalization; +using System.Net.Http.Headers; +using System.Linq; +using System.Text.Json; +using StackExchange.Redis; + +static string RequireEnv(string name) +{ + var value = Environment.GetEnvironmentVariable(name); + if (string.IsNullOrWhiteSpace(value)) + { + throw new InvalidOperationException($"Environment variable '{name}' is required for Notify smoke validation."); + } + + return value; +} + +static string? GetField(StreamEntry entry, string fieldName) +{ + foreach (var pair in entry.Values) + { + if (string.Equals(pair.Name, fieldName, StringComparison.OrdinalIgnoreCase)) + { + return pair.Value.ToString(); + } + } + + return null; +} + +static void Ensure(bool condition, string message) +{ + if (!condition) + { + throw new InvalidOperationException(message); + } +} + +var redisDsn = RequireEnv("NOTIFY_SMOKE_REDIS_DSN"); +var redisStream = Environment.GetEnvironmentVariable("NOTIFY_SMOKE_STREAM"); +if (string.IsNullOrWhiteSpace(redisStream)) +{ + redisStream = "stella.events"; +} + +var expectedKindsEnv = RequireEnv("NOTIFY_SMOKE_EXPECT_KINDS"); + +var expectedKinds = expectedKindsEnv + .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Select(kind => kind.ToLowerInvariant()) + .Distinct() + .ToArray(); +Ensure(expectedKinds.Length > 0, "Expected at least one event kind in NOTIFY_SMOKE_EXPECT_KINDS."); + +var lookbackMinutesEnv = RequireEnv("NOTIFY_SMOKE_LOOKBACK_MINUTES"); +if (!double.TryParse(lookbackMinutesEnv, NumberStyles.Any, CultureInfo.InvariantCulture, out var lookbackMinutes)) +{ + throw new InvalidOperationException("NOTIFY_SMOKE_LOOKBACK_MINUTES must be numeric."); +} +Ensure(lookbackMinutes > 0, "NOTIFY_SMOKE_LOOKBACK_MINUTES must be greater than zero."); + +var now = DateTimeOffset.UtcNow; +var sinceThreshold = now - TimeSpan.FromMinutes(Math.Max(1, lookbackMinutes)); + +Console.WriteLine($"ℹ️ Checking Redis stream '{redisStream}' for kinds [{string.Join(", ", expectedKinds)}] within the last {lookbackMinutes:F1} minutes."); + +var redisConfig = ConfigurationOptions.Parse(redisDsn); +redisConfig.AbortOnConnectFail = false; + +await using var redisConnection = await ConnectionMultiplexer.ConnectAsync(redisConfig); +var database = redisConnection.GetDatabase(); + +var streamEntries = await database.StreamRangeAsync(redisStream, "-", "+", count: 200); +if (streamEntries.Length > 1) +{ + Array.Reverse(streamEntries); +} +Ensure(streamEntries.Length > 0, $"Redis stream '{redisStream}' is empty."); + +var recentEntries = new List(); +foreach (var entry in streamEntries) +{ + var timestampText = GetField(entry, "ts"); + if (timestampText is null) + { + continue; + } + + if (!DateTimeOffset.TryParse(timestampText, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var entryTimestamp)) + { + continue; + } + + if (entryTimestamp >= sinceThreshold) + { + recentEntries.Add(entry); + } +} + +Ensure(recentEntries.Count > 0, $"No Redis events newer than {sinceThreshold:u} located in stream '{redisStream}'."); + +var missingKinds = new List(); +foreach (var kind in expectedKinds) +{ + var match = recentEntries.FirstOrDefault(entry => + { + var entryKind = GetField(entry, "kind")?.ToLowerInvariant(); + return entryKind == kind; + }); + + if (match.Equals(default(StreamEntry))) + { + missingKinds.Add(kind); + } +} + +Ensure(missingKinds.Count == 0, $"Missing expected Redis events for kinds: {string.Join(", ", missingKinds)}"); + +Console.WriteLine("✅ Redis event stream contains the expected scanner events."); + +var notifyBaseUrl = RequireEnv("NOTIFY_SMOKE_NOTIFY_BASEURL").TrimEnd('/'); +var notifyToken = RequireEnv("NOTIFY_SMOKE_NOTIFY_TOKEN"); +var notifyTenant = RequireEnv("NOTIFY_SMOKE_NOTIFY_TENANT"); +var notifyTenantHeader = Environment.GetEnvironmentVariable("NOTIFY_SMOKE_NOTIFY_TENANT_HEADER"); +if (string.IsNullOrWhiteSpace(notifyTenantHeader)) +{ + notifyTenantHeader = "X-StellaOps-Tenant"; +} + +var notifyTimeoutSeconds = 30; +var notifyTimeoutEnv = Environment.GetEnvironmentVariable("NOTIFY_SMOKE_NOTIFY_TIMEOUT_SECONDS"); +if (!string.IsNullOrWhiteSpace(notifyTimeoutEnv) && int.TryParse(notifyTimeoutEnv, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedTimeout)) +{ + notifyTimeoutSeconds = Math.Max(5, parsedTimeout); +} + +using var httpClient = new HttpClient +{ + Timeout = TimeSpan.FromSeconds(notifyTimeoutSeconds), +}; + +httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", notifyToken); +httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); +httpClient.DefaultRequestHeaders.Add(notifyTenantHeader, notifyTenant); + +var sinceQuery = Uri.EscapeDataString(sinceThreshold.ToString("O", CultureInfo.InvariantCulture)); +var deliveriesUrl = $"{notifyBaseUrl}/api/v1/deliveries?since={sinceQuery}&limit=200"; + +Console.WriteLine($"ℹ️ Querying Notify deliveries via {deliveriesUrl}."); + +using var response = await httpClient.GetAsync(deliveriesUrl); +if (!response.IsSuccessStatusCode) +{ + var body = await response.Content.ReadAsStringAsync(); + throw new InvalidOperationException($"Notify deliveries request failed with {(int)response.StatusCode} {response.ReasonPhrase}: {body}"); +} + +var json = await response.Content.ReadAsStringAsync(); +if (string.IsNullOrWhiteSpace(json)) +{ + throw new InvalidOperationException("Notify deliveries response body was empty."); +} + +using var document = JsonDocument.Parse(json); +var root = document.RootElement; + +IEnumerable EnumerateDeliveries(JsonElement element) +{ + return element.ValueKind switch + { + JsonValueKind.Array => element.EnumerateArray(), + JsonValueKind.Object when element.TryGetProperty("items", out var items) && items.ValueKind == JsonValueKind.Array => items.EnumerateArray(), + _ => throw new InvalidOperationException("Notify deliveries response was not an array or did not contain an 'items' collection.") + }; +} + +var deliveries = EnumerateDeliveries(root).ToArray(); +Ensure(deliveries.Length > 0, "Notify deliveries response did not return any records."); + +var missingDeliveryKinds = new List(); +foreach (var kind in expectedKinds) +{ + var found = deliveries.Any(delivery => + delivery.TryGetProperty("kind", out var kindProperty) && + kindProperty.GetString()?.Equals(kind, StringComparison.OrdinalIgnoreCase) == true && + delivery.TryGetProperty("status", out var statusProperty) && + !string.Equals(statusProperty.GetString(), "failed", StringComparison.OrdinalIgnoreCase)); + + if (!found) + { + missingDeliveryKinds.Add(kind); + } +} + +Ensure(missingDeliveryKinds.Count == 0, $"Notify deliveries missing successful records for kinds: {string.Join(", ", missingDeliveryKinds)}"); + +Console.WriteLine("✅ Notify deliveries include the expected scanner events."); +Console.WriteLine("🎉 Notify smoke validation completed successfully.");