feat: Implement runner execution pipeline with planner dispatch and execution services

- Introduced RunnerBackgroundService to handle execution of runner segments. - Added RunnerExecutionService for processing segments and aggregating results. - Implemented PlannerQueueDispatchService to manage dispatching of planner messages. - Created PlannerQueueDispatcherBackgroundService for leasing and processing planner queue messages. - Developed ScannerReportClient for interacting with the scanner service. - Enhanced observability with SchedulerWorkerMetrics for tracking planner and runner performance. - Added comprehensive documentation for the new runner execution pipeline and observability metrics. - Implemented event emission for rescan activity and scanner report readiness.
2025-10-27 18:57:35 +02:00
parent 14617e9c3b
commit 92ff5a6011
42 changed files with 3981 additions and 184 deletions
--- a/deploy/compose/README.md
+++ b/deploy/compose/README.md
@@ -66,6 +66,21 @@ Scanner WebService can emit signed `scanner.report.*` events to Redis Streams wh

 Helm values mirror the same knobs under each service’s `env` map (see `deploy/helm/stellaops/values-*.yaml`).

+### Scheduler worker configuration
+
+Every Compose profile now provisions the `scheduler-worker` container (backed by the
+`StellaOps.Scheduler.Worker.Host` entrypoint). The environment placeholders exposed
+in the `.env` samples match the options bound by `AddSchedulerWorker`:
+
+- `SCHEDULER_QUEUE_KIND` – queue transport (`Nats` or `Redis`).
+- `SCHEDULER_QUEUE_NATS_URL` – NATS connection string used by planner/runner consumers.
+- `SCHEDULER_STORAGE_DATABASE` – MongoDB database name for scheduler state.
+- `SCHEDULER_SCANNER_BASEADDRESS` – base URL the runner uses when invoking Scanner’s
+  `/api/v1/reports` (defaults to the in-cluster `http://scanner-web:8444`).
+
+Helm deployments inherit the same defaults from `services.scheduler-worker.env` in
+`values.yaml`; override them per environment as needed.
+
 ### Front-door network hand-off

 `docker-compose.prod.yaml` adds a `frontdoor` network so operators can attach Traefik, Envoy, or an on-prem load balancer that terminates TLS. Override `FRONTDOOR_NETWORK` in `prod.env` if your reverse proxy uses a different bridge name. Attach only the externally reachable services (Authority, Signer, Attestor, Concelier, Scanner Web, Notify Web, UI) to that network—internal infrastructure (Mongo, MinIO, RustFS, NATS) stays on the private `stellaops` network.
--- a/deploy/compose/docker-compose.airgap.yaml
+++ b/deploy/compose/docker-compose.airgap.yaml
@@ -170,8 +170,8 @@ services:
    labels: *release-labels

  scanner-worker:
-    image: registry.stella-ops.org/stellaops/scanner-worker@sha256:eea5d6cfe7835950c5ec7a735a651f2f0d727d3e470cf9027a4a402ea89c4fb5
-    restart: unless-stopped
+    image: registry.stella-ops.org/stellaops/scanner-worker@sha256:eea5d6cfe7835950c5ec7a735a651f2f0d727d3e470cf9027a4a402ea89c4fb5
+    restart: unless-stopped
    depends_on:
      - scanner-web
      - rustfs
@@ -182,10 +182,30 @@ services:
      SCANNER__ARTIFACTSTORE__ENDPOINT: "http://rustfs:8080/api/v1"
      SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts"
      SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30"
-      SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}"
-    networks:
-      - stellaops
-    labels: *release-labels
+      SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}"
+    networks:
+      - stellaops
+    labels: *release-labels
+
+  scheduler-worker:
+    image: registry.stella-ops.org/stellaops/scheduler-worker:2025.10.0-edge
+    restart: unless-stopped
+    depends_on:
+      - mongo
+      - nats
+      - scanner-web
+    command:
+      - "dotnet"
+      - "StellaOps.Scheduler.Worker.Host.dll"
+    environment:
+      SCHEDULER__QUEUE__KIND: "${SCHEDULER_QUEUE_KIND:-Nats}"
+      SCHEDULER__QUEUE__NATS__URL: "${SCHEDULER_QUEUE_NATS_URL:-nats://nats:4222}"
+      SCHEDULER__STORAGE__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017"
+      SCHEDULER__STORAGE__DATABASE: "${SCHEDULER_STORAGE_DATABASE:-stellaops_scheduler}"
+      SCHEDULER__WORKER__RUNNER__SCANNER__BASEADDRESS: "${SCHEDULER_SCANNER_BASEADDRESS:-http://scanner-web:8444}"
+    networks:
+      - stellaops
+    labels: *release-labels

  notify-web:
    image: ${NOTIFY_WEB_IMAGE:-registry.stella-ops.org/stellaops/notify-web:2025.09.2}
--- a/deploy/compose/docker-compose.dev.yaml
+++ b/deploy/compose/docker-compose.dev.yaml
@@ -168,8 +168,8 @@ services:
    labels: *release-labels

  scanner-worker:
-    image: registry.stella-ops.org/stellaops/scanner-worker@sha256:92dda42f6f64b2d9522104a5c9ffb61d37b34dd193132b68457a259748008f37
-    restart: unless-stopped
+    image: registry.stella-ops.org/stellaops/scanner-worker@sha256:92dda42f6f64b2d9522104a5c9ffb61d37b34dd193132b68457a259748008f37
+    restart: unless-stopped
    depends_on:
      - scanner-web
      - rustfs
@@ -181,9 +181,29 @@ services:
      SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts"
      SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30"
      SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}"
-    networks:
-      - stellaops
-    labels: *release-labels
+    networks:
+      - stellaops
+    labels: *release-labels
+
+  scheduler-worker:
+    image: registry.stella-ops.org/stellaops/scheduler-worker:2025.10.0-edge
+    restart: unless-stopped
+    depends_on:
+      - mongo
+      - nats
+      - scanner-web
+    command:
+      - "dotnet"
+      - "StellaOps.Scheduler.Worker.Host.dll"
+    environment:
+      SCHEDULER__QUEUE__KIND: "${SCHEDULER_QUEUE_KIND:-Nats}"
+      SCHEDULER__QUEUE__NATS__URL: "${SCHEDULER_QUEUE_NATS_URL:-nats://nats:4222}"
+      SCHEDULER__STORAGE__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017"
+      SCHEDULER__STORAGE__DATABASE: "${SCHEDULER_STORAGE_DATABASE:-stellaops_scheduler}"
+      SCHEDULER__WORKER__RUNNER__SCANNER__BASEADDRESS: "${SCHEDULER_SCANNER_BASEADDRESS:-http://scanner-web:8444}"
+    networks:
+      - stellaops
+    labels: *release-labels

  notify-web:
    image: ${NOTIFY_WEB_IMAGE:-registry.stella-ops.org/stellaops/notify-web:2025.10.0-edge}
--- a/deploy/compose/docker-compose.prod.yaml
+++ b/deploy/compose/docker-compose.prod.yaml
@@ -193,6 +193,26 @@ services:
      - stellaops
    labels: *release-labels

+  scheduler-worker:
+    image: registry.stella-ops.org/stellaops/scheduler-worker:2025.10.0-edge
+    restart: unless-stopped
+    depends_on:
+      - mongo
+      - nats
+      - scanner-web
+    command:
+      - "dotnet"
+      - "StellaOps.Scheduler.Worker.Host.dll"
+    environment:
+      SCHEDULER__QUEUE__KIND: "${SCHEDULER_QUEUE_KIND:-Nats}"
+      SCHEDULER__QUEUE__NATS__URL: "${SCHEDULER_QUEUE_NATS_URL:-nats://nats:4222}"
+      SCHEDULER__STORAGE__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017"
+      SCHEDULER__STORAGE__DATABASE: "${SCHEDULER_STORAGE_DATABASE:-stellaops_scheduler}"
+      SCHEDULER__WORKER__RUNNER__SCANNER__BASEADDRESS: "${SCHEDULER_SCANNER_BASEADDRESS:-http://scanner-web:8444}"
+    networks:
+      - stellaops
+    labels: *release-labels
+
  notify-web:
    image: ${NOTIFY_WEB_IMAGE:-registry.stella-ops.org/stellaops/notify-web:2025.09.2}
    restart: unless-stopped
--- a/deploy/compose/docker-compose.stage.yaml
+++ b/deploy/compose/docker-compose.stage.yaml
@@ -168,8 +168,8 @@ services:
    labels: *release-labels

  scanner-worker:
-    image: registry.stella-ops.org/stellaops/scanner-worker@sha256:32e25e76386eb9ea8bee0a1ad546775db9a2df989fab61ac877e351881960dab
-    restart: unless-stopped
+    image: registry.stella-ops.org/stellaops/scanner-worker@sha256:32e25e76386eb9ea8bee0a1ad546775db9a2df989fab61ac877e351881960dab
+    restart: unless-stopped
    depends_on:
      - scanner-web
      - rustfs
@@ -180,10 +180,30 @@ services:
      SCANNER__ARTIFACTSTORE__ENDPOINT: "http://rustfs:8080/api/v1"
      SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts"
      SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30"
-      SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}"
-    networks:
-      - stellaops
-    labels: *release-labels
+      SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}"
+    networks:
+      - stellaops
+    labels: *release-labels
+
+  scheduler-worker:
+    image: registry.stella-ops.org/stellaops/scheduler-worker:2025.10.0-edge
+    restart: unless-stopped
+    depends_on:
+      - mongo
+      - nats
+      - scanner-web
+    command:
+      - "dotnet"
+      - "StellaOps.Scheduler.Worker.Host.dll"
+    environment:
+      SCHEDULER__QUEUE__KIND: "${SCHEDULER_QUEUE_KIND:-Nats}"
+      SCHEDULER__QUEUE__NATS__URL: "${SCHEDULER_QUEUE_NATS_URL:-nats://nats:4222}"
+      SCHEDULER__STORAGE__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017"
+      SCHEDULER__STORAGE__DATABASE: "${SCHEDULER_STORAGE_DATABASE:-stellaops_scheduler}"
+      SCHEDULER__WORKER__RUNNER__SCANNER__BASEADDRESS: "${SCHEDULER_SCANNER_BASEADDRESS:-http://scanner-web:8444}"
+    networks:
+      - stellaops
+    labels: *release-labels

  notify-web:
    image: ${NOTIFY_WEB_IMAGE:-registry.stella-ops.org/stellaops/notify-web:2025.09.2}
--- a/deploy/compose/env/airgap.env.example
+++ b/deploy/compose/env/airgap.env.example
@@ -23,3 +23,7 @@ SCANNER_EVENTS_DSN=
 SCANNER_EVENTS_STREAM=stella.events
 SCANNER_EVENTS_PUBLISH_TIMEOUT_SECONDS=5
 SCANNER_EVENTS_MAX_STREAM_LENGTH=10000
+SCHEDULER_QUEUE_KIND=Nats
+SCHEDULER_QUEUE_NATS_URL=nats://nats:4222
+SCHEDULER_STORAGE_DATABASE=stellaops_scheduler
+SCHEDULER_SCANNER_BASEADDRESS=http://scanner-web:8444
--- a/deploy/compose/env/dev.env.example
+++ b/deploy/compose/env/dev.env.example
@@ -22,3 +22,7 @@ SCANNER_EVENTS_DSN=
 SCANNER_EVENTS_STREAM=stella.events
 SCANNER_EVENTS_PUBLISH_TIMEOUT_SECONDS=5
 SCANNER_EVENTS_MAX_STREAM_LENGTH=10000
+SCHEDULER_QUEUE_KIND=Nats
+SCHEDULER_QUEUE_NATS_URL=nats://nats:4222
+SCHEDULER_STORAGE_DATABASE=stellaops_scheduler
+SCHEDULER_SCANNER_BASEADDRESS=http://scanner-web:8444
--- a/deploy/compose/env/prod.env.example
+++ b/deploy/compose/env/prod.env.example
@@ -25,5 +25,9 @@ SCANNER_EVENTS_DSN=
 SCANNER_EVENTS_STREAM=stella.events
 SCANNER_EVENTS_PUBLISH_TIMEOUT_SECONDS=5
 SCANNER_EVENTS_MAX_STREAM_LENGTH=10000
+SCHEDULER_QUEUE_KIND=Nats
+SCHEDULER_QUEUE_NATS_URL=nats://nats:4222
+SCHEDULER_STORAGE_DATABASE=stellaops_scheduler
+SCHEDULER_SCANNER_BASEADDRESS=http://scanner-web:8444
 # External reverse proxy (Traefik, Envoy, etc.) that terminates TLS.
 FRONTDOOR_NETWORK=stellaops_frontdoor
--- a/deploy/compose/env/stage.env.example
+++ b/deploy/compose/env/stage.env.example
@@ -22,3 +22,7 @@ SCANNER_EVENTS_DSN=
 SCANNER_EVENTS_STREAM=stella.events
 SCANNER_EVENTS_PUBLISH_TIMEOUT_SECONDS=5
 SCANNER_EVENTS_MAX_STREAM_LENGTH=10000
+SCHEDULER_QUEUE_KIND=Nats
+SCHEDULER_QUEUE_NATS_URL=nats://nats:4222
+SCHEDULER_STORAGE_DATABASE=stellaops_scheduler
+SCHEDULER_SCANNER_BASEADDRESS=http://scanner-web:8444
--- a/deploy/helm/stellaops/values.yaml
+++ b/deploy/helm/stellaops/values.yaml
@@ -34,4 +34,16 @@ telemetry:
      metricsPort: 9464
    resources: {}

-services: {}
+services:
+  scheduler-worker:
+    image: registry.stella-ops.org/stellaops/scheduler-worker:2025.10.0-edge
+    replicas: 1
+    command:
+      - dotnet
+      - StellaOps.Scheduler.Worker.Host.dll
+    env:
+      SCHEDULER__QUEUE__KIND: Nats
+      SCHEDULER__QUEUE__NATS__URL: nats://nats:4222
+      SCHEDULER__STORAGE__CONNECTIONSTRING: mongodb://scheduler-mongo:27017
+      SCHEDULER__STORAGE__DATABASE: stellaops_scheduler
+      SCHEDULER__WORKER__RUNNER__SCANNER__BASEADDRESS: http://scanner-web:8444