feat: scheduler web+worker merge + audit Batch 1 (68 endpoints annotated)
Scheduler: - Merge scheduler-worker into scheduler-web with Worker:Embedded flag - Default embedded=true (compose), false available for K8s split - Upgrade to resources-heavy, comment out scheduler-worker container Audit Batch 1 (first real audit emission): - Create AuditedRouteGroupExtensions convention helper - EvidenceLocker: 7 endpoints (store/snapshot/verify/hold/export/verdict) - Integrations: 6 endpoints (CRUD + test + discover) - Scanner: 55 endpoints across 25 files - Sprint 005 FILTER-001/002/003 marked DONE Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
master
@@ -115,17 +115,17 @@ services:
|
||||
com.stellaops.crypto.profile: "china"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scheduler Worker - China crypto overlay
|
||||
# Scheduler Worker - MERGED into scheduler-web (Scheduler:Worker:Embedded=true)
|
||||
# ---------------------------------------------------------------------------
|
||||
scheduler-worker:
|
||||
image: registry.stella-ops.org/stellaops/scheduler-worker:china
|
||||
environment:
|
||||
<<: *crypto-env
|
||||
volumes:
|
||||
- ../../etc/appsettings.crypto.china.yaml:/app/etc/appsettings.crypto.yaml:ro
|
||||
- ../../etc/crypto-plugins-manifest.json:/app/etc/crypto-plugins-manifest.json:ro
|
||||
labels:
|
||||
com.stellaops.crypto.profile: "china"
|
||||
# scheduler-worker:
|
||||
# image: registry.stella-ops.org/stellaops/scheduler-worker:china
|
||||
# environment:
|
||||
# <<: *crypto-env
|
||||
# volumes:
|
||||
# - ../../etc/appsettings.crypto.china.yaml:/app/etc/appsettings.crypto.yaml:ro
|
||||
# - ../../etc/crypto-plugins-manifest.json:/app/etc/crypto-plugins-manifest.json:ro
|
||||
# labels:
|
||||
# com.stellaops.crypto.profile: "china"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Notify Web - China crypto overlay
|
||||
|
||||
@@ -121,18 +121,18 @@ services:
|
||||
com.stellaops.compliance: "eidas"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scheduler Worker - EU crypto overlay
|
||||
# Scheduler Worker - MERGED into scheduler-web (Scheduler:Worker:Embedded=true)
|
||||
# ---------------------------------------------------------------------------
|
||||
scheduler-worker:
|
||||
image: registry.stella-ops.org/stellaops/scheduler-worker:eu
|
||||
environment:
|
||||
<<: *crypto-env
|
||||
volumes:
|
||||
- ../../etc/appsettings.crypto.eu.yaml:/app/etc/appsettings.crypto.yaml:ro
|
||||
- ../../etc/crypto-plugins-manifest.json:/app/etc/crypto-plugins-manifest.json:ro
|
||||
labels:
|
||||
com.stellaops.crypto.profile: "eu"
|
||||
com.stellaops.compliance: "eidas"
|
||||
# scheduler-worker:
|
||||
# image: registry.stella-ops.org/stellaops/scheduler-worker:eu
|
||||
# environment:
|
||||
# <<: *crypto-env
|
||||
# volumes:
|
||||
# - ../../etc/appsettings.crypto.eu.yaml:/app/etc/appsettings.crypto.yaml:ro
|
||||
# - ../../etc/crypto-plugins-manifest.json:/app/etc/crypto-plugins-manifest.json:ro
|
||||
# labels:
|
||||
# com.stellaops.crypto.profile: "eu"
|
||||
# com.stellaops.compliance: "eidas"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Notify Web - EU crypto overlay
|
||||
|
||||
@@ -129,18 +129,18 @@ services:
|
||||
com.stellaops.crypto.provider: "openssl.gost,pkcs11.gost,cryptopro.gost"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scheduler Worker - Russia crypto overlay
|
||||
# Scheduler Worker - MERGED into scheduler-web (Scheduler:Worker:Embedded=true)
|
||||
# ---------------------------------------------------------------------------
|
||||
scheduler-worker:
|
||||
image: registry.stella-ops.org/stellaops/scheduler-worker:russia
|
||||
environment:
|
||||
<<: *crypto-env
|
||||
volumes:
|
||||
- ../../etc/appsettings.crypto.russia.yaml:/app/etc/appsettings.crypto.yaml:ro
|
||||
- ../../etc/crypto-plugins-manifest.json:/app/etc/crypto-plugins-manifest.json:ro
|
||||
labels:
|
||||
com.stellaops.crypto.profile: "russia"
|
||||
com.stellaops.crypto.provider: "openssl.gost,pkcs11.gost,cryptopro.gost"
|
||||
# scheduler-worker:
|
||||
# image: registry.stella-ops.org/stellaops/scheduler-worker:russia
|
||||
# environment:
|
||||
# <<: *crypto-env
|
||||
# volumes:
|
||||
# - ../../etc/appsettings.crypto.russia.yaml:/app/etc/appsettings.crypto.yaml:ro
|
||||
# - ../../etc/crypto-plugins-manifest.json:/app/etc/crypto-plugins-manifest.json:ro
|
||||
# labels:
|
||||
# com.stellaops.crypto.profile: "russia"
|
||||
# com.stellaops.crypto.provider: "openssl.gost,pkcs11.gost,cryptopro.gost"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Notify Web - Russia crypto overlay
|
||||
|
||||
@@ -1193,14 +1193,16 @@ services:
|
||||
# jobengine and jobengine-worker removed.
|
||||
# Release endpoints → release-orchestrator service (Slot 47)
|
||||
# Workflow orchestration → workflow service (Slot 46)
|
||||
# Scheduler remains in Slot 14 (scheduler-web / scheduler-worker)
|
||||
# Scheduler remains in Slot 19 (scheduler-web; worker merged in)
|
||||
|
||||
# --- Slot 18: TaskRunner (REMOVED) ------------------------------------------
|
||||
# taskrunner-web and taskrunner-worker deleted; task_runner_id DB columns left as nullable legacy
|
||||
|
||||
# --- Slot 19: Scheduler ----------------------------------------------------
|
||||
# --- Slot 19: Scheduler (web + embedded worker) ----------------------------
|
||||
# Worker BackgroundServices now run embedded in the web process (Scheduler:Worker:Embedded=true).
|
||||
# Set Scheduler__Worker__Embedded=false and restore scheduler-worker for K8s split deployments.
|
||||
scheduler-web:
|
||||
<<: *resources-medium
|
||||
<<: *resources-heavy
|
||||
image: stellaops/scheduler-web:dev
|
||||
container_name: stellaops-scheduler-web
|
||||
restart: unless-stopped
|
||||
@@ -1211,69 +1213,72 @@ services:
|
||||
ConnectionStrings__Default: *postgres-connection
|
||||
ConnectionStrings__Redis: "cache.stella-ops.local:6379"
|
||||
Scheduler__Authority__Enabled: "false"
|
||||
# Worker options are validated even in web mode
|
||||
# Embedded worker mode (all 8 BackgroundServices in this process)
|
||||
Scheduler__Worker__Embedded: "true"
|
||||
scheduler__queue__Kind: "Redis"
|
||||
scheduler__queue__Redis__ConnectionString: "cache.stella-ops.local:6379"
|
||||
Scheduler__Storage__Postgres__Scheduler__ConnectionString: *postgres-connection
|
||||
Scheduler__Storage__Postgres__Scheduler__SchemaName: "scheduler"
|
||||
Scheduler__Worker__Runner__Scanner__BaseAddress: "http://scanner.stella-ops.local"
|
||||
Scheduler__Worker__Runner__Scanner__BaseAddress: "${SCHEDULER_SCANNER_BASEADDRESS:-http://scanner.stella-ops.local}"
|
||||
Scheduler__Worker__Graph__Cartographer__BaseAddress: "http://graph.stella-ops.local"
|
||||
Scheduler__Worker__Graph__SchedulerApi__BaseAddress: "http://scheduler.stella-ops.local"
|
||||
Scheduler__Worker__Policy__Api__BaseAddress: "http://policy.stella-ops.local"
|
||||
# Surface environment (merged from scheduler-worker)
|
||||
SURFACE_FS_ENDPOINT: "http://s3.stella-ops.local:8333"
|
||||
Router__Enabled: "${SCHEDULER_ROUTER_ENABLED:-true}"
|
||||
Router__Messaging__ConsumerGroup: "scheduler"
|
||||
volumes:
|
||||
- *cert-volume
|
||||
tmpfs:
|
||||
- /plugins:mode=1777
|
||||
- /var/lib/stellaops/surface:mode=1777
|
||||
ports:
|
||||
- "127.1.0.19:80:80"
|
||||
networks:
|
||||
stellaops:
|
||||
aliases:
|
||||
- scheduler.stella-ops.local
|
||||
- scheduler-worker.stella-ops.local
|
||||
frontdoor: {}
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "bash -c 'echo > /dev/tcp/$(hostname)/80'"]
|
||||
<<: *healthcheck-tcp
|
||||
labels: *release-labels
|
||||
|
||||
scheduler-worker:
|
||||
<<: *resources-medium
|
||||
image: stellaops/scheduler-worker:dev
|
||||
container_name: stellaops-scheduler-worker
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
<<: [*kestrel-cert, *gc-medium]
|
||||
# Queue config (Redis transport)
|
||||
scheduler__queue__Kind: "Redis"
|
||||
scheduler__queue__Redis__ConnectionString: "cache.stella-ops.local:6379"
|
||||
# Persistence config (section: Scheduler:Storage, subsection: Postgres:Scheduler)
|
||||
Scheduler__Storage__Postgres__Scheduler__ConnectionString: *postgres-connection
|
||||
Scheduler__Storage__Postgres__Scheduler__SchemaName: "scheduler"
|
||||
# Worker config
|
||||
Scheduler__Worker__Runner__Scanner__BaseAddress: "${SCHEDULER_SCANNER_BASEADDRESS:-http://scanner.stella-ops.local}"
|
||||
Scheduler__Worker__Graph__Cartographer__BaseAddress: "http://graph.stella-ops.local"
|
||||
Scheduler__Worker__Graph__SchedulerApi__BaseAddress: "http://scheduler.stella-ops.local"
|
||||
Scheduler__Worker__Policy__Api__BaseAddress: "http://policy.stella-ops.local"
|
||||
# Surface environment
|
||||
SURFACE_FS_ENDPOINT: "http://s3.stella-ops.local:8333"
|
||||
volumes:
|
||||
- *cert-volume
|
||||
tmpfs:
|
||||
- /var/lib/stellaops/surface:mode=1777
|
||||
networks:
|
||||
stellaops:
|
||||
aliases:
|
||||
- scheduler-worker.stella-ops.local
|
||||
healthcheck:
|
||||
<<: *healthcheck-worker
|
||||
labels: *release-labels
|
||||
# scheduler-worker: MERGED into scheduler-web (Scheduler:Worker:Embedded=true)
|
||||
# Uncomment and set Scheduler__Worker__Embedded=false on scheduler-web for K8s split.
|
||||
# scheduler-worker:
|
||||
# <<: *resources-medium
|
||||
# image: stellaops/scheduler-worker:dev
|
||||
# container_name: stellaops-scheduler-worker
|
||||
# restart: unless-stopped
|
||||
# depends_on:
|
||||
# postgres:
|
||||
# condition: service_healthy
|
||||
# valkey:
|
||||
# condition: service_healthy
|
||||
# environment:
|
||||
# <<: [*kestrel-cert, *gc-medium]
|
||||
# scheduler__queue__Kind: "Redis"
|
||||
# scheduler__queue__Redis__ConnectionString: "cache.stella-ops.local:6379"
|
||||
# Scheduler__Storage__Postgres__Scheduler__ConnectionString: *postgres-connection
|
||||
# Scheduler__Storage__Postgres__Scheduler__SchemaName: "scheduler"
|
||||
# Scheduler__Worker__Runner__Scanner__BaseAddress: "${SCHEDULER_SCANNER_BASEADDRESS:-http://scanner.stella-ops.local}"
|
||||
# Scheduler__Worker__Graph__Cartographer__BaseAddress: "http://graph.stella-ops.local"
|
||||
# Scheduler__Worker__Graph__SchedulerApi__BaseAddress: "http://scheduler.stella-ops.local"
|
||||
# Scheduler__Worker__Policy__Api__BaseAddress: "http://policy.stella-ops.local"
|
||||
# SURFACE_FS_ENDPOINT: "http://s3.stella-ops.local:8333"
|
||||
# volumes:
|
||||
# - *cert-volume
|
||||
# tmpfs:
|
||||
# - /var/lib/stellaops/surface:mode=1777
|
||||
# networks:
|
||||
# stellaops:
|
||||
# aliases:
|
||||
# - scheduler-worker.stella-ops.local
|
||||
# healthcheck:
|
||||
# <<: *healthcheck-worker
|
||||
# labels: *release-labels
|
||||
|
||||
# --- Slot 20: Graph API ----------------------------------------------------
|
||||
graph-api:
|
||||
|
||||
@@ -13,7 +13,8 @@ mkdir -p "$OUT_DIR"
|
||||
docker compose ps >/dev/null
|
||||
|
||||
echo "Pausing worker containers for consistency..."
|
||||
docker compose pause scanner-worker scheduler-worker taskrunner-worker || true
|
||||
docker compose pause scanner-worker || true
|
||||
# scheduler-worker merged into scheduler-web; taskrunner-worker removed
|
||||
|
||||
echo "Backing up volumes..."
|
||||
docker run --rm \
|
||||
@@ -23,6 +24,6 @@ docker run --rm \
|
||||
-v "$PWD/$OUT_DIR":/out \
|
||||
alpine sh -c "cd / && tar czf /out/stellaops-backup-$TS.tar.gz data"
|
||||
|
||||
docker compose unpause scanner-worker scheduler-worker taskrunner-worker || true
|
||||
docker compose unpause scanner-worker || true
|
||||
|
||||
echo "Backup written to $OUT_DIR/stellaops-backup-$TS.tar.gz"
|
||||
|
||||
@@ -43,7 +43,8 @@ riskengine-worker|devops/docker/Dockerfile.hardened.template|src/Findings/Stella
|
||||
# ── Slot 18: TaskRunner (REMOVED) ───────────────────────────────────────────────
|
||||
# ── Slot 19: Scheduler ──────────────────────────────────────────────────────────
|
||||
scheduler-web|devops/docker/Dockerfile.hardened.template|src/JobEngine/StellaOps.Scheduler.WebService/StellaOps.Scheduler.WebService.csproj|StellaOps.Scheduler.WebService|8080
|
||||
scheduler-worker|devops/docker/Dockerfile.hardened.template|src/JobEngine/StellaOps.Scheduler.Worker.Host/StellaOps.Scheduler.Worker.Host.csproj|StellaOps.Scheduler.Worker.Host|8080
|
||||
# scheduler-worker: MERGED into scheduler-web (Scheduler:Worker:Embedded=true)
|
||||
# scheduler-worker|devops/docker/Dockerfile.hardened.template|src/JobEngine/StellaOps.Scheduler.Worker.Host/StellaOps.Scheduler.Worker.Host.csproj|StellaOps.Scheduler.Worker.Host|8080
|
||||
# ── Slot 20: Graph ──────────────────────────────────────────────────────────────
|
||||
graph-api|devops/docker/Dockerfile.hardened.template|src/Graph/StellaOps.Graph.Api/StellaOps.Graph.Api.csproj|StellaOps.Graph.Api|8080
|
||||
# ── Slot 21: Cartographer (RETIRED -- merged into graph-api Slot 20) ──────────
|
||||
|
||||
@@ -165,7 +165,7 @@ Sprint 208 consolidated Scheduler, TaskRunner, and PacksRegistry source trees un
|
||||
|
||||
The Scheduler service re-evaluates already-cataloged images when intelligence changes (Concelier/Excititor/policy), orchestrates nightly and ad-hoc runs, targets only impacted images using the BOM-Index, and emits report-ready events for downstream Notify. Default mode is analysis-only (no image pull); optional content-refresh can be enabled per schedule.
|
||||
|
||||
**Deployables:** `StellaOps.Scheduler.WebService` (stateless), `StellaOps.Scheduler.Worker.Host` (scale-out).
|
||||
**Deployables:** `StellaOps.Scheduler.WebService` (stateless API + embedded worker BackgroundServices). Worker processes run in the same host by default (`Scheduler:Worker:Embedded=true`). For K8s scale-out, set `Embedded=false` and deploy `StellaOps.Scheduler.Worker.Host` separately.
|
||||
|
||||
**Database:** `SchedulerDbContext` (schema `scheduler`, 11 entities). Owns `schedules`, `runs`, `impact_cursors`, `locks`, `audit` tables. See archived docs: `docs-archived/modules/scheduler/architecture.md`.
|
||||
|
||||
|
||||
@@ -41,6 +41,8 @@ using StellaOps.Scheduler.Worker.Options;
|
||||
using StellaOps.Scheduler.Plugin;
|
||||
using StellaOps.Scheduler.Plugin.Scan;
|
||||
using StellaOps.Scheduler.Plugin.Doctor;
|
||||
using StellaOps.Scheduler.Queue;
|
||||
using StellaOps.Scheduler.Worker.DependencyInjection;
|
||||
using System.Linq;
|
||||
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
@@ -161,11 +163,32 @@ builder.Services.AddScoped<IGraphJobService, GraphJobService>();
|
||||
builder.Services.AddImpactIndex();
|
||||
builder.Services.AddResolverJobServices();
|
||||
|
||||
// Exception lifecycle workers (SCHED-WORKER-25-101/25-102)
|
||||
var workerOptions = builder.Configuration.GetSection("Scheduler:Worker").Get<SchedulerWorkerOptions>() ?? new SchedulerWorkerOptions();
|
||||
workerOptions.Validate();
|
||||
builder.Services.AddSingleton(workerOptions);
|
||||
builder.Services.AddSingleton<SchedulerWorkerMetrics>();
|
||||
// Embedded worker mode: when Scheduler:Worker:Embedded is true (default),
|
||||
// all 8 BackgroundServices (6 heavy workers + 2 exception workers) run in this
|
||||
// process, eliminating the need for a separate scheduler-worker container.
|
||||
// Set to false for K8s deployments that scale workers independently.
|
||||
var embeddedWorker = builder.Configuration.GetValue("Scheduler:Worker:Embedded", true);
|
||||
|
||||
if (embeddedWorker)
|
||||
{
|
||||
// Register queue transport (Redis/NATS) – required by worker background services
|
||||
builder.Services.AddSchedulerQueues(builder.Configuration);
|
||||
|
||||
// Register all worker background services (Planner, Runner, PolicyRun,
|
||||
// GraphBuild, GraphOverlay, PlannerQueueDispatcher) plus supporting services
|
||||
// (Surface FS, crypto, HTTP clients for Scanner/Policy/Cartographer).
|
||||
builder.Services.AddSchedulerWorker(builder.Configuration.GetSection("Scheduler:Worker"));
|
||||
}
|
||||
else
|
||||
{
|
||||
// Standalone web mode: only exception lifecycle workers run here.
|
||||
var workerOptions = builder.Configuration.GetSection("Scheduler:Worker").Get<SchedulerWorkerOptions>() ?? new SchedulerWorkerOptions();
|
||||
workerOptions.Validate();
|
||||
builder.Services.AddSingleton(workerOptions);
|
||||
builder.Services.AddSingleton<SchedulerWorkerMetrics>();
|
||||
}
|
||||
|
||||
// Exception workers and bootstrap always run in the web process regardless of embedded mode
|
||||
builder.Services.AddSingleton<IExceptionRepository, PostgresExceptionRepository>();
|
||||
builder.Services.AddSingleton<IExceptionEventPublisher>(NullExceptionEventPublisher.Instance);
|
||||
builder.Services.AddSingleton<IExpiringDigestService>(NullExpiringDigestService.Instance);
|
||||
|
||||
Reference in New Issue
Block a user