From 786d09b88f9bebcf68844444a01b17785703b3da Mon Sep 17 00:00:00 2001 From: master <> Date: Wed, 15 Apr 2026 11:14:41 +0300 Subject: [PATCH] feat(policy): persist gate evaluation queue, snapshots, orchestrator jobs Policy Engine: moves gate evaluation, snapshots, orchestrator job tracking, and ledger export from in-memory state to Postgres-backed stores. - New persistence migrations 007 (runtime state), 008 (snapshot artifact identity), 009 (orchestrator jobs). - New repositories: PolicyEngineSnapshotRepository, PolicyEngineLedgerExportRepository, PolicyEngineOrchestratorJobRepository, WorkerResultRepository. - Gateway services: GateEvaluationJobDispatchService, GateEvaluationJobStatusService, GateEvaluationJobWorker, SchedulerBackedGateEvaluationQueue (plus Unsupported fallback), GateTargetSnapshotMaterializer, PersistedKnowledgeSnapshotStore, GateBaselineBootstrapper, PolicyGateEvaluationJobExecutor. - New endpoints: GateJobEndpoints for job status + dispatch. - Worker host: PolicyOrchestratorJobWorkerHost to drain the persistent queue. - PersistedOrchestratorStores + DeltaSnapshotServiceAdapter swap in the persistent implementations via DI. Tests: PersistedDeltaRuntimeTests, PolicyEngineGateTargetSnapshotRuntimeTests, PolicyEngineRegistryWebhookRuntimeTests, PostgresLedgerExportStoreTests, PostgresSnapshotStoreTests, PolicyGatewayPersistedDeltaRuntimeTests, RegistryWebhookQueueRuntimeTests. Archives the old S001 demo seed. Docs: policy API + architecture pages updated. Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/api/policy.md | 52 ++ docs/modules/policy/architecture.md | 18 +- .../Endpoints/Gateway/GateEndpoints.cs | 30 +- .../Endpoints/Gateway/GateJobEndpoints.cs | 39 ++ .../Endpoints/Gateway/GovernanceEndpoints.cs | 32 +- .../Gateway/RegistryWebhookEndpoints.cs | 202 +++--- .../Endpoints/SnapshotEndpoint.cs | 6 +- .../Ledger/LedgerExportService.cs | 2 +- .../Ledger/LedgerExportStore.cs | 65 +- .../Orchestration/OrchestratorJobService.cs | 9 +- .../Orchestration/OrchestratorJobStore.cs | 29 + .../PersistedOrchestratorStores.cs | 197 ++++++ .../Orchestration/PolicyWorkerService.cs | 47 +- src/Policy/StellaOps.Policy.Engine/Program.cs | 41 +- .../Properties/AssemblyInfo.cs | 1 + .../Gateway/DeltaSnapshotServiceAdapter.cs | 57 +- .../Gateway/GateBaselineBootstrapper.cs | 120 ++++ .../GateEvaluationJobDispatchService.cs | 210 +++++++ .../Gateway/GateEvaluationJobStatusService.cs | 144 +++++ .../Gateway/GateEvaluationJobWorker.cs | 44 ++ .../Gateway/GateEvaluationQueueContracts.cs | 48 ++ .../Gateway/GateEvaluationQueueOptions.cs | 10 + .../Gateway/GateTargetSnapshotMaterializer.cs | 123 ++++ .../Gateway/InMemoryGateEvaluationQueue.cs | 185 ------ .../PersistedKnowledgeSnapshotStore.cs | 356 +++++++++++ ...icyAsyncGateEvaluationRuntimeExtensions.cs | 100 +++ .../PolicyGateEvaluationJobExecutor.cs | 287 +++++++++ .../SchedulerBackedGateEvaluationQueue.cs | 137 +++++ .../Gateway/UnsupportedGateEvaluationQueue.cs | 24 + .../Snapshots/SnapshotModels.cs | 15 +- .../Snapshots/SnapshotService.cs | 20 +- .../Snapshots/SnapshotStore.cs | 69 +++ .../StellaOps.Policy.Engine.csproj | 1 + src/Policy/StellaOps.Policy.Engine/TASKS.md | 6 + .../PolicyOrchestratorJobWorkerHost.cs | 100 +++ .../Endpoints/GateEndpoints.cs | 29 +- .../Endpoints/RegistryWebhookEndpoints.cs | 202 +++--- .../StellaOps.Policy.Gateway/Program.cs | 33 +- .../Services/InMemoryGateEvaluationQueue.cs | 185 ------ src/Policy/StellaOps.Policy.Gateway/TASKS.md | 3 + .../Extensions/PolicyPersistenceExtensions.cs | 14 + .../Migrations/001_initial_schema.sql | 239 ++++---- .../007_policy_engine_runtime_state.sql | 34 ++ ...licy_engine_snapshot_artifact_identity.sql | 13 + .../009_policy_engine_orchestrator_jobs.sql | 22 + .../{ => _archived}/S001_demo_seed.sql | 0 .../PolicyEngineLedgerExportDocument.cs | 18 + .../PolicyEngineOrchestratorJobDocument.cs | 22 + .../Models/PolicyEngineSnapshotDocument.cs | 19 + .../IPolicyEngineLedgerExportRepository.cs | 13 + .../IPolicyEngineSnapshotRepository.cs | 13 + .../Repositories/IWorkerResultRepository.cs | 7 + .../PolicyEngineLedgerExportRepository.cs | 197 ++++++ .../PolicyEngineOrchestratorJobRepository.cs | 273 +++++++++ .../PolicyEngineSnapshotRepository.cs | 204 +++++++ .../Repositories/WorkerResultRepository.cs | 54 ++ .../Postgres/ServiceCollectionExtensions.cs | 18 + .../StellaOps.Policy.Persistence/TASKS.md | 4 + .../Deltas/PersistedDeltaRuntimeTests.cs | 182 ++++++ .../Integration/PolicyEngineApiHostTests.cs | 299 ++++++++- ...icyEngineGateTargetSnapshotRuntimeTests.cs | 214 +++++++ ...PolicyEngineRegistryWebhookRuntimeTests.cs | 578 ++++++++++++++++++ .../Ledger/PostgresLedgerExportStoreTests.cs | 112 ++++ .../SnapshotServiceTests.cs | 10 +- .../Snapshots/PostgresSnapshotStoreTests.cs | 118 ++++ .../StellaOps.Policy.Engine.Tests/TASKS.md | 3 + ...PolicyGatewayPersistedDeltaRuntimeTests.cs | 220 +++++++ .../RegistryWebhookQueueRuntimeTests.cs | 567 +++++++++++++++++ .../StellaOps.Policy.Gateway.Tests/TASKS.md | 3 + .../TestPolicyGatewayFactory.cs | 14 +- 70 files changed, 5994 insertions(+), 768 deletions(-) create mode 100644 src/Policy/StellaOps.Policy.Engine/Endpoints/Gateway/GateJobEndpoints.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Orchestration/PersistedOrchestratorStores.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateBaselineBootstrapper.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateEvaluationJobDispatchService.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateEvaluationJobStatusService.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateEvaluationJobWorker.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateEvaluationQueueContracts.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateEvaluationQueueOptions.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateTargetSnapshotMaterializer.cs delete mode 100644 src/Policy/StellaOps.Policy.Engine/Services/Gateway/InMemoryGateEvaluationQueue.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Services/Gateway/PersistedKnowledgeSnapshotStore.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Services/Gateway/PolicyAsyncGateEvaluationRuntimeExtensions.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Services/Gateway/PolicyGateEvaluationJobExecutor.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Services/Gateway/SchedulerBackedGateEvaluationQueue.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Services/Gateway/UnsupportedGateEvaluationQueue.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Workers/PolicyOrchestratorJobWorkerHost.cs delete mode 100644 src/Policy/StellaOps.Policy.Gateway/Services/InMemoryGateEvaluationQueue.cs create mode 100644 src/Policy/__Libraries/StellaOps.Policy.Persistence/Migrations/007_policy_engine_runtime_state.sql create mode 100644 src/Policy/__Libraries/StellaOps.Policy.Persistence/Migrations/008_policy_engine_snapshot_artifact_identity.sql create mode 100644 src/Policy/__Libraries/StellaOps.Policy.Persistence/Migrations/009_policy_engine_orchestrator_jobs.sql rename src/Policy/__Libraries/StellaOps.Policy.Persistence/Migrations/{ => _archived}/S001_demo_seed.sql (100%) create mode 100644 src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Models/PolicyEngineLedgerExportDocument.cs create mode 100644 src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Models/PolicyEngineOrchestratorJobDocument.cs create mode 100644 src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Models/PolicyEngineSnapshotDocument.cs create mode 100644 src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/IPolicyEngineLedgerExportRepository.cs create mode 100644 src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/IPolicyEngineSnapshotRepository.cs create mode 100644 src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/PolicyEngineLedgerExportRepository.cs create mode 100644 src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/PolicyEngineOrchestratorJobRepository.cs create mode 100644 src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/PolicyEngineSnapshotRepository.cs create mode 100644 src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Deltas/PersistedDeltaRuntimeTests.cs create mode 100644 src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Integration/PolicyEngineGateTargetSnapshotRuntimeTests.cs create mode 100644 src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Integration/PolicyEngineRegistryWebhookRuntimeTests.cs create mode 100644 src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Ledger/PostgresLedgerExportStoreTests.cs create mode 100644 src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Snapshots/PostgresSnapshotStoreTests.cs create mode 100644 src/Policy/__Tests/StellaOps.Policy.Gateway.Tests/PolicyGatewayPersistedDeltaRuntimeTests.cs create mode 100644 src/Policy/__Tests/StellaOps.Policy.Gateway.Tests/RegistryWebhookQueueRuntimeTests.cs diff --git a/docs/api/policy.md b/docs/api/policy.md index 70f867318..066b568e0 100644 --- a/docs/api/policy.md +++ b/docs/api/policy.md @@ -407,6 +407,58 @@ Scopes: policy:runs, policy:simulate Produces sealed bundle for determinism verification; returns location of bundle. +### 6.6 Orchestrator Job Producer Runtime + +The engine-owned orchestration surface is exposed directly under `/policy/*`. Unlike the stateless batch evaluator below, this path persists orchestrator and worker state in Policy storage. + +``` +POST /policy/orchestrator/jobs +Scopes: policy:run +``` + +**Request** + +```json +{ + "tenantId": "acme", + "policyVersion": "sha256:1fb2...", + "items": [ + { + "findingId": "acme::artifact-1::CVE-2024-12345", + "eventId": "5d1fcc61-6903-42ef-9285-7f4d3d8f7f69", + "event": { "...": "canonical ledger payload" } + } + ] +} +``` + +**Response 200** + +```json +{ + "jobId": "01HSR2M1D0BP7V2QY3QGJ31Z8V", + "status": "queued", + "requestedAt": "2026-04-15T08:30:00Z", + "completedAt": null, + "resultHash": null +} +``` + +Related endpoints: + +``` +POST /policy/orchestrator/jobs/preview +GET /policy/orchestrator/jobs/{jobId} +GET /policy/worker/jobs/{jobId} +``` + +Runtime contract: + +- Submitting `/policy/orchestrator/jobs` now signals the engine background worker, which leases queued jobs, marks them `running`, executes `PolicyWorkerService`, persists `policy.worker_results`, and records terminal `completed` or `failed` state on the orchestrator job. +- Clients should poll `GET /policy/orchestrator/jobs/{jobId}` for status and read the final materialized worker payload from `GET /policy/worker/jobs/{jobId}`. +- `POST /policy/orchestrator/jobs/preview` remains non-mutating. +- `POST /api/policy/eval/batch` remains explicitly stateless and must not be used as a persistence side channel for orchestrator or worker rows. + --- ## 7 · Batch Evaluation API diff --git a/docs/modules/policy/architecture.md b/docs/modules/policy/architecture.md index f5e8ecf70..aa300a6d1 100644 --- a/docs/modules/policy/architecture.md +++ b/docs/modules/policy/architecture.md @@ -450,12 +450,28 @@ POST /api/v1/webhooks/registry/harbor POST /api/v1/webhooks/registry/generic ``` -Webhook handlers enqueue async gate evaluation jobs in the Scheduler via `GateEvaluationJob`. +Webhook push handlers now use a runtime-selected async gate-evaluation path. When `Postgres:Scheduler` is not configured, both Policy hosts resolve `IGateEvaluationQueue` to an explicit unsupported runtime adapter and return `501 problem+json` instead of fabricating queued work. When scheduler persistence is configured, both hosts register the shared scheduler-backed queue runtime, auto-migrate the scheduler schema, enqueue deduplicated `policy.gate-evaluation` jobs, dispatch them through the worker service, and expose persisted request/decision status from `GET /api/v1/policy/gate/jobs/{jobId}`. The previous process-local `InMemoryGateEvaluationQueue` and background worker path was removed because it fabricated "no drift" gate contexts and fake job IDs instead of dispatching real work. #### Gate Bypass Auditing Bypass attempts are logged to `policy.gate_bypass_audit`: +- The live `POST /api/v1/policy/gate/evaluate` runtime now resolves gate-bypass auditing through the PostgreSQL-backed `PostgresGateBypassAuditRepository`, scoped by the current tenant context. When a tenant context is genuinely absent, the adapter falls back deterministically to tenant `public` rather than using a process-local in-memory store. +- `StellaOps.Policy.Gateway` now uses the same PostgreSQL-backed gate-bypass audit path through the unified `IStellaOpsTenantAccessor`, so the standalone gateway no longer keeps a separate in-memory audit repository for compatibility routes. + +#### Runtime Snapshots And Ledger Exports + +- The live snapshot surface now persists engine runtime state in PostgreSQL-owned tables `policy.engine_ledger_exports` and `policy.engine_snapshots`, reached through `PostgresLedgerExportStore` and `PostgresSnapshotStore` rather than the previous process-local in-memory stores. +- Sync and async gate evaluation now materialize a tenant-scoped target snapshot in `policy.engine_snapshots` before delta computation. The runtime derives that target snapshot from the latest persisted `policy.engine_ledger_exports` document (or the baseline snapshot's export), stamps the artifact digest/repository/tag onto the snapshot row, and then passes the real snapshot identifier into `DeltaComputer`. +- `IOrchestratorJobStore` and `IWorkerResultStore` now resolve to persisted adapters over `policy.orchestrator_jobs` and `policy.worker_results`, so Policy export/bootstrap logic survives host recreates instead of depending on process-local completed-job state. +- Direct `/policy/orchestrator/jobs` submissions now use a real producer runtime. `OrchestratorJobService.SubmitAsync` signals `PolicyOrchestratorJobWorkerHost`, the host leases the next queued job from `IOrchestratorJobStore`, marks it `running`, executes `PolicyWorkerService`, persists `policy.worker_results`, and records terminal `completed` or `failed` state instead of requiring a separate manual `/policy/worker/run` call. +- The deterministic `/api/policy/eval/batch` surface remains stateless by contract. It returns evaluation results to callers but does not populate `policy.orchestrator_jobs` or `policy.worker_results`. +- When a gate request omits an explicit baseline reference and the tenant has no persisted baseline snapshot yet, the engine now auto-builds the first ledger export from completed persisted Policy results and auto-creates a baseline snapshot before materializing the target snapshot. Explicit baseline references remain strict: if the caller asks for a missing snapshot ID, the runtime fails instead of inventing one. +- `StellaOps.Policy.Persistence` now applies startup migrations for the Policy schema on `policy-engine` boot, and `001_initial_schema.sql` is idempotent on reused local volumes so snapshot/export runtime convergence does not depend on a fresh database. +- The merged gateway compatibility routes now register the unified StellaOps tenant accessor and middleware alongside the Policy-specific tenant context middleware. This keeps copied `RequireTenant()` filters from failing pre-handler with `500` and allows the persisted delta compatibility path to reach the real `DeltaComputer`. +- The live delta compatibility surface now projects persisted engine snapshots through `PersistedKnowledgeSnapshotStore` and `DeltaSnapshotServiceAdapter`, so tenant-scoped `/api/policy/deltas/compute` requests fail only on normal contract/data issues rather than process-local tenant or snapshot-store gaps. +- `StellaOps.Policy.Gateway` now uses the same persisted delta projection path for its standalone compatibility host: `ISnapshotStore` resolves to `PersistedKnowledgeSnapshotStore` and `StellaOps.Policy.Deltas.ISnapshotService` resolves to the engine-owned `DeltaSnapshotServiceAdapter`, replacing the old `InMemorySnapshotStore` path that fabricated mostly-empty compatibility input. + ```json { "bypassId": "bypass-uuid", diff --git a/src/Policy/StellaOps.Policy.Engine/Endpoints/Gateway/GateEndpoints.cs b/src/Policy/StellaOps.Policy.Engine/Endpoints/Gateway/GateEndpoints.cs index 57c96ea33..d5dac8ade 100644 --- a/src/Policy/StellaOps.Policy.Engine/Endpoints/Gateway/GateEndpoints.cs +++ b/src/Policy/StellaOps.Policy.Engine/Endpoints/Gateway/GateEndpoints.cs @@ -11,6 +11,7 @@ using StellaOps.Policy.Audit; using StellaOps.Policy.Deltas; using StellaOps.Policy.Engine.Gates; using StellaOps.Policy.Engine.Services; +using StellaOps.Policy.Engine.Services.Gateway; using StellaOps.Policy.Engine.Contracts.Gateway; namespace StellaOps.Policy.Engine.Endpoints.Gateway; @@ -39,6 +40,8 @@ public static class GateEndpoints IDriftGateEvaluator gateEvaluator, IDeltaComputer deltaComputer, IBaselineSelector baselineSelector, + GateBaselineBootstrapper gateBaselineBootstrapper, + GateTargetSnapshotMaterializer targetSnapshotMaterializer, IGateBypassAuditor bypassAuditor, IMemoryCache cache, [FromServices] TimeProvider timeProvider, @@ -73,6 +76,22 @@ public static class GateEndpoints baselineSelector, cancellationToken); + if (!baselineResult.IsFound && string.IsNullOrWhiteSpace(request.BaselineRef)) + { + var bootstrapped = await gateBaselineBootstrapper + .TryEnsureBaselineAsync(cancellationToken) + .ConfigureAwait(false); + if (bootstrapped) + { + baselineResult = await ResolveBaselineAsync( + request.ImageDigest, + request.BaselineRef, + baselineSelector, + cancellationToken) + .ConfigureAwait(false); + } + } + if (!baselineResult.IsFound) { // If no baseline, allow with a note (first build scenario) @@ -94,10 +113,17 @@ public static class GateEndpoints } // Step 2: Compute delta between baseline and current + var targetSnapshot = await targetSnapshotMaterializer.MaterializeAsync( + request.ImageDigest, + request.Repository, + request.Tag, + baselineResult.Snapshot!.SnapshotId, + cancellationToken); + var delta = await deltaComputer.ComputeDeltaAsync( baselineResult.Snapshot!.SnapshotId, - request.ImageDigest, // Use image digest as target snapshot ID - new ArtifactRef(request.ImageDigest, null, null), + targetSnapshot.SnapshotId, + new ArtifactRef(request.ImageDigest, request.Repository, request.Tag), cancellationToken); // Cache the delta for audit diff --git a/src/Policy/StellaOps.Policy.Engine/Endpoints/Gateway/GateJobEndpoints.cs b/src/Policy/StellaOps.Policy.Engine/Endpoints/Gateway/GateJobEndpoints.cs new file mode 100644 index 000000000..76016f8e1 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Endpoints/Gateway/GateJobEndpoints.cs @@ -0,0 +1,39 @@ +using Microsoft.AspNetCore.Mvc; +using StellaOps.Auth.Abstractions; +using StellaOps.Auth.ServerIntegration; +using StellaOps.Auth.ServerIntegration.Tenancy; +using StellaOps.Policy.Engine.Services.Gateway; + +namespace StellaOps.Policy.Engine.Endpoints.Gateway; + +public static class GateJobEndpoints +{ + public static void MapGateJobEndpoints(this WebApplication app) + { + var gates = app.MapGroup("/api/v1/policy/gate") + .WithTags("Gates") + .RequireTenant(); + + gates.MapGet("/jobs/{jobId}", async Task( + string jobId, + [FromServices] IGateEvaluationJobStatusService statusService, + CancellationToken cancellationToken) => + { + var status = await statusService.GetAsync(jobId, cancellationToken).ConfigureAwait(false); + if (status is null) + { + return Results.NotFound(new ProblemDetails + { + Title = "Job not found", + Status = StatusCodes.Status404NotFound, + Detail = $"No async gate evaluation job found with ID: {jobId}" + }); + } + + return Results.Ok(status); + }) + .RequireAuthorization(policy => policy.RequireStellaOpsScopes(StellaOpsScopes.PolicyRead)) + .WithName("GetGateJobStatus") + .WithDescription("Returns the persisted scheduler-backed status for an async registry webhook gate evaluation job."); + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Endpoints/Gateway/GovernanceEndpoints.cs b/src/Policy/StellaOps.Policy.Engine/Endpoints/Gateway/GovernanceEndpoints.cs index bdd45971f..5a0a9ff74 100644 --- a/src/Policy/StellaOps.Policy.Engine/Endpoints/Gateway/GovernanceEndpoints.cs +++ b/src/Policy/StellaOps.Policy.Engine/Endpoints/Gateway/GovernanceEndpoints.cs @@ -37,93 +37,93 @@ public static class GovernanceEndpoints // Sealed Mode endpoints governance.MapGet("/sealed-mode/status", GetSealedModeStatusAsync) .RequireAuthorization(policy => policy.RequireStellaOpsScopes(StellaOpsScopes.AirgapStatusRead)) - .WithName("GetSealedModeStatus") + .WithName("Governance.GetSealedModeStatus") .WithDescription("Retrieve the current sealed mode status for the tenant, including whether the environment is sealed, when it was sealed, by whom, configured trust roots, allowed sources, and any active override entries. Returns HTTP 400 when no tenant can be resolved from the request context."); governance.MapGet("/sealed-mode/overrides", GetSealedModeOverridesAsync) .RequireAuthorization(policy => policy.RequireStellaOpsScopes(StellaOpsScopes.AirgapStatusRead)) - .WithName("GetSealedModeOverrides") + .WithName("Governance.GetSealedModeOverrides") .WithDescription("List all sealed mode overrides for the tenant, including override type, target resource, approver IDs, expiry timestamp, and active status. Used by operators to audit active bypass grants and verify sealed posture integrity."); governance.MapPost("/sealed-mode/toggle", ToggleSealedModeAsync) .RequireAuthorization(policy => policy.RequireStellaOpsScopes(StellaOpsScopes.AirgapSeal)) - .WithName("ToggleSealedMode") + .WithName("Governance.ToggleSealedMode") .WithDescription("Enable or disable sealed mode for the tenant. When enabling, records the sealing actor, timestamp, reason, trust roots, and allowed sources. When disabling, records the unseal timestamp. Every toggle is recorded as a governance audit event.") .Audited(AuditModules.Policy, AuditActions.Policy.ToggleSealedMode); governance.MapPost("/sealed-mode/overrides", CreateSealedModeOverrideAsync) .RequireAuthorization(policy => policy.RequireStellaOpsScopes(StellaOpsScopes.AirgapSeal)) - .WithName("CreateSealedModeOverride") + .WithName("Governance.CreateSealedModeOverride") .WithDescription("Create a time-limited override to allow a specific operation or target to bypass sealed mode restrictions. The override expires after the configured duration (defaulting to 24 hours) and is recorded in the governance audit log with the approving actor.") .Audited(AuditModules.Policy, AuditActions.Policy.EmergencyUnseal); governance.MapPost("/sealed-mode/overrides/{overrideId}/revoke", RevokeSealedModeOverrideAsync) .RequireAuthorization(policy => policy.RequireStellaOpsScopes(StellaOpsScopes.AirgapSeal)) - .WithName("RevokeSealedModeOverride") + .WithName("Governance.RevokeSealedModeOverride") .WithDescription("Revoke an active sealed mode override before its natural expiry, providing an optional reason. The override is marked inactive immediately, preventing further bypass use. The revocation is recorded in the governance audit log.") .Audited(AuditModules.Policy, AuditActions.Policy.RevokeSealedOverride); // Risk Profile endpoints governance.MapGet("/risk-profiles", ListRiskProfilesAsync) .RequireAuthorization(policy => policy.RequireStellaOpsScopes(StellaOpsScopes.PolicyRead)) - .WithName("ListRiskProfiles") + .WithName("Governance.ListRiskProfiles") .WithDescription("List risk profiles for the tenant with optional status filtering (draft, active, deprecated). Each profile includes its signal configuration, severity overrides, action overrides, and lifecycle metadata. The default risk profile is always included in the response."); governance.MapGet("/risk-profiles/{profileId}", GetRiskProfileAsync) .RequireAuthorization(policy => policy.RequireStellaOpsScopes(StellaOpsScopes.PolicyRead)) - .WithName("GetRiskProfile") + .WithName("Governance.GetRiskProfile") .WithDescription("Retrieve the full configuration of a specific risk profile by its identifier, including all signals with weights and enabled state, severity and action overrides, and the profile version and lifecycle metadata."); governance.MapPost("/risk-profiles", CreateRiskProfileAsync) .RequireAuthorization(policy => policy.RequireStellaOpsScopes(StellaOpsScopes.PolicyAuthor)) - .WithName("CreateRiskProfile") + .WithName("Governance.CreateRiskProfile") .WithDescription("Create a new risk profile in draft state with the specified signal configuration, severity overrides, and action overrides. The profile can optionally extend an existing base profile. Audit events are recorded for all profile changes.") .Audited(AuditModules.Policy, AuditActions.Policy.CreateRiskProfile); governance.MapPut("/risk-profiles/{profileId}", UpdateRiskProfileAsync) .RequireAuthorization(policy => policy.RequireStellaOpsScopes(StellaOpsScopes.PolicyAuthor)) - .WithName("UpdateRiskProfile") + .WithName("Governance.UpdateRiskProfile") .WithDescription("Update the name, description, signals, severity overrides, or action overrides of an existing risk profile. Partial updates are supported: only supplied fields are changed. The modified-at timestamp and actor are updated on every successful write.") .Audited(AuditModules.Policy, AuditActions.Policy.UpdateRiskProfile); governance.MapDelete("/risk-profiles/{profileId}", DeleteRiskProfileAsync) .RequireAuthorization(policy => policy.RequireStellaOpsScopes(StellaOpsScopes.PolicyAuthor)) - .WithName("DeleteRiskProfile") + .WithName("Governance.DeleteRiskProfile") .WithDescription("Permanently delete a risk profile by its identifier, removing it from the tenant's profile registry. Returns HTTP 404 when the profile does not exist. Deletion is recorded as a governance audit event.") .Audited(AuditModules.Policy, AuditActions.Policy.DeleteRiskProfile); governance.MapPost("/risk-profiles/{profileId}/activate", ActivateRiskProfileAsync) .RequireAuthorization(policy => policy.RequireStellaOpsScopes(StellaOpsScopes.PolicyActivate)) - .WithName("ActivateRiskProfile") + .WithName("Governance.ActivateRiskProfile") .WithDescription("Transition a risk profile to the active state, making it the candidate for policy evaluation use. Records the activating actor and timestamp. Activation is an audit-logged, irreversible state transition from draft.") .Audited(AuditModules.Policy, AuditActions.Policy.ActivateRiskProfile); governance.MapPost("/risk-profiles/{profileId}/deprecate", DeprecateRiskProfileAsync) .RequireAuthorization(policy => policy.RequireStellaOpsScopes(StellaOpsScopes.PolicyActivate)) - .WithName("DeprecateRiskProfile") + .WithName("Governance.DeprecateRiskProfile") .WithDescription("Transition a risk profile to the deprecated state with an optional deprecation reason. Deprecated profiles remain visible for audit and historical reference but should not be assigned to new policy evaluations.") .Audited(AuditModules.Policy, AuditActions.Policy.DeprecateRiskProfile); governance.MapPost("/risk-profiles/validate", ValidateRiskProfileAsync) .RequireAuthorization(policy => policy.RequireStellaOpsScopes(StellaOpsScopes.PolicyRead)) - .WithName("ValidateRiskProfile") + .WithName("Governance.ValidateRiskProfile") .WithDescription("Validate a candidate risk profile configuration without persisting it. Checks for required fields (name, at least one signal) and emits warnings when signal weights do not sum to 1.0. Used by policy authoring tools to provide inline validation feedback before profile creation."); // Risk Budget endpoints governance.MapGet("/risk-budget/dashboard", GetRiskBudgetDashboardAsync) .RequireAuthorization(policy => policy.RequireStellaOpsScopes(StellaOpsScopes.PolicyRead)) - .WithName("GetRiskBudgetDashboard") + .WithName("Governance.GetRiskBudgetDashboard") .WithDescription("Retrieve the current risk budget dashboard including utilization, headroom, top contributors, active alerts, and KPIs for the tenant."); // Audit endpoints governance.MapGet("/audit/events", GetAuditEventsAsync) .RequireAuthorization(policy => policy.RequireStellaOpsScopes(StellaOpsScopes.PolicyAudit)) - .WithName("GetGovernanceAuditEvents") + .WithName("Governance.GetAuditEvents") .WithDescription("Retrieve paginated governance audit events for the tenant, ordered by most recent first. Events cover sealed mode changes, override grants and revocations, and risk profile lifecycle actions. Requires tenant ID via header or query parameter."); governance.MapGet("/audit/events/{eventId}", GetAuditEventAsync) .RequireAuthorization(policy => policy.RequireStellaOpsScopes(StellaOpsScopes.PolicyAudit)) - .WithName("GetGovernanceAuditEvent") + .WithName("Governance.GetAuditEvent") .WithDescription("Retrieve a single governance audit event by its identifier, including event type, actor, target resource, timestamp, and human-readable summary. Returns HTTP 404 when the event does not exist or belongs to a different tenant."); // Initialize default profiles diff --git a/src/Policy/StellaOps.Policy.Engine/Endpoints/Gateway/RegistryWebhookEndpoints.cs b/src/Policy/StellaOps.Policy.Engine/Endpoints/Gateway/RegistryWebhookEndpoints.cs index 843a55bea..b11467c97 100644 --- a/src/Policy/StellaOps.Policy.Engine/Endpoints/Gateway/RegistryWebhookEndpoints.cs +++ b/src/Policy/StellaOps.Policy.Engine/Endpoints/Gateway/RegistryWebhookEndpoints.cs @@ -10,6 +10,7 @@ using Microsoft.AspNetCore.Http.HttpResults; using Microsoft.AspNetCore.Mvc; using StellaOps.Auth.ServerIntegration.Tenancy; using StellaOps.Policy.Engine.Gates; +using StellaOps.Policy.Engine.Services.Gateway; using System.Text.Json; using System.Text.Json.Serialization; @@ -30,23 +31,26 @@ internal static class RegistryWebhookEndpoints group.MapPost("/docker", HandleDockerRegistryWebhook) .WithName("DockerRegistryWebhook") .WithSummary("Handle Docker Registry v2 webhook events") - .WithDescription("Receive Docker Registry v2 notification events and enqueue a gate evaluation job for each push event that includes a valid image digest. Returns a 202 Accepted response with the list of queued job IDs that can be polled for evaluation status.") + .WithDescription("Receive Docker Registry v2 notification events and enqueue a gate evaluation job for each push event that includes a valid image digest. Returns 202 only when a scheduler-backed async queue is available; otherwise returns 501 to indicate the runtime cannot truthfully accept deferred gate evaluation.") .Produces(StatusCodes.Status202Accepted) - .Produces(StatusCodes.Status400BadRequest); + .Produces(StatusCodes.Status400BadRequest) + .Produces(StatusCodes.Status501NotImplemented); group.MapPost("/harbor", HandleHarborWebhook) .WithName("HarborWebhook") .WithSummary("Handle Harbor registry webhook events") - .WithDescription("Receive Harbor registry webhook events and enqueue a gate evaluation job for each PUSH_ARTIFACT or pushImage event that contains a resource with a valid digest. Non-push event types are silently acknowledged without queuing any jobs.") + .WithDescription("Receive Harbor registry webhook events and enqueue a gate evaluation job for each PUSH_ARTIFACT or pushImage event that contains a resource with a valid digest. Non-push event types are silently acknowledged without queuing any jobs. Push events return 501 when the runtime lacks a scheduler-backed async queue.") .Produces(StatusCodes.Status202Accepted) - .Produces(StatusCodes.Status400BadRequest); + .Produces(StatusCodes.Status400BadRequest) + .Produces(StatusCodes.Status501NotImplemented); group.MapPost("/generic", HandleGenericWebhook) .WithName("GenericRegistryWebhook") .WithSummary("Handle generic registry webhook events with image digest") - .WithDescription("Receive a generic registry webhook payload containing an image digest and enqueue a single gate evaluation job. Supports any registry that can POST a JSON body with imageDigest, repository, tag, and optional baselineRef fields.") + .WithDescription("Receive a generic registry webhook payload containing an image digest and enqueue a single gate evaluation job. Supports any registry that can POST a JSON body with imageDigest, repository, tag, and optional baselineRef fields. Returns 501 when the runtime lacks a scheduler-backed async queue.") .Produces(StatusCodes.Status202Accepted) - .Produces(StatusCodes.Status400BadRequest); + .Produces(StatusCodes.Status400BadRequest) + .Produces(StatusCodes.Status501NotImplemented); return endpoints; } @@ -70,31 +74,39 @@ internal static class RegistryWebhookEndpoints var jobs = new List(); - foreach (var evt in notification.Events.Where(e => e.Action == "push")) + try { - if (string.IsNullOrEmpty(evt.Target?.Digest)) + foreach (var evt in notification.Events.Where(e => e.Action == "push")) { - logger.LogWarning("Skipping push event without digest: {Repository}", evt.Target?.Repository); - continue; + if (string.IsNullOrEmpty(evt.Target?.Digest)) + { + logger.LogWarning("Skipping push event without digest: {Repository}", evt.Target?.Repository); + continue; + } + + var jobId = await evaluationQueue.EnqueueAsync(new GateEvaluationRequest + { + ImageDigest = evt.Target.Digest, + Repository = evt.Target.Repository ?? "unknown", + Tag = evt.Target.Tag, + RegistryUrl = evt.Request?.Host, + Source = "docker-registry", + Timestamp = evt.Timestamp ?? timeProvider.GetUtcNow() + }, ct); + + jobs.Add(jobId); + + logger.LogInformation( + "Queued gate evaluation for {Repository}@{Digest}, job: {JobId}", + evt.Target.Repository, + evt.Target.Digest, + jobId); } - - var jobId = await evaluationQueue.EnqueueAsync(new GateEvaluationRequest - { - ImageDigest = evt.Target.Digest, - Repository = evt.Target.Repository ?? "unknown", - Tag = evt.Target.Tag, - RegistryUrl = evt.Request?.Host, - Source = "docker-registry", - Timestamp = evt.Timestamp ?? timeProvider.GetUtcNow() - }, ct); - - jobs.Add(jobId); - - logger.LogInformation( - "Queued gate evaluation for {Repository}@{Digest}, job: {JobId}", - evt.Target.Repository, - evt.Target.Digest, - jobId); + } + catch (GateEvaluationQueueUnavailableException ex) + { + logger.LogWarning(ex, "Registry webhook async gate evaluation is unavailable for Docker push events."); + return QueueUnavailableProblem(); } return TypedResults.Accepted( @@ -130,31 +142,39 @@ internal static class RegistryWebhookEndpoints var jobs = new List(); - foreach (var resource in notification.EventData.Resources) + try { - if (string.IsNullOrEmpty(resource.Digest)) + foreach (var resource in notification.EventData.Resources) { - logger.LogWarning("Skipping resource without digest: {ResourceUrl}", resource.ResourceUrl); - continue; + if (string.IsNullOrEmpty(resource.Digest)) + { + logger.LogWarning("Skipping resource without digest: {ResourceUrl}", resource.ResourceUrl); + continue; + } + + var jobId = await evaluationQueue.EnqueueAsync(new GateEvaluationRequest + { + ImageDigest = resource.Digest, + Repository = notification.EventData.Repository?.Name ?? "unknown", + Tag = resource.Tag, + RegistryUrl = notification.EventData.Repository?.RepoFullName, + Source = "harbor", + Timestamp = notification.OccurAt ?? timeProvider.GetUtcNow() + }, ct); + + jobs.Add(jobId); + + logger.LogInformation( + "Queued gate evaluation for {Repository}@{Digest}, job: {JobId}", + notification.EventData.Repository?.Name, + resource.Digest, + jobId); } - - var jobId = await evaluationQueue.EnqueueAsync(new GateEvaluationRequest - { - ImageDigest = resource.Digest, - Repository = notification.EventData.Repository?.Name ?? "unknown", - Tag = resource.Tag, - RegistryUrl = notification.EventData.Repository?.RepoFullName, - Source = "harbor", - Timestamp = notification.OccurAt ?? timeProvider.GetUtcNow() - }, ct); - - jobs.Add(jobId); - - logger.LogInformation( - "Queued gate evaluation for {Repository}@{Digest}, job: {JobId}", - notification.EventData.Repository?.Name, - resource.Digest, - jobId); + } + catch (GateEvaluationQueueUnavailableException ex) + { + logger.LogWarning(ex, "Registry webhook async gate evaluation is unavailable for Harbor push events."); + return QueueUnavailableProblem(); } return TypedResults.Accepted( @@ -179,26 +199,42 @@ internal static class RegistryWebhookEndpoints statusCode: StatusCodes.Status400BadRequest); } - var jobId = await evaluationQueue.EnqueueAsync(new GateEvaluationRequest + try { - ImageDigest = notification.ImageDigest, - Repository = notification.Repository ?? "unknown", - Tag = notification.Tag, - RegistryUrl = notification.RegistryUrl, - BaselineRef = notification.BaselineRef, - Source = notification.Source ?? "generic", - Timestamp = timeProvider.GetUtcNow() - }, ct); + var jobId = await evaluationQueue.EnqueueAsync(new GateEvaluationRequest + { + ImageDigest = notification.ImageDigest, + Repository = notification.Repository ?? "unknown", + Tag = notification.Tag, + RegistryUrl = notification.RegistryUrl, + BaselineRef = notification.BaselineRef, + Source = notification.Source ?? "generic", + Timestamp = timeProvider.GetUtcNow() + }, ct); - logger.LogInformation( - "Queued gate evaluation for {Repository}@{Digest}, job: {JobId}", - notification.Repository, - notification.ImageDigest, - jobId); + logger.LogInformation( + "Queued gate evaluation for {Repository}@{Digest}, job: {JobId}", + notification.Repository, + notification.ImageDigest, + jobId); - return TypedResults.Accepted( - $"/api/v1/policy/gate/jobs/{jobId}", - new WebhookAcceptedResponse(1, [jobId])); + return TypedResults.Accepted( + $"/api/v1/policy/gate/jobs/{jobId}", + new WebhookAcceptedResponse(1, [jobId])); + } + catch (GateEvaluationQueueUnavailableException ex) + { + logger.LogWarning(ex, "Registry webhook async gate evaluation is unavailable for generic push events."); + return QueueUnavailableProblem(); + } + } + + private static ProblemHttpResult QueueUnavailableProblem() + { + return TypedResults.Problem( + title: "Async gate evaluation queue unavailable", + detail: "Registry webhook push events require a scheduler-backed async gate evaluation queue. This runtime does not provide one.", + statusCode: StatusCodes.Status501NotImplemented); } } @@ -379,35 +415,3 @@ public sealed record GenericRegistryWebhook public sealed record WebhookAcceptedResponse( int JobsQueued, IReadOnlyList JobIds); - -// ============================================================================ -// Gate Evaluation Queue Interface -// ============================================================================ - -/// -/// Interface for queuing gate evaluation jobs. -/// -public interface IGateEvaluationQueue -{ - /// - /// Enqueues a gate evaluation request. - /// - /// The evaluation request. - /// Cancellation token. - /// The job ID for tracking. - Task EnqueueAsync(GateEvaluationRequest request, CancellationToken cancellationToken = default); -} - -/// -/// Request to evaluate a gate for an image. -/// -public sealed record GateEvaluationRequest -{ - public required string ImageDigest { get; init; } - public required string Repository { get; init; } - public string? Tag { get; init; } - public string? RegistryUrl { get; init; } - public string? BaselineRef { get; init; } - public required string Source { get; init; } - public required DateTimeOffset Timestamp { get; init; } -} diff --git a/src/Policy/StellaOps.Policy.Engine/Endpoints/SnapshotEndpoint.cs b/src/Policy/StellaOps.Policy.Engine/Endpoints/SnapshotEndpoint.cs index 0306a1d6e..adb8fa661 100644 --- a/src/Policy/StellaOps.Policy.Engine/Endpoints/SnapshotEndpoint.cs +++ b/src/Policy/StellaOps.Policy.Engine/Endpoints/SnapshotEndpoint.cs @@ -11,17 +11,17 @@ public static class SnapshotEndpoint { routes.MapPost("/policy/snapshots", CreateAsync) .WithName("PolicyEngine.Snapshots.Create") - .WithDescription("Create a new in-memory policy evaluation snapshot from a component manifest, capturing the component graph, PURL set, and applicable advisory signals at a point in time for subsequent policy decision evaluation.") + .WithDescription("Create a new persisted policy evaluation snapshot from a component manifest, capturing the component graph, PURL set, and applicable advisory signals at a point in time for subsequent policy decision evaluation.") .RequireAuthorization(policy => policy.RequireStellaOpsScopes(StellaOpsScopes.PolicyEdit)); routes.MapGet("/policy/snapshots", ListAsync) .WithName("PolicyEngine.Snapshots.List") - .WithDescription("List all active in-memory policy snapshots for a given tenant, returning snapshot identifiers, creation timestamps, and summary component counts for use in batch evaluation workflows.") + .WithDescription("List persisted policy snapshots for a given tenant, returning snapshot identifiers, creation timestamps, and summary component counts for use in batch evaluation workflows.") .RequireAuthorization(policy => policy.RequireStellaOpsScopes(StellaOpsScopes.PolicyRead)); routes.MapGet("/policy/snapshots/{snapshotId}", GetAsync) .WithName("PolicyEngine.Snapshots.Get") - .WithDescription("Retrieve a specific in-memory policy evaluation snapshot by identifier, including its full component graph, resolved advisory signals, and any cached partial evaluation state.") + .WithDescription("Retrieve a specific persisted policy evaluation snapshot by identifier, including its full component graph, resolved advisory signals, and any cached partial evaluation state.") .RequireAuthorization(policy => policy.RequireStellaOpsScopes(StellaOpsScopes.PolicyRead)); return routes; diff --git a/src/Policy/StellaOps.Policy.Engine/Ledger/LedgerExportService.cs b/src/Policy/StellaOps.Policy.Engine/Ledger/LedgerExportService.cs index dea6ac40a..a6b53f897 100644 --- a/src/Policy/StellaOps.Policy.Engine/Ledger/LedgerExportService.cs +++ b/src/Policy/StellaOps.Policy.Engine/Ledger/LedgerExportService.cs @@ -88,7 +88,7 @@ internal sealed class LedgerExportService lines.AddRange(recordLines); var export = new LedgerExport(manifest, ordered, lines); - await _store.SaveAsync(export, cancellationToken).ConfigureAwait(false); + await _store.SaveAsync(request.TenantId, export, cancellationToken).ConfigureAwait(false); return export; } diff --git a/src/Policy/StellaOps.Policy.Engine/Ledger/LedgerExportStore.cs b/src/Policy/StellaOps.Policy.Engine/Ledger/LedgerExportStore.cs index c35d2fe52..734f97df0 100644 --- a/src/Policy/StellaOps.Policy.Engine/Ledger/LedgerExportStore.cs +++ b/src/Policy/StellaOps.Policy.Engine/Ledger/LedgerExportStore.cs @@ -1,10 +1,14 @@ using System.Collections.Concurrent; +using System.Globalization; +using System.Text.Json; +using StellaOps.Policy.Persistence.Postgres.Models; +using StellaOps.Policy.Persistence.Postgres.Repositories; namespace StellaOps.Policy.Engine.Ledger; internal interface ILedgerExportStore { - Task SaveAsync(LedgerExport export, CancellationToken cancellationToken = default); + Task SaveAsync(string tenantId, LedgerExport export, CancellationToken cancellationToken = default); Task GetAsync(string exportId, CancellationToken cancellationToken = default); Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default); } @@ -13,8 +17,9 @@ internal sealed class InMemoryLedgerExportStore : ILedgerExportStore { private readonly ConcurrentDictionary _exports = new(StringComparer.Ordinal); - public Task SaveAsync(LedgerExport export, CancellationToken cancellationToken = default) + public Task SaveAsync(string tenantId, LedgerExport export, CancellationToken cancellationToken = default) { + ArgumentException.ThrowIfNullOrWhiteSpace(tenantId); ArgumentNullException.ThrowIfNull(export); _exports[export.Manifest.ExportId] = export; return Task.CompletedTask; @@ -42,3 +47,59 @@ internal sealed class InMemoryLedgerExportStore : ILedgerExportStore return Task.FromResult>(ordered); } } + +internal sealed class PostgresLedgerExportStore : ILedgerExportStore +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + private readonly IPolicyEngineLedgerExportRepository _repository; + + public PostgresLedgerExportStore(IPolicyEngineLedgerExportRepository repository) + { + _repository = repository ?? throw new ArgumentNullException(nameof(repository)); + } + + public async Task SaveAsync(string tenantId, LedgerExport export, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(tenantId); + ArgumentNullException.ThrowIfNull(export); + + var document = new PolicyEngineLedgerExportDocument + { + ExportId = export.Manifest.ExportId, + TenantId = tenantId, + SchemaVersion = export.Manifest.SchemaVersion, + GeneratedAt = DateTimeOffset.Parse(export.Manifest.GeneratedAt, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind), + RecordCount = export.Manifest.RecordCount, + Sha256 = export.Manifest.Sha256, + ManifestJson = JsonSerializer.Serialize(export.Manifest, SerializerOptions), + RecordsJson = JsonSerializer.Serialize(export.Records, SerializerOptions), + LinesJson = JsonSerializer.Serialize(export.Lines, SerializerOptions) + }; + + await _repository.SaveAsync(document, cancellationToken).ConfigureAwait(false); + } + + public async Task GetAsync(string exportId, CancellationToken cancellationToken = default) + { + var document = await _repository.GetAsync(exportId, cancellationToken).ConfigureAwait(false); + return document is null ? null : Deserialize(document); + } + + public async Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default) + { + var documents = await _repository.ListAsync(tenantId, cancellationToken).ConfigureAwait(false); + return documents.Select(Deserialize).ToList(); + } + + private static LedgerExport Deserialize(PolicyEngineLedgerExportDocument document) + { + var manifest = JsonSerializer.Deserialize(document.ManifestJson, SerializerOptions) + ?? throw new InvalidOperationException($"Persisted ledger export {document.ExportId} is missing a manifest."); + var records = JsonSerializer.Deserialize>(document.RecordsJson, SerializerOptions) + ?? throw new InvalidOperationException($"Persisted ledger export {document.ExportId} is missing records."); + var lines = JsonSerializer.Deserialize>(document.LinesJson, SerializerOptions) + ?? throw new InvalidOperationException($"Persisted ledger export {document.ExportId} is missing lines."); + + return new LedgerExport(manifest, records, lines); + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Orchestration/OrchestratorJobService.cs b/src/Policy/StellaOps.Policy.Engine/Orchestration/OrchestratorJobService.cs index 0195af3da..9e9d07ddd 100644 --- a/src/Policy/StellaOps.Policy.Engine/Orchestration/OrchestratorJobService.cs +++ b/src/Policy/StellaOps.Policy.Engine/Orchestration/OrchestratorJobService.cs @@ -1,5 +1,6 @@ using System.Globalization; using System.Text; +using StellaOps.Policy.Engine.Workers; namespace StellaOps.Policy.Engine.Orchestration; @@ -9,11 +10,16 @@ internal sealed class OrchestratorJobService private readonly TimeProvider _timeProvider; private readonly IOrchestratorJobStore _store; + private readonly IOrchestratorJobExecutionSignal? _executionSignal; - public OrchestratorJobService(TimeProvider timeProvider, IOrchestratorJobStore store) + public OrchestratorJobService( + TimeProvider timeProvider, + IOrchestratorJobStore store, + IOrchestratorJobExecutionSignal? executionSignal = null) { _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); _store = store ?? throw new ArgumentNullException(nameof(store)); + _executionSignal = executionSignal; } public async Task SubmitAsync( @@ -22,6 +28,7 @@ internal sealed class OrchestratorJobService { var job = BuildJob(request); await _store.SaveAsync(job, cancellationToken).ConfigureAwait(false); + _executionSignal?.NotifyWorkAvailable(); return job; } diff --git a/src/Policy/StellaOps.Policy.Engine/Orchestration/OrchestratorJobStore.cs b/src/Policy/StellaOps.Policy.Engine/Orchestration/OrchestratorJobStore.cs index 9d5d8e23b..41d2107c9 100644 --- a/src/Policy/StellaOps.Policy.Engine/Orchestration/OrchestratorJobStore.cs +++ b/src/Policy/StellaOps.Policy.Engine/Orchestration/OrchestratorJobStore.cs @@ -8,11 +8,13 @@ internal interface IOrchestratorJobStore Task GetAsync(string jobId, CancellationToken cancellationToken = default); Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default); Task UpdateAsync(string jobId, Func update, CancellationToken cancellationToken = default); + Task TryLeaseNextQueuedAsync(CancellationToken cancellationToken = default); } internal sealed class InMemoryOrchestratorJobStore : IOrchestratorJobStore { private readonly ConcurrentDictionary _jobs = new(StringComparer.Ordinal); + private readonly object _sync = new(); public Task SaveAsync(OrchestratorJob job, CancellationToken cancellationToken = default) { @@ -55,4 +57,31 @@ internal sealed class InMemoryOrchestratorJobStore : IOrchestratorJobStore return Task.CompletedTask; } + + public Task TryLeaseNextQueuedAsync(CancellationToken cancellationToken = default) + { + lock (_sync) + { + var next = _jobs.Values + .Where(job => string.Equals(job.Status, "queued", StringComparison.Ordinal)) + .OrderBy(job => job.RequestedAt) + .ThenBy(job => job.JobId, StringComparer.Ordinal) + .FirstOrDefault(); + + if (next is null) + { + return Task.FromResult(null); + } + + var leased = next with + { + Status = "running", + CompletedAt = null, + ResultHash = null + }; + + _jobs[leased.JobId] = leased; + return Task.FromResult(leased); + } + } } diff --git a/src/Policy/StellaOps.Policy.Engine/Orchestration/PersistedOrchestratorStores.cs b/src/Policy/StellaOps.Policy.Engine/Orchestration/PersistedOrchestratorStores.cs new file mode 100644 index 000000000..767654bf9 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Orchestration/PersistedOrchestratorStores.cs @@ -0,0 +1,197 @@ +using System.Text.Json; +using StellaOps.Policy.Persistence.Postgres.Models; +using StellaOps.Policy.Persistence.Postgres.Repositories; + +namespace StellaOps.Policy.Engine.Orchestration; + +internal sealed class PersistedOrchestratorJobStore : IOrchestratorJobStore +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + private readonly IPolicyEngineOrchestratorJobRepository _repository; + + public PersistedOrchestratorJobStore(IPolicyEngineOrchestratorJobRepository repository) + { + _repository = repository ?? throw new ArgumentNullException(nameof(repository)); + } + + public async Task SaveAsync(OrchestratorJob job, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(job); + await _repository.SaveAsync(Serialize(job), cancellationToken).ConfigureAwait(false); + } + + public async Task GetAsync(string jobId, CancellationToken cancellationToken = default) + { + var document = await _repository.GetAsync(jobId, cancellationToken).ConfigureAwait(false); + return document is null ? null : Deserialize(document); + } + + public async Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default) + { + var documents = await _repository.ListAsync(tenantId, cancellationToken).ConfigureAwait(false); + return documents.Select(Deserialize).ToList(); + } + + public async Task UpdateAsync(string jobId, Func update, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(update); + + var existing = await GetAsync(jobId, cancellationToken).ConfigureAwait(false) + ?? throw new KeyNotFoundException($"Job {jobId} not found"); + + await SaveAsync(update(existing), cancellationToken).ConfigureAwait(false); + } + + public async Task TryLeaseNextQueuedAsync(CancellationToken cancellationToken = default) + { + var document = await _repository.LeaseNextQueuedAsync(cancellationToken).ConfigureAwait(false); + return document is null ? null : Deserialize(document); + } + + private static PolicyEngineOrchestratorJobDocument Serialize(OrchestratorJob job) + { + return new PolicyEngineOrchestratorJobDocument + { + JobId = job.JobId, + TenantId = job.TenantId, + ContextId = job.ContextId, + PolicyProfileHash = job.PolicyProfileHash, + RequestedAt = job.RequestedAt, + Priority = job.Priority, + BatchItemsJson = JsonSerializer.Serialize(job.BatchItems, SerializerOptions), + CallbacksJson = job.Callbacks is null ? null : JsonSerializer.Serialize(job.Callbacks, SerializerOptions), + TraceRef = job.TraceRef, + Status = job.Status, + DeterminismHash = job.DeterminismHash, + CompletedAt = job.CompletedAt, + ResultHash = job.ResultHash, + CreatedAt = job.RequestedAt + }; + } + + private static OrchestratorJob Deserialize(PolicyEngineOrchestratorJobDocument document) + { + var batchItems = JsonSerializer.Deserialize>(document.BatchItemsJson, SerializerOptions) + ?? throw new InvalidOperationException($"Persisted orchestrator job {document.JobId} is missing batch items."); + var callbacks = string.IsNullOrWhiteSpace(document.CallbacksJson) + ? null + : JsonSerializer.Deserialize(document.CallbacksJson, SerializerOptions); + + return new OrchestratorJob( + JobId: document.JobId, + TenantId: document.TenantId, + ContextId: document.ContextId, + PolicyProfileHash: document.PolicyProfileHash, + RequestedAt: document.RequestedAt, + Priority: document.Priority, + BatchItems: batchItems, + Callbacks: callbacks, + TraceRef: document.TraceRef, + Status: document.Status, + DeterminismHash: document.DeterminismHash, + CompletedAt: document.CompletedAt, + ResultHash: document.ResultHash); + } +} + +internal sealed class PersistedWorkerResultStore : IWorkerResultStore +{ + private const string JobType = "policy-orchestrator"; + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + private readonly IServiceScopeFactory _scopeFactory; + private readonly IOrchestratorJobStore _jobs; + + public PersistedWorkerResultStore(IServiceScopeFactory scopeFactory, IOrchestratorJobStore jobs) + { + _scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory)); + _jobs = jobs ?? throw new ArgumentNullException(nameof(jobs)); + } + + public async Task SaveAsync(WorkerRunResult result, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(result); + + var job = await _jobs.GetAsync(result.JobId, cancellationToken).ConfigureAwait(false) + ?? throw new KeyNotFoundException($"Job {result.JobId} not found"); + + using var scope = _scopeFactory.CreateScope(); + var repository = scope.ServiceProvider.GetRequiredService(); + var existing = await repository.GetByJobAsync(job.TenantId, JobType, result.JobId, cancellationToken).ConfigureAwait(false); + var serialized = JsonSerializer.Serialize(result, SerializerOptions); + + if (existing is null) + { + await repository.CreateAsync( + new WorkerResultEntity + { + Id = CreateDeterministicGuid(job.TenantId, result.JobId), + TenantId = job.TenantId, + JobType = JobType, + JobId = result.JobId, + Status = "completed", + OutputHash = result.ResultHash, + Progress = 100, + Result = serialized, + RetryCount = 0, + MaxRetries = 0, + ScheduledAt = result.StartedAt, + StartedAt = result.StartedAt, + CompletedAt = result.CompletedAt, + Metadata = "{}", + CreatedAt = result.StartedAt, + CreatedBy = result.WorkerId + }, + cancellationToken).ConfigureAwait(false); + + return; + } + + await repository.CompleteAsync( + job.TenantId, + existing.Id, + serialized, + result.ResultHash, + cancellationToken).ConfigureAwait(false); + } + + public async Task GetByJobIdAsync(string jobId, CancellationToken cancellationToken = default) + { + var job = await _jobs.GetAsync(jobId, cancellationToken).ConfigureAwait(false); + if (job is null) + { + return null; + } + + using var scope = _scopeFactory.CreateScope(); + var repository = scope.ServiceProvider.GetRequiredService(); + var entity = await repository.GetByJobAsync(job.TenantId, JobType, jobId, cancellationToken).ConfigureAwait(false); + return entity is null ? null : Deserialize(entity); + } + + public async Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default) + { + using var scope = _scopeFactory.CreateScope(); + var repository = scope.ServiceProvider.GetRequiredService(); + var entities = await repository.ListAsync(tenantId, cancellationToken).ConfigureAwait(false); + + return entities + .Where(entity => string.Equals(entity.JobType, JobType, StringComparison.Ordinal)) + .Select(Deserialize) + .OrderBy(result => result.JobId, StringComparer.Ordinal) + .ToList(); + } + + private static WorkerRunResult Deserialize(WorkerResultEntity entity) + { + return JsonSerializer.Deserialize(entity.Result ?? string.Empty, SerializerOptions) + ?? throw new InvalidOperationException($"Persisted worker result {entity.Id} is missing a valid result payload."); + } + + private static Guid CreateDeterministicGuid(string tenantId, string jobId) + { + var hash = StableIdGenerator.Sha256Hex($"{tenantId}|{jobId}"); + return Guid.ParseExact( + $"{hash[..8]}-{hash[8..12]}-{hash[12..16]}-{hash[16..20]}-{hash[20..32]}", + "D"); + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Orchestration/PolicyWorkerService.cs b/src/Policy/StellaOps.Policy.Engine/Orchestration/PolicyWorkerService.cs index 846da05f8..d10c7e0f8 100644 --- a/src/Policy/StellaOps.Policy.Engine/Orchestration/PolicyWorkerService.cs +++ b/src/Policy/StellaOps.Policy.Engine/Orchestration/PolicyWorkerService.cs @@ -41,24 +41,43 @@ internal sealed class PolicyWorkerService var workerId = string.IsNullOrWhiteSpace(request.WorkerId) ? "worker-stub" : request.WorkerId; var startedAt = _timeProvider.GetUtcNow(); - var results = BuildResults(job); + try + { + var results = BuildResults(job); - var completedAt = _timeProvider.GetUtcNow(); - var resultHash = StableIdGenerator.Sha256Hex(BuildSeed(job.JobId, results)); + var completedAt = _timeProvider.GetUtcNow(); + var resultHash = StableIdGenerator.Sha256Hex(BuildSeed(job.JobId, results)); - var runResult = new WorkerRunResult( - JobId: job.JobId, - WorkerId: workerId, - StartedAt: startedAt, - CompletedAt: completedAt, - Results: results, - ResultHash: resultHash); + var runResult = new WorkerRunResult( + JobId: job.JobId, + WorkerId: workerId, + StartedAt: startedAt, + CompletedAt: completedAt, + Results: results, + ResultHash: resultHash); - await _results.SaveAsync(runResult, cancellationToken).ConfigureAwait(false); - await _jobs.UpdateAsync(job.JobId, j => j with { Status = "completed", CompletedAt = completedAt, ResultHash = resultHash }, cancellationToken) - .ConfigureAwait(false); + await _results.SaveAsync(runResult, cancellationToken).ConfigureAwait(false); + await _jobs.UpdateAsync(job.JobId, j => j with { Status = "completed", CompletedAt = completedAt, ResultHash = resultHash }, cancellationToken) + .ConfigureAwait(false); - return runResult; + return runResult; + } + catch + { + var failedAt = _timeProvider.GetUtcNow(); + await _jobs.UpdateAsync( + job.JobId, + j => j with + { + Status = "failed", + CompletedAt = failedAt, + ResultHash = null + }, + cancellationToken) + .ConfigureAwait(false); + + throw; + } } private static IReadOnlyList BuildResults(OrchestratorJob job) diff --git a/src/Policy/StellaOps.Policy.Engine/Program.cs b/src/Policy/StellaOps.Policy.Engine/Program.cs index 137030213..f670a53f8 100644 --- a/src/Policy/StellaOps.Policy.Engine/Program.cs +++ b/src/Policy/StellaOps.Policy.Engine/Program.cs @@ -7,6 +7,7 @@ using StellaOps.AirGap.Policy; using StellaOps.Auth.Abstractions; using StellaOps.Auth.Client; using StellaOps.Auth.ServerIntegration; +using StellaOps.Auth.ServerIntegration.Tenancy; using StellaOps.Configuration; using StellaOps.Cryptography.DependencyInjection; using StellaOps.Policy.Engine.BatchEvaluation; @@ -202,11 +203,13 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); -builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(); -builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(); -builder.Services.AddSingleton(); +builder.Services.AddHostedService(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(); // Console export jobs per CONTRACT-EXPORT-BUNDLE-009 @@ -237,8 +240,10 @@ builder.Services.AddSingleton(sp => (StellaOps.Policy.Engine.AirGap.AirGapNotificationService)sp.GetRequiredService()); -builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); @@ -254,6 +259,7 @@ builder.Services.AddStellaOpsCrypto(); builder.Services.AddSingleton(TimeProvider.System); builder.Services.AddSystemGuidProvider(); builder.Services.AddHttpContextAccessor(); +builder.Services.AddStellaOpsTenantServices(); builder.Services.AddTenantContext(builder.Configuration); builder.Services.AddStellaOpsCors(builder.Environment, builder.Configuration); builder.Services.AddRouting(options => options.LowercaseUrls = true); @@ -371,24 +377,37 @@ builder.Services.AddHostedService(); // Delta services (from gateway) builder.Services.AddScoped(); builder.Services.AddScoped(); -builder.Services.AddScoped(); +builder.Services.AddScoped(); builder.Services.AddScoped(); // Gate services (from gateway) builder.Services.Configure( builder.Configuration.GetSection(StellaOps.Policy.Engine.Gates.DriftGateOptions.SectionName)); builder.Services.AddScoped(); -builder.Services.AddSingleton(); -builder.Services.AddSingleton(sp => sp.GetRequiredService()); -builder.Services.AddHostedService(); +builder.Services.AddPolicyAsyncGateEvaluationRuntime(builder.Configuration, PolicyEngineOptions.SectionName); // Unknowns gate services (from gateway) builder.Services.Configure(_ => { }); builder.Services.AddHttpClient(); // Gate bypass audit services (from gateway) -builder.Services.AddSingleton(); +builder.Services.AddScoped(sp => +{ + var persistence = sp.GetRequiredService(); + var logger = sp.GetRequiredService>(); + var tenantAccessor = sp.GetRequiredService(); + var tenantId = tenantAccessor.TenantContext?.TenantId; + + if (string.IsNullOrWhiteSpace(tenantId)) + { + tenantId = TenantContextConstants.DefaultTenantId; + } + + return new StellaOps.Policy.Persistence.Postgres.PostgresGateBypassAuditRepository( + persistence, + logger, + tenantId); +}); builder.Services.AddSingleton(); builder.Services.AddScoped(); @@ -439,6 +458,7 @@ app.LogStellaOpsLocalHostname("policy-engine"); app.UseStellaOpsCors(); app.UseAuthentication(); app.UseAuthorization(); +app.UseStellaOpsTenantMiddleware(); app.UseTenantContext(); // POL-TEN-01: tenant enforcement middleware app.TryUseStellaRouter(routerEnabled); @@ -508,6 +528,7 @@ app.MapDeltasEndpoints(); app.MapPolicySimulationEndpoints(); // Gate evaluation endpoints app.MapGateEndpoints(); +app.MapGateJobEndpoints(); // Unknowns gate endpoints app.MapGatesEndpoints(); // Score-based gate endpoints diff --git a/src/Policy/StellaOps.Policy.Engine/Properties/AssemblyInfo.cs b/src/Policy/StellaOps.Policy.Engine/Properties/AssemblyInfo.cs index 826700d82..739921d05 100644 --- a/src/Policy/StellaOps.Policy.Engine/Properties/AssemblyInfo.cs +++ b/src/Policy/StellaOps.Policy.Engine/Properties/AssemblyInfo.cs @@ -1,3 +1,4 @@ using System.Runtime.CompilerServices; +[assembly: InternalsVisibleTo("StellaOps.Policy.Gateway")] [assembly: InternalsVisibleTo("StellaOps.Policy.Engine.Tests")] diff --git a/src/Policy/StellaOps.Policy.Engine/Services/Gateway/DeltaSnapshotServiceAdapter.cs b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/DeltaSnapshotServiceAdapter.cs index e0dda4ba6..23cbe62f8 100644 --- a/src/Policy/StellaOps.Policy.Engine/Services/Gateway/DeltaSnapshotServiceAdapter.cs +++ b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/DeltaSnapshotServiceAdapter.cs @@ -7,17 +7,19 @@ using StellaOps.Policy.Snapshots; namespace StellaOps.Policy.Engine.Services.Gateway; +using EngineSnapshotStore = StellaOps.Policy.Engine.Snapshots.ISnapshotStore; + /// -/// Adapter that bridges between the KnowledgeSnapshotManifest-based snapshot store -/// and the SnapshotData interface required by the DeltaComputer. +/// Adapter that projects persisted policy-engine snapshots into the data shape +/// required by the delta computer. /// -public sealed class DeltaSnapshotServiceAdapter : StellaOps.Policy.Deltas.ISnapshotService +internal sealed class DeltaSnapshotServiceAdapter : StellaOps.Policy.Deltas.ISnapshotService { - private readonly ISnapshotStore _snapshotStore; + private readonly EngineSnapshotStore _snapshotStore; private readonly ILogger _logger; public DeltaSnapshotServiceAdapter( - ISnapshotStore snapshotStore, + EngineSnapshotStore snapshotStore, ILogger logger) { _snapshotStore = snapshotStore ?? throw new ArgumentNullException(nameof(snapshotStore)); @@ -25,7 +27,7 @@ public sealed class DeltaSnapshotServiceAdapter : StellaOps.Policy.Deltas.ISnaps } /// - /// Gets snapshot data by ID, converting from KnowledgeSnapshotManifest. + /// Gets snapshot data by ID from the persisted runtime snapshot store. /// public async Task GetSnapshotAsync(string snapshotId, CancellationToken ct = default) { @@ -34,34 +36,43 @@ public sealed class DeltaSnapshotServiceAdapter : StellaOps.Policy.Deltas.ISnaps return null; } - var manifest = await _snapshotStore.GetAsync(snapshotId, ct).ConfigureAwait(false); - if (manifest is null) + var snapshot = await _snapshotStore.GetAsync(snapshotId, ct).ConfigureAwait(false); + if (snapshot is null) { _logger.LogDebug("Snapshot {SnapshotId} not found in store", snapshotId); return null; } - return ConvertToSnapshotData(manifest); + return ConvertToSnapshotData(snapshot); } - private static SnapshotData ConvertToSnapshotData(KnowledgeSnapshotManifest manifest) + private static SnapshotData ConvertToSnapshotData(StellaOps.Policy.Engine.Snapshots.SnapshotDetail snapshot) { - // Get policy version from manifest sources - var policySource = manifest.Sources.FirstOrDefault(s => s.Type == KnowledgeSourceTypes.Policy); - var policyVersion = policySource?.Digest; + var packages = PersistedKnowledgeSnapshotStore.ProjectPackages(snapshot.Records) + .Select(package => new PackageData(package.Purl, package.Version, package.License)) + .ToList(); + var reachability = PersistedKnowledgeSnapshotStore.ProjectReachability(snapshot.Records) + .Select(item => new ReachabilityData(item.CveId, item.Purl, item.IsReachable)) + .ToList(); + var vexStatements = PersistedKnowledgeSnapshotStore.ProjectVexStatements(snapshot.Records) + .Select(item => new VexStatementData(item.CveId, item.Status, item.Justification)) + .ToList(); + var policyViolations = PersistedKnowledgeSnapshotStore.ProjectPolicyViolations(snapshot.Records) + .Select(item => new PolicyViolationData(item.RuleId, item.Severity, item.Message)) + .ToList(); + var unknowns = PersistedKnowledgeSnapshotStore.ProjectUnknowns(snapshot.Records) + .Select(item => new UnknownData(item.Id, item.ReasonCode, item.Description)) + .ToList(); - // Note: In a full implementation, we would fetch and parse the bundled content - // from each source to extract packages, reachability, VEX statements, etc. - // For now, we return the manifest metadata only. return new SnapshotData { - SnapshotId = manifest.SnapshotId, - Packages = [], - Reachability = [], - VexStatements = [], - PolicyViolations = [], - Unknowns = [], - PolicyVersion = policyVersion + SnapshotId = snapshot.SnapshotId, + Packages = packages, + Reachability = reachability, + VexStatements = vexStatements, + PolicyViolations = policyViolations, + Unknowns = unknowns, + PolicyVersion = snapshot.OverlayHash }; } } diff --git a/src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateBaselineBootstrapper.cs b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateBaselineBootstrapper.cs new file mode 100644 index 000000000..9b883da63 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateBaselineBootstrapper.cs @@ -0,0 +1,120 @@ +using StellaOps.Auth.ServerIntegration.Tenancy; +using StellaOps.Policy.Engine.Ledger; +using StellaOps.Policy.Engine.Orchestration; +using StellaOps.Policy.Engine.Snapshots; +using StellaOps.Policy.Engine.Tenancy; + +namespace StellaOps.Policy.Engine.Services.Gateway; + +internal sealed class GateBaselineBootstrapper +{ + private const string BootstrapOverlayHash = "bootstrap"; + + private readonly ILedgerExportStore _ledgerExportStore; + private readonly LedgerExportService _ledgerExportService; + private readonly ISnapshotStore _snapshotStore; + private readonly SnapshotService _snapshotService; + private readonly IOrchestratorJobStore _jobStore; + private readonly IWorkerResultStore _workerResultStore; + private readonly IStellaOpsTenantAccessor _tenantAccessor; + private readonly ILogger _logger; + + public GateBaselineBootstrapper( + ILedgerExportStore ledgerExportStore, + LedgerExportService ledgerExportService, + ISnapshotStore snapshotStore, + SnapshotService snapshotService, + IOrchestratorJobStore jobStore, + IWorkerResultStore workerResultStore, + IStellaOpsTenantAccessor tenantAccessor, + ILogger logger) + { + _ledgerExportStore = ledgerExportStore ?? throw new ArgumentNullException(nameof(ledgerExportStore)); + _ledgerExportService = ledgerExportService ?? throw new ArgumentNullException(nameof(ledgerExportService)); + _snapshotStore = snapshotStore ?? throw new ArgumentNullException(nameof(snapshotStore)); + _snapshotService = snapshotService ?? throw new ArgumentNullException(nameof(snapshotService)); + _jobStore = jobStore ?? throw new ArgumentNullException(nameof(jobStore)); + _workerResultStore = workerResultStore ?? throw new ArgumentNullException(nameof(workerResultStore)); + _tenantAccessor = tenantAccessor ?? throw new ArgumentNullException(nameof(tenantAccessor)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task TryEnsureBaselineAsync(CancellationToken cancellationToken = default) + { + var tenantId = string.IsNullOrWhiteSpace(_tenantAccessor.TenantId) + ? TenantContextConstants.DefaultTenantId + : _tenantAccessor.TenantId!; + + var latestSnapshot = await GetLatestSnapshotAsync(tenantId, cancellationToken).ConfigureAwait(false); + if (latestSnapshot is not null) + { + return true; + } + + var exports = await _ledgerExportStore.ListAsync(tenantId, cancellationToken).ConfigureAwait(false); + var latestExport = exports + .OrderByDescending(export => export.Manifest.GeneratedAt, StringComparer.Ordinal) + .ThenBy(export => export.Manifest.ExportId, StringComparer.Ordinal) + .FirstOrDefault(); + + if (latestExport is null || latestExport.Records.Count == 0) + { + if (!await HasCompletedPolicyDataAsync(tenantId, cancellationToken).ConfigureAwait(false)) + { + _logger.LogInformation( + "Skipping baseline bootstrap for tenant {TenantId}: no completed policy result data exists yet.", + tenantId); + return false; + } + + latestExport = await _ledgerExportService.BuildAsync( + new LedgerExportRequest(tenantId), + cancellationToken).ConfigureAwait(false); + + if (latestExport.Records.Count == 0) + { + _logger.LogWarning( + "Skipping baseline bootstrap for tenant {TenantId}: built ledger export {ExportId} with zero records.", + tenantId, + latestExport.Manifest.ExportId); + return false; + } + } + + var snapshot = await _snapshotService.CreateAsync( + new SnapshotRequest(tenantId, BootstrapOverlayHash), + cancellationToken).ConfigureAwait(false); + + _logger.LogInformation( + "Bootstrapped baseline snapshot {SnapshotId} for tenant {TenantId} from ledger export {ExportId}.", + snapshot.SnapshotId, + tenantId, + snapshot.LedgerExportId); + + return true; + } + + private async Task GetLatestSnapshotAsync(string tenantId, CancellationToken cancellationToken) + { + return (await _snapshotStore.ListAsync(tenantId, cancellationToken).ConfigureAwait(false)) + .OrderByDescending(snapshot => snapshot.GeneratedAt, StringComparer.Ordinal) + .ThenBy(snapshot => snapshot.SnapshotId, StringComparer.Ordinal) + .FirstOrDefault(); + } + + private async Task HasCompletedPolicyDataAsync(string tenantId, CancellationToken cancellationToken) + { + var completedJobIds = (await _jobStore.ListAsync(tenantId, cancellationToken).ConfigureAwait(false)) + .Where(job => string.Equals(job.Status, "completed", StringComparison.Ordinal)) + .Select(job => job.JobId) + .ToHashSet(StringComparer.Ordinal); + + if (completedJobIds.Count == 0) + { + return false; + } + + return (await _workerResultStore.ListAsync(tenantId, cancellationToken).ConfigureAwait(false)) + .Any(result => completedJobIds.Contains(result.JobId) && result.Results.Count > 0); + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateEvaluationJobDispatchService.cs b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateEvaluationJobDispatchService.cs new file mode 100644 index 000000000..21c3b7dc6 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateEvaluationJobDispatchService.cs @@ -0,0 +1,210 @@ +using StellaOps.Auth.ServerIntegration.Tenancy; +using StellaOps.Policy.Engine.Contracts.Gateway; +using StellaOps.Policy.Engine.Tenancy; +using StellaOps.Policy.Persistence.Postgres.Models; +using StellaOps.Policy.Persistence.Postgres.Repositories; +using StellaOps.Scheduler.Persistence.Postgres.Repositories; +using System.Text.Json; + +namespace StellaOps.Policy.Engine.Services.Gateway; + +internal interface IGateEvaluationJobExecutor +{ + Task ExecuteAsync(GateEvaluationRequest request, CancellationToken cancellationToken = default); +} + +public sealed class GateEvaluationJobDispatchService +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + private readonly IServiceScopeFactory _scopeFactory; + private readonly GateEvaluationQueueOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public GateEvaluationJobDispatchService( + IServiceScopeFactory scopeFactory, + GateEvaluationQueueOptions options, + TimeProvider timeProvider, + ILogger logger) + { + _scopeFactory = scopeFactory; + _options = options; + _timeProvider = timeProvider; + _logger = logger; + } + + public async Task RunPendingOnceAsync(CancellationToken cancellationToken = default) + { + using var scope = _scopeFactory.CreateScope(); + var workerResults = scope.ServiceProvider.GetRequiredService(); + var pending = await workerResults + .GetPendingAsync(_options.JobType, _options.BatchSize, cancellationToken) + .ConfigureAwait(false); + + var processed = 0; + foreach (var result in pending) + { + if (await TryProcessAsync(result, cancellationToken).ConfigureAwait(false)) + { + processed++; + } + } + + return processed; + } + + internal async Task TryProcessAsync( + WorkerResultEntity workerResult, + CancellationToken cancellationToken = default) + { + using var scope = _scopeFactory.CreateScope(); + var jobRepository = scope.ServiceProvider.GetRequiredService(); + var workerResults = scope.ServiceProvider.GetRequiredService(); + var executor = scope.ServiceProvider.GetRequiredService(); + var tenantAccessor = scope.ServiceProvider.GetRequiredService(); + var requestTenantAccessor = scope.ServiceProvider.GetService(); + + if (!Guid.TryParse(workerResult.JobId, out var jobId)) + { + await workerResults.FailAsync( + workerResult.TenantId, + workerResult.Id, + $"Invalid scheduler job identifier '{workerResult.JobId}'.", + cancellationToken).ConfigureAwait(false); + return true; + } + + var leasedJob = await jobRepository.TryLeaseJobAsync( + workerResult.TenantId, + jobId, + $"{Environment.MachineName}:policy-gate-worker", + TimeSpan.FromSeconds(Math.Max(5, _options.LeaseDurationSeconds)), + cancellationToken).ConfigureAwait(false); + + if (leasedJob is null || leasedJob.LeaseId is null) + { + return false; + } + + var actorId = ResolveActorId(workerResult.Metadata); + tenantAccessor.TenantContext = new StellaOpsTenantContext + { + TenantId = workerResult.TenantId, + ActorId = actorId ?? "policy-gate-worker", + Source = TenantSource.Unknown + }; + + if (requestTenantAccessor is not null) + { + requestTenantAccessor.TenantContext = TenantContext.ForTenant( + workerResult.TenantId, + canWrite: true, + actorId: actorId ?? "policy-gate-worker", + timeProvider: _timeProvider); + } + + try + { + var request = DeserializeRequest(leasedJob.Payload, workerResult.Metadata); + var response = await executor.ExecuteAsync(request, cancellationToken).ConfigureAwait(false); + var resultJson = JsonSerializer.Serialize(response, SerializerOptions); + var outputHash = SchedulerBackedGateEvaluationQueue.ComputeSha256(resultJson); + + await jobRepository.CompleteAsync( + workerResult.TenantId, + leasedJob.Id, + leasedJob.LeaseId.Value, + resultJson, + cancellationToken).ConfigureAwait(false); + + await workerResults.CompleteAsync( + workerResult.TenantId, + workerResult.Id, + resultJson, + outputHash, + cancellationToken).ConfigureAwait(false); + + _logger.LogInformation( + "Completed async gate evaluation job {JobId} for tenant {TenantId}.", + leasedJob.Id, + workerResult.TenantId); + + return true; + } + catch (Exception ex) + { + var canRetry = workerResult.RetryCount + 1 < workerResult.MaxRetries; + + await jobRepository.FailAsync( + workerResult.TenantId, + leasedJob.Id, + leasedJob.LeaseId.Value, + ex.Message, + retry: canRetry, + cancellationToken: cancellationToken).ConfigureAwait(false); + + if (canRetry) + { + await workerResults.IncrementRetryAsync( + workerResult.TenantId, + workerResult.Id, + cancellationToken).ConfigureAwait(false); + } + else + { + await workerResults.FailAsync( + workerResult.TenantId, + workerResult.Id, + ex.Message, + cancellationToken).ConfigureAwait(false); + } + + _logger.LogError( + ex, + "Async gate evaluation job {JobId} failed for tenant {TenantId}. Retry={Retry}.", + leasedJob.Id, + workerResult.TenantId, + canRetry); + + return true; + } + finally + { + tenantAccessor.TenantContext = null; + if (requestTenantAccessor is not null) + { + requestTenantAccessor.TenantContext = null; + } + } + } + + private static GateEvaluationRequest DeserializeRequest(string payload, string? metadata) + { + var request = JsonSerializer.Deserialize(payload, SerializerOptions); + if (request is not null) + { + return request; + } + + if (!string.IsNullOrWhiteSpace(metadata)) + { + var jobMetadata = JsonSerializer.Deserialize(metadata, SerializerOptions); + if (jobMetadata?.Request is not null) + { + return jobMetadata.Request; + } + } + + throw new InvalidOperationException("Gate evaluation job payload is missing a valid request."); + } + + private static string? ResolveActorId(string? metadata) + { + if (string.IsNullOrWhiteSpace(metadata)) + { + return null; + } + + return JsonSerializer.Deserialize(metadata, SerializerOptions)?.ActorId; + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateEvaluationJobStatusService.cs b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateEvaluationJobStatusService.cs new file mode 100644 index 000000000..326a6b77b --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateEvaluationJobStatusService.cs @@ -0,0 +1,144 @@ +using StellaOps.Auth.ServerIntegration.Tenancy; +using StellaOps.Policy.Engine.Contracts.Gateway; +using StellaOps.Policy.Engine.Tenancy; +using StellaOps.Policy.Persistence.Postgres.Models; +using StellaOps.Policy.Persistence.Postgres.Repositories; +using StellaOps.Scheduler.Persistence.Postgres.Models; +using StellaOps.Scheduler.Persistence.Postgres.Repositories; +using System.Text.Json; + +namespace StellaOps.Policy.Engine.Services.Gateway; + +public interface IGateEvaluationJobStatusService +{ + Task GetAsync(string jobId, CancellationToken cancellationToken = default); +} + +public sealed class GateEvaluationJobStatusService : IGateEvaluationJobStatusService +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + private readonly IWorkerResultRepository _workerResults; + private readonly IJobRepository _jobs; + private readonly IStellaOpsTenantAccessor _tenantAccessor; + private readonly GateEvaluationQueueOptions _options; + + public GateEvaluationJobStatusService( + IWorkerResultRepository workerResults, + IJobRepository jobs, + IStellaOpsTenantAccessor tenantAccessor, + GateEvaluationQueueOptions options) + { + _workerResults = workerResults; + _jobs = jobs; + _tenantAccessor = tenantAccessor; + _options = options; + } + + public async Task GetAsync(string jobId, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(jobId)) + { + return null; + } + + var tenantId = string.IsNullOrWhiteSpace(_tenantAccessor.TenantId) + ? TenantContextConstants.DefaultTenantId + : _tenantAccessor.TenantId!; + var workerResult = await _workerResults + .GetByJobAsync(tenantId, _options.JobType, jobId, cancellationToken) + .ConfigureAwait(false); + + JobEntity? job = null; + if (Guid.TryParse(jobId, out var schedulerJobId)) + { + job = await _jobs.GetByIdAsync(tenantId, schedulerJobId, cancellationToken).ConfigureAwait(false); + } + + if (workerResult is null && job is null) + { + return null; + } + + return new GateEvaluationJobStatusResponse + { + JobId = jobId, + Status = ResolveStatus(workerResult, job), + SchedulerStatus = job?.Status.ToString().ToLowerInvariant(), + Progress = ResolveProgress(workerResult, job), + RetryCount = workerResult?.RetryCount ?? 0, + MaxRetries = workerResult?.MaxRetries ?? Math.Max(1, _options.MaxAttempts), + QueuedAt = workerResult?.CreatedAt ?? job?.CreatedAt, + StartedAt = workerResult?.StartedAt ?? job?.StartedAt ?? job?.LeasedAt, + CompletedAt = workerResult?.CompletedAt ?? job?.CompletedAt, + ErrorMessage = workerResult?.ErrorMessage ?? job?.Reason, + Request = DeserializeRequest(workerResult?.Metadata, job?.Payload), + Decision = DeserializeDecision(workerResult?.Result, job?.Result) + }; + } + + private static string ResolveStatus(WorkerResultEntity? workerResult, JobEntity? job) + { + if (workerResult?.Status is "completed" or "failed") + { + return workerResult.Status; + } + + return job?.Status switch + { + JobStatus.Leased or JobStatus.Running => "running", + JobStatus.Succeeded => "completed", + JobStatus.Failed or JobStatus.Canceled or JobStatus.TimedOut => "failed", + JobStatus.Pending or JobStatus.Scheduled => "pending", + _ => workerResult?.Status ?? "pending" + }; + } + + private static int ResolveProgress(WorkerResultEntity? workerResult, JobEntity? job) + { + if (workerResult is not null) + { + return workerResult.Status == "completed" ? 100 : workerResult.Progress; + } + + return job?.Status switch + { + JobStatus.Succeeded => 100, + JobStatus.Leased or JobStatus.Running => 50, + _ => 0 + }; + } + + private static GateEvaluationRequest? DeserializeRequest(string? metadata, string? payload) + { + if (!string.IsNullOrWhiteSpace(metadata)) + { + var jobMetadata = JsonSerializer.Deserialize(metadata, SerializerOptions); + if (jobMetadata?.Request is not null) + { + return jobMetadata.Request; + } + } + + if (!string.IsNullOrWhiteSpace(payload)) + { + return JsonSerializer.Deserialize(payload, SerializerOptions); + } + + return null; + } + + private static GateEvaluateResponse? DeserializeDecision(string? result, string? schedulerResult) + { + if (!string.IsNullOrWhiteSpace(result)) + { + return JsonSerializer.Deserialize(result, SerializerOptions); + } + + if (!string.IsNullOrWhiteSpace(schedulerResult)) + { + return JsonSerializer.Deserialize(schedulerResult, SerializerOptions); + } + + return null; + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateEvaluationJobWorker.cs b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateEvaluationJobWorker.cs new file mode 100644 index 000000000..1322f7469 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateEvaluationJobWorker.cs @@ -0,0 +1,44 @@ +namespace StellaOps.Policy.Engine.Services.Gateway; + +public sealed class GateEvaluationJobWorker : BackgroundService +{ + private readonly GateEvaluationJobDispatchService _dispatchService; + private readonly GateEvaluationQueueOptions _options; + private readonly ILogger _logger; + + public GateEvaluationJobWorker( + GateEvaluationJobDispatchService dispatchService, + GateEvaluationQueueOptions options, + ILogger logger) + { + _dispatchService = dispatchService; + _options = options; + _logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + var delay = TimeSpan.FromSeconds(Math.Max(1, _options.PollIntervalSeconds)); + + while (!stoppingToken.IsCancellationRequested) + { + try + { + var processed = await _dispatchService.RunPendingOnceAsync(stoppingToken).ConfigureAwait(false); + if (processed == 0) + { + await Task.Delay(delay, stoppingToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + return; + } + catch (Exception ex) + { + _logger.LogError(ex, "Policy gate evaluation worker loop failed."); + await Task.Delay(delay, stoppingToken).ConfigureAwait(false); + } + } + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateEvaluationQueueContracts.cs b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateEvaluationQueueContracts.cs new file mode 100644 index 000000000..8b00dc5b2 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateEvaluationQueueContracts.cs @@ -0,0 +1,48 @@ +using System.Text.Json.Serialization; +using StellaOps.Policy.Engine.Contracts.Gateway; + +namespace StellaOps.Policy.Engine.Services.Gateway; + +public interface IGateEvaluationQueue +{ + Task EnqueueAsync(GateEvaluationRequest request, CancellationToken cancellationToken = default); +} + +public sealed record GateEvaluationRequest +{ + public required string ImageDigest { get; init; } + public required string Repository { get; init; } + public string? Tag { get; init; } + public string? RegistryUrl { get; init; } + public string? BaselineRef { get; init; } + public required string Source { get; init; } + public required DateTimeOffset Timestamp { get; init; } +} + +public sealed record GateEvaluationJobStatusResponse +{ + public required string JobId { get; init; } + public required string Status { get; init; } + public string? SchedulerStatus { get; init; } + public int Progress { get; init; } + public int RetryCount { get; init; } + public int MaxRetries { get; init; } + public DateTimeOffset? QueuedAt { get; init; } + public DateTimeOffset? StartedAt { get; init; } + public DateTimeOffset? CompletedAt { get; init; } + public string? ErrorMessage { get; init; } + public GateEvaluationRequest? Request { get; init; } + public GateEvaluateResponse? Decision { get; init; } +} + +internal sealed record GateEvaluationJobMetadata +{ + [JsonPropertyName("request")] + public required GateEvaluationRequest Request { get; init; } + + [JsonPropertyName("actorId")] + public string? ActorId { get; init; } + + [JsonPropertyName("enqueuedAt")] + public DateTimeOffset EnqueuedAt { get; init; } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateEvaluationQueueOptions.cs b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateEvaluationQueueOptions.cs new file mode 100644 index 000000000..63ab7840e --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateEvaluationQueueOptions.cs @@ -0,0 +1,10 @@ +namespace StellaOps.Policy.Engine.Services.Gateway; + +public sealed class GateEvaluationQueueOptions +{ + public int PollIntervalSeconds { get; set; } = 5; + public int LeaseDurationSeconds { get; set; } = 30; + public int MaxAttempts { get; set; } = 3; + public int BatchSize { get; set; } = 10; + public string JobType { get; set; } = "policy.gate-evaluation"; +} diff --git a/src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateTargetSnapshotMaterializer.cs b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateTargetSnapshotMaterializer.cs new file mode 100644 index 000000000..3ebeb509c --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/GateTargetSnapshotMaterializer.cs @@ -0,0 +1,123 @@ +using System.Globalization; +using StellaOps.Auth.ServerIntegration.Tenancy; +using StellaOps.Policy.Engine.Ledger; +using StellaOps.Policy.Engine.Orchestration; +using StellaOps.Policy.Engine.Snapshots; +using StellaOps.Policy.Engine.Tenancy; + +namespace StellaOps.Policy.Engine.Services.Gateway; + +internal sealed class GateTargetSnapshotMaterializer +{ + private readonly TimeProvider _timeProvider; + private readonly ILedgerExportStore _ledgerExportStore; + private readonly ISnapshotStore _snapshotStore; + private readonly IStellaOpsTenantAccessor _tenantAccessor; + private readonly ILogger _logger; + + public GateTargetSnapshotMaterializer( + TimeProvider timeProvider, + ILedgerExportStore ledgerExportStore, + ISnapshotStore snapshotStore, + IStellaOpsTenantAccessor tenantAccessor, + ILogger logger) + { + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _ledgerExportStore = ledgerExportStore ?? throw new ArgumentNullException(nameof(ledgerExportStore)); + _snapshotStore = snapshotStore ?? throw new ArgumentNullException(nameof(snapshotStore)); + _tenantAccessor = tenantAccessor ?? throw new ArgumentNullException(nameof(tenantAccessor)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + internal async Task MaterializeAsync( + string imageDigest, + string? repository, + string? tag, + string? baselineSnapshotId, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(imageDigest); + + var tenantId = string.IsNullOrWhiteSpace(_tenantAccessor.TenantId) + ? TenantContextConstants.DefaultTenantId + : _tenantAccessor.TenantId!; + + var baselineSnapshot = await ResolveBaselineSnapshotAsync(tenantId, baselineSnapshotId, cancellationToken).ConfigureAwait(false); + var export = await ResolveLedgerExportAsync(tenantId, baselineSnapshot, cancellationToken).ConfigureAwait(false); + + var generatedAt = _timeProvider.GetUtcNow().ToString("O", CultureInfo.InvariantCulture); + var statusCounts = export.Records + .GroupBy(record => record.Status) + .ToDictionary(group => group.Key, group => group.Count(), StringComparer.Ordinal); + var snapshotId = StableIdGenerator.CreateUlid( + $"{tenantId}|{export.Manifest.ExportId}|{baselineSnapshot.OverlayHash}|{imageDigest}|{repository ?? string.Empty}|{tag ?? string.Empty}"); + + var snapshot = new SnapshotDetail( + SnapshotId: snapshotId, + TenantId: tenantId, + LedgerExportId: export.Manifest.ExportId, + GeneratedAt: generatedAt, + OverlayHash: baselineSnapshot.OverlayHash, + StatusCounts: statusCounts, + Records: export.Records, + ArtifactDigest: imageDigest, + ArtifactRepository: repository, + ArtifactTag: tag); + + await _snapshotStore.SaveAsync(snapshot, cancellationToken).ConfigureAwait(false); + + _logger.LogInformation( + "Materialized target snapshot {SnapshotId} for {Repository}@{Digest} using ledger export {ExportId}.", + snapshot.SnapshotId, + repository ?? "", + imageDigest, + export.Manifest.ExportId); + + return snapshot; + } + + private async Task ResolveBaselineSnapshotAsync( + string tenantId, + string? baselineSnapshotId, + CancellationToken cancellationToken) + { + if (!string.IsNullOrWhiteSpace(baselineSnapshotId)) + { + var snapshot = await _snapshotStore.GetAsync(baselineSnapshotId, cancellationToken).ConfigureAwait(false); + if (snapshot is not null) + { + return snapshot; + } + + throw new InvalidOperationException($"Baseline snapshot {baselineSnapshotId} not found."); + } + + var latestSnapshot = (await _snapshotStore.ListAsync(tenantId, cancellationToken).ConfigureAwait(false)) + .OrderByDescending(snapshot => snapshot.GeneratedAt, StringComparer.Ordinal) + .ThenBy(snapshot => snapshot.SnapshotId, StringComparer.Ordinal) + .FirstOrDefault(); + + return latestSnapshot + ?? throw new InvalidOperationException($"No persisted policy snapshot found for tenant {tenantId}."); + } + + private async Task ResolveLedgerExportAsync( + string tenantId, + SnapshotDetail baselineSnapshot, + CancellationToken cancellationToken) + { + var latestExport = (await _ledgerExportStore.ListAsync(tenantId, cancellationToken).ConfigureAwait(false)) + .OrderByDescending(export => export.Manifest.GeneratedAt, StringComparer.Ordinal) + .ThenBy(export => export.Manifest.ExportId, StringComparer.Ordinal) + .FirstOrDefault(); + + if (latestExport is not null) + { + return latestExport; + } + + var baselineExport = await _ledgerExportStore.GetAsync(baselineSnapshot.LedgerExportId, cancellationToken).ConfigureAwait(false); + return baselineExport + ?? throw new InvalidOperationException($"No persisted ledger export found for tenant {tenantId}."); + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Services/Gateway/InMemoryGateEvaluationQueue.cs b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/InMemoryGateEvaluationQueue.cs deleted file mode 100644 index 9c673c286..000000000 --- a/src/Policy/StellaOps.Policy.Engine/Services/Gateway/InMemoryGateEvaluationQueue.cs +++ /dev/null @@ -1,185 +0,0 @@ -// ----------------------------------------------------------------------------- -// InMemoryGateEvaluationQueue.cs -// Sprint: SPRINT_20251226_001_BE_cicd_gate_integration -// Task: CICD-GATE-02 - Gate evaluation queue implementation -// Description: In-memory queue for gate evaluation jobs with background processing -// ----------------------------------------------------------------------------- - - -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using StellaOps.Policy.Engine.Gates; -using StellaOps.Policy.Engine.Endpoints.Gateway; -using System.Threading.Channels; - -namespace StellaOps.Policy.Engine.Services.Gateway; - -/// -/// In-memory implementation of the gate evaluation queue. -/// Uses System.Threading.Channels for async producer-consumer pattern. -/// -public sealed class InMemoryGateEvaluationQueue : IGateEvaluationQueue -{ - private readonly Channel _channel; - private readonly ILogger _logger; - private readonly TimeProvider _timeProvider; - - public InMemoryGateEvaluationQueue( - ILogger logger, - TimeProvider? timeProvider = null) - { - ArgumentNullException.ThrowIfNull(logger); - _logger = logger; - _timeProvider = timeProvider ?? TimeProvider.System; - - // Bounded channel to prevent unbounded memory growth - _channel = Channel.CreateBounded(new BoundedChannelOptions(1000) - { - FullMode = BoundedChannelFullMode.Wait, - SingleReader = false, - SingleWriter = false - }); - } - - /// - public async Task EnqueueAsync(GateEvaluationRequest request, CancellationToken cancellationToken = default) - { - ArgumentNullException.ThrowIfNull(request); - - var jobId = GenerateJobId(); - var job = new GateEvaluationJob - { - JobId = jobId, - Request = request, - QueuedAt = _timeProvider.GetUtcNow() - }; - - await _channel.Writer.WriteAsync(job, cancellationToken).ConfigureAwait(false); - - _logger.LogDebug( - "Enqueued gate evaluation job {JobId} for {Repository}@{Digest}", - jobId, - request.Repository, - request.ImageDigest); - - return jobId; - } - - /// - /// Gets the channel reader for consuming jobs. - /// - public ChannelReader Reader => _channel.Reader; - - private string GenerateJobId() - { - // Format: gate-{timestamp}-{random} - var timestamp = _timeProvider.GetUtcNow().ToUnixTimeMilliseconds(); - var random = Guid.NewGuid().ToString("N")[..8]; - return $"gate-{timestamp}-{random}"; - } -} - -/// -/// A gate evaluation job in the queue. -/// -public sealed record GateEvaluationJob -{ - public required string JobId { get; init; } - public required GateEvaluationRequest Request { get; init; } - public required DateTimeOffset QueuedAt { get; init; } -} - -/// -/// Background service that processes gate evaluation jobs from the queue. -/// Orchestrates: image analysis -> drift delta computation -> gate evaluation. -/// -public sealed class GateEvaluationWorker : BackgroundService -{ - private readonly InMemoryGateEvaluationQueue _queue; - private readonly IServiceScopeFactory _scopeFactory; - private readonly ILogger _logger; - - public GateEvaluationWorker( - InMemoryGateEvaluationQueue queue, - IServiceScopeFactory scopeFactory, - ILogger logger) - { - ArgumentNullException.ThrowIfNull(queue); - ArgumentNullException.ThrowIfNull(scopeFactory); - ArgumentNullException.ThrowIfNull(logger); - - _queue = queue; - _scopeFactory = scopeFactory; - _logger = logger; - } - - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - _logger.LogInformation("Gate evaluation worker starting"); - - await foreach (var job in _queue.Reader.ReadAllAsync(stoppingToken)) - { - try - { - await ProcessJobAsync(job, stoppingToken).ConfigureAwait(false); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - _logger.LogError(ex, - "Error processing gate evaluation job {JobId} for {Repository}@{Digest}", - job.JobId, - job.Request.Repository, - job.Request.ImageDigest); - } - } - - _logger.LogInformation("Gate evaluation worker stopping"); - } - - private async Task ProcessJobAsync(GateEvaluationJob job, CancellationToken cancellationToken) - { - _logger.LogInformation( - "Processing gate evaluation job {JobId} for {Repository}@{Digest}", - job.JobId, - job.Request.Repository, - job.Request.ImageDigest); - - using var scope = _scopeFactory.CreateScope(); - var evaluator = scope.ServiceProvider.GetRequiredService(); - - // Build a minimal context for the gate evaluation. - // In production, this would involve: - // 1. Fetching or triggering a scan of the image - // 2. Computing the reachability delta against the baseline - // 3. Building the DriftGateContext with actual metrics - // - // For now, we create a placeholder context that represents "no drift detected" - // which allows the gate to pass. The full implementation requires Scanner integration. - var driftContext = new DriftGateContext - { - DeltaReachable = 0, - DeltaUnreachable = 0, - HasKevReachable = false, - BaseScanId = job.Request.BaselineRef, - HeadScanId = job.Request.ImageDigest - }; - - var evalRequest = new DriftGateRequest - { - Context = driftContext, - PolicyId = null, // Use default policy - AllowOverride = false - }; - - var result = await evaluator.EvaluateAsync(evalRequest, cancellationToken).ConfigureAwait(false); - - _logger.LogInformation( - "Gate evaluation {JobId} completed: Decision={Decision}, GateCount={GateCount}", - job.JobId, - result.Decision, - result.Gates.Length); - - // TODO: Store result and notify via webhook/event - // This will be implemented in CICD-GATE-03 - } -} diff --git a/src/Policy/StellaOps.Policy.Engine/Services/Gateway/PersistedKnowledgeSnapshotStore.cs b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/PersistedKnowledgeSnapshotStore.cs new file mode 100644 index 000000000..8bb69778a --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/PersistedKnowledgeSnapshotStore.cs @@ -0,0 +1,356 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using StellaOps.Auth.ServerIntegration.Tenancy; +using StellaOps.Policy.Engine.Ledger; +using StellaOps.Policy.Engine.Tenancy; +using StellaOps.Policy.Snapshots; + +namespace StellaOps.Policy.Engine.Services.Gateway; + +using EngineSnapshotStore = StellaOps.Policy.Engine.Snapshots.ISnapshotStore; + +internal sealed class PersistedKnowledgeSnapshotStore : ISnapshotStore +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + private readonly EngineSnapshotStore _snapshotStore; + private readonly IStellaOpsTenantAccessor _tenantAccessor; + + public PersistedKnowledgeSnapshotStore( + EngineSnapshotStore snapshotStore, + IStellaOpsTenantAccessor tenantAccessor) + { + _snapshotStore = snapshotStore ?? throw new ArgumentNullException(nameof(snapshotStore)); + _tenantAccessor = tenantAccessor ?? throw new ArgumentNullException(nameof(tenantAccessor)); + } + + public Task SaveAsync(KnowledgeSnapshotManifest manifest, CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(manifest); + throw new NotSupportedException("Policy engine compatibility snapshots are projected from persisted runtime snapshots."); + } + + public async Task GetAsync(string snapshotId, CancellationToken ct = default) + { + var snapshot = await _snapshotStore.GetAsync(snapshotId, ct).ConfigureAwait(false); + return snapshot is null ? null : CreateManifest(snapshot); + } + + public async Task> ListAsync(int skip = 0, int take = 100, CancellationToken ct = default) + { + var tenantId = string.IsNullOrWhiteSpace(_tenantAccessor.TenantId) + ? TenantContextConstants.DefaultTenantId + : _tenantAccessor.TenantId!; + var snapshots = await _snapshotStore.ListAsync(tenantId, ct).ConfigureAwait(false); + return snapshots + .OrderByDescending(snapshot => snapshot.GeneratedAt, StringComparer.Ordinal) + .ThenBy(snapshot => snapshot.SnapshotId, StringComparer.Ordinal) + .Skip(skip) + .Take(take) + .Select(CreateManifest) + .ToList(); + } + + public Task DeleteAsync(string snapshotId, CancellationToken ct = default) + { + return Task.FromResult(false); + } + + public async Task GetBundledContentAsync(string bundlePath, CancellationToken ct = default) + { + if (string.IsNullOrWhiteSpace(bundlePath)) + { + return null; + } + + var parts = bundlePath.Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (parts.Length != 2) + { + return null; + } + + var snapshot = await _snapshotStore.GetAsync(parts[0], ct).ConfigureAwait(false); + if (snapshot is null) + { + return null; + } + + return parts[1] switch + { + "packages.json" => SerializeBytes(ProjectPackages(snapshot.Records)), + "reachability.json" => SerializeBytes(ProjectReachability(snapshot.Records)), + "vex.json" => SerializeBytes(ProjectVexStatements(snapshot.Records)), + "policy-violations.json" => SerializeBytes(ProjectPolicyViolations(snapshot.Records)), + "unknowns.json" => SerializeBytes(ProjectUnknowns(snapshot.Records)), + _ => null + }; + } + + public async Task GetByDigestAsync(string digest, CancellationToken ct = default) + { + if (string.IsNullOrWhiteSpace(digest)) + { + return null; + } + + var snapshots = await _snapshotStore.ListAsync(cancellationToken: ct).ConfigureAwait(false); + foreach (var snapshot in snapshots) + { + var manifest = CreateManifest(snapshot); + foreach (var source in manifest.Sources) + { + if (!string.Equals(source.Digest, digest, StringComparison.Ordinal) || string.IsNullOrWhiteSpace(source.BundlePath)) + { + continue; + } + + return await GetBundledContentAsync(source.BundlePath, ct).ConfigureAwait(false); + } + } + + return null; + } + + private static KnowledgeSnapshotManifest CreateManifest(StellaOps.Policy.Engine.Snapshots.SnapshotDetail snapshot) + { + var packages = ProjectPackages(snapshot.Records); + var reachability = ProjectReachability(snapshot.Records); + var vexStatements = ProjectVexStatements(snapshot.Records); + var policyViolations = ProjectPolicyViolations(snapshot.Records); + var unknowns = ProjectUnknowns(snapshot.Records); + var sources = new List + { + new() + { + Name = "policy-overlay", + Type = KnowledgeSourceTypes.Policy, + Epoch = snapshot.GeneratedAt, + Digest = snapshot.OverlayHash, + RecordCount = snapshot.StatusCounts.Values.Sum(), + InclusionMode = SourceInclusionMode.Referenced + }, + CreateBundledSource(snapshot, "packages", KnowledgeSourceTypes.Sbom, "packages.json", packages), + }; + + if (reachability.Count > 0) + { + sources.Add(CreateBundledSource(snapshot, "reachability", KnowledgeSourceTypes.Reachability, "reachability.json", reachability)); + } + + if (vexStatements.Count > 0) + { + sources.Add(CreateBundledSource(snapshot, "vex", KnowledgeSourceTypes.Vex, "vex.json", vexStatements)); + } + + if (policyViolations.Count > 0) + { + sources.Add(CreateBundledSource(snapshot, "policy-violations", KnowledgeSourceTypes.Policy, "policy-violations.json", policyViolations)); + } + + if (unknowns.Count > 0) + { + sources.Add(CreateBundledSource(snapshot, "unknowns", KnowledgeSourceTypes.AdvisoryFeed, "unknowns.json", unknowns)); + } + + return new KnowledgeSnapshotManifest + { + SnapshotId = snapshot.SnapshotId, + CreatedAt = DateTimeOffset.Parse(snapshot.GeneratedAt, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind), + Engine = new EngineInfo("policy-engine-runtime", "persisted", "local"), + Policy = new PolicyBundleRef("policy-runtime", snapshot.OverlayHash, null), + Scoring = new ScoringRulesRef("policy-runtime", snapshot.OverlayHash, null), + Sources = sources + }; + } + + private static KnowledgeSourceDescriptor CreateBundledSource( + StellaOps.Policy.Engine.Snapshots.SnapshotDetail snapshot, + string name, + string type, + string fileName, + IReadOnlyList content) + { + var bytes = SerializeBytes(content); + return new KnowledgeSourceDescriptor + { + Name = name, + Type = type, + Epoch = snapshot.GeneratedAt, + Digest = $"sha256:{Convert.ToHexStringLower(SHA256.HashData(bytes))}", + RecordCount = content.Count, + InclusionMode = SourceInclusionMode.Bundled, + BundlePath = $"{snapshot.SnapshotId}/{fileName}" + }; + } + + private static byte[] SerializeBytes(IReadOnlyList value) + { + return JsonSerializer.SerializeToUtf8Bytes(value, SerializerOptions); + } + + internal static IReadOnlyList ProjectPackages(IReadOnlyList records) + { + return records + .Select(record => record.ComponentPurl) + .Distinct(StringComparer.Ordinal) + .OrderBy(purl => purl, StringComparer.Ordinal) + .Select(purl => new PackageProjection(purl, ExtractVersion(purl), null)) + .ToList(); + } + + internal static IReadOnlyList ProjectReachability(IReadOnlyList records) + { + return records + .Select(record => TryProjectReachability(record, out var projection) ? projection : null) + .Where(projection => projection is not null) + .Distinct() + .OrderBy(projection => projection!.CveId, StringComparer.Ordinal) + .ThenBy(projection => projection!.Purl, StringComparer.Ordinal) + .Cast() + .ToList(); + } + + internal static IReadOnlyList ProjectVexStatements(IReadOnlyList records) + { + return records + .Select(record => TryNormalizeVexStatus(record.Status, out var status) + ? new { record.AdvisoryId, Status = status } + : null) + .Where(item => item is not null) + .GroupBy(item => item!.AdvisoryId, StringComparer.Ordinal) + .Select(group => + { + var status = group + .Select(item => item!.Status) + .OrderBy(status => GetVexSeverityRank(status)) + .ThenBy(status => status, StringComparer.Ordinal) + .First(); + return new VexProjection(group.Key, status, null); + }) + .OrderBy(item => item.CveId, StringComparer.Ordinal) + .ToList(); + } + + internal static IReadOnlyList ProjectPolicyViolations(IReadOnlyList records) + { + return records + .Where(record => IsPolicyViolationStatus(record.Status)) + .Select(record => new PolicyViolationProjection( + RuleId: $"advisory:{record.AdvisoryId}:{record.ComponentPurl}", + Severity: MapPolicySeverity(record.Status), + Message: $"{NormalizeStatus(record.Status)} for {record.AdvisoryId} on {record.ComponentPurl}")) + .OrderBy(item => item.RuleId, StringComparer.Ordinal) + .ToList(); + } + + internal static IReadOnlyList ProjectUnknowns(IReadOnlyList records) + { + return records + .Where(record => IsUnknownStatus(record.Status)) + .Select(record => new UnknownProjection( + Id: $"{record.AdvisoryId}|{record.ComponentPurl}", + ReasonCode: NormalizeStatus(record.Status), + Description: $"Unknown policy state for {record.AdvisoryId} on {record.ComponentPurl}")) + .OrderBy(item => item.Id, StringComparer.Ordinal) + .ToList(); + } + + private static bool TryProjectReachability(LedgerExportRecord record, out ReachabilityProjection projection) + { + if (TryNormalizeReachability(record.Status, out var isReachable)) + { + projection = new ReachabilityProjection(record.AdvisoryId, record.ComponentPurl, isReachable); + return true; + } + + projection = null!; + return false; + } + + private static bool TryNormalizeReachability(string status, out bool isReachable) + { + switch (NormalizeStatus(status)) + { + case "reachable": + case "new-reachable": + case "new_reachable": + case "affected": + case "under_investigation": + isReachable = true; + return true; + case "unreachable": + case "not-reachable": + case "not_reachable": + case "not_affected": + case "fixed": + isReachable = false; + return true; + default: + isReachable = false; + return false; + } + } + + private static bool TryNormalizeVexStatus(string status, out string normalizedStatus) + { + normalizedStatus = NormalizeStatus(status); + return normalizedStatus is "affected" or "not_affected" or "fixed" or "under_investigation"; + } + + private static bool IsPolicyViolationStatus(string status) + { + return NormalizeStatus(status) is "blocked" or "deny" or "denied" or "warn" or "warning" or "fail" or "failed" or "violation"; + } + + private static bool IsUnknownStatus(string status) + { + return NormalizeStatus(status) is "unknown" or "pending" or "error" or "indeterminate"; + } + + private static string MapPolicySeverity(string status) + { + return NormalizeStatus(status) switch + { + "blocked" or "deny" or "denied" or "fail" or "failed" => "high", + "warn" or "warning" => "medium", + _ => "low" + }; + } + + private static int GetVexSeverityRank(string status) + { + return status switch + { + "affected" => 0, + "under_investigation" => 1, + "fixed" => 2, + "not_affected" => 3, + _ => 4 + }; + } + + private static string NormalizeStatus(string status) + { + return status.Trim().Replace('-', '_').ToLowerInvariant(); + } + + private static string? ExtractVersion(string purl) + { + var atIndex = purl.LastIndexOf('@'); + if (atIndex < 0 || atIndex == purl.Length - 1) + { + return null; + } + + var version = purl[(atIndex + 1)..]; + var delimiterIndex = version.IndexOfAny(['?', '#']); + return delimiterIndex >= 0 ? version[..delimiterIndex] : version; + } + + internal sealed record PackageProjection(string Purl, string? Version, string? License); + internal sealed record ReachabilityProjection(string CveId, string Purl, bool IsReachable); + internal sealed record VexProjection(string CveId, string Status, string? Justification); + internal sealed record PolicyViolationProjection(string RuleId, string Severity, string? Message); + internal sealed record UnknownProjection(string Id, string ReasonCode, string? Description); +} diff --git a/src/Policy/StellaOps.Policy.Engine/Services/Gateway/PolicyAsyncGateEvaluationRuntimeExtensions.cs b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/PolicyAsyncGateEvaluationRuntimeExtensions.cs new file mode 100644 index 000000000..0149606b1 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/PolicyAsyncGateEvaluationRuntimeExtensions.cs @@ -0,0 +1,100 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Infrastructure.Postgres.Migrations; +using StellaOps.Infrastructure.Postgres.Options; +using StellaOps.Scheduler.Persistence.Postgres; +using StellaOps.Scheduler.Persistence.Postgres.Repositories; +using System.Reflection; + +namespace StellaOps.Policy.Engine.Services.Gateway; + +public static class PolicyAsyncGateEvaluationRuntimeExtensions +{ + public static IServiceCollection AddPolicyAsyncGateEvaluationRuntime( + this IServiceCollection services, + IConfiguration configuration, + string hostOptionsSectionName) + { + services.AddOptions() + .Bind(configuration.GetSection($"{hostOptionsSectionName}:GateEvaluationQueue")); + services.AddSingleton(sp => sp.GetRequiredService>().Value); + + var schedulerOptions = configuration.GetSection("Postgres:Scheduler").Get(); + if (schedulerOptions is null || string.IsNullOrWhiteSpace(schedulerOptions.ConnectionString)) + { + services.AddSingleton(); + return services; + } + + var normalizedOptions = CloneSchedulerOptions(schedulerOptions); + + services.AddSingleton(sp => + { + var logger = sp.GetRequiredService>(); + return new SchedulerDataSource(Microsoft.Extensions.Options.Options.Create(normalizedOptions), logger); + }); + + services.AddSingleton(sp => + { + var logger = sp.GetRequiredService().CreateLogger("Migration.Scheduler.Persistence"); + var lifetime = sp.GetRequiredService(); + + return new PolicySchedulerStartupMigrationHost( + normalizedOptions.ConnectionString, + SchedulerDataSource.DefaultSchemaName, + typeof(SchedulerDataSource).Assembly, + logger, + lifetime); + }); + + services.AddScoped(); + services.AddSingleton(); + services.AddSingleton(); + services.AddHostedService(); + services.AddScoped(); + services.AddScoped(); + + return services; + } + + private static PostgresOptions CloneSchedulerOptions(PostgresOptions options) + { + return new PostgresOptions + { + ConnectionString = options.ConnectionString, + CommandTimeoutSeconds = options.CommandTimeoutSeconds, + MaxPoolSize = options.MaxPoolSize, + ApplicationName = options.ApplicationName, + MinPoolSize = options.MinPoolSize, + ConnectionIdleLifetimeSeconds = options.ConnectionIdleLifetimeSeconds, + Pooling = options.Pooling, + SchemaName = string.IsNullOrWhiteSpace(options.SchemaName) + ? SchedulerDataSource.DefaultSchemaName + : options.SchemaName, + AutoMigrate = options.AutoMigrate, + MigrationsPath = options.MigrationsPath + }; + } + + private sealed class PolicySchedulerStartupMigrationHost : StartupMigrationHost + { + public PolicySchedulerStartupMigrationHost( + string connectionString, + string schemaName, + Assembly migrationsAssembly, + ILogger logger, + IHostApplicationLifetime lifetime) + : base( + connectionString, + schemaName, + "Scheduler.Persistence", + migrationsAssembly, + logger, + lifetime) + { + } + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Services/Gateway/PolicyGateEvaluationJobExecutor.cs b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/PolicyGateEvaluationJobExecutor.cs new file mode 100644 index 000000000..092594436 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/PolicyGateEvaluationJobExecutor.cs @@ -0,0 +1,287 @@ +using Microsoft.Extensions.Caching.Memory; +using StellaOps.Policy.Deltas; +using StellaOps.Policy.Engine.Contracts.Gateway; +using StellaOps.Policy.Engine.Gates; + +namespace StellaOps.Policy.Engine.Services.Gateway; + +internal sealed class PolicyGateEvaluationJobExecutor : IGateEvaluationJobExecutor +{ + private const string DeltaCachePrefix = "delta:"; + private const string DecisionCachePrefix = "gate:decision:"; + private static readonly TimeSpan CacheDuration = TimeSpan.FromMinutes(30); + private readonly IDriftGateEvaluator _gateEvaluator; + private readonly IDeltaComputer _deltaComputer; + private readonly IBaselineSelector _baselineSelector; + private readonly GateBaselineBootstrapper _baselineBootstrapper; + private readonly GateTargetSnapshotMaterializer _targetSnapshotMaterializer; + private readonly IMemoryCache _cache; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public PolicyGateEvaluationJobExecutor( + IDriftGateEvaluator gateEvaluator, + IDeltaComputer deltaComputer, + IBaselineSelector baselineSelector, + GateBaselineBootstrapper baselineBootstrapper, + GateTargetSnapshotMaterializer targetSnapshotMaterializer, + IMemoryCache cache, + TimeProvider timeProvider, + ILogger logger) + { + _gateEvaluator = gateEvaluator; + _deltaComputer = deltaComputer; + _baselineSelector = baselineSelector; + _baselineBootstrapper = baselineBootstrapper; + _targetSnapshotMaterializer = targetSnapshotMaterializer; + _cache = cache; + _timeProvider = timeProvider; + _logger = logger; + } + + public async Task ExecuteAsync( + GateEvaluationRequest request, + CancellationToken cancellationToken = default) + { + var gateRequest = new GateEvaluateRequest + { + ImageDigest = request.ImageDigest, + Repository = request.Repository, + Tag = request.Tag, + BaselineRef = request.BaselineRef, + Source = request.Source + }; + + var baselineResult = await ResolveBaselineAsync( + request.ImageDigest, + request.BaselineRef, + _baselineSelector, + cancellationToken).ConfigureAwait(false); + + if (!baselineResult.IsFound && string.IsNullOrWhiteSpace(request.BaselineRef)) + { + var bootstrapped = await _baselineBootstrapper + .TryEnsureBaselineAsync(cancellationToken) + .ConfigureAwait(false); + if (bootstrapped) + { + baselineResult = await ResolveBaselineAsync( + request.ImageDigest, + request.BaselineRef, + _baselineSelector, + cancellationToken).ConfigureAwait(false); + } + } + + GateEvaluateResponse response; + if (!baselineResult.IsFound) + { + response = new GateEvaluateResponse + { + DecisionId = $"gate:{_timeProvider.GetUtcNow():yyyyMMddHHmmss}:{Guid.NewGuid():N}", + Status = GateStatus.Pass, + ExitCode = GateExitCodes.Pass, + ImageDigest = request.ImageDigest, + BaselineRef = request.BaselineRef, + DecidedAt = _timeProvider.GetUtcNow(), + Summary = "First build - no baseline for comparison", + Advisory = "This appears to be a first build. Future builds will be compared against this baseline." + }; + } + else + { + var targetSnapshot = await _targetSnapshotMaterializer.MaterializeAsync( + request.ImageDigest, + request.Repository, + request.Tag, + baselineResult.Snapshot!.SnapshotId, + cancellationToken).ConfigureAwait(false); + + var delta = await _deltaComputer.ComputeDeltaAsync( + baselineResult.Snapshot!.SnapshotId, + targetSnapshot.SnapshotId, + new ArtifactRef(request.ImageDigest, request.Repository, request.Tag), + cancellationToken).ConfigureAwait(false); + + _cache.Set(DeltaCachePrefix + delta.DeltaId, delta, CacheDuration); + + var decision = await _gateEvaluator.EvaluateAsync( + new DriftGateRequest + { + Context = BuildGateContext(delta) + }, + cancellationToken).ConfigureAwait(false); + + response = BuildResponse(gateRequest, decision, delta); + } + + _cache.Set(DecisionCachePrefix + response.DecisionId, response, CacheDuration); + _logger.LogInformation( + "Executed async gate evaluation for {Repository}@{Digest}; decision {DecisionId}.", + request.Repository, + request.ImageDigest, + response.DecisionId); + + return response; + } + + private static async Task ResolveBaselineAsync( + string imageDigest, + string? baselineRef, + IBaselineSelector baselineSelector, + CancellationToken cancellationToken) + { + if (!string.IsNullOrWhiteSpace(baselineRef)) + { + if (baselineRef.StartsWith("snapshot:", StringComparison.Ordinal) || Guid.TryParse(baselineRef, out _)) + { + return await baselineSelector.SelectExplicitAsync( + baselineRef.Replace("snapshot:", "", StringComparison.Ordinal), + cancellationToken).ConfigureAwait(false); + } + + var strategy = baselineRef.ToLowerInvariant() switch + { + "last-approved" or "lastapproved" => BaselineSelectionStrategy.LastApproved, + "previous-build" or "previousbuild" => BaselineSelectionStrategy.PreviousBuild, + "production" or "production-deployed" => BaselineSelectionStrategy.ProductionDeployed, + "branch-base" or "branchbase" => BaselineSelectionStrategy.BranchBase, + _ => BaselineSelectionStrategy.LastApproved + }; + + return await baselineSelector.SelectBaselineAsync(imageDigest, strategy, cancellationToken) + .ConfigureAwait(false); + } + + return await baselineSelector.SelectBaselineAsync( + imageDigest, + BaselineSelectionStrategy.LastApproved, + cancellationToken).ConfigureAwait(false); + } + + private static DriftGateContext BuildGateContext(SecurityStateDelta delta) + { + var newlyReachableVexStatuses = new List(); + var newlyReachableSinkIds = new List(); + var newlyUnreachableSinkIds = new List(); + double? maxCvss = null; + double? maxEpss = null; + var hasKev = false; + var deltaReachable = 0; + var deltaUnreachable = 0; + + foreach (var driver in delta.Drivers) + { + if (driver.Type is "new-reachable-cve" or "new-reachable-path") + { + deltaReachable++; + if (driver.CveId is not null) + { + newlyReachableSinkIds.Add(driver.CveId); + } + if (driver.Details.TryGetValue("vex_status", out var vexStatus)) + { + newlyReachableVexStatuses.Add(vexStatus); + } + if (driver.Details.TryGetValue("cvss", out var cvssStr) && + double.TryParse(cvssStr, out var cvss) && + (!maxCvss.HasValue || cvss > maxCvss.Value)) + { + maxCvss = cvss; + } + if (driver.Details.TryGetValue("epss", out var epssStr) && + double.TryParse(epssStr, out var epss) && + (!maxEpss.HasValue || epss > maxEpss.Value)) + { + maxEpss = epss; + } + if (driver.Details.TryGetValue("is_kev", out var kevStr) && + bool.TryParse(kevStr, out var isKev) && + isKev) + { + hasKev = true; + } + } + else if (driver.Type is "removed-reachable-cve" or "removed-reachable-path") + { + deltaUnreachable++; + if (driver.CveId is not null) + { + newlyUnreachableSinkIds.Add(driver.CveId); + } + } + } + + return new DriftGateContext + { + DeltaReachable = deltaReachable, + DeltaUnreachable = deltaUnreachable, + HasKevReachable = hasKev, + NewlyReachableVexStatuses = newlyReachableVexStatuses, + MaxCvss = maxCvss, + MaxEpss = maxEpss, + BaseScanId = delta.BaselineSnapshotId, + HeadScanId = delta.TargetSnapshotId, + NewlyReachableSinkIds = newlyReachableSinkIds, + NewlyUnreachableSinkIds = newlyUnreachableSinkIds + }; + } + + private static GateEvaluateResponse BuildResponse( + GateEvaluateRequest request, + DriftGateDecision decision, + SecurityStateDelta delta) + { + var status = decision.Decision switch + { + DriftGateDecisionType.Allow => GateStatus.Pass, + DriftGateDecisionType.Warn => GateStatus.Warn, + DriftGateDecisionType.Block => GateStatus.Fail, + _ => GateStatus.Pass + }; + + var exitCode = decision.Decision switch + { + DriftGateDecisionType.Allow => GateExitCodes.Pass, + DriftGateDecisionType.Warn => GateExitCodes.Warn, + DriftGateDecisionType.Block => GateExitCodes.Fail, + _ => GateExitCodes.Pass + }; + + return new GateEvaluateResponse + { + DecisionId = decision.DecisionId, + Status = status, + ExitCode = exitCode, + ImageDigest = request.ImageDigest, + BaselineRef = request.BaselineRef, + DecidedAt = decision.DecidedAt, + Summary = BuildSummary(decision), + Advisory = decision.Advisory, + Gates = decision.Gates.Select(g => new GateResultDto + { + Name = g.Name, + Result = g.Result.ToString(), + Reason = g.Reason, + Note = g.Note, + Condition = g.Condition + }).ToList(), + BlockedBy = decision.BlockedBy, + BlockReason = decision.BlockReason, + Suggestion = decision.Suggestion, + OverrideApplied = false, + DeltaSummary = DeltaSummaryDto.FromModel(delta.Summary) + }; + } + + private static string BuildSummary(DriftGateDecision decision) + { + return decision.Decision switch + { + DriftGateDecisionType.Allow => "Gate passed - release may proceed", + DriftGateDecisionType.Warn => $"Gate passed with warnings - review recommended{(decision.Advisory is not null ? $": {decision.Advisory}" : "")}", + DriftGateDecisionType.Block => $"Gate blocked - {decision.BlockReason ?? "release cannot proceed"}", + _ => "Gate evaluation complete" + }; + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Services/Gateway/SchedulerBackedGateEvaluationQueue.cs b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/SchedulerBackedGateEvaluationQueue.cs new file mode 100644 index 000000000..7064ab22b --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/SchedulerBackedGateEvaluationQueue.cs @@ -0,0 +1,137 @@ +using StellaOps.Auth.ServerIntegration.Tenancy; +using StellaOps.Determinism; +using StellaOps.Policy.Engine.Tenancy; +using StellaOps.Policy.Persistence.Postgres.Models; +using StellaOps.Policy.Persistence.Postgres.Repositories; +using StellaOps.Scheduler.Persistence.Postgres.Models; +using StellaOps.Scheduler.Persistence.Postgres.Repositories; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; + +namespace StellaOps.Policy.Engine.Services.Gateway; + +public sealed class SchedulerBackedGateEvaluationQueue : IGateEvaluationQueue +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + private readonly IServiceScopeFactory _scopeFactory; + private readonly IStellaOpsTenantAccessor _tenantAccessor; + private readonly IGuidProvider _guidProvider; + private readonly TimeProvider _timeProvider; + private readonly GateEvaluationQueueOptions _options; + private readonly ILogger _logger; + + public SchedulerBackedGateEvaluationQueue( + IServiceScopeFactory scopeFactory, + IStellaOpsTenantAccessor tenantAccessor, + IGuidProvider guidProvider, + TimeProvider timeProvider, + GateEvaluationQueueOptions options, + ILogger logger) + { + _scopeFactory = scopeFactory; + _tenantAccessor = tenantAccessor; + _guidProvider = guidProvider; + _timeProvider = timeProvider; + _options = options; + _logger = logger; + } + + public async Task EnqueueAsync(GateEvaluationRequest request, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + + var now = _timeProvider.GetUtcNow(); + var tenantContext = _tenantAccessor.TenantContext; + var tenantId = string.IsNullOrWhiteSpace(tenantContext?.TenantId) + ? TenantContextConstants.DefaultTenantId + : tenantContext!.TenantId; + var normalizedRequest = request with + { + Timestamp = request.Timestamp == default ? now : request.Timestamp + }; + var payload = JsonSerializer.Serialize(normalizedRequest, SerializerOptions); + var payloadHash = ComputeSha256(payload); + var idempotencyKey = $"policy-gate:{payloadHash}"; + + using var scope = _scopeFactory.CreateScope(); + var jobRepository = scope.ServiceProvider.GetRequiredService(); + var workerResults = scope.ServiceProvider.GetRequiredService(); + var existing = await jobRepository + .GetByIdempotencyKeyAsync(tenantId, idempotencyKey, cancellationToken) + .ConfigureAwait(false); + + if (existing is not null) + { + _logger.LogInformation( + "Deduplicated gate evaluation job {JobId} for tenant {TenantId}.", + existing.Id, + tenantId); + return existing.Id.ToString(); + } + + var jobId = _guidProvider.NewGuid(); + var workerResultId = _guidProvider.NewGuid(); + var actorId = tenantContext?.ActorId ?? "registry-webhook"; + DateTimeOffset? notBefore = normalizedRequest.Timestamp > now ? normalizedRequest.Timestamp : null; + + await jobRepository.CreateAsync( + new JobEntity + { + Id = jobId, + TenantId = tenantId, + JobType = _options.JobType, + Status = JobStatus.Scheduled, + Priority = 0, + Payload = payload, + PayloadDigest = payloadHash, + IdempotencyKey = idempotencyKey, + CorrelationId = workerResultId.ToString("D"), + MaxAttempts = Math.Max(1, _options.MaxAttempts), + NotBefore = notBefore, + CreatedBy = actorId + }, + cancellationToken).ConfigureAwait(false); + + await workerResults.CreateAsync( + new WorkerResultEntity + { + Id = workerResultId, + TenantId = tenantId, + JobType = _options.JobType, + JobId = jobId.ToString(), + Status = "pending", + InputHash = payloadHash, + Progress = 0, + RetryCount = 0, + MaxRetries = Math.Max(1, _options.MaxAttempts), + ScheduledAt = notBefore ?? now, + Metadata = JsonSerializer.Serialize( + new GateEvaluationJobMetadata + { + Request = normalizedRequest, + ActorId = actorId, + EnqueuedAt = now + }, + SerializerOptions), + CreatedAt = now, + CreatedBy = actorId + }, + cancellationToken).ConfigureAwait(false); + + _logger.LogInformation( + "Queued async gate evaluation job {JobId} for tenant {TenantId} ({Repository}@{Digest}).", + jobId, + tenantId, + normalizedRequest.Repository, + normalizedRequest.ImageDigest); + + return jobId.ToString(); + } + + internal static string ComputeSha256(string value) + { + var bytes = Encoding.UTF8.GetBytes(value); + return $"sha256:{Convert.ToHexStringLower(SHA256.HashData(bytes))}"; + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Services/Gateway/UnsupportedGateEvaluationQueue.cs b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/UnsupportedGateEvaluationQueue.cs new file mode 100644 index 000000000..49e670a11 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Services/Gateway/UnsupportedGateEvaluationQueue.cs @@ -0,0 +1,24 @@ +namespace StellaOps.Policy.Engine.Services.Gateway; + +/// +/// Honest runtime implementation for environments that do not yet provide +/// a scheduler-backed async gate-evaluation dispatcher. +/// +public sealed class UnsupportedGateEvaluationQueue : IGateEvaluationQueue +{ + public Task EnqueueAsync(GateEvaluationRequest request, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + + throw new GateEvaluationQueueUnavailableException( + "Async registry webhook gate evaluation requires a scheduler-backed queue. This runtime does not provide one."); + } +} + +public sealed class GateEvaluationQueueUnavailableException : InvalidOperationException +{ + public GateEvaluationQueueUnavailableException(string message) + : base(message) + { + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Snapshots/SnapshotModels.cs b/src/Policy/StellaOps.Policy.Engine/Snapshots/SnapshotModels.cs index e59946be0..5f6b7107f 100644 --- a/src/Policy/StellaOps.Policy.Engine/Snapshots/SnapshotModels.cs +++ b/src/Policy/StellaOps.Policy.Engine/Snapshots/SnapshotModels.cs @@ -9,7 +9,10 @@ internal sealed record SnapshotSummary( [property: JsonPropertyName("tenant_id")] string TenantId, [property: JsonPropertyName("ledger_export_id")] string LedgerExportId, [property: JsonPropertyName("generated_at")] string GeneratedAt, - [property: JsonPropertyName("status_counts")] IReadOnlyDictionary StatusCounts); + [property: JsonPropertyName("status_counts")] IReadOnlyDictionary StatusCounts, + [property: JsonPropertyName("artifact_digest")] string? ArtifactDigest = null, + [property: JsonPropertyName("artifact_repository")] string? ArtifactRepository = null, + [property: JsonPropertyName("artifact_tag")] string? ArtifactTag = null); internal sealed record SnapshotDetail( [property: JsonPropertyName("snapshot_id")] string SnapshotId, @@ -18,8 +21,14 @@ internal sealed record SnapshotDetail( [property: JsonPropertyName("generated_at")] string GeneratedAt, [property: JsonPropertyName("overlay_hash")] string OverlayHash, [property: JsonPropertyName("status_counts")] IReadOnlyDictionary StatusCounts, - [property: JsonPropertyName("records")] IReadOnlyList Records); + [property: JsonPropertyName("records")] IReadOnlyList Records, + [property: JsonPropertyName("artifact_digest")] string? ArtifactDigest = null, + [property: JsonPropertyName("artifact_repository")] string? ArtifactRepository = null, + [property: JsonPropertyName("artifact_tag")] string? ArtifactTag = null); internal sealed record SnapshotRequest( [property: JsonPropertyName("tenant_id")] string TenantId, - [property: JsonPropertyName("overlay_hash")] string OverlayHash); + [property: JsonPropertyName("overlay_hash")] string OverlayHash, + [property: JsonPropertyName("artifact_digest")] string? ArtifactDigest = null, + [property: JsonPropertyName("artifact_repository")] string? ArtifactRepository = null, + [property: JsonPropertyName("artifact_tag")] string? ArtifactTag = null); diff --git a/src/Policy/StellaOps.Policy.Engine/Snapshots/SnapshotService.cs b/src/Policy/StellaOps.Policy.Engine/Snapshots/SnapshotService.cs index 73cfa0a07..4d2c1ad7d 100644 --- a/src/Policy/StellaOps.Policy.Engine/Snapshots/SnapshotService.cs +++ b/src/Policy/StellaOps.Policy.Engine/Snapshots/SnapshotService.cs @@ -6,7 +6,7 @@ using System.Globalization; namespace StellaOps.Policy.Engine.Snapshots; /// -/// Snapshot API stub (POLICY-ENGINE-35-201) built on ledger exports. +/// Persisted snapshot API built on ledger exports. /// internal sealed class SnapshotService { @@ -43,7 +43,8 @@ internal sealed class SnapshotService .ToDictionary(g => g.Key, g => g.Count(), StringComparer.Ordinal); var generatedAt = _timeProvider.GetUtcNow().ToString("O", CultureInfo.InvariantCulture); - var snapshotId = StableIdGenerator.CreateUlid($"{export.Manifest.ExportId}|{request.OverlayHash}"); + var snapshotId = StableIdGenerator.CreateUlid( + $"{export.Manifest.ExportId}|{request.OverlayHash}|{request.ArtifactDigest ?? string.Empty}|{request.ArtifactRepository ?? string.Empty}|{request.ArtifactTag ?? string.Empty}"); var snapshot = new SnapshotDetail( SnapshotId: snapshotId, @@ -52,7 +53,10 @@ internal sealed class SnapshotService GeneratedAt: generatedAt, OverlayHash: request.OverlayHash, StatusCounts: statusCounts, - Records: export.Records); + Records: export.Records, + ArtifactDigest: request.ArtifactDigest, + ArtifactRepository: request.ArtifactRepository, + ArtifactTag: request.ArtifactTag); await _store.SaveAsync(snapshot, cancellationToken).ConfigureAwait(false); return snapshot; @@ -70,7 +74,15 @@ internal sealed class SnapshotService var snapshots = await _store.ListAsync(tenantId, cancellationToken).ConfigureAwait(false); var summaries = snapshots .OrderByDescending(s => s.GeneratedAt, StringComparer.Ordinal) - .Select(s => new SnapshotSummary(s.SnapshotId, s.TenantId, s.LedgerExportId, s.GeneratedAt, s.StatusCounts)) + .Select(s => new SnapshotSummary( + s.SnapshotId, + s.TenantId, + s.LedgerExportId, + s.GeneratedAt, + s.StatusCounts, + s.ArtifactDigest, + s.ArtifactRepository, + s.ArtifactTag)) .ToList(); return (summaries, null); diff --git a/src/Policy/StellaOps.Policy.Engine/Snapshots/SnapshotStore.cs b/src/Policy/StellaOps.Policy.Engine/Snapshots/SnapshotStore.cs index bc7011485..a3f625495 100644 --- a/src/Policy/StellaOps.Policy.Engine/Snapshots/SnapshotStore.cs +++ b/src/Policy/StellaOps.Policy.Engine/Snapshots/SnapshotStore.cs @@ -1,4 +1,9 @@ using System.Collections.Concurrent; +using System.Globalization; +using System.Text.Json; +using StellaOps.Policy.Engine.Ledger; +using StellaOps.Policy.Persistence.Postgres.Models; +using StellaOps.Policy.Persistence.Postgres.Repositories; namespace StellaOps.Policy.Engine.Snapshots; @@ -42,3 +47,67 @@ internal sealed class InMemorySnapshotStore : ISnapshotStore return Task.FromResult>(ordered); } } + +internal sealed class PostgresSnapshotStore : ISnapshotStore +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + private readonly IPolicyEngineSnapshotRepository _repository; + + public PostgresSnapshotStore(IPolicyEngineSnapshotRepository repository) + { + _repository = repository ?? throw new ArgumentNullException(nameof(repository)); + } + + public async Task SaveAsync(SnapshotDetail snapshot, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(snapshot); + + var document = new PolicyEngineSnapshotDocument + { + SnapshotId = snapshot.SnapshotId, + TenantId = snapshot.TenantId, + LedgerExportId = snapshot.LedgerExportId, + GeneratedAt = DateTimeOffset.Parse(snapshot.GeneratedAt, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind), + OverlayHash = snapshot.OverlayHash, + ArtifactDigest = snapshot.ArtifactDigest, + ArtifactRepository = snapshot.ArtifactRepository, + ArtifactTag = snapshot.ArtifactTag, + StatusCountsJson = JsonSerializer.Serialize(snapshot.StatusCounts, SerializerOptions), + RecordsJson = JsonSerializer.Serialize(snapshot.Records, SerializerOptions) + }; + + await _repository.SaveAsync(document, cancellationToken).ConfigureAwait(false); + } + + public async Task GetAsync(string snapshotId, CancellationToken cancellationToken = default) + { + var document = await _repository.GetAsync(snapshotId, cancellationToken).ConfigureAwait(false); + return document is null ? null : Deserialize(document); + } + + public async Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default) + { + var documents = await _repository.ListAsync(tenantId, cancellationToken).ConfigureAwait(false); + return documents.Select(Deserialize).ToList(); + } + + private static SnapshotDetail Deserialize(PolicyEngineSnapshotDocument document) + { + var statusCounts = JsonSerializer.Deserialize>(document.StatusCountsJson, SerializerOptions) + ?? throw new InvalidOperationException($"Persisted snapshot {document.SnapshotId} is missing status counts."); + var records = JsonSerializer.Deserialize>(document.RecordsJson, SerializerOptions) + ?? throw new InvalidOperationException($"Persisted snapshot {document.SnapshotId} is missing records."); + + return new SnapshotDetail( + SnapshotId: document.SnapshotId, + TenantId: document.TenantId, + LedgerExportId: document.LedgerExportId, + GeneratedAt: document.GeneratedAt.ToString("O", CultureInfo.InvariantCulture), + OverlayHash: document.OverlayHash, + StatusCounts: statusCounts, + Records: records, + ArtifactDigest: document.ArtifactDigest, + ArtifactRepository: document.ArtifactRepository, + ArtifactTag: document.ArtifactTag); + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/StellaOps.Policy.Engine.csproj b/src/Policy/StellaOps.Policy.Engine/StellaOps.Policy.Engine.csproj index 9980774da..fd619fb3f 100644 --- a/src/Policy/StellaOps.Policy.Engine/StellaOps.Policy.Engine.csproj +++ b/src/Policy/StellaOps.Policy.Engine/StellaOps.Policy.Engine.csproj @@ -52,6 +52,7 @@ + diff --git a/src/Policy/StellaOps.Policy.Engine/TASKS.md b/src/Policy/StellaOps.Policy.Engine/TASKS.md index 18ae93b49..2a50f7c79 100644 --- a/src/Policy/StellaOps.Policy.Engine/TASKS.md +++ b/src/Policy/StellaOps.Policy.Engine/TASKS.md @@ -13,3 +13,9 @@ Source of truth: `docs/implplan/SPRINT_20260119_021_Policy_license_compliance.md | TASK-021-009 | BLOCKED | License compliance integrated into runtime evaluation; CLI overrides need API contract. | | TASK-021-011 | DOING | Engine-level tests updated for license compliance gating; suite stability pending. | | TASK-021-012 | DONE | Real SBOM integration tests added (npm-monorepo, alpine-busybox, python-venv, java-multi-license); filtered integration runs passed. | +| NOMOCK-014 | DONE | `docs/implplan/SPRINT_20260410_001_Web_runtime_no_mocks_real_backend.md`: live gate-bypass audit now resolves to PostgreSQL; governance endpoint namespaced to stop `/api/v1/governance/*` vs `/api/risk/*` host collisions. | +| NOMOCK-015 | DONE | `docs/implplan/SPRINT_20260410_001_Web_runtime_no_mocks_real_backend.md`: live Policy snapshot and ledger-export runtime now persist to PostgreSQL and survive `policy-engine` recreates; startup migrations for `001_initial_schema.sql` were made idempotent for reused local volumes. | +| NOMOCK-016 | DONE | `docs/implplan/SPRINT_20260410_001_Web_runtime_no_mocks_real_backend.md`: merged gateway routes now register the unified StellaOps tenant accessor/middleware in `policy-engine`, so tenant-scoped `RequireTenant()` endpoints like `/api/policy/deltas/compute` no longer fail pre-handler with `500`. | +| NOMOCK-019 | DONE | `SPRINT_20260410_001_Web_runtime_no_mocks_real_backend.md`: removed the merged gateway's fictional async gate-evaluation queue, added the shared runtime-selected scheduler-backed dispatcher and `/api/v1/policy/gate/jobs/{id}` status surface, and now materializes persisted target snapshots for webhook/sync gate evaluation so live jobs complete against real `policy.engine_ledger_exports` + `policy.engine_snapshots` data instead of failing on digest-as-snapshot-ID assumptions. | +| NOMOCK-020 | DONE | `SPRINT_20260410_001_Web_runtime_no_mocks_real_backend.md`: live Policy first-run bootstrap now reads completed persisted `policy.orchestrator_jobs` + `policy.worker_results`, auto-builds the first ledger export/baseline snapshot when none exist, and proves the webhook path succeeds for a brand-new tenant without manual export/snapshot seed rows. | +| NOMOCK-021 | DONE | `SPRINT_20260410_001_Web_runtime_no_mocks_real_backend.md`: `/policy/orchestrator/jobs` is now the real upstream producer path, signaling a background host that leases queued jobs, persists `policy.worker_results`, and records terminal orchestrator status without a separate manual `/policy/worker/run`; the slice now also has a reusable live proof harness at `src/Web/StellaOps.Web/tests/e2e/integrations/policy-orchestrator.e2e.spec.ts`. | diff --git a/src/Policy/StellaOps.Policy.Engine/Workers/PolicyOrchestratorJobWorkerHost.cs b/src/Policy/StellaOps.Policy.Engine/Workers/PolicyOrchestratorJobWorkerHost.cs new file mode 100644 index 000000000..7c284adf1 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Workers/PolicyOrchestratorJobWorkerHost.cs @@ -0,0 +1,100 @@ +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using StellaOps.Policy.Engine.Orchestration; + +namespace StellaOps.Policy.Engine.Workers; + +internal interface IOrchestratorJobExecutionSignal +{ + void NotifyWorkAvailable(); + Task WaitAsync(CancellationToken cancellationToken); +} + +internal sealed class OrchestratorJobExecutionSignal : IOrchestratorJobExecutionSignal, IDisposable +{ + private readonly SemaphoreSlim _signal = new(initialCount: 1, maxCount: 1); + + public void NotifyWorkAvailable() + { + try + { + _signal.Release(); + } + catch (SemaphoreFullException) + { + } + } + + public Task WaitAsync(CancellationToken cancellationToken) + { + return _signal.WaitAsync(cancellationToken); + } + + public void Dispose() + { + _signal.Dispose(); + } +} + +internal sealed class PolicyOrchestratorJobWorkerHost : BackgroundService +{ + private const string WorkerId = "policy-orchestrator-worker"; + + private readonly IOrchestratorJobExecutionSignal _signal; + private readonly IOrchestratorJobStore _jobs; + private readonly PolicyWorkerService _workerService; + private readonly ILogger _logger; + + public PolicyOrchestratorJobWorkerHost( + IOrchestratorJobExecutionSignal signal, + IOrchestratorJobStore jobs, + PolicyWorkerService workerService, + ILogger logger) + { + _signal = signal ?? throw new ArgumentNullException(nameof(signal)); + _jobs = jobs ?? throw new ArgumentNullException(nameof(jobs)); + _workerService = workerService ?? throw new ArgumentNullException(nameof(workerService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _logger.LogInformation("Policy orchestrator job worker host started."); + + while (!stoppingToken.IsCancellationRequested) + { + await _signal.WaitAsync(stoppingToken).ConfigureAwait(false); + + while (!stoppingToken.IsCancellationRequested) + { + var job = await _jobs.TryLeaseNextQueuedAsync(stoppingToken).ConfigureAwait(false); + if (job is null) + { + break; + } + + try + { + _logger.LogInformation( + "Executing orchestrator job {JobId} for tenant {TenantId} with {ItemCount} items.", + job.JobId, + job.TenantId, + job.BatchItems.Count); + + await _workerService.ExecuteAsync( + new WorkerRunRequest(job.JobId, WorkerId), + stoppingToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + _logger.LogError(ex, "Policy orchestrator job {JobId} failed during automatic execution.", job.JobId); + } + } + } + } +} diff --git a/src/Policy/StellaOps.Policy.Gateway/Endpoints/GateEndpoints.cs b/src/Policy/StellaOps.Policy.Gateway/Endpoints/GateEndpoints.cs index 155d840ff..cce7792fd 100644 --- a/src/Policy/StellaOps.Policy.Gateway/Endpoints/GateEndpoints.cs +++ b/src/Policy/StellaOps.Policy.Gateway/Endpoints/GateEndpoints.cs @@ -39,6 +39,8 @@ public static class GateEndpoints IDriftGateEvaluator gateEvaluator, IDeltaComputer deltaComputer, IBaselineSelector baselineSelector, + StellaOps.Policy.Engine.Services.Gateway.GateBaselineBootstrapper gateBaselineBootstrapper, + StellaOps.Policy.Engine.Services.Gateway.GateTargetSnapshotMaterializer targetSnapshotMaterializer, IGateBypassAuditor bypassAuditor, IMemoryCache cache, [FromServices] TimeProvider timeProvider, @@ -73,6 +75,22 @@ public static class GateEndpoints baselineSelector, cancellationToken); + if (!baselineResult.IsFound && string.IsNullOrWhiteSpace(request.BaselineRef)) + { + var bootstrapped = await gateBaselineBootstrapper + .TryEnsureBaselineAsync(cancellationToken) + .ConfigureAwait(false); + if (bootstrapped) + { + baselineResult = await ResolveBaselineAsync( + request.ImageDigest, + request.BaselineRef, + baselineSelector, + cancellationToken) + .ConfigureAwait(false); + } + } + if (!baselineResult.IsFound) { // If no baseline, allow with a note (first build scenario) @@ -94,10 +112,17 @@ public static class GateEndpoints } // Step 2: Compute delta between baseline and current + var targetSnapshot = await targetSnapshotMaterializer.MaterializeAsync( + request.ImageDigest, + request.Repository, + request.Tag, + baselineResult.Snapshot!.SnapshotId, + cancellationToken); + var delta = await deltaComputer.ComputeDeltaAsync( baselineResult.Snapshot!.SnapshotId, - request.ImageDigest, // Use image digest as target snapshot ID - new ArtifactRef(request.ImageDigest, null, null), + targetSnapshot.SnapshotId, + new ArtifactRef(request.ImageDigest, request.Repository, request.Tag), cancellationToken); // Cache the delta for audit diff --git a/src/Policy/StellaOps.Policy.Gateway/Endpoints/RegistryWebhookEndpoints.cs b/src/Policy/StellaOps.Policy.Gateway/Endpoints/RegistryWebhookEndpoints.cs index fc5a25fee..0cdbb1f2f 100644 --- a/src/Policy/StellaOps.Policy.Gateway/Endpoints/RegistryWebhookEndpoints.cs +++ b/src/Policy/StellaOps.Policy.Gateway/Endpoints/RegistryWebhookEndpoints.cs @@ -10,6 +10,7 @@ using Microsoft.AspNetCore.Http.HttpResults; using Microsoft.AspNetCore.Mvc; using StellaOps.Auth.ServerIntegration.Tenancy; using StellaOps.Policy.Engine.Gates; +using StellaOps.Policy.Engine.Services.Gateway; using System.Text.Json; using System.Text.Json.Serialization; @@ -30,23 +31,26 @@ internal static class RegistryWebhookEndpoints group.MapPost("/docker", HandleDockerRegistryWebhook) .WithName("DockerRegistryWebhook") .WithSummary("Handle Docker Registry v2 webhook events") - .WithDescription("Receive Docker Registry v2 notification events and enqueue a gate evaluation job for each push event that includes a valid image digest. Returns a 202 Accepted response with the list of queued job IDs that can be polled for evaluation status.") + .WithDescription("Receive Docker Registry v2 notification events and enqueue a gate evaluation job for each push event that includes a valid image digest. Returns 202 only when a scheduler-backed async queue is available; otherwise returns 501 to indicate the runtime cannot truthfully accept deferred gate evaluation.") .Produces(StatusCodes.Status202Accepted) - .Produces(StatusCodes.Status400BadRequest); + .Produces(StatusCodes.Status400BadRequest) + .Produces(StatusCodes.Status501NotImplemented); group.MapPost("/harbor", HandleHarborWebhook) .WithName("HarborWebhook") .WithSummary("Handle Harbor registry webhook events") - .WithDescription("Receive Harbor registry webhook events and enqueue a gate evaluation job for each PUSH_ARTIFACT or pushImage event that contains a resource with a valid digest. Non-push event types are silently acknowledged without queuing any jobs.") + .WithDescription("Receive Harbor registry webhook events and enqueue a gate evaluation job for each PUSH_ARTIFACT or pushImage event that contains a resource with a valid digest. Non-push event types are silently acknowledged without queuing any jobs. Push events return 501 when the runtime lacks a scheduler-backed async queue.") .Produces(StatusCodes.Status202Accepted) - .Produces(StatusCodes.Status400BadRequest); + .Produces(StatusCodes.Status400BadRequest) + .Produces(StatusCodes.Status501NotImplemented); group.MapPost("/generic", HandleGenericWebhook) .WithName("GenericRegistryWebhook") .WithSummary("Handle generic registry webhook events with image digest") - .WithDescription("Receive a generic registry webhook payload containing an image digest and enqueue a single gate evaluation job. Supports any registry that can POST a JSON body with imageDigest, repository, tag, and optional baselineRef fields.") + .WithDescription("Receive a generic registry webhook payload containing an image digest and enqueue a single gate evaluation job. Supports any registry that can POST a JSON body with imageDigest, repository, tag, and optional baselineRef fields. Returns 501 when the runtime lacks a scheduler-backed async queue.") .Produces(StatusCodes.Status202Accepted) - .Produces(StatusCodes.Status400BadRequest); + .Produces(StatusCodes.Status400BadRequest) + .Produces(StatusCodes.Status501NotImplemented); return endpoints; } @@ -70,31 +74,39 @@ internal static class RegistryWebhookEndpoints var jobs = new List(); - foreach (var evt in notification.Events.Where(e => e.Action == "push")) + try { - if (string.IsNullOrEmpty(evt.Target?.Digest)) + foreach (var evt in notification.Events.Where(e => e.Action == "push")) { - logger.LogWarning("Skipping push event without digest: {Repository}", evt.Target?.Repository); - continue; + if (string.IsNullOrEmpty(evt.Target?.Digest)) + { + logger.LogWarning("Skipping push event without digest: {Repository}", evt.Target?.Repository); + continue; + } + + var jobId = await evaluationQueue.EnqueueAsync(new GateEvaluationRequest + { + ImageDigest = evt.Target.Digest, + Repository = evt.Target.Repository ?? "unknown", + Tag = evt.Target.Tag, + RegistryUrl = evt.Request?.Host, + Source = "docker-registry", + Timestamp = evt.Timestamp ?? timeProvider.GetUtcNow() + }, ct); + + jobs.Add(jobId); + + logger.LogInformation( + "Queued gate evaluation for {Repository}@{Digest}, job: {JobId}", + evt.Target.Repository, + evt.Target.Digest, + jobId); } - - var jobId = await evaluationQueue.EnqueueAsync(new GateEvaluationRequest - { - ImageDigest = evt.Target.Digest, - Repository = evt.Target.Repository ?? "unknown", - Tag = evt.Target.Tag, - RegistryUrl = evt.Request?.Host, - Source = "docker-registry", - Timestamp = evt.Timestamp ?? timeProvider.GetUtcNow() - }, ct); - - jobs.Add(jobId); - - logger.LogInformation( - "Queued gate evaluation for {Repository}@{Digest}, job: {JobId}", - evt.Target.Repository, - evt.Target.Digest, - jobId); + } + catch (GateEvaluationQueueUnavailableException ex) + { + logger.LogWarning(ex, "Registry webhook async gate evaluation is unavailable for Docker push events."); + return QueueUnavailableProblem(); } return TypedResults.Accepted( @@ -130,31 +142,39 @@ internal static class RegistryWebhookEndpoints var jobs = new List(); - foreach (var resource in notification.EventData.Resources) + try { - if (string.IsNullOrEmpty(resource.Digest)) + foreach (var resource in notification.EventData.Resources) { - logger.LogWarning("Skipping resource without digest: {ResourceUrl}", resource.ResourceUrl); - continue; + if (string.IsNullOrEmpty(resource.Digest)) + { + logger.LogWarning("Skipping resource without digest: {ResourceUrl}", resource.ResourceUrl); + continue; + } + + var jobId = await evaluationQueue.EnqueueAsync(new GateEvaluationRequest + { + ImageDigest = resource.Digest, + Repository = notification.EventData.Repository?.Name ?? "unknown", + Tag = resource.Tag, + RegistryUrl = notification.EventData.Repository?.RepoFullName, + Source = "harbor", + Timestamp = notification.OccurAt ?? timeProvider.GetUtcNow() + }, ct); + + jobs.Add(jobId); + + logger.LogInformation( + "Queued gate evaluation for {Repository}@{Digest}, job: {JobId}", + notification.EventData.Repository?.Name, + resource.Digest, + jobId); } - - var jobId = await evaluationQueue.EnqueueAsync(new GateEvaluationRequest - { - ImageDigest = resource.Digest, - Repository = notification.EventData.Repository?.Name ?? "unknown", - Tag = resource.Tag, - RegistryUrl = notification.EventData.Repository?.RepoFullName, - Source = "harbor", - Timestamp = notification.OccurAt ?? timeProvider.GetUtcNow() - }, ct); - - jobs.Add(jobId); - - logger.LogInformation( - "Queued gate evaluation for {Repository}@{Digest}, job: {JobId}", - notification.EventData.Repository?.Name, - resource.Digest, - jobId); + } + catch (GateEvaluationQueueUnavailableException ex) + { + logger.LogWarning(ex, "Registry webhook async gate evaluation is unavailable for Harbor push events."); + return QueueUnavailableProblem(); } return TypedResults.Accepted( @@ -179,26 +199,42 @@ internal static class RegistryWebhookEndpoints statusCode: StatusCodes.Status400BadRequest); } - var jobId = await evaluationQueue.EnqueueAsync(new GateEvaluationRequest + try { - ImageDigest = notification.ImageDigest, - Repository = notification.Repository ?? "unknown", - Tag = notification.Tag, - RegistryUrl = notification.RegistryUrl, - BaselineRef = notification.BaselineRef, - Source = notification.Source ?? "generic", - Timestamp = timeProvider.GetUtcNow() - }, ct); + var jobId = await evaluationQueue.EnqueueAsync(new GateEvaluationRequest + { + ImageDigest = notification.ImageDigest, + Repository = notification.Repository ?? "unknown", + Tag = notification.Tag, + RegistryUrl = notification.RegistryUrl, + BaselineRef = notification.BaselineRef, + Source = notification.Source ?? "generic", + Timestamp = timeProvider.GetUtcNow() + }, ct); - logger.LogInformation( - "Queued gate evaluation for {Repository}@{Digest}, job: {JobId}", - notification.Repository, - notification.ImageDigest, - jobId); + logger.LogInformation( + "Queued gate evaluation for {Repository}@{Digest}, job: {JobId}", + notification.Repository, + notification.ImageDigest, + jobId); - return TypedResults.Accepted( - $"/api/v1/policy/gate/jobs/{jobId}", - new WebhookAcceptedResponse(1, [jobId])); + return TypedResults.Accepted( + $"/api/v1/policy/gate/jobs/{jobId}", + new WebhookAcceptedResponse(1, [jobId])); + } + catch (GateEvaluationQueueUnavailableException ex) + { + logger.LogWarning(ex, "Registry webhook async gate evaluation is unavailable for generic push events."); + return QueueUnavailableProblem(); + } + } + + private static ProblemHttpResult QueueUnavailableProblem() + { + return TypedResults.Problem( + title: "Async gate evaluation queue unavailable", + detail: "Registry webhook push events require a scheduler-backed async gate evaluation queue. This runtime does not provide one.", + statusCode: StatusCodes.Status501NotImplemented); } } @@ -379,35 +415,3 @@ public sealed record GenericRegistryWebhook public sealed record WebhookAcceptedResponse( int JobsQueued, IReadOnlyList JobIds); - -// ============================================================================ -// Gate Evaluation Queue Interface -// ============================================================================ - -/// -/// Interface for queuing gate evaluation jobs. -/// -public interface IGateEvaluationQueue -{ - /// - /// Enqueues a gate evaluation request. - /// - /// The evaluation request. - /// Cancellation token. - /// The job ID for tracking. - Task EnqueueAsync(GateEvaluationRequest request, CancellationToken cancellationToken = default); -} - -/// -/// Request to evaluate a gate for an image. -/// -public sealed record GateEvaluationRequest -{ - public required string ImageDigest { get; init; } - public required string Repository { get; init; } - public string? Tag { get; init; } - public string? RegistryUrl { get; init; } - public string? BaselineRef { get; init; } - public required string Source { get; init; } - public required DateTimeOffset Timestamp { get; init; } -} diff --git a/src/Policy/StellaOps.Policy.Gateway/Program.cs b/src/Policy/StellaOps.Policy.Gateway/Program.cs index 93dd5ad1a..c3eedcbb6 100644 --- a/src/Policy/StellaOps.Policy.Gateway/Program.cs +++ b/src/Policy/StellaOps.Policy.Gateway/Program.cs @@ -159,24 +159,42 @@ builder.Services.AddHostedService(); // Delta services builder.Services.AddScoped(); builder.Services.AddScoped(); -builder.Services.AddScoped(); -builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); // Gate services (Sprint: SPRINT_20251226_001_BE_cicd_gate_integration) builder.Services.Configure( builder.Configuration.GetSection(DriftGateOptions.SectionName)); builder.Services.AddScoped(); -builder.Services.AddSingleton(); -builder.Services.AddSingleton(sp => sp.GetRequiredService()); -builder.Services.AddHostedService(); +StellaOps.Policy.Engine.Services.Gateway.PolicyAsyncGateEvaluationRuntimeExtensions + .AddPolicyAsyncGateEvaluationRuntime(builder.Services, builder.Configuration, PolicyGatewayOptions.SectionName); // Unknowns gate services (Sprint: SPRINT_20260118_018_Unknowns_queue_enhancement) builder.Services.Configure(_ => { }); builder.Services.AddHttpClient(); // Gate bypass audit services (Sprint: SPRINT_20251226_001_BE_cicd_gate_integration, Task: CICD-GATE-06) -builder.Services.AddSingleton(); +builder.Services.AddScoped(sp => +{ + var persistence = sp.GetRequiredService(); + var logger = sp.GetRequiredService>(); + var tenantAccessor = sp.GetRequiredService(); + var tenantId = tenantAccessor.TenantId; + + if (string.IsNullOrWhiteSpace(tenantId)) + { + tenantId = StellaOps.Policy.Engine.Tenancy.TenantContextConstants.DefaultTenantId; + } + + return new StellaOps.Policy.Persistence.Postgres.PostgresGateBypassAuditRepository( + persistence, + logger, + tenantId); +}); builder.Services.AddSingleton(); builder.Services.AddScoped(); @@ -681,6 +699,7 @@ app.MapPolicySimulationEndpoints(); // Gate evaluation endpoints (Sprint: SPRINT_20251226_001_BE_cicd_gate_integration) app.MapGateEndpoints(); +StellaOps.Policy.Engine.Endpoints.Gateway.GateJobEndpoints.MapGateJobEndpoints(app); // Unknowns gate endpoints (Sprint: SPRINT_20260118_030_LIB_verdict_rekor_gate_api) app.MapGatesEndpoints(); diff --git a/src/Policy/StellaOps.Policy.Gateway/Services/InMemoryGateEvaluationQueue.cs b/src/Policy/StellaOps.Policy.Gateway/Services/InMemoryGateEvaluationQueue.cs deleted file mode 100644 index a65fc2698..000000000 --- a/src/Policy/StellaOps.Policy.Gateway/Services/InMemoryGateEvaluationQueue.cs +++ /dev/null @@ -1,185 +0,0 @@ -// ----------------------------------------------------------------------------- -// InMemoryGateEvaluationQueue.cs -// Sprint: SPRINT_20251226_001_BE_cicd_gate_integration -// Task: CICD-GATE-02 - Gate evaluation queue implementation -// Description: In-memory queue for gate evaluation jobs with background processing -// ----------------------------------------------------------------------------- - - -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using StellaOps.Policy.Engine.Gates; -using StellaOps.Policy.Gateway.Endpoints; -using System.Threading.Channels; - -namespace StellaOps.Policy.Gateway.Services; - -/// -/// In-memory implementation of the gate evaluation queue. -/// Uses System.Threading.Channels for async producer-consumer pattern. -/// -public sealed class InMemoryGateEvaluationQueue : IGateEvaluationQueue -{ - private readonly Channel _channel; - private readonly ILogger _logger; - private readonly TimeProvider _timeProvider; - - public InMemoryGateEvaluationQueue( - ILogger logger, - TimeProvider? timeProvider = null) - { - ArgumentNullException.ThrowIfNull(logger); - _logger = logger; - _timeProvider = timeProvider ?? TimeProvider.System; - - // Bounded channel to prevent unbounded memory growth - _channel = Channel.CreateBounded(new BoundedChannelOptions(1000) - { - FullMode = BoundedChannelFullMode.Wait, - SingleReader = false, - SingleWriter = false - }); - } - - /// - public async Task EnqueueAsync(GateEvaluationRequest request, CancellationToken cancellationToken = default) - { - ArgumentNullException.ThrowIfNull(request); - - var jobId = GenerateJobId(); - var job = new GateEvaluationJob - { - JobId = jobId, - Request = request, - QueuedAt = _timeProvider.GetUtcNow() - }; - - await _channel.Writer.WriteAsync(job, cancellationToken).ConfigureAwait(false); - - _logger.LogDebug( - "Enqueued gate evaluation job {JobId} for {Repository}@{Digest}", - jobId, - request.Repository, - request.ImageDigest); - - return jobId; - } - - /// - /// Gets the channel reader for consuming jobs. - /// - public ChannelReader Reader => _channel.Reader; - - private string GenerateJobId() - { - // Format: gate-{timestamp}-{random} - var timestamp = _timeProvider.GetUtcNow().ToUnixTimeMilliseconds(); - var random = Guid.NewGuid().ToString("N")[..8]; - return $"gate-{timestamp}-{random}"; - } -} - -/// -/// A gate evaluation job in the queue. -/// -public sealed record GateEvaluationJob -{ - public required string JobId { get; init; } - public required GateEvaluationRequest Request { get; init; } - public required DateTimeOffset QueuedAt { get; init; } -} - -/// -/// Background service that processes gate evaluation jobs from the queue. -/// Orchestrates: image analysis -> drift delta computation -> gate evaluation. -/// -public sealed class GateEvaluationWorker : BackgroundService -{ - private readonly InMemoryGateEvaluationQueue _queue; - private readonly IServiceScopeFactory _scopeFactory; - private readonly ILogger _logger; - - public GateEvaluationWorker( - InMemoryGateEvaluationQueue queue, - IServiceScopeFactory scopeFactory, - ILogger logger) - { - ArgumentNullException.ThrowIfNull(queue); - ArgumentNullException.ThrowIfNull(scopeFactory); - ArgumentNullException.ThrowIfNull(logger); - - _queue = queue; - _scopeFactory = scopeFactory; - _logger = logger; - } - - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - _logger.LogInformation("Gate evaluation worker starting"); - - await foreach (var job in _queue.Reader.ReadAllAsync(stoppingToken)) - { - try - { - await ProcessJobAsync(job, stoppingToken).ConfigureAwait(false); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - _logger.LogError(ex, - "Error processing gate evaluation job {JobId} for {Repository}@{Digest}", - job.JobId, - job.Request.Repository, - job.Request.ImageDigest); - } - } - - _logger.LogInformation("Gate evaluation worker stopping"); - } - - private async Task ProcessJobAsync(GateEvaluationJob job, CancellationToken cancellationToken) - { - _logger.LogInformation( - "Processing gate evaluation job {JobId} for {Repository}@{Digest}", - job.JobId, - job.Request.Repository, - job.Request.ImageDigest); - - using var scope = _scopeFactory.CreateScope(); - var evaluator = scope.ServiceProvider.GetRequiredService(); - - // Build a minimal context for the gate evaluation. - // In production, this would involve: - // 1. Fetching or triggering a scan of the image - // 2. Computing the reachability delta against the baseline - // 3. Building the DriftGateContext with actual metrics - // - // For now, we create a placeholder context that represents "no drift detected" - // which allows the gate to pass. The full implementation requires Scanner integration. - var driftContext = new DriftGateContext - { - DeltaReachable = 0, - DeltaUnreachable = 0, - HasKevReachable = false, - BaseScanId = job.Request.BaselineRef, - HeadScanId = job.Request.ImageDigest - }; - - var evalRequest = new DriftGateRequest - { - Context = driftContext, - PolicyId = null, // Use default policy - AllowOverride = false - }; - - var result = await evaluator.EvaluateAsync(evalRequest, cancellationToken).ConfigureAwait(false); - - _logger.LogInformation( - "Gate evaluation {JobId} completed: Decision={Decision}, GateCount={GateCount}", - job.JobId, - result.Decision, - result.Gates.Length); - - // TODO: Store result and notify via webhook/event - // This will be implemented in CICD-GATE-03 - } -} diff --git a/src/Policy/StellaOps.Policy.Gateway/TASKS.md b/src/Policy/StellaOps.Policy.Gateway/TASKS.md index c4c11a1ce..6208893e6 100644 --- a/src/Policy/StellaOps.Policy.Gateway/TASKS.md +++ b/src/Policy/StellaOps.Policy.Gateway/TASKS.md @@ -10,3 +10,6 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | AUDIT-0445-A | TODO | Revalidated 2026-01-07 (open findings). | | TASK-033-013 | DONE | Fixed ScoreGateEndpoints duplication, DeltaVerdict references, and Policy.Gateway builds (SPRINT_20260120_033). | | SPRINT-20260224-002-LOC-101 | DONE | `SPRINT_20260224_002_Platform_translation_rollout_phase3_phase4.md`: adopted StellaOps localization runtime bundle loading in Policy Gateway and localized selected validation/error response strings (`en-US`/`de-DE`). | +| NOMOCK-017 | DONE | `SPRINT_20260410_001_Web_runtime_no_mocks_real_backend.md`: replaced the live delta compatibility `InMemorySnapshotStore` binding with the persisted engine snapshot projection in `StellaOps.Policy.Gateway`. | +| NOMOCK-018 | DONE | `SPRINT_20260410_001_Web_runtime_no_mocks_real_backend.md`: replaced the live gateway `InMemoryGateBypassAuditRepository` binding with the tenant-aware PostgreSQL audit adapter. | +| NOMOCK-019 | DONE | `SPRINT_20260410_001_Web_runtime_no_mocks_real_backend.md`: replaced the fictional live async gate-evaluation queue with the shared runtime-selected Policy async dispatcher, so the standalone gateway now returns truthful `501` without scheduler persistence and real queued job IDs plus persisted status when the scheduler-backed path is configured. | diff --git a/src/Policy/__Libraries/StellaOps.Policy.Persistence/Extensions/PolicyPersistenceExtensions.cs b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Extensions/PolicyPersistenceExtensions.cs index 5fd8b1097..2601fb7df 100644 --- a/src/Policy/__Libraries/StellaOps.Policy.Persistence/Extensions/PolicyPersistenceExtensions.cs +++ b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Extensions/PolicyPersistenceExtensions.cs @@ -5,6 +5,7 @@ using ILocalPolicyAuditRepository = StellaOps.Policy.Persistence.Postgres.Reposi using ILocalRiskProfileRepository = StellaOps.Policy.Persistence.Postgres.Repositories.IRiskProfileRepository; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using StellaOps.Infrastructure.Postgres.Migrations; using StellaOps.Infrastructure.Postgres.Options; using StellaOps.Policy.Persistence.Postgres; using StellaOps.Policy.Persistence.Postgres.Repositories; @@ -29,8 +30,13 @@ public static class PolicyPersistenceExtensions IConfiguration configuration, string sectionName = "Postgres:Policy") { + services.Configure(configuration.GetSection(sectionName)); services.Configure(sectionName, configuration.GetSection(sectionName)); services.AddSingleton(); + services.AddStartupMigrations( + PolicyDataSource.DefaultSchemaName, + "Policy.Persistence", + typeof(PolicyDataSource).Assembly); // Register repositories services.AddScoped(); @@ -49,6 +55,8 @@ public static class PolicyPersistenceExtensions services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddSingleton(); + services.AddSingleton(); return services; } @@ -65,6 +73,10 @@ public static class PolicyPersistenceExtensions { services.Configure(configureOptions); services.AddSingleton(); + services.AddStartupMigrations( + PolicyDataSource.DefaultSchemaName, + "Policy.Persistence", + typeof(PolicyDataSource).Assembly); // Register repositories services.AddScoped(); @@ -83,6 +95,8 @@ public static class PolicyPersistenceExtensions services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddSingleton(); + services.AddSingleton(); return services; } diff --git a/src/Policy/__Libraries/StellaOps.Policy.Persistence/Migrations/001_initial_schema.sql b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Migrations/001_initial_schema.sql index 0956dabe9..81b7c546b 100644 --- a/src/Policy/__Libraries/StellaOps.Policy.Persistence/Migrations/001_initial_schema.sql +++ b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Migrations/001_initial_schema.sql @@ -74,8 +74,8 @@ CREATE TABLE IF NOT EXISTS policy.packs ( UNIQUE(tenant_id, name) ); -CREATE INDEX idx_packs_tenant ON policy.packs(tenant_id); -CREATE INDEX idx_packs_builtin ON policy.packs(is_builtin); +CREATE INDEX IF NOT EXISTS idx_packs_tenant ON policy.packs(tenant_id); +CREATE INDEX IF NOT EXISTS idx_packs_builtin ON policy.packs(is_builtin); -- Pack versions table (immutable versions) CREATE TABLE IF NOT EXISTS policy.pack_versions ( @@ -92,8 +92,8 @@ CREATE TABLE IF NOT EXISTS policy.pack_versions ( UNIQUE(pack_id, version) ); -CREATE INDEX idx_pack_versions_pack ON policy.pack_versions(pack_id); -CREATE INDEX idx_pack_versions_published ON policy.pack_versions(pack_id, is_published); +CREATE INDEX IF NOT EXISTS idx_pack_versions_pack ON policy.pack_versions(pack_id); +CREATE INDEX IF NOT EXISTS idx_pack_versions_published ON policy.pack_versions(pack_id, is_published); -- Rules table (OPA/Rego rules) CREATE TABLE IF NOT EXISTS policy.rules ( @@ -112,10 +112,10 @@ CREATE TABLE IF NOT EXISTS policy.rules ( UNIQUE(pack_version_id, name) ); -CREATE INDEX idx_rules_pack_version ON policy.rules(pack_version_id); -CREATE INDEX idx_rules_severity ON policy.rules(severity); -CREATE INDEX idx_rules_category ON policy.rules(category); -CREATE INDEX idx_rules_tags ON policy.rules USING GIN(tags); +CREATE INDEX IF NOT EXISTS idx_rules_pack_version ON policy.rules(pack_version_id); +CREATE INDEX IF NOT EXISTS idx_rules_severity ON policy.rules(severity); +CREATE INDEX IF NOT EXISTS idx_rules_category ON policy.rules(category); +CREATE INDEX IF NOT EXISTS idx_rules_tags ON policy.rules USING GIN(tags); -- ============================================================================ -- Risk Profile Tables @@ -139,8 +139,8 @@ CREATE TABLE IF NOT EXISTS policy.risk_profiles ( UNIQUE(tenant_id, name, version) ); -CREATE INDEX idx_risk_profiles_tenant ON policy.risk_profiles(tenant_id); -CREATE INDEX idx_risk_profiles_active ON policy.risk_profiles(tenant_id, name, is_active) +CREATE INDEX IF NOT EXISTS idx_risk_profiles_tenant ON policy.risk_profiles(tenant_id); +CREATE INDEX IF NOT EXISTS idx_risk_profiles_active ON policy.risk_profiles(tenant_id, name, is_active) WHERE is_active = TRUE; CREATE TABLE IF NOT EXISTS policy.risk_profile_history ( @@ -155,7 +155,7 @@ CREATE TABLE IF NOT EXISTS policy.risk_profile_history ( change_reason TEXT ); -CREATE INDEX idx_risk_profile_history_profile ON policy.risk_profile_history(risk_profile_id); +CREATE INDEX IF NOT EXISTS idx_risk_profile_history_profile ON policy.risk_profile_history(risk_profile_id); -- ============================================================================ -- Evaluation Tables @@ -187,11 +187,11 @@ CREATE TABLE IF NOT EXISTS policy.evaluation_runs ( created_by TEXT ); -CREATE INDEX idx_evaluation_runs_tenant ON policy.evaluation_runs(tenant_id); -CREATE INDEX idx_evaluation_runs_project ON policy.evaluation_runs(tenant_id, project_id); -CREATE INDEX idx_evaluation_runs_artifact ON policy.evaluation_runs(tenant_id, artifact_id); -CREATE INDEX idx_evaluation_runs_created ON policy.evaluation_runs(tenant_id, created_at); -CREATE INDEX idx_evaluation_runs_status ON policy.evaluation_runs(status); +CREATE INDEX IF NOT EXISTS idx_evaluation_runs_tenant ON policy.evaluation_runs(tenant_id); +CREATE INDEX IF NOT EXISTS idx_evaluation_runs_project ON policy.evaluation_runs(tenant_id, project_id); +CREATE INDEX IF NOT EXISTS idx_evaluation_runs_artifact ON policy.evaluation_runs(tenant_id, artifact_id); +CREATE INDEX IF NOT EXISTS idx_evaluation_runs_created ON policy.evaluation_runs(tenant_id, created_at); +CREATE INDEX IF NOT EXISTS idx_evaluation_runs_status ON policy.evaluation_runs(status); CREATE TABLE IF NOT EXISTS policy.explanations ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), @@ -208,8 +208,8 @@ CREATE TABLE IF NOT EXISTS policy.explanations ( created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); -CREATE INDEX idx_explanations_run ON policy.explanations(evaluation_run_id); -CREATE INDEX idx_explanations_result ON policy.explanations(evaluation_run_id, result); +CREATE INDEX IF NOT EXISTS idx_explanations_run ON policy.explanations(evaluation_run_id); +CREATE INDEX IF NOT EXISTS idx_explanations_result ON policy.explanations(evaluation_run_id, result); -- ============================================================================ -- CVSS & Risk Scoring Tables @@ -251,11 +251,11 @@ CREATE TABLE IF NOT EXISTS policy.cvss_receipts ( CONSTRAINT cvss_receipts_input_hash_key UNIQUE (tenant_id, input_hash) ); -CREATE INDEX idx_cvss_receipts_tenant_created ON policy.cvss_receipts (tenant_id, created_at DESC, id); -CREATE INDEX idx_cvss_receipts_tenant_vuln ON policy.cvss_receipts (tenant_id, vulnerability_id); -CREATE INDEX idx_cvss_receipts_version ON policy.cvss_receipts(cvss_version); -CREATE INDEX idx_cvss_receipts_severity ON policy.cvss_receipts(tenant_id, severity); -CREATE INDEX idx_cvss_receipts_version_severity ON policy.cvss_receipts(tenant_id, cvss_version, severity); +CREATE INDEX IF NOT EXISTS idx_cvss_receipts_tenant_created ON policy.cvss_receipts (tenant_id, created_at DESC, id); +CREATE INDEX IF NOT EXISTS idx_cvss_receipts_tenant_vuln ON policy.cvss_receipts (tenant_id, vulnerability_id); +CREATE INDEX IF NOT EXISTS idx_cvss_receipts_version ON policy.cvss_receipts(cvss_version); +CREATE INDEX IF NOT EXISTS idx_cvss_receipts_severity ON policy.cvss_receipts(tenant_id, severity); +CREATE INDEX IF NOT EXISTS idx_cvss_receipts_version_severity ON policy.cvss_receipts(tenant_id, cvss_version, severity); CREATE TABLE IF NOT EXISTS policy.epss_scores ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), @@ -269,10 +269,10 @@ CREATE TABLE IF NOT EXISTS policy.epss_scores ( UNIQUE(cve_id, model_version) ); -CREATE INDEX idx_epss_scores_cve ON policy.epss_scores(cve_id); -CREATE INDEX idx_epss_scores_percentile ON policy.epss_scores(percentile DESC); -CREATE INDEX idx_epss_scores_expires ON policy.epss_scores(expires_at); -CREATE INDEX idx_epss_scores_model ON policy.epss_scores(model_version); +CREATE INDEX IF NOT EXISTS idx_epss_scores_cve ON policy.epss_scores(cve_id); +CREATE INDEX IF NOT EXISTS idx_epss_scores_percentile ON policy.epss_scores(percentile DESC); +CREATE INDEX IF NOT EXISTS idx_epss_scores_expires ON policy.epss_scores(expires_at); +CREATE INDEX IF NOT EXISTS idx_epss_scores_model ON policy.epss_scores(model_version); CREATE TABLE IF NOT EXISTS policy.epss_history ( id BIGSERIAL PRIMARY KEY, @@ -283,8 +283,8 @@ CREATE TABLE IF NOT EXISTS policy.epss_history ( recorded_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); -CREATE INDEX idx_epss_history_cve ON policy.epss_history(cve_id); -CREATE INDEX idx_epss_history_recorded ON policy.epss_history(cve_id, recorded_at DESC); +CREATE INDEX IF NOT EXISTS idx_epss_history_cve ON policy.epss_history(cve_id); +CREATE INDEX IF NOT EXISTS idx_epss_history_recorded ON policy.epss_history(cve_id, recorded_at DESC); CREATE TABLE IF NOT EXISTS policy.risk_scores ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), @@ -310,12 +310,12 @@ CREATE TABLE IF NOT EXISTS policy.risk_scores ( UNIQUE(tenant_id, vulnerability_id, input_hash) ); -CREATE INDEX idx_risk_scores_tenant ON policy.risk_scores(tenant_id); -CREATE INDEX idx_risk_scores_vuln ON policy.risk_scores(tenant_id, vulnerability_id); -CREATE INDEX idx_risk_scores_combined ON policy.risk_scores(tenant_id, combined_risk_score DESC); -CREATE INDEX idx_risk_scores_kev ON policy.risk_scores(kev_flag) WHERE kev_flag = TRUE; -CREATE INDEX idx_risk_scores_epss ON policy.risk_scores(epss_percentile DESC) WHERE epss_percentile IS NOT NULL; -CREATE INDEX idx_risk_scores_created ON policy.risk_scores(tenant_id, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_risk_scores_tenant ON policy.risk_scores(tenant_id); +CREATE INDEX IF NOT EXISTS idx_risk_scores_vuln ON policy.risk_scores(tenant_id, vulnerability_id); +CREATE INDEX IF NOT EXISTS idx_risk_scores_combined ON policy.risk_scores(tenant_id, combined_risk_score DESC); +CREATE INDEX IF NOT EXISTS idx_risk_scores_kev ON policy.risk_scores(kev_flag) WHERE kev_flag = TRUE; +CREATE INDEX IF NOT EXISTS idx_risk_scores_epss ON policy.risk_scores(epss_percentile DESC) WHERE epss_percentile IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_risk_scores_created ON policy.risk_scores(tenant_id, created_at DESC); CREATE TABLE IF NOT EXISTS policy.epss_thresholds ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), @@ -331,8 +331,8 @@ CREATE TABLE IF NOT EXISTS policy.epss_thresholds ( UNIQUE(tenant_id, name) ); -CREATE INDEX idx_epss_thresholds_tenant ON policy.epss_thresholds(tenant_id); -CREATE INDEX idx_epss_thresholds_default ON policy.epss_thresholds(tenant_id, is_default) +CREATE INDEX IF NOT EXISTS idx_epss_thresholds_tenant ON policy.epss_thresholds(tenant_id); +CREATE INDEX IF NOT EXISTS idx_epss_thresholds_default ON policy.epss_thresholds(tenant_id, is_default) WHERE is_default = TRUE; CREATE TABLE IF NOT EXISTS policy.risk_score_history ( @@ -348,8 +348,8 @@ CREATE TABLE IF NOT EXISTS policy.risk_score_history ( change_reason TEXT ); -CREATE INDEX idx_risk_score_history_score ON policy.risk_score_history(risk_score_id); -CREATE INDEX idx_risk_score_history_changed ON policy.risk_score_history(changed_at); +CREATE INDEX IF NOT EXISTS idx_risk_score_history_score ON policy.risk_score_history(risk_score_id); +CREATE INDEX IF NOT EXISTS idx_risk_score_history_changed ON policy.risk_score_history(changed_at); -- ============================================================================ -- Policy Snapshots & Events Tables @@ -368,9 +368,9 @@ CREATE TABLE IF NOT EXISTS policy.snapshots ( UNIQUE(tenant_id, policy_id, version) ); -CREATE INDEX idx_snapshots_tenant ON policy.snapshots(tenant_id); -CREATE INDEX idx_snapshots_policy ON policy.snapshots(tenant_id, policy_id); -CREATE INDEX idx_snapshots_digest ON policy.snapshots(content_digest); +CREATE INDEX IF NOT EXISTS idx_snapshots_tenant ON policy.snapshots(tenant_id); +CREATE INDEX IF NOT EXISTS idx_snapshots_policy ON policy.snapshots(tenant_id, policy_id); +CREATE INDEX IF NOT EXISTS idx_snapshots_digest ON policy.snapshots(content_digest); CREATE TABLE IF NOT EXISTS policy.violation_events ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), @@ -387,12 +387,12 @@ CREATE TABLE IF NOT EXISTS policy.violation_events ( created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); -CREATE INDEX idx_violation_events_tenant ON policy.violation_events(tenant_id); -CREATE INDEX idx_violation_events_policy ON policy.violation_events(tenant_id, policy_id); -CREATE INDEX idx_violation_events_rule ON policy.violation_events(rule_id); -CREATE INDEX idx_violation_events_severity ON policy.violation_events(severity); -CREATE INDEX idx_violation_events_purl ON policy.violation_events(subject_purl) WHERE subject_purl IS NOT NULL; -CREATE INDEX idx_violation_events_occurred ON policy.violation_events(tenant_id, occurred_at); +CREATE INDEX IF NOT EXISTS idx_violation_events_tenant ON policy.violation_events(tenant_id); +CREATE INDEX IF NOT EXISTS idx_violation_events_policy ON policy.violation_events(tenant_id, policy_id); +CREATE INDEX IF NOT EXISTS idx_violation_events_rule ON policy.violation_events(rule_id); +CREATE INDEX IF NOT EXISTS idx_violation_events_severity ON policy.violation_events(severity); +CREATE INDEX IF NOT EXISTS idx_violation_events_purl ON policy.violation_events(subject_purl) WHERE subject_purl IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_violation_events_occurred ON policy.violation_events(tenant_id, occurred_at); CREATE TABLE IF NOT EXISTS policy.conflicts ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), @@ -412,9 +412,9 @@ CREATE TABLE IF NOT EXISTS policy.conflicts ( created_by TEXT ); -CREATE INDEX idx_conflicts_tenant ON policy.conflicts(tenant_id); -CREATE INDEX idx_conflicts_status ON policy.conflicts(tenant_id, status); -CREATE INDEX idx_conflicts_type ON policy.conflicts(conflict_type); +CREATE INDEX IF NOT EXISTS idx_conflicts_tenant ON policy.conflicts(tenant_id); +CREATE INDEX IF NOT EXISTS idx_conflicts_status ON policy.conflicts(tenant_id, status); +CREATE INDEX IF NOT EXISTS idx_conflicts_type ON policy.conflicts(conflict_type); CREATE TABLE IF NOT EXISTS policy.ledger_exports ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), @@ -434,10 +434,10 @@ CREATE TABLE IF NOT EXISTS policy.ledger_exports ( created_by TEXT ); -CREATE INDEX idx_ledger_exports_tenant ON policy.ledger_exports(tenant_id); -CREATE INDEX idx_ledger_exports_status ON policy.ledger_exports(status); -CREATE INDEX idx_ledger_exports_digest ON policy.ledger_exports(content_digest) WHERE content_digest IS NOT NULL; -CREATE INDEX idx_ledger_exports_created ON policy.ledger_exports(tenant_id, created_at); +CREATE INDEX IF NOT EXISTS idx_ledger_exports_tenant ON policy.ledger_exports(tenant_id); +CREATE INDEX IF NOT EXISTS idx_ledger_exports_status ON policy.ledger_exports(status); +CREATE INDEX IF NOT EXISTS idx_ledger_exports_digest ON policy.ledger_exports(content_digest) WHERE content_digest IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_ledger_exports_created ON policy.ledger_exports(tenant_id, created_at); CREATE TABLE IF NOT EXISTS policy.worker_results ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), @@ -461,10 +461,10 @@ CREATE TABLE IF NOT EXISTS policy.worker_results ( UNIQUE(tenant_id, job_type, job_id) ); -CREATE INDEX idx_worker_results_tenant ON policy.worker_results(tenant_id); -CREATE INDEX idx_worker_results_status ON policy.worker_results(status); -CREATE INDEX idx_worker_results_job_type ON policy.worker_results(job_type); -CREATE INDEX idx_worker_results_scheduled ON policy.worker_results(scheduled_at) WHERE status = 'pending'; +CREATE INDEX IF NOT EXISTS idx_worker_results_tenant ON policy.worker_results(tenant_id); +CREATE INDEX IF NOT EXISTS idx_worker_results_status ON policy.worker_results(status); +CREATE INDEX IF NOT EXISTS idx_worker_results_job_type ON policy.worker_results(job_type); +CREATE INDEX IF NOT EXISTS idx_worker_results_scheduled ON policy.worker_results(scheduled_at) WHERE status = 'pending'; -- ============================================================================ -- Unknowns Management Table @@ -498,11 +498,11 @@ CREATE TABLE IF NOT EXISTS policy.unknowns ( UNIQUE(tenant_id, package_id, package_version) ); -CREATE INDEX idx_unknowns_tenant_band ON policy.unknowns(tenant_id, band); -CREATE INDEX idx_unknowns_tenant_score ON policy.unknowns(tenant_id, score DESC); -CREATE INDEX idx_unknowns_last_evaluated ON policy.unknowns(last_evaluated_at); -CREATE INDEX idx_unknowns_package ON policy.unknowns(package_id, package_version); -CREATE INDEX idx_unknowns_reason_code ON policy.unknowns(reason_code) WHERE reason_code IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_unknowns_tenant_band ON policy.unknowns(tenant_id, band); +CREATE INDEX IF NOT EXISTS idx_unknowns_tenant_score ON policy.unknowns(tenant_id, score DESC); +CREATE INDEX IF NOT EXISTS idx_unknowns_last_evaluated ON policy.unknowns(last_evaluated_at); +CREATE INDEX IF NOT EXISTS idx_unknowns_package ON policy.unknowns(package_id, package_version); +CREATE INDEX IF NOT EXISTS idx_unknowns_reason_code ON policy.unknowns(reason_code) WHERE reason_code IS NOT NULL; -- ============================================================================ -- Recheck Policies & Evidence Tables @@ -519,7 +519,7 @@ CREATE TABLE IF NOT EXISTS policy.recheck_policies ( updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); -CREATE INDEX idx_recheck_policies_tenant ON policy.recheck_policies (tenant_id, is_active); +CREATE INDEX IF NOT EXISTS idx_recheck_policies_tenant ON policy.recheck_policies (tenant_id, is_active); CREATE TABLE IF NOT EXISTS policy.evidence_hooks ( hook_id TEXT PRIMARY KEY, @@ -533,7 +533,7 @@ CREATE TABLE IF NOT EXISTS policy.evidence_hooks ( created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); -CREATE INDEX idx_evidence_hooks_tenant_type ON policy.evidence_hooks (tenant_id, type); +CREATE INDEX IF NOT EXISTS idx_evidence_hooks_tenant_type ON policy.evidence_hooks (tenant_id, type); -- ============================================================================ -- Exception Management Tables @@ -581,16 +581,16 @@ CREATE TABLE IF NOT EXISTS policy.exceptions ( UNIQUE(tenant_id, name) ); -CREATE INDEX idx_exceptions_tenant ON policy.exceptions(tenant_id); -CREATE INDEX idx_exceptions_status ON policy.exceptions(tenant_id, status); -CREATE INDEX idx_exceptions_expires ON policy.exceptions(expires_at) WHERE status = 'active'; -CREATE INDEX idx_exceptions_project ON policy.exceptions(tenant_id, project_id); -CREATE INDEX idx_exceptions_vuln_id ON policy.exceptions(vulnerability_id) WHERE vulnerability_id IS NOT NULL; -CREATE INDEX idx_exceptions_purl ON policy.exceptions(purl_pattern) WHERE purl_pattern IS NOT NULL; -CREATE INDEX idx_exceptions_artifact ON policy.exceptions(artifact_digest) WHERE artifact_digest IS NOT NULL; -CREATE INDEX idx_exceptions_policy_rule ON policy.exceptions(policy_rule_id) WHERE policy_rule_id IS NOT NULL; -CREATE INDEX idx_exceptions_owner ON policy.exceptions(owner_id) WHERE owner_id IS NOT NULL; -CREATE INDEX idx_exceptions_recheck_policy ON policy.exceptions(tenant_id, recheck_policy_id) WHERE recheck_policy_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_exceptions_tenant ON policy.exceptions(tenant_id); +CREATE INDEX IF NOT EXISTS idx_exceptions_status ON policy.exceptions(tenant_id, status); +CREATE INDEX IF NOT EXISTS idx_exceptions_expires ON policy.exceptions(expires_at) WHERE status = 'active'; +CREATE INDEX IF NOT EXISTS idx_exceptions_project ON policy.exceptions(tenant_id, project_id); +CREATE INDEX IF NOT EXISTS idx_exceptions_vuln_id ON policy.exceptions(vulnerability_id) WHERE vulnerability_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_exceptions_purl ON policy.exceptions(purl_pattern) WHERE purl_pattern IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_exceptions_artifact ON policy.exceptions(artifact_digest) WHERE artifact_digest IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_exceptions_policy_rule ON policy.exceptions(policy_rule_id) WHERE policy_rule_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_exceptions_owner ON policy.exceptions(owner_id) WHERE owner_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_exceptions_recheck_policy ON policy.exceptions(tenant_id, recheck_policy_id) WHERE recheck_policy_id IS NOT NULL; CREATE TABLE IF NOT EXISTS policy.submitted_evidence ( evidence_id TEXT PRIMARY KEY, @@ -609,9 +609,9 @@ CREATE TABLE IF NOT EXISTS policy.submitted_evidence ( validation_error TEXT ); -CREATE INDEX idx_submitted_evidence_exception ON policy.submitted_evidence (tenant_id, exception_id); -CREATE INDEX idx_submitted_evidence_hook ON policy.submitted_evidence (tenant_id, hook_id); -CREATE INDEX idx_submitted_evidence_status ON policy.submitted_evidence (tenant_id, validation_status); +CREATE INDEX IF NOT EXISTS idx_submitted_evidence_exception ON policy.submitted_evidence (tenant_id, exception_id); +CREATE INDEX IF NOT EXISTS idx_submitted_evidence_hook ON policy.submitted_evidence (tenant_id, hook_id); +CREATE INDEX IF NOT EXISTS idx_submitted_evidence_status ON policy.submitted_evidence (tenant_id, validation_status); CREATE TABLE IF NOT EXISTS policy.exception_events ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), @@ -629,8 +629,8 @@ CREATE TABLE IF NOT EXISTS policy.exception_events ( UNIQUE (exception_id, sequence_number) ); -CREATE INDEX idx_exception_events_exception ON policy.exception_events(exception_id); -CREATE INDEX idx_exception_events_time ON policy.exception_events USING BRIN (occurred_at); +CREATE INDEX IF NOT EXISTS idx_exception_events_exception ON policy.exception_events(exception_id); +CREATE INDEX IF NOT EXISTS idx_exception_events_time ON policy.exception_events USING BRIN (occurred_at); CREATE TABLE IF NOT EXISTS policy.exception_applications ( id UUID NOT NULL PRIMARY KEY, @@ -648,12 +648,12 @@ CREATE TABLE IF NOT EXISTS policy.exception_applications ( metadata JSONB NOT NULL DEFAULT '{}' ); -CREATE INDEX ix_exception_applications_exception_id ON policy.exception_applications (tenant_id, exception_id); -CREATE INDEX ix_exception_applications_finding_id ON policy.exception_applications (tenant_id, finding_id); -CREATE INDEX ix_exception_applications_vulnerability_id ON policy.exception_applications (tenant_id, vulnerability_id) WHERE vulnerability_id IS NOT NULL; -CREATE INDEX ix_exception_applications_evaluation_run_id ON policy.exception_applications (tenant_id, evaluation_run_id) WHERE evaluation_run_id IS NOT NULL; -CREATE INDEX ix_exception_applications_applied_at ON policy.exception_applications (tenant_id, applied_at DESC); -CREATE INDEX ix_exception_applications_stats ON policy.exception_applications (tenant_id, effect_type, applied_status); +CREATE INDEX IF NOT EXISTS ix_exception_applications_exception_id ON policy.exception_applications (tenant_id, exception_id); +CREATE INDEX IF NOT EXISTS ix_exception_applications_finding_id ON policy.exception_applications (tenant_id, finding_id); +CREATE INDEX IF NOT EXISTS ix_exception_applications_vulnerability_id ON policy.exception_applications (tenant_id, vulnerability_id) WHERE vulnerability_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS ix_exception_applications_evaluation_run_id ON policy.exception_applications (tenant_id, evaluation_run_id) WHERE evaluation_run_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS ix_exception_applications_applied_at ON policy.exception_applications (tenant_id, applied_at DESC); +CREATE INDEX IF NOT EXISTS ix_exception_applications_stats ON policy.exception_applications (tenant_id, effect_type, applied_status); -- ============================================================================ -- Budget Management Tables @@ -673,10 +673,10 @@ CREATE TABLE IF NOT EXISTS policy.budget_ledger ( CONSTRAINT uq_budget_ledger_service_window UNIQUE (service_id, "window") ); -CREATE INDEX idx_budget_ledger_service_id ON policy.budget_ledger (service_id); -CREATE INDEX idx_budget_ledger_tenant_id ON policy.budget_ledger (tenant_id); -CREATE INDEX idx_budget_ledger_window ON policy.budget_ledger ("window"); -CREATE INDEX idx_budget_ledger_status ON policy.budget_ledger (status); +CREATE INDEX IF NOT EXISTS idx_budget_ledger_service_id ON policy.budget_ledger (service_id); +CREATE INDEX IF NOT EXISTS idx_budget_ledger_tenant_id ON policy.budget_ledger (tenant_id); +CREATE INDEX IF NOT EXISTS idx_budget_ledger_window ON policy.budget_ledger ("window"); +CREATE INDEX IF NOT EXISTS idx_budget_ledger_status ON policy.budget_ledger (status); CREATE TABLE IF NOT EXISTS policy.budget_entries ( entry_id VARCHAR(64) PRIMARY KEY, @@ -693,9 +693,9 @@ CREATE TABLE IF NOT EXISTS policy.budget_entries ( REFERENCES policy.budget_ledger (service_id, "window") ON DELETE CASCADE ); -CREATE INDEX idx_budget_entries_service_window ON policy.budget_entries (service_id, "window"); -CREATE INDEX idx_budget_entries_release_id ON policy.budget_entries (release_id); -CREATE INDEX idx_budget_entries_consumed_at ON policy.budget_entries (consumed_at); +CREATE INDEX IF NOT EXISTS idx_budget_entries_service_window ON policy.budget_entries (service_id, "window"); +CREATE INDEX IF NOT EXISTS idx_budget_entries_release_id ON policy.budget_entries (release_id); +CREATE INDEX IF NOT EXISTS idx_budget_entries_consumed_at ON policy.budget_entries (consumed_at); -- ============================================================================ -- Approval Workflow Tables @@ -734,12 +734,12 @@ CREATE TABLE IF NOT EXISTS policy.exception_approval_requests ( updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); -CREATE INDEX idx_approval_requests_tenant ON policy.exception_approval_requests(tenant_id); -CREATE INDEX idx_approval_requests_status ON policy.exception_approval_requests(tenant_id, status); -CREATE INDEX idx_approval_requests_requestor ON policy.exception_approval_requests(requestor_id); -CREATE INDEX idx_approval_requests_pending ON policy.exception_approval_requests(tenant_id, status) WHERE status IN ('pending', 'partial'); -CREATE INDEX idx_approval_requests_expiry ON policy.exception_approval_requests(request_expires_at) WHERE status IN ('pending', 'partial'); -CREATE INDEX idx_approval_requests_vuln ON policy.exception_approval_requests(vulnerability_id) WHERE vulnerability_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_approval_requests_tenant ON policy.exception_approval_requests(tenant_id); +CREATE INDEX IF NOT EXISTS idx_approval_requests_status ON policy.exception_approval_requests(tenant_id, status); +CREATE INDEX IF NOT EXISTS idx_approval_requests_requestor ON policy.exception_approval_requests(requestor_id); +CREATE INDEX IF NOT EXISTS idx_approval_requests_pending ON policy.exception_approval_requests(tenant_id, status) WHERE status IN ('pending', 'partial'); +CREATE INDEX IF NOT EXISTS idx_approval_requests_expiry ON policy.exception_approval_requests(request_expires_at) WHERE status IN ('pending', 'partial'); +CREATE INDEX IF NOT EXISTS idx_approval_requests_vuln ON policy.exception_approval_requests(vulnerability_id) WHERE vulnerability_id IS NOT NULL; CREATE TABLE IF NOT EXISTS policy.exception_approval_audit ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), @@ -757,8 +757,8 @@ CREATE TABLE IF NOT EXISTS policy.exception_approval_audit ( UNIQUE (request_id, sequence_number) ); -CREATE INDEX idx_approval_audit_request ON policy.exception_approval_audit(request_id); -CREATE INDEX idx_approval_audit_time ON policy.exception_approval_audit USING BRIN (occurred_at); +CREATE INDEX IF NOT EXISTS idx_approval_audit_request ON policy.exception_approval_audit(request_id); +CREATE INDEX IF NOT EXISTS idx_approval_audit_time ON policy.exception_approval_audit USING BRIN (occurred_at); CREATE TABLE IF NOT EXISTS policy.exception_approval_rules ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), @@ -780,7 +780,7 @@ CREATE TABLE IF NOT EXISTS policy.exception_approval_rules ( UNIQUE (tenant_id, gate_level, name) ); -CREATE INDEX idx_approval_rules_lookup ON policy.exception_approval_rules(tenant_id, gate_level, enabled); +CREATE INDEX IF NOT EXISTS idx_approval_rules_lookup ON policy.exception_approval_rules(tenant_id, gate_level, enabled); -- ============================================================================ -- Audit Log Table @@ -799,9 +799,9 @@ CREATE TABLE IF NOT EXISTS policy.audit ( created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); -CREATE INDEX idx_audit_tenant ON policy.audit(tenant_id); -CREATE INDEX idx_audit_resource ON policy.audit(resource_type, resource_id); -CREATE INDEX idx_audit_created ON policy.audit(tenant_id, created_at); +CREATE INDEX IF NOT EXISTS idx_audit_tenant ON policy.audit(tenant_id); +CREATE INDEX IF NOT EXISTS idx_audit_resource ON policy.audit(resource_type, resource_id); +CREATE INDEX IF NOT EXISTS idx_audit_created ON policy.audit(tenant_id, created_at); -- ============================================================================ -- CVSS Helper Functions @@ -1006,22 +1006,27 @@ $$; -- Triggers -- ============================================================================ +DROP TRIGGER IF EXISTS trg_packs_updated_at ON policy.packs; CREATE TRIGGER trg_packs_updated_at BEFORE UPDATE ON policy.packs FOR EACH ROW EXECUTE FUNCTION policy.update_updated_at(); +DROP TRIGGER IF EXISTS trg_risk_profiles_updated_at ON policy.risk_profiles; CREATE TRIGGER trg_risk_profiles_updated_at BEFORE UPDATE ON policy.risk_profiles FOR EACH ROW EXECUTE FUNCTION policy.update_updated_at(); +DROP TRIGGER IF EXISTS trg_risk_scores_updated_at ON policy.risk_scores; CREATE TRIGGER trg_risk_scores_updated_at BEFORE UPDATE ON policy.risk_scores FOR EACH ROW EXECUTE FUNCTION policy.update_updated_at(); +DROP TRIGGER IF EXISTS trg_epss_thresholds_updated_at ON policy.epss_thresholds; CREATE TRIGGER trg_epss_thresholds_updated_at BEFORE UPDATE ON policy.epss_thresholds FOR EACH ROW EXECUTE FUNCTION policy.update_updated_at(); +DROP TRIGGER IF EXISTS trg_unknowns_updated_at ON policy.unknowns; CREATE TRIGGER trg_unknowns_updated_at BEFORE UPDATE ON policy.unknowns FOR EACH ROW EXECUTE FUNCTION policy.unknowns_set_updated_at(); @@ -1170,50 +1175,63 @@ ALTER TABLE policy.exception_approval_rules ENABLE ROW LEVEL SECURITY; ALTER TABLE policy.audit ENABLE ROW LEVEL SECURITY; -- Direct Tenant Isolation Policies +DROP POLICY IF EXISTS packs_tenant_isolation ON policy.packs; CREATE POLICY packs_tenant_isolation ON policy.packs FOR ALL USING (tenant_id = policy_app.require_current_tenant()) WITH CHECK (tenant_id = policy_app.require_current_tenant()); +DROP POLICY IF EXISTS risk_profiles_tenant_isolation ON policy.risk_profiles; CREATE POLICY risk_profiles_tenant_isolation ON policy.risk_profiles FOR ALL USING (tenant_id = policy_app.require_current_tenant()) WITH CHECK (tenant_id = policy_app.require_current_tenant()); +DROP POLICY IF EXISTS evaluation_runs_tenant_isolation ON policy.evaluation_runs; CREATE POLICY evaluation_runs_tenant_isolation ON policy.evaluation_runs FOR ALL USING (tenant_id = policy_app.require_current_tenant()) WITH CHECK (tenant_id = policy_app.require_current_tenant()); +DROP POLICY IF EXISTS exceptions_tenant_isolation ON policy.exceptions; CREATE POLICY exceptions_tenant_isolation ON policy.exceptions FOR ALL USING (tenant_id = policy_app.require_current_tenant()) WITH CHECK (tenant_id = policy_app.require_current_tenant()); +DROP POLICY IF EXISTS audit_tenant_isolation ON policy.audit; CREATE POLICY audit_tenant_isolation ON policy.audit FOR ALL USING (tenant_id = policy_app.require_current_tenant()) WITH CHECK (tenant_id = policy_app.require_current_tenant()); +DROP POLICY IF EXISTS recheck_policies_tenant_isolation ON policy.recheck_policies; CREATE POLICY recheck_policies_tenant_isolation ON policy.recheck_policies FOR ALL USING (tenant_id = policy_app.require_current_tenant()) WITH CHECK (tenant_id = policy_app.require_current_tenant()); +DROP POLICY IF EXISTS evidence_hooks_tenant_isolation ON policy.evidence_hooks; CREATE POLICY evidence_hooks_tenant_isolation ON policy.evidence_hooks FOR ALL USING (tenant_id = policy_app.require_current_tenant()) WITH CHECK (tenant_id = policy_app.require_current_tenant()); +DROP POLICY IF EXISTS submitted_evidence_tenant_isolation ON policy.submitted_evidence; CREATE POLICY submitted_evidence_tenant_isolation ON policy.submitted_evidence FOR ALL USING (tenant_id = policy_app.require_current_tenant()) WITH CHECK (tenant_id = policy_app.require_current_tenant()); +DROP POLICY IF EXISTS approval_requests_tenant_isolation ON policy.exception_approval_requests; CREATE POLICY approval_requests_tenant_isolation ON policy.exception_approval_requests FOR ALL USING (tenant_id = current_setting('app.current_tenant', true)); +DROP POLICY IF EXISTS approval_audit_tenant_isolation ON policy.exception_approval_audit; CREATE POLICY approval_audit_tenant_isolation ON policy.exception_approval_audit FOR ALL USING (tenant_id = current_setting('app.current_tenant', true)); +DROP POLICY IF EXISTS approval_rules_tenant_isolation ON policy.exception_approval_rules; CREATE POLICY approval_rules_tenant_isolation ON policy.exception_approval_rules FOR ALL USING (tenant_id = current_setting('app.current_tenant', true)); +DROP POLICY IF EXISTS budget_ledger_tenant_isolation ON policy.budget_ledger; CREATE POLICY budget_ledger_tenant_isolation ON policy.budget_ledger FOR ALL USING (tenant_id = current_setting('app.tenant_id', TRUE) OR tenant_id IS NULL); +DROP POLICY IF EXISTS budget_entries_tenant_isolation ON policy.budget_entries; CREATE POLICY budget_entries_tenant_isolation ON policy.budget_entries FOR ALL USING ( EXISTS ( @@ -1225,11 +1243,13 @@ CREATE POLICY budget_entries_tenant_isolation ON policy.budget_entries ); -- FK-Based Tenant Isolation Policies +DROP POLICY IF EXISTS pack_versions_tenant_isolation ON policy.pack_versions; CREATE POLICY pack_versions_tenant_isolation ON policy.pack_versions FOR ALL USING ( pack_id IN (SELECT id FROM policy.packs WHERE tenant_id = policy_app.require_current_tenant()) ); +DROP POLICY IF EXISTS rules_tenant_isolation ON policy.rules; CREATE POLICY rules_tenant_isolation ON policy.rules FOR ALL USING ( pack_version_id IN ( @@ -1239,25 +1259,30 @@ CREATE POLICY rules_tenant_isolation ON policy.rules ) ); +DROP POLICY IF EXISTS risk_profile_history_tenant_isolation ON policy.risk_profile_history; CREATE POLICY risk_profile_history_tenant_isolation ON policy.risk_profile_history FOR ALL USING ( risk_profile_id IN (SELECT id FROM policy.risk_profiles WHERE tenant_id = policy_app.require_current_tenant()) ); +DROP POLICY IF EXISTS explanations_tenant_isolation ON policy.explanations; CREATE POLICY explanations_tenant_isolation ON policy.explanations FOR ALL USING ( evaluation_run_id IN (SELECT id FROM policy.evaluation_runs WHERE tenant_id = policy_app.require_current_tenant()) ); +DROP POLICY IF EXISTS exception_events_tenant_isolation ON policy.exception_events; CREATE POLICY exception_events_tenant_isolation ON policy.exception_events FOR ALL USING ( EXISTS (SELECT 1 FROM policy.exceptions e WHERE e.exception_id = exception_events.exception_id) ); +DROP POLICY IF EXISTS exception_applications_tenant_isolation ON policy.exception_applications; CREATE POLICY exception_applications_tenant_isolation ON policy.exception_applications FOR ALL USING (tenant_id = current_setting('app.tenant_id', true)::uuid); -- Unknowns RLS (uses UUID tenant_id) +DROP POLICY IF EXISTS unknowns_tenant_isolation ON policy.unknowns; CREATE POLICY unknowns_tenant_isolation ON policy.unknowns USING (tenant_id::text = current_setting('app.current_tenant', true)) WITH CHECK (tenant_id::text = current_setting('app.current_tenant', true)); @@ -1275,6 +1300,7 @@ $$; DO $$ BEGIN IF EXISTS (SELECT FROM pg_roles WHERE rolname = 'stellaops_service') THEN + DROP POLICY IF EXISTS unknowns_service_bypass ON policy.unknowns; CREATE POLICY unknowns_service_bypass ON policy.unknowns TO stellaops_service USING (true) WITH CHECK (true); END IF; @@ -1318,3 +1344,4 @@ VALUES 'Critical severity - executive approval required', 4, 3, ARRAY['ciso', 'delivery-manager', 'product-manager'], 7, false, true, 200, 100) ON CONFLICT DO NOTHING; + diff --git a/src/Policy/__Libraries/StellaOps.Policy.Persistence/Migrations/007_policy_engine_runtime_state.sql b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Migrations/007_policy_engine_runtime_state.sql new file mode 100644 index 000000000..9e16b075c --- /dev/null +++ b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Migrations/007_policy_engine_runtime_state.sql @@ -0,0 +1,34 @@ +-- Policy Schema Migration 007: Policy Engine runtime snapshot and ledger persistence + +CREATE TABLE IF NOT EXISTS policy.engine_ledger_exports ( + export_id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + schema_version TEXT NOT NULL, + generated_at TIMESTAMPTZ NOT NULL, + record_count INT NOT NULL, + sha256 TEXT NOT NULL, + manifest_json JSONB NOT NULL, + records_json JSONB NOT NULL, + lines_json JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_engine_ledger_exports_tenant_generated + ON policy.engine_ledger_exports (tenant_id, generated_at DESC, export_id); + +CREATE TABLE IF NOT EXISTS policy.engine_snapshots ( + snapshot_id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + ledger_export_id TEXT NOT NULL REFERENCES policy.engine_ledger_exports(export_id) ON DELETE RESTRICT, + generated_at TIMESTAMPTZ NOT NULL, + overlay_hash TEXT NOT NULL, + status_counts_json JSONB NOT NULL, + records_json JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_engine_snapshots_tenant_generated + ON policy.engine_snapshots (tenant_id, generated_at DESC, snapshot_id); + +CREATE INDEX IF NOT EXISTS idx_engine_snapshots_ledger_export + ON policy.engine_snapshots (ledger_export_id); diff --git a/src/Policy/__Libraries/StellaOps.Policy.Persistence/Migrations/008_policy_engine_snapshot_artifact_identity.sql b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Migrations/008_policy_engine_snapshot_artifact_identity.sql new file mode 100644 index 000000000..50da1f819 --- /dev/null +++ b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Migrations/008_policy_engine_snapshot_artifact_identity.sql @@ -0,0 +1,13 @@ +-- Policy Schema Migration 008: Persist artifact identity for engine snapshots + +ALTER TABLE IF EXISTS policy.engine_snapshots + ADD COLUMN IF NOT EXISTS artifact_digest TEXT NULL; + +ALTER TABLE IF EXISTS policy.engine_snapshots + ADD COLUMN IF NOT EXISTS artifact_repository TEXT NULL; + +ALTER TABLE IF EXISTS policy.engine_snapshots + ADD COLUMN IF NOT EXISTS artifact_tag TEXT NULL; + +CREATE INDEX IF NOT EXISTS idx_engine_snapshots_tenant_artifact_generated + ON policy.engine_snapshots (tenant_id, artifact_digest, generated_at DESC, snapshot_id); diff --git a/src/Policy/__Libraries/StellaOps.Policy.Persistence/Migrations/009_policy_engine_orchestrator_jobs.sql b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Migrations/009_policy_engine_orchestrator_jobs.sql new file mode 100644 index 000000000..08a5e8f9d --- /dev/null +++ b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Migrations/009_policy_engine_orchestrator_jobs.sql @@ -0,0 +1,22 @@ +CREATE TABLE IF NOT EXISTS policy.orchestrator_jobs ( + job_id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + context_id TEXT NOT NULL, + policy_profile_hash TEXT NOT NULL, + requested_at TIMESTAMPTZ NOT NULL, + priority TEXT NOT NULL, + batch_items_json JSONB NOT NULL, + callbacks_json JSONB, + trace_ref TEXT NOT NULL, + status TEXT NOT NULL, + determinism_hash TEXT NOT NULL, + completed_at TIMESTAMPTZ, + result_hash TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_orchestrator_jobs_tenant_requested + ON policy.orchestrator_jobs (tenant_id, requested_at DESC, job_id); + +CREATE INDEX IF NOT EXISTS idx_orchestrator_jobs_status + ON policy.orchestrator_jobs (tenant_id, status, requested_at DESC, job_id); diff --git a/src/Policy/__Libraries/StellaOps.Policy.Persistence/Migrations/S001_demo_seed.sql b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Migrations/_archived/S001_demo_seed.sql similarity index 100% rename from src/Policy/__Libraries/StellaOps.Policy.Persistence/Migrations/S001_demo_seed.sql rename to src/Policy/__Libraries/StellaOps.Policy.Persistence/Migrations/_archived/S001_demo_seed.sql diff --git a/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Models/PolicyEngineLedgerExportDocument.cs b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Models/PolicyEngineLedgerExportDocument.cs new file mode 100644 index 000000000..c64f7e03a --- /dev/null +++ b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Models/PolicyEngineLedgerExportDocument.cs @@ -0,0 +1,18 @@ +namespace StellaOps.Policy.Persistence.Postgres.Models; + +/// +/// Persisted runtime document for Policy Engine ledger exports. +/// +public sealed record PolicyEngineLedgerExportDocument +{ + public required string ExportId { get; init; } + public required string TenantId { get; init; } + public required string SchemaVersion { get; init; } + public required DateTimeOffset GeneratedAt { get; init; } + public int RecordCount { get; init; } + public required string Sha256 { get; init; } + public required string ManifestJson { get; init; } + public required string RecordsJson { get; init; } + public required string LinesJson { get; init; } + public DateTimeOffset CreatedAt { get; init; } +} diff --git a/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Models/PolicyEngineOrchestratorJobDocument.cs b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Models/PolicyEngineOrchestratorJobDocument.cs new file mode 100644 index 000000000..4abdbed14 --- /dev/null +++ b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Models/PolicyEngineOrchestratorJobDocument.cs @@ -0,0 +1,22 @@ +namespace StellaOps.Policy.Persistence.Postgres.Models; + +/// +/// Persisted Policy Engine orchestrator job document for runtime bootstrap and worker replay. +/// +public sealed record PolicyEngineOrchestratorJobDocument +{ + public required string JobId { get; init; } + public required string TenantId { get; init; } + public required string ContextId { get; init; } + public required string PolicyProfileHash { get; init; } + public DateTimeOffset RequestedAt { get; init; } + public required string Priority { get; init; } + public required string BatchItemsJson { get; init; } + public string? CallbacksJson { get; init; } + public required string TraceRef { get; init; } + public required string Status { get; init; } + public required string DeterminismHash { get; init; } + public DateTimeOffset? CompletedAt { get; init; } + public string? ResultHash { get; init; } + public DateTimeOffset CreatedAt { get; init; } +} diff --git a/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Models/PolicyEngineSnapshotDocument.cs b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Models/PolicyEngineSnapshotDocument.cs new file mode 100644 index 000000000..c6cd3c623 --- /dev/null +++ b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Models/PolicyEngineSnapshotDocument.cs @@ -0,0 +1,19 @@ +namespace StellaOps.Policy.Persistence.Postgres.Models; + +/// +/// Persisted runtime document for Policy Engine snapshots. +/// +public sealed record PolicyEngineSnapshotDocument +{ + public required string SnapshotId { get; init; } + public required string TenantId { get; init; } + public required string LedgerExportId { get; init; } + public required DateTimeOffset GeneratedAt { get; init; } + public required string OverlayHash { get; init; } + public string? ArtifactDigest { get; init; } + public string? ArtifactRepository { get; init; } + public string? ArtifactTag { get; init; } + public required string StatusCountsJson { get; init; } + public required string RecordsJson { get; init; } + public DateTimeOffset CreatedAt { get; init; } +} diff --git a/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/IPolicyEngineLedgerExportRepository.cs b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/IPolicyEngineLedgerExportRepository.cs new file mode 100644 index 000000000..2f7219e6e --- /dev/null +++ b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/IPolicyEngineLedgerExportRepository.cs @@ -0,0 +1,13 @@ +using StellaOps.Policy.Persistence.Postgres.Models; + +namespace StellaOps.Policy.Persistence.Postgres.Repositories; + +/// +/// Persistence contract for Policy Engine runtime ledger export documents. +/// +public interface IPolicyEngineLedgerExportRepository +{ + Task SaveAsync(PolicyEngineLedgerExportDocument document, CancellationToken cancellationToken = default); + Task GetAsync(string exportId, CancellationToken cancellationToken = default); + Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default); +} diff --git a/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/IPolicyEngineSnapshotRepository.cs b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/IPolicyEngineSnapshotRepository.cs new file mode 100644 index 000000000..831dc4b3e --- /dev/null +++ b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/IPolicyEngineSnapshotRepository.cs @@ -0,0 +1,13 @@ +using StellaOps.Policy.Persistence.Postgres.Models; + +namespace StellaOps.Policy.Persistence.Postgres.Repositories; + +/// +/// Persistence contract for Policy Engine runtime snapshot documents. +/// +public interface IPolicyEngineSnapshotRepository +{ + Task SaveAsync(PolicyEngineSnapshotDocument document, CancellationToken cancellationToken = default); + Task GetAsync(string snapshotId, CancellationToken cancellationToken = default); + Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default); +} diff --git a/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/IWorkerResultRepository.cs b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/IWorkerResultRepository.cs index 4505e0d6f..4d0014231 100644 --- a/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/IWorkerResultRepository.cs +++ b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/IWorkerResultRepository.cs @@ -26,6 +26,13 @@ public interface IWorkerResultRepository string jobId, CancellationToken cancellationToken = default); + /// + /// Lists worker results, optionally filtered by tenant. + /// + Task> ListAsync( + string? tenantId = null, + CancellationToken cancellationToken = default); + /// /// Gets worker results by status. /// diff --git a/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/PolicyEngineLedgerExportRepository.cs b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/PolicyEngineLedgerExportRepository.cs new file mode 100644 index 000000000..e36942fbc --- /dev/null +++ b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/PolicyEngineLedgerExportRepository.cs @@ -0,0 +1,197 @@ +using Microsoft.Extensions.Logging; +using Npgsql; +using StellaOps.Infrastructure.Postgres.Repositories; +using StellaOps.Policy.Persistence.Postgres.Models; + +namespace StellaOps.Policy.Persistence.Postgres.Repositories; + +/// +/// PostgreSQL repository for persisted Policy Engine ledger export runtime documents. +/// +public sealed class PolicyEngineLedgerExportRepository + : RepositoryBase, IPolicyEngineLedgerExportRepository +{ + public PolicyEngineLedgerExportRepository( + PolicyDataSource dataSource, + ILogger logger) + : base(dataSource, logger) + { + } + + public async Task SaveAsync(PolicyEngineLedgerExportDocument document, CancellationToken cancellationToken = default) + { + const string sql = """ + INSERT INTO policy.engine_ledger_exports ( + export_id, + tenant_id, + schema_version, + generated_at, + record_count, + sha256, + manifest_json, + records_json, + lines_json + ) + VALUES ( + @export_id, + @tenant_id, + @schema_version, + @generated_at, + @record_count, + @sha256, + @manifest_json, + @records_json, + @lines_json + ) + ON CONFLICT (export_id) DO UPDATE + SET tenant_id = EXCLUDED.tenant_id, + schema_version = EXCLUDED.schema_version, + generated_at = EXCLUDED.generated_at, + record_count = EXCLUDED.record_count, + sha256 = EXCLUDED.sha256, + manifest_json = EXCLUDED.manifest_json, + records_json = EXCLUDED.records_json, + lines_json = EXCLUDED.lines_json + """; + + await ExecuteAsync( + document.TenantId, + sql, + command => + { + AddParameter(command, "export_id", document.ExportId); + AddParameter(command, "tenant_id", document.TenantId); + AddParameter(command, "schema_version", document.SchemaVersion); + AddParameter(command, "generated_at", document.GeneratedAt); + AddParameter(command, "record_count", document.RecordCount); + AddParameter(command, "sha256", document.Sha256); + AddJsonbParameter(command, "manifest_json", document.ManifestJson); + AddJsonbParameter(command, "records_json", document.RecordsJson); + AddJsonbParameter(command, "lines_json", document.LinesJson); + }, + cancellationToken).ConfigureAwait(false); + } + + public async Task GetAsync(string exportId, CancellationToken cancellationToken = default) + { + const string sql = """ + SELECT + export_id, + tenant_id, + schema_version, + generated_at, + record_count, + sha256, + manifest_json::text, + records_json::text, + lines_json::text, + created_at + FROM policy.engine_ledger_exports + WHERE export_id = @export_id + """; + + return await QuerySystemSingleAsync( + sql, + command => AddParameter(command, "export_id", exportId), + Map, + cancellationToken).ConfigureAwait(false); + } + + public async Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default) + { + const string baseSql = """ + SELECT + export_id, + tenant_id, + schema_version, + generated_at, + record_count, + sha256, + manifest_json::text, + records_json::text, + lines_json::text, + created_at + FROM policy.engine_ledger_exports + """; + + const string orderBy = """ + ORDER BY generated_at ASC, export_id ASC + """; + + if (string.IsNullOrWhiteSpace(tenantId)) + { + return await QuerySystemAsync( + $"{baseSql}\n{orderBy}", + null, + Map, + cancellationToken).ConfigureAwait(false); + } + + const string filter = """ + WHERE tenant_id = @tenant_id + """; + + return await QueryAsync( + tenantId, + $"{baseSql}\n{filter}\n{orderBy}", + command => AddParameter(command, "tenant_id", tenantId), + Map, + cancellationToken).ConfigureAwait(false); + } + + private async Task QuerySystemSingleAsync( + string sql, + Action? configureCommand, + Func mapRow, + CancellationToken cancellationToken) + { + await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false); + await using var command = CreateCommand(sql, connection); + configureCommand?.Invoke(command); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + return null; + } + + return mapRow(reader); + } + + private async Task> QuerySystemAsync( + string sql, + Action? configureCommand, + Func mapRow, + CancellationToken cancellationToken) + { + await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false); + await using var command = CreateCommand(sql, connection); + configureCommand?.Invoke(command); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + var results = new List(); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + results.Add(mapRow(reader)); + } + + return results; + } + + private static PolicyEngineLedgerExportDocument Map(NpgsqlDataReader reader) + { + return new PolicyEngineLedgerExportDocument + { + ExportId = reader.GetString(0), + TenantId = reader.GetString(1), + SchemaVersion = reader.GetString(2), + GeneratedAt = reader.GetFieldValue(3), + RecordCount = reader.GetInt32(4), + Sha256 = reader.GetString(5), + ManifestJson = reader.GetString(6), + RecordsJson = reader.GetString(7), + LinesJson = reader.GetString(8), + CreatedAt = reader.GetFieldValue(9) + }; + } +} diff --git a/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/PolicyEngineOrchestratorJobRepository.cs b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/PolicyEngineOrchestratorJobRepository.cs new file mode 100644 index 000000000..26bae1137 --- /dev/null +++ b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/PolicyEngineOrchestratorJobRepository.cs @@ -0,0 +1,273 @@ +using Microsoft.Extensions.Logging; +using Npgsql; +using StellaOps.Infrastructure.Postgres.Repositories; +using StellaOps.Policy.Persistence.Postgres.Models; + +namespace StellaOps.Policy.Persistence.Postgres.Repositories; + +/// +/// PostgreSQL repository for persisted Policy Engine orchestrator jobs. +/// +public interface IPolicyEngineOrchestratorJobRepository +{ + Task SaveAsync(PolicyEngineOrchestratorJobDocument document, CancellationToken cancellationToken = default); + Task GetAsync(string jobId, CancellationToken cancellationToken = default); + Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default); + Task LeaseNextQueuedAsync(CancellationToken cancellationToken = default); +} + +/// +/// PostgreSQL repository for persisted Policy Engine orchestrator job documents. +/// +public sealed class PolicyEngineOrchestratorJobRepository + : RepositoryBase, IPolicyEngineOrchestratorJobRepository +{ + public PolicyEngineOrchestratorJobRepository( + PolicyDataSource dataSource, + ILogger logger) + : base(dataSource, logger) + { + } + + public async Task SaveAsync(PolicyEngineOrchestratorJobDocument document, CancellationToken cancellationToken = default) + { + const string sql = """ + INSERT INTO policy.orchestrator_jobs ( + job_id, + tenant_id, + context_id, + policy_profile_hash, + requested_at, + priority, + batch_items_json, + callbacks_json, + trace_ref, + status, + determinism_hash, + completed_at, + result_hash + ) + VALUES ( + @job_id, + @tenant_id, + @context_id, + @policy_profile_hash, + @requested_at, + @priority, + @batch_items_json, + @callbacks_json, + @trace_ref, + @status, + @determinism_hash, + @completed_at, + @result_hash + ) + ON CONFLICT (job_id) DO UPDATE + SET tenant_id = EXCLUDED.tenant_id, + context_id = EXCLUDED.context_id, + policy_profile_hash = EXCLUDED.policy_profile_hash, + requested_at = EXCLUDED.requested_at, + priority = EXCLUDED.priority, + batch_items_json = EXCLUDED.batch_items_json, + callbacks_json = EXCLUDED.callbacks_json, + trace_ref = EXCLUDED.trace_ref, + status = EXCLUDED.status, + determinism_hash = EXCLUDED.determinism_hash, + completed_at = EXCLUDED.completed_at, + result_hash = EXCLUDED.result_hash + """; + + await ExecuteAsync( + document.TenantId, + sql, + command => + { + AddParameter(command, "job_id", document.JobId); + AddParameter(command, "tenant_id", document.TenantId); + AddParameter(command, "context_id", document.ContextId); + AddParameter(command, "policy_profile_hash", document.PolicyProfileHash); + AddParameter(command, "requested_at", document.RequestedAt); + AddParameter(command, "priority", document.Priority); + AddJsonbParameter(command, "batch_items_json", document.BatchItemsJson); + AddJsonbParameter(command, "callbacks_json", document.CallbacksJson); + AddParameter(command, "trace_ref", document.TraceRef); + AddParameter(command, "status", document.Status); + AddParameter(command, "determinism_hash", document.DeterminismHash); + AddParameter(command, "completed_at", (object?)document.CompletedAt ?? DBNull.Value); + AddParameter(command, "result_hash", (object?)document.ResultHash ?? DBNull.Value); + }, + cancellationToken).ConfigureAwait(false); + } + + public async Task GetAsync(string jobId, CancellationToken cancellationToken = default) + { + const string sql = """ + SELECT + job_id, + tenant_id, + context_id, + policy_profile_hash, + requested_at, + priority, + batch_items_json::text, + callbacks_json::text, + trace_ref, + status, + determinism_hash, + completed_at, + result_hash, + created_at + FROM policy.orchestrator_jobs + WHERE job_id = @job_id + """; + + return await QuerySystemSingleAsync( + sql, + command => AddParameter(command, "job_id", jobId), + Map, + cancellationToken).ConfigureAwait(false); + } + + public async Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default) + { + const string baseSql = """ + SELECT + job_id, + tenant_id, + context_id, + policy_profile_hash, + requested_at, + priority, + batch_items_json::text, + callbacks_json::text, + trace_ref, + status, + determinism_hash, + completed_at, + result_hash, + created_at + FROM policy.orchestrator_jobs + """; + + const string orderBy = """ + ORDER BY requested_at ASC, job_id ASC + """; + + if (string.IsNullOrWhiteSpace(tenantId)) + { + return await QuerySystemAsync( + $"{baseSql}\n{orderBy}", + null, + Map, + cancellationToken).ConfigureAwait(false); + } + + const string filter = """ + WHERE tenant_id = @tenant_id + """; + + return await QueryAsync( + tenantId, + $"{baseSql}\n{filter}\n{orderBy}", + command => AddParameter(command, "tenant_id", tenantId), + Map, + cancellationToken).ConfigureAwait(false); + } + + public async Task LeaseNextQueuedAsync(CancellationToken cancellationToken = default) + { + const string sql = """ + WITH next_job AS ( + SELECT job_id + FROM policy.orchestrator_jobs + WHERE status = 'queued' + ORDER BY requested_at ASC, job_id ASC + LIMIT 1 + FOR UPDATE SKIP LOCKED + ) + UPDATE policy.orchestrator_jobs AS jobs + SET status = 'running', + completed_at = NULL, + result_hash = NULL + FROM next_job + WHERE jobs.job_id = next_job.job_id + RETURNING + jobs.job_id, + jobs.tenant_id, + jobs.context_id, + jobs.policy_profile_hash, + jobs.requested_at, + jobs.priority, + jobs.batch_items_json::text, + jobs.callbacks_json::text, + jobs.trace_ref, + jobs.status, + jobs.determinism_hash, + jobs.completed_at, + jobs.result_hash, + jobs.created_at + """; + + return await QuerySystemSingleAsync(sql, null, Map, cancellationToken).ConfigureAwait(false); + } + + private async Task QuerySystemSingleAsync( + string sql, + Action? configureCommand, + Func mapRow, + CancellationToken cancellationToken) + { + await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false); + await using var command = CreateCommand(sql, connection); + configureCommand?.Invoke(command); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + return null; + } + + return mapRow(reader); + } + + private async Task> QuerySystemAsync( + string sql, + Action? configureCommand, + Func mapRow, + CancellationToken cancellationToken) + { + await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false); + await using var command = CreateCommand(sql, connection); + configureCommand?.Invoke(command); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + var results = new List(); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + results.Add(mapRow(reader)); + } + + return results; + } + + private static PolicyEngineOrchestratorJobDocument Map(NpgsqlDataReader reader) + { + return new PolicyEngineOrchestratorJobDocument + { + JobId = reader.GetString(0), + TenantId = reader.GetString(1), + ContextId = reader.GetString(2), + PolicyProfileHash = reader.GetString(3), + RequestedAt = reader.GetFieldValue(4), + Priority = reader.GetString(5), + BatchItemsJson = reader.GetString(6), + CallbacksJson = reader.IsDBNull(7) ? null : reader.GetString(7), + TraceRef = reader.GetString(8), + Status = reader.GetString(9), + DeterminismHash = reader.GetString(10), + CompletedAt = reader.IsDBNull(11) ? null : reader.GetFieldValue(11), + ResultHash = reader.IsDBNull(12) ? null : reader.GetString(12), + CreatedAt = reader.GetFieldValue(13) + }; + } +} diff --git a/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/PolicyEngineSnapshotRepository.cs b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/PolicyEngineSnapshotRepository.cs new file mode 100644 index 000000000..439afdf16 --- /dev/null +++ b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/PolicyEngineSnapshotRepository.cs @@ -0,0 +1,204 @@ +using Microsoft.Extensions.Logging; +using Npgsql; +using StellaOps.Infrastructure.Postgres.Repositories; +using StellaOps.Policy.Persistence.Postgres.Models; + +namespace StellaOps.Policy.Persistence.Postgres.Repositories; + +/// +/// PostgreSQL repository for persisted Policy Engine runtime snapshot documents. +/// +public sealed class PolicyEngineSnapshotRepository + : RepositoryBase, IPolicyEngineSnapshotRepository +{ + public PolicyEngineSnapshotRepository( + PolicyDataSource dataSource, + ILogger logger) + : base(dataSource, logger) + { + } + + public async Task SaveAsync(PolicyEngineSnapshotDocument document, CancellationToken cancellationToken = default) + { + const string sql = """ + INSERT INTO policy.engine_snapshots ( + snapshot_id, + tenant_id, + ledger_export_id, + generated_at, + overlay_hash, + artifact_digest, + artifact_repository, + artifact_tag, + status_counts_json, + records_json + ) + VALUES ( + @snapshot_id, + @tenant_id, + @ledger_export_id, + @generated_at, + @overlay_hash, + @artifact_digest, + @artifact_repository, + @artifact_tag, + @status_counts_json, + @records_json + ) + ON CONFLICT (snapshot_id) DO UPDATE + SET tenant_id = EXCLUDED.tenant_id, + ledger_export_id = EXCLUDED.ledger_export_id, + generated_at = EXCLUDED.generated_at, + overlay_hash = EXCLUDED.overlay_hash, + artifact_digest = EXCLUDED.artifact_digest, + artifact_repository = EXCLUDED.artifact_repository, + artifact_tag = EXCLUDED.artifact_tag, + status_counts_json = EXCLUDED.status_counts_json, + records_json = EXCLUDED.records_json + """; + + await ExecuteAsync( + document.TenantId, + sql, + command => + { + AddParameter(command, "snapshot_id", document.SnapshotId); + AddParameter(command, "tenant_id", document.TenantId); + AddParameter(command, "ledger_export_id", document.LedgerExportId); + AddParameter(command, "generated_at", document.GeneratedAt); + AddParameter(command, "overlay_hash", document.OverlayHash); + AddParameter(command, "artifact_digest", (object?)document.ArtifactDigest ?? DBNull.Value); + AddParameter(command, "artifact_repository", (object?)document.ArtifactRepository ?? DBNull.Value); + AddParameter(command, "artifact_tag", (object?)document.ArtifactTag ?? DBNull.Value); + AddJsonbParameter(command, "status_counts_json", document.StatusCountsJson); + AddJsonbParameter(command, "records_json", document.RecordsJson); + }, + cancellationToken).ConfigureAwait(false); + } + + public async Task GetAsync(string snapshotId, CancellationToken cancellationToken = default) + { + const string sql = """ + SELECT + snapshot_id, + tenant_id, + ledger_export_id, + generated_at, + overlay_hash, + artifact_digest, + artifact_repository, + artifact_tag, + status_counts_json::text, + records_json::text, + created_at + FROM policy.engine_snapshots + WHERE snapshot_id = @snapshot_id + """; + + return await QuerySystemSingleAsync( + sql, + command => AddParameter(command, "snapshot_id", snapshotId), + Map, + cancellationToken).ConfigureAwait(false); + } + + public async Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default) + { + const string baseSql = """ + SELECT + snapshot_id, + tenant_id, + ledger_export_id, + generated_at, + overlay_hash, + artifact_digest, + artifact_repository, + artifact_tag, + status_counts_json::text, + records_json::text, + created_at + FROM policy.engine_snapshots + """; + + const string orderBy = """ + ORDER BY generated_at ASC, snapshot_id ASC + """; + + if (string.IsNullOrWhiteSpace(tenantId)) + { + return await QuerySystemAsync( + $"{baseSql}\n{orderBy}", + null, + Map, + cancellationToken).ConfigureAwait(false); + } + + const string filter = """ + WHERE tenant_id = @tenant_id + """; + + return await QueryAsync( + tenantId, + $"{baseSql}\n{filter}\n{orderBy}", + command => AddParameter(command, "tenant_id", tenantId), + Map, + cancellationToken).ConfigureAwait(false); + } + + private async Task QuerySystemSingleAsync( + string sql, + Action? configureCommand, + Func mapRow, + CancellationToken cancellationToken) + { + await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false); + await using var command = CreateCommand(sql, connection); + configureCommand?.Invoke(command); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + return null; + } + + return mapRow(reader); + } + + private async Task> QuerySystemAsync( + string sql, + Action? configureCommand, + Func mapRow, + CancellationToken cancellationToken) + { + await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false); + await using var command = CreateCommand(sql, connection); + configureCommand?.Invoke(command); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + var results = new List(); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + results.Add(mapRow(reader)); + } + + return results; + } + + private static PolicyEngineSnapshotDocument Map(NpgsqlDataReader reader) + { + return new PolicyEngineSnapshotDocument + { + SnapshotId = reader.GetString(0), + TenantId = reader.GetString(1), + LedgerExportId = reader.GetString(2), + GeneratedAt = reader.GetFieldValue(3), + OverlayHash = reader.GetString(4), + ArtifactDigest = reader.IsDBNull(5) ? null : reader.GetString(5), + ArtifactRepository = reader.IsDBNull(6) ? null : reader.GetString(6), + ArtifactTag = reader.IsDBNull(7) ? null : reader.GetString(7), + StatusCountsJson = reader.GetString(8), + RecordsJson = reader.GetString(9), + CreatedAt = reader.GetFieldValue(10) + }; + } +} diff --git a/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/WorkerResultRepository.cs b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/WorkerResultRepository.cs index 263bc1924..b40fc5da9 100644 --- a/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/WorkerResultRepository.cs +++ b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/Repositories/WorkerResultRepository.cs @@ -67,6 +67,40 @@ public sealed class WorkerResultRepository : RepositoryBase, I .ConfigureAwait(false); } + /// + public async Task> ListAsync( + string? tenantId = null, + CancellationToken cancellationToken = default) + { + const string baseSql = """ + SELECT * FROM policy.worker_results + """; + + const string orderBy = """ + ORDER BY created_at ASC, job_id ASC + """; + + if (string.IsNullOrWhiteSpace(tenantId)) + { + return await QuerySystemAsync( + $"{baseSql}\n{orderBy}", + null, + MapResult, + cancellationToken).ConfigureAwait(false); + } + + const string filter = """ + WHERE tenant_id = @tenant_id + """; + + return await QueryAsync( + tenantId, + $"{baseSql}\n{filter}\n{orderBy}", + command => AddParameter(command, "tenant_id", tenantId), + MapResult, + cancellationToken).ConfigureAwait(false); + } + /// public async Task> GetByStatusAsync( string tenantId, @@ -279,4 +313,24 @@ public sealed class WorkerResultRepository : RepositoryBase, I CreatedAt = reader.GetFieldValue(reader.GetOrdinal("created_at")), CreatedBy = GetNullableString(reader, reader.GetOrdinal("created_by")) }; + + private async Task> QuerySystemAsync( + string sql, + Action? configureCommand, + Func mapRow, + CancellationToken cancellationToken) + { + await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false); + await using var command = CreateCommand(sql, connection); + configureCommand?.Invoke(command); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + var results = new List(); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + results.Add(mapRow(reader)); + } + + return results; + } } diff --git a/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/ServiceCollectionExtensions.cs b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/ServiceCollectionExtensions.cs index 767f93bd7..712a33ce8 100644 --- a/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/ServiceCollectionExtensions.cs +++ b/src/Policy/__Libraries/StellaOps.Policy.Persistence/Postgres/ServiceCollectionExtensions.cs @@ -6,6 +6,7 @@ using ILocalRiskProfileRepository = StellaOps.Policy.Persistence.Postgres.Reposi using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using StellaOps.Infrastructure.Postgres; +using StellaOps.Infrastructure.Postgres.Migrations; using StellaOps.Infrastructure.Postgres.Options; using StellaOps.Policy.Audit; using StellaOps.Policy.Gates.Attestation; @@ -31,8 +32,13 @@ public static class ServiceCollectionExtensions IConfiguration configuration, string sectionName = "Postgres:Policy") { + services.Configure(configuration.GetSection(sectionName)); services.Configure(sectionName, configuration.GetSection(sectionName)); services.AddSingleton(); + services.AddStartupMigrations( + PolicyDataSource.DefaultSchemaName, + "Policy.Persistence", + typeof(PolicyDataSource).Assembly); // Register repositories services.AddScoped(); @@ -51,6 +57,11 @@ public static class ServiceCollectionExtensions services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); return services; } @@ -67,6 +78,10 @@ public static class ServiceCollectionExtensions { services.Configure(configureOptions); services.AddSingleton(); + services.AddStartupMigrations( + PolicyDataSource.DefaultSchemaName, + "Policy.Persistence", + typeof(PolicyDataSource).Assembly); // Register repositories services.AddScoped(); @@ -89,6 +104,9 @@ public static class ServiceCollectionExtensions // Sprint 017: Trusted key registry and gate bypass audit services.AddScoped(); services.AddScoped(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); return services; } diff --git a/src/Policy/__Libraries/StellaOps.Policy.Persistence/TASKS.md b/src/Policy/__Libraries/StellaOps.Policy.Persistence/TASKS.md index 0440fdb11..1f9f25b7c 100644 --- a/src/Policy/__Libraries/StellaOps.Policy.Persistence/TASKS.md +++ b/src/Policy/__Libraries/StellaOps.Policy.Persistence/TASKS.md @@ -13,3 +13,7 @@ Source of truth: `docs/implplan/SPRINT_20260222_089_Policy_dal_to_efcore.md`. | POLICY-EF-03 | DONE | 14 repositories converted to EF Core (partial or full); 8 complex repositories retained as raw SQL. Build passes 0W/0E. | | POLICY-EF-04 | DONE | Compiled model stubs verified; runtime factory uses UseModel on default schema; non-default schema uses reflection fallback. | | POLICY-EF-05 | DONE | Sequential build validated; AGENTS.md and TASKS.md updated; architecture doc paths corrected. | +| NOMOCK-014 | DONE | `docs/implplan/SPRINT_20260410_001_Web_runtime_no_mocks_real_backend.md`: config-driven Policy persistence now registers `IGateBypassAuditPersistence` for the live gate-bypass audit path. | +| NOMOCK-015 | DONE | `docs/implplan/SPRINT_20260410_001_Web_runtime_no_mocks_real_backend.md`: Policy persistence now owns startup-migrated `policy.engine_ledger_exports` and `policy.engine_snapshots` for the live snapshot/export runtime, and `001_initial_schema.sql` is idempotent on reused local databases. | +| NOMOCK-016 | DONE | `docs/implplan/SPRINT_20260410_001_Web_runtime_no_mocks_real_backend.md`: the persisted Policy snapshot runtime now backs merged gateway delta routes without the old pre-handler tenant failure once `policy-engine` registers the unified tenant accessor required by `RequireTenant()`. | +| NOMOCK-020 | DONE | `docs/implplan/SPRINT_20260410_001_Web_runtime_no_mocks_real_backend.md`: Policy persistence now owns runtime-migrated `policy.orchestrator_jobs` for persisted Policy upstream results, and the live first-run bootstrap path reads real `policy.orchestrator_jobs` + `policy.worker_results` instead of process-local completed-job state. | diff --git a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Deltas/PersistedDeltaRuntimeTests.cs b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Deltas/PersistedDeltaRuntimeTests.cs new file mode 100644 index 000000000..8985809d5 --- /dev/null +++ b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Deltas/PersistedDeltaRuntimeTests.cs @@ -0,0 +1,182 @@ +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Auth.ServerIntegration.Tenancy; +using StellaOps.Policy.Deltas; +using StellaOps.Policy.Engine.Ledger; +using StellaOps.Policy.Engine.Services.Gateway; +using StellaOps.Policy.Engine.Snapshots; +using Xunit; + +namespace StellaOps.Policy.Engine.Tests.Deltas; + +public sealed class PersistedKnowledgeSnapshotStoreTests +{ + [Fact] + public async Task GetAsync_ProjectsPersistedSnapshotIntoKnowledgeManifest() + { + var engineStore = new FakeEngineSnapshotStore( + [ + CreateSnapshot( + snapshotId: "snap-policy-001", + tenantId: "tenant-a", + generatedAt: "2026-04-14T12:00:00.0000000Z", + overlayHash: "overlay-a", + records: + [ + CreateRecord("tenant-a", "pkg:oci/demo/api@1.0.0", "CVE-2026-1000", "blocked"), + CreateRecord("tenant-a", "pkg:oci/demo/api@1.0.0", "CVE-2026-1000", "affected"), + CreateRecord("tenant-a", "pkg:oci/demo/api@1.0.0", "CVE-2026-1001", "unknown") + ]) + ]); + var store = new PersistedKnowledgeSnapshotStore(engineStore, new FakeTenantAccessor("tenant-a")); + + var manifest = await store.GetAsync("snap-policy-001"); + var vexBundle = await store.GetBundledContentAsync("snap-policy-001/vex.json"); + + Assert.NotNull(manifest); + Assert.Equal("snap-policy-001", manifest.SnapshotId); + Assert.Equal("overlay-a", manifest.Policy.Digest); + Assert.Contains(manifest.Sources, source => source.Name == "packages" && source.Type == StellaOps.Policy.Snapshots.KnowledgeSourceTypes.Sbom); + Assert.Contains(manifest.Sources, source => source.Name == "reachability" && source.Type == StellaOps.Policy.Snapshots.KnowledgeSourceTypes.Reachability); + Assert.Contains(manifest.Sources, source => source.Name == "vex" && source.Type == StellaOps.Policy.Snapshots.KnowledgeSourceTypes.Vex); + Assert.Contains(manifest.Sources, source => source.Name == "policy-violations"); + Assert.Contains(manifest.Sources, source => source.Name == "unknowns"); + Assert.NotNull(vexBundle); + Assert.NotEmpty(vexBundle); + } + + [Fact] + public async Task ListAsync_ReturnsMostRecentSnapshotsFirst() + { + var engineStore = new FakeEngineSnapshotStore( + [ + CreateSnapshot("snap-older", "tenant-a", "2026-04-14T11:00:00.0000000Z", "overlay-1", [CreateRecord("tenant-a", "pkg:oci/demo/api@1.0.0", "CVE-2026-1000", "allow")]), + CreateSnapshot("snap-newer", "tenant-a", "2026-04-14T12:00:00.0000000Z", "overlay-2", [CreateRecord("tenant-a", "pkg:oci/demo/api@1.1.0", "CVE-2026-1000", "allow")]) + ]); + var store = new PersistedKnowledgeSnapshotStore(engineStore, new FakeTenantAccessor("tenant-a")); + + var manifests = await store.ListAsync(); + + Assert.Collection( + manifests, + item => Assert.Equal("snap-newer", item.SnapshotId), + item => Assert.Equal("snap-older", item.SnapshotId)); + } + + internal static SnapshotDetail CreateSnapshot( + string snapshotId, + string tenantId, + string generatedAt, + string overlayHash, + IReadOnlyList records) + { + return new SnapshotDetail( + SnapshotId: snapshotId, + TenantId: tenantId, + LedgerExportId: $"exp-{snapshotId}", + GeneratedAt: generatedAt, + OverlayHash: overlayHash, + StatusCounts: records + .GroupBy(record => record.Status, StringComparer.Ordinal) + .ToDictionary(group => group.Key, group => group.Count(), StringComparer.Ordinal), + Records: records); + } + + internal static LedgerExportRecord CreateRecord(string tenantId, string componentPurl, string advisoryId, string status) + { + return new LedgerExportRecord( + TenantId: tenantId, + JobId: $"job-{advisoryId}", + ContextId: "ctx-001", + ComponentPurl: componentPurl, + AdvisoryId: advisoryId, + Status: status, + TraceRef: $"trace-{advisoryId}", + OccurredAt: "2026-04-14T12:00:00.0000000Z"); + } + + internal sealed class FakeEngineSnapshotStore : StellaOps.Policy.Engine.Snapshots.ISnapshotStore + { + private readonly Dictionary _snapshots; + + public FakeEngineSnapshotStore(IEnumerable snapshots) + { + _snapshots = snapshots.ToDictionary(snapshot => snapshot.SnapshotId, StringComparer.Ordinal); + } + + public Task SaveAsync(SnapshotDetail snapshot, CancellationToken cancellationToken = default) + { + _snapshots[snapshot.SnapshotId] = snapshot; + return Task.CompletedTask; + } + + public Task GetAsync(string snapshotId, CancellationToken cancellationToken = default) + { + _snapshots.TryGetValue(snapshotId, out var snapshot); + return Task.FromResult(snapshot); + } + + public Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default) + { + IEnumerable snapshots = _snapshots.Values; + if (!string.IsNullOrWhiteSpace(tenantId)) + { + snapshots = snapshots.Where(snapshot => string.Equals(snapshot.TenantId, tenantId, StringComparison.Ordinal)); + } + + return Task.FromResult>(snapshots.ToList()); + } + } + + private sealed class FakeTenantAccessor : IStellaOpsTenantAccessor + { + public FakeTenantAccessor(string tenantId) + { + TenantContext = new StellaOpsTenantContext + { + TenantId = tenantId, + ActorId = "test-user", + Source = TenantSource.Unknown + }; + } + + public StellaOpsTenantContext? TenantContext { get; set; } + } +} + +public sealed class DeltaSnapshotServiceAdapterTests +{ + [Fact] + public async Task GetSnapshotAsync_ProjectsPersistedRecordsIntoRealDeltaInputs() + { + var snapshot = PersistedKnowledgeSnapshotStoreTests.CreateSnapshot( + snapshotId: "snap-delta-001", + tenantId: "tenant-a", + generatedAt: "2026-04-14T12:00:00.0000000Z", + overlayHash: "overlay-delta", + records: + [ + PersistedKnowledgeSnapshotStoreTests.CreateRecord("tenant-a", "pkg:oci/demo/api@1.2.3", "CVE-2026-2000", "affected"), + PersistedKnowledgeSnapshotStoreTests.CreateRecord("tenant-a", "pkg:oci/demo/api@1.2.3", "CVE-2026-2001", "blocked"), + PersistedKnowledgeSnapshotStoreTests.CreateRecord("tenant-a", "pkg:oci/demo/api@1.2.3", "CVE-2026-2002", "unknown") + ]); + var engineStore = new PersistedKnowledgeSnapshotStoreTests.FakeEngineSnapshotStore([snapshot]); + var adapter = new DeltaSnapshotServiceAdapter(engineStore, NullLogger.Instance); + + var data = await adapter.GetSnapshotAsync("snap-delta-001"); + + Assert.NotNull(data); + Assert.Equal("snap-delta-001", data.SnapshotId); + Assert.Equal("overlay-delta", data.PolicyVersion); + Assert.Collection( + data.Packages, + package => + { + Assert.Equal("pkg:oci/demo/api@1.2.3", package.Purl); + Assert.Equal("1.2.3", package.Version); + }); + Assert.Contains(data.Reachability, item => item.CveId == "CVE-2026-2000" && item.IsReachable); + Assert.Contains(data.VexStatements, item => item.CveId == "CVE-2026-2000" && item.Status == "affected"); + Assert.Contains(data.PolicyViolations, item => item.RuleId.Contains("CVE-2026-2001", StringComparison.Ordinal)); + Assert.Contains(data.Unknowns, item => item.Id.Contains("CVE-2026-2002", StringComparison.Ordinal)); + } +} diff --git a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Integration/PolicyEngineApiHostTests.cs b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Integration/PolicyEngineApiHostTests.cs index 9a5a79142..5ed1c4d75 100644 --- a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Integration/PolicyEngineApiHostTests.cs +++ b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Integration/PolicyEngineApiHostTests.cs @@ -1,4 +1,6 @@ using System.Net; +using System.Net.Http.Json; +using System.Globalization; using System.Security.Claims; using System.Text.Encodings.Web; using Microsoft.AspNetCore.Authentication; @@ -11,7 +13,16 @@ using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using StellaOps.Auth.Abstractions; +using StellaOps.Auth.ServerIntegration.Tenancy; +using StellaOps.Policy.Audit; +using StellaOps.Policy.Engine.Contracts.Gateway; using StellaOps.Policy.Engine.Attestation; +using StellaOps.Policy.Engine.Ledger; +using StellaOps.Policy.Engine.Orchestration; +using StellaOps.Policy.Engine.Snapshots; +using StellaOps.Policy.Engine.Tenancy; +using StellaOps.Policy.Engine.Workers; +using StellaOps.Policy.Persistence.Postgres; using StellaOps.TestKit.Fixtures; using Xunit; @@ -78,6 +89,244 @@ public sealed class PolicyEngineApiHostTests : IClassFixture +{ + private readonly PolicyEngineWebServiceFixture _factory; + + public PolicyEngineGateBypassAuditRegistrationTests(PolicyEngineWebServiceFixture factory) + { + _factory = factory; + } + + [Fact] + public void GateBypassAuditRepository_ResolvesToPostgresAdapter_WithDefaultTenantFallback() + { + using var scope = _factory.Services.CreateScope(); + var tenantAccessor = scope.ServiceProvider.GetRequiredService(); + tenantAccessor.TenantContext = null; + + var repository = scope.ServiceProvider.GetRequiredService(); + + var postgresRepository = Assert.IsType(repository); + Assert.Equal(TenantContextConstants.DefaultTenantId, ReadTenantId(postgresRepository)); + } + + [Fact] + public void GateBypassAuditRepository_ResolvesToPostgresAdapter_WithCurrentTenant() + { + using var scope = _factory.Services.CreateScope(); + var tenantAccessor = scope.ServiceProvider.GetRequiredService(); + tenantAccessor.TenantContext = TenantContext.ForTenant("test-tenant"); + + var repository = scope.ServiceProvider.GetRequiredService(); + + var postgresRepository = Assert.IsType(repository); + Assert.Equal("test-tenant", ReadTenantId(postgresRepository)); + } + + private static string ReadTenantId(PostgresGateBypassAuditRepository repository) + { + var tenantIdField = typeof(PostgresGateBypassAuditRepository).GetField( + "_tenantId", + System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic); + + Assert.NotNull(tenantIdField); + return Assert.IsType(tenantIdField!.GetValue(repository)); + } +} + +public sealed class PolicyEngineRuntimeStoreRegistrationTests : IClassFixture +{ + private readonly PolicyEngineWebServiceFixture _factory; + + public PolicyEngineRuntimeStoreRegistrationTests(PolicyEngineWebServiceFixture factory) + { + _factory = factory; + } + + [Fact] + public void LedgerExportStore_ResolvesToPostgresAdapter() + { + var repository = _factory.Services.GetRequiredService(); + Assert.IsType(repository); + } + + [Fact] + public void SnapshotStore_ResolvesToPostgresAdapter() + { + var repository = _factory.Services.GetRequiredService(); + Assert.IsType(repository); + } + + [Fact] + public void OrchestratorJobStore_ResolvesToPersistedAdapter() + { + var repository = _factory.Services.GetRequiredService(); + Assert.IsType(repository); + } + + [Fact] + public void WorkerResultStore_ResolvesToPersistedAdapter() + { + var repository = _factory.Services.GetRequiredService(); + Assert.IsType(repository); + } + + [Fact] + public void SharedSnapshotStore_ResolvesToPersistedCompatibilityAdapter() + { + var repository = _factory.Services.GetRequiredService(); + Assert.IsType(repository); + } + + [Fact] + public void StellaOpsTenantAccessor_ResolvesForMergedGatewayRouteFilters() + { + var accessor = _factory.Services.GetRequiredService(); + Assert.NotNull(accessor); + } +} + +public sealed class PolicyEngineOrchestratorProducerRuntimeTests : IClassFixture +{ + private readonly PolicyEngineOrchestratorProducerFixture _factory; + + public PolicyEngineOrchestratorProducerRuntimeTests(PolicyEngineOrchestratorProducerFixture factory) + { + _factory = factory; + } + + [Fact] + public async Task SubmitJob_AutoExecutesQueuedWork_AndPersistsWorkerResult() + { + using var client = _factory.CreateClient(); + client.DefaultRequestHeaders.Add(TestAuthHandler.HeaderName, TestAuthHandler.HeaderValue); + + var tenantId = $"producer-auto-{Guid.NewGuid():N}"; + var request = new OrchestratorJobRequest( + TenantId: tenantId, + ContextId: "ctx-bootstrap", + PolicyProfileHash: "profile-demo", + BatchItems: + [ + new OrchestratorJobItem("pkg:oci/demo/api@2.0.0", "CVE-2026-1000"), + new OrchestratorJobItem("pkg:oci/demo/api@2.0.0", "CVE-2026-1001") + ], + RequestedAt: DateTimeOffset.Parse("2026-04-15T12:00:00Z", CultureInfo.InvariantCulture)); + + var submitResponse = await client.PostAsJsonAsync("/policy/orchestrator/jobs", request); + submitResponse.EnsureSuccessStatusCode(); + + var submitted = await submitResponse.Content.ReadFromJsonAsync(); + Assert.NotNull(submitted); + Assert.Equal("queued", submitted.Status); + + var completed = await WaitForCompletedJobAsync(client, submitted.JobId); + Assert.NotNull(completed); + Assert.Equal("completed", completed.Status); + Assert.NotNull(completed.CompletedAt); + Assert.False(string.IsNullOrWhiteSpace(completed.ResultHash)); + + var resultResponse = await client.GetAsync($"/policy/worker/jobs/{submitted.JobId}"); + resultResponse.EnsureSuccessStatusCode(); + + var result = await resultResponse.Content.ReadFromJsonAsync(); + Assert.NotNull(result); + Assert.Equal(submitted.JobId, result.JobId); + Assert.Equal("policy-orchestrator-worker", result.WorkerId); + Assert.Equal(completed.ResultHash, result.ResultHash); + Assert.Equal(2, result.Results.Count); + Assert.All(result.Results, item => Assert.False(string.IsNullOrWhiteSpace(item.Status))); + } + + private static async Task WaitForCompletedJobAsync(HttpClient client, string jobId) + { + for (var attempt = 0; attempt < 40; attempt++) + { + var response = await client.GetAsync($"/policy/orchestrator/jobs/{jobId}"); + response.EnsureSuccessStatusCode(); + + var job = await response.Content.ReadFromJsonAsync(); + Assert.NotNull(job); + + if (string.Equals(job.Status, "completed", StringComparison.Ordinal)) + { + return job; + } + + await Task.Delay(TimeSpan.FromMilliseconds(250)); + } + + throw new TimeoutException($"Timed out waiting for orchestrator job {jobId} to complete."); + } +} + +public sealed class PolicyEngineDeltaApiTests : IClassFixture +{ + private readonly PolicyEngineDeltaApiFixture _factory; + + public PolicyEngineDeltaApiTests(PolicyEngineDeltaApiFixture factory) + { + _factory = factory; + } + + [Fact] + public async Task ComputeDelta_UsesPersistedSnapshotsInsteadOfInMemoryCompatibilityStore() + { + var baselineSnapshot = Deltas.PersistedKnowledgeSnapshotStoreTests.CreateSnapshot( + snapshotId: "snap-delta-baseline", + tenantId: "test-tenant", + generatedAt: "2026-04-14T10:00:00.0000000Z", + overlayHash: "overlay-base", + records: + [ + Deltas.PersistedKnowledgeSnapshotStoreTests.CreateRecord("test-tenant", "pkg:oci/demo/api@1.0.0", "CVE-2026-3000", "allow") + ]); + var targetSnapshot = Deltas.PersistedKnowledgeSnapshotStoreTests.CreateSnapshot( + snapshotId: "snap-delta-target", + tenantId: "test-tenant", + generatedAt: "2026-04-14T11:00:00.0000000Z", + overlayHash: "overlay-target", + records: + [ + Deltas.PersistedKnowledgeSnapshotStoreTests.CreateRecord("test-tenant", "pkg:oci/demo/api@1.0.0", "CVE-2026-3000", "blocked") + ]); + + var snapshotStore = _factory.Services.GetRequiredService(); + await snapshotStore.SaveAsync(baselineSnapshot); + await snapshotStore.SaveAsync(targetSnapshot); + + using var client = _factory.CreateClient(); + client.DefaultRequestHeaders.Add(TestAuthHandler.HeaderName, TestAuthHandler.HeaderValue); + + var response = await client.PostAsJsonAsync( + "/api/policy/deltas/compute", + new ComputeDeltaRequest + { + ArtifactDigest = "sha256:test-artifact", + ArtifactName = "demo-api", + TargetSnapshotId = targetSnapshot.SnapshotId, + BaselineSnapshotId = baselineSnapshot.SnapshotId + }); + + var responseBody = await response.Content.ReadAsStringAsync(); + if (!response.IsSuccessStatusCode) + { + throw new InvalidOperationException( + $"ComputeDelta returned {(int)response.StatusCode} {response.StatusCode}:{Environment.NewLine}{responseBody}"); + } + + var payload = await response.Content.ReadFromJsonAsync(); + + Assert.NotNull(payload); + Assert.Equal(baselineSnapshot.SnapshotId, payload.BaselineSnapshotId); + Assert.Equal(targetSnapshot.SnapshotId, payload.TargetSnapshotId); + Assert.True(payload.DriverCount >= 1); + Assert.True(payload.Summary.RiskIncreasing >= 1); + } } public sealed class PolicyEngineWebServiceFixture : WebServiceFixture @@ -87,7 +336,7 @@ public sealed class PolicyEngineWebServiceFixture : WebServiceFixture(); services.RemoveAll>(); @@ -108,7 +357,7 @@ public sealed class PolicyEngineWebServiceFixture : WebServiceFixture { }); } - private static void ConfigureTestWebHost(IWebHostBuilder builder) + internal static void ConfigureTestWebHost(IWebHostBuilder builder) { builder.ConfigureAppConfiguration((_, config) => { @@ -116,6 +365,7 @@ public sealed class PolicyEngineWebServiceFixture : WebServiceFixture +{ + public PolicyEngineDeltaApiFixture() + : base(ConfigureDeltaTestServices, PolicyEngineWebServiceFixture.ConfigureTestWebHost) + { + } + + private static void ConfigureDeltaTestServices(IServiceCollection services) + { + PolicyEngineWebServiceFixture.ConfigureTestServices(services); + + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + + services.AddSingleton(); + services.AddScoped(); + services.AddScoped(); + } +} + +public sealed class PolicyEngineOrchestratorProducerFixture : WebServiceFixture +{ + public PolicyEngineOrchestratorProducerFixture() + : base(ConfigureProducerServices, PolicyEngineWebServiceFixture.ConfigureTestWebHost) + { + } + + private static void ConfigureProducerServices(IServiceCollection services) + { + PolicyEngineWebServiceFixture.ConfigureTestServices(services); + + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + + services.AddSingleton(); + services.AddSingleton(); + services.AddHostedService(); + } +} + internal sealed class TestAuthHandler : AuthenticationHandler { public const string SchemeName = "Test"; @@ -152,7 +444,8 @@ internal sealed class TestAuthHandler : AuthenticationHandler(); + var snapshots = await snapshotStore.ListAsync(tenantId); + var targetSnapshot = snapshots + .Single(snapshot => string.Equals(snapshot.ArtifactDigest, "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", StringComparison.Ordinal)); + + Assert.Equal("demo/api", targetSnapshot.ArtifactRepository); + Assert.Equal("2.0.0", targetSnapshot.ArtifactTag); + Assert.NotEqual(baselineSnapshotId, targetSnapshot.SnapshotId); + } + + [Fact] + public async Task EvaluateGate_AutoBootstrapsBaselineSnapshot_WhenNoBaselineExists() + { + using var fixture = new PolicyEngineGateEvaluationFixture(); + const string tenantId = "test-tenant"; + await SeedCompletedPolicyResultAsync(fixture, tenantId); + + using var client = CreateAuthenticatedClient(fixture); + var response = await client.PostAsJsonAsync( + "/api/v1/policy/gate/evaluate", + new + { + imageDigest = "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + repository = "demo/api", + tag = "3.0.0", + source = "integration-test" + }); + + var body = await response.Content.ReadAsStringAsync(); + Assert.True( + response.StatusCode is HttpStatusCode.OK or HttpStatusCode.Forbidden, + $"Expected bootstrapped gate evaluation, got {(int)response.StatusCode} {response.StatusCode}:{Environment.NewLine}{body}"); + + var payload = await response.Content.ReadFromJsonAsync(); + Assert.NotNull(payload); + Assert.NotEqual("First build - no baseline for comparison", payload!.Summary); + + var snapshotStore = fixture.Services.GetRequiredService(); + var snapshots = await snapshotStore.ListAsync(tenantId); + var baselineSnapshot = snapshots.Single(snapshot => + string.Equals(snapshot.OverlayHash, "bootstrap", StringComparison.Ordinal) && + string.IsNullOrWhiteSpace(snapshot.ArtifactDigest)); + var targetSnapshot = snapshots.Single(snapshot => + string.Equals(snapshot.ArtifactDigest, "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", StringComparison.Ordinal)); + + Assert.Equal(2, snapshots.Count); + Assert.Equal("demo/api", targetSnapshot.ArtifactRepository); + Assert.Equal("3.0.0", targetSnapshot.ArtifactTag); + Assert.Equal(baselineSnapshot.LedgerExportId, targetSnapshot.LedgerExportId); + + var ledgerStore = fixture.Services.GetRequiredService(); + var exports = await ledgerStore.ListAsync(tenantId); + var export = Assert.Single(exports); + Assert.NotEmpty(export.Records); + } + + [Fact] + public async Task EvaluateGate_WithoutPolicyDataAndWithoutBaseline_ReturnsFirstBuildWithoutPersistingRuntimeState() + { + using var fixture = new PolicyEngineGateEvaluationFixture(); + const string tenantId = "test-tenant"; + + using var client = CreateAuthenticatedClient(fixture); + var response = await client.PostAsJsonAsync( + "/api/v1/policy/gate/evaluate", + new + { + imageDigest = "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + repository = "demo/api", + tag = "4.0.0", + source = "integration-test" + }); + + var body = await response.Content.ReadAsStringAsync(); + Assert.Equal( + HttpStatusCode.OK, + response.StatusCode); + + var payload = await response.Content.ReadFromJsonAsync(); + Assert.NotNull(payload); + Assert.Equal("First build - no baseline for comparison", payload!.Summary); + Assert.Contains("Future builds will be compared against this baseline.", payload.Advisory, StringComparison.Ordinal); + + var snapshotStore = fixture.Services.GetRequiredService(); + Assert.Empty(await snapshotStore.ListAsync(tenantId)); + + var ledgerStore = fixture.Services.GetRequiredService(); + Assert.Empty(await ledgerStore.ListAsync(tenantId)); + } + + private static HttpClient CreateAuthenticatedClient(PolicyEngineGateEvaluationFixture fixture) + { + var client = fixture.CreateClient(); + client.DefaultRequestHeaders.Add(TestAuthHandler.HeaderName, TestAuthHandler.HeaderValue); + return client; + } + + private static async Task SeedBaselineSnapshotAsync(PolicyEngineGateEvaluationFixture fixture, string tenantId) + { + await SeedCompletedPolicyResultAsync(fixture, tenantId); + + var snapshotService = fixture.Services.GetRequiredService(); + var snapshot = await snapshotService.CreateAsync(new SnapshotRequest( + tenantId, + "overlay-gate-runtime", + ArtifactDigest: "sha256:baseline0000000000000000000000000000000000000000000000000000000000", + ArtifactRepository: "demo/api", + ArtifactTag: "1.0.0")); + + return snapshot.SnapshotId; + } + + private static async Task SeedCompletedPolicyResultAsync(PolicyEngineGateEvaluationFixture fixture, string tenantId) + { + var jobStore = fixture.Services.GetRequiredService(); + var resultStore = fixture.Services.GetRequiredService(); + + var requestedAt = DateTimeOffset.Parse("2026-04-15T13:00:00Z"); + var job = new OrchestratorJob( + JobId: $"job-baseline-{tenantId}", + TenantId: tenantId, + ContextId: "ctx-gate-001", + PolicyProfileHash: "profile-hash", + RequestedAt: requestedAt, + Priority: "normal", + BatchItems: [new OrchestratorJobItem("pkg:oci/demo/api@1.0.0", "CVE-2026-5000")], + Callbacks: null, + TraceRef: "trace-gate-001", + Status: "completed", + DeterminismHash: "det-hash", + CompletedAt: requestedAt, + ResultHash: "result-hash"); + + await jobStore.SaveAsync(job); + await resultStore.SaveAsync(new WorkerRunResult( + job.JobId, + "worker", + requestedAt, + requestedAt, + [new WorkerResultItem("pkg:oci/demo/api@1.0.0", "CVE-2026-5000", "blocked", "trace-gate-001")], + "worker-hash")); + } +} + +public sealed class PolicyEngineGateEvaluationFixture : WebServiceFixture +{ + public PolicyEngineGateEvaluationFixture() + : base(ConfigureServices, PolicyEngineWebServiceFixture.ConfigureTestWebHost) + { + } + + private static void ConfigureServices(IServiceCollection services) + { + PolicyEngineWebServiceFixture.ConfigureTestServices(services); + + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddScoped(); + services.AddScoped(); + } +} diff --git a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Integration/PolicyEngineRegistryWebhookRuntimeTests.cs b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Integration/PolicyEngineRegistryWebhookRuntimeTests.cs new file mode 100644 index 000000000..b54563c6f --- /dev/null +++ b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Integration/PolicyEngineRegistryWebhookRuntimeTests.cs @@ -0,0 +1,578 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using StellaOps.Auth.ServerIntegration.Tenancy; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; +using StellaOps.Determinism; +using StellaOps.Policy.Engine.Contracts.Gateway; +using StellaOps.Policy.Engine.Services.Gateway; +using StellaOps.Policy.Persistence.Postgres.Models; +using StellaOps.Policy.Persistence.Postgres.Repositories; +using StellaOps.Scheduler.Persistence.Postgres.Models; +using StellaOps.Scheduler.Persistence.Postgres.Repositories; +using StellaOps.TestKit.Fixtures; +using Xunit; + +namespace StellaOps.Policy.Engine.Tests.Integration; + +public sealed class PolicyEngineRegistryWebhookRuntimeTests : IClassFixture +{ + private readonly PolicyEngineWebServiceFixture _factory; + + public PolicyEngineRegistryWebhookRuntimeTests(PolicyEngineWebServiceFixture factory) + { + _factory = factory; + } + + [Fact] + public void GateEvaluationQueue_ResolvesToUnsupportedRuntime() + { + var queue = _factory.Services.GetRequiredService(); + + Assert.IsType(queue); + } + + [Fact] + public async Task GenericRegistryWebhook_ReturnsNotImplemented_WhenAsyncQueueIsUnavailable() + { + using var client = _factory.CreateClient(); + client.DefaultRequestHeaders.Add(TestAuthHandler.HeaderName, TestAuthHandler.HeaderValue); + client.DefaultRequestHeaders.Add("X-StellaOps-Tenant", "test-tenant"); + + var response = await client.PostAsJsonAsync( + "/api/v1/webhooks/registry/generic", + new + { + imageDigest = "sha256:2222222222222222222222222222222222222222222222222222222222222222", + repository = "demo/api", + tag = "2.0.0", + baselineRef = "baseline-002", + source = "test" + }); + + var body = await response.Content.ReadAsStringAsync(); + using var json = JsonDocument.Parse(body); + + Assert.Equal(HttpStatusCode.NotImplemented, response.StatusCode); + Assert.Equal("Async gate evaluation queue unavailable", json.RootElement.GetProperty("title").GetString()); + Assert.Contains( + "scheduler-backed async gate evaluation queue", + json.RootElement.GetProperty("detail").GetString(), + StringComparison.Ordinal); + } +} + +public sealed class PolicyEngineSchedulerWebhookRuntimeTests +{ + [Fact] + public async Task SchedulerBackedRuntime_CanCompleteQueuedJob_AndResolveCompletedStatus_InFocusedHost() + { + var jobs = new FakeJobRepository(); + var workerResults = new FakeWorkerResultRepository(); + + await using var factory = CreateSchedulerFactory(jobs, workerResults); + using var scope = factory.Services.CreateScope(); + var tenantAccessor = scope.ServiceProvider.GetRequiredService(); + tenantAccessor.TenantContext = new StellaOpsTenantContext + { + TenantId = "test-tenant", + ActorId = "test-user", + Source = TenantSource.Unknown + }; + + var queue = scope.ServiceProvider.GetRequiredService(); + Assert.IsType(queue); + + var jobId = await queue.EnqueueAsync(new GateEvaluationRequest + { + ImageDigest = "sha256:4444444444444444444444444444444444444444444444444444444444444444", + Repository = "demo/api", + Tag = "4.0.0", + BaselineRef = "baseline-004", + Source = "test", + Timestamp = DateTimeOffset.Parse("2026-04-15T12:00:00Z") + }); + + Assert.NotNull(await jobs.GetByIdAsync("test-tenant", Guid.Parse(jobId), CancellationToken.None)); + Assert.NotNull(await workerResults.GetByJobAsync("test-tenant", "policy.gate-evaluation", jobId, CancellationToken.None)); + + var dispatcher = scope.ServiceProvider.GetRequiredService(); + var processed = await dispatcher.RunPendingOnceAsync(); + + Assert.Equal(1, processed); + + var statusService = scope.ServiceProvider.GetRequiredService(); + var status = await statusService.GetAsync(jobId); + + Assert.NotNull(status); + Assert.Equal("completed", status!.Status); + Assert.Equal("succeeded", status.SchedulerStatus); + Assert.Equal("decision:demo/api", status.Decision!.DecisionId); + Assert.Equal("Gate passed - release may proceed", status.Decision.Summary); + Assert.Equal("sha256:4444444444444444444444444444444444444444444444444444444444444444", status.Request!.ImageDigest); + } + + private static WebServiceFixture CreateSchedulerFactory( + FakeJobRepository jobs, + FakeWorkerResultRepository workerResults) + { + return new WebServiceFixture( + services => + { + PolicyEngineWebServiceFixture.ConfigureTestServices(services); + + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + + services.AddSingleton(new GateEvaluationQueueOptions + { + LeaseDurationSeconds = 30, + MaxAttempts = 2, + BatchSize = 10 + }); + services.AddSingleton(jobs); + services.AddSingleton(workerResults); + services.AddSingleton(new SequentialGuidProvider(Guid.Empty)); + services.AddSingleton(); + services.AddSingleton(); + services.AddScoped(); + services.AddScoped(); + }, + PolicyEngineWebServiceFixture.ConfigureTestWebHost); + } + + private sealed class FakeGateEvaluationJobExecutor : IGateEvaluationJobExecutor + { + public Task ExecuteAsync(GateEvaluationRequest request, CancellationToken cancellationToken = default) + { + return Task.FromResult(new GateEvaluateResponse + { + DecisionId = $"decision:{request.Repository}", + Status = GateStatus.Pass, + ExitCode = GateExitCodes.Pass, + ImageDigest = request.ImageDigest, + BaselineRef = request.BaselineRef, + DecidedAt = DateTimeOffset.Parse("2026-04-15T12:02:00Z"), + Summary = "Gate passed - release may proceed", + Advisory = "Queued policy gate completed successfully." + }); + } + } + + private sealed class FakeJobRepository : IJobRepository + { + private static readonly DateTimeOffset DefaultTimestamp = DateTimeOffset.Parse("2026-04-15T12:00:00Z"); + private readonly object _gate = new(); + private readonly Dictionary<(string TenantId, Guid JobId), StoredJob> _jobs = []; + + public Task CreateAsync(JobEntity job, CancellationToken cancellationToken = default) + { + lock (_gate) + { + var stored = StoredJob.From(job); + if (stored.CreatedAt == default) + { + stored.CreatedAt = DefaultTimestamp; + } + + _jobs[(stored.TenantId, stored.Id)] = stored; + return Task.FromResult(stored.ToEntity()); + } + } + + public Task GetByIdAsync(string tenantId, Guid id, CancellationToken cancellationToken = default) + { + lock (_gate) + { + return Task.FromResult(_jobs.TryGetValue((tenantId, id), out var stored) ? stored.ToEntity() : null); + } + } + + public Task GetByIdempotencyKeyAsync(string tenantId, string idempotencyKey, CancellationToken cancellationToken = default) + { + lock (_gate) + { + var stored = _jobs.Values.FirstOrDefault(job => + string.Equals(job.TenantId, tenantId, StringComparison.Ordinal) && + string.Equals(job.IdempotencyKey, idempotencyKey, StringComparison.Ordinal)); + return Task.FromResult(stored?.ToEntity()); + } + } + + public Task> GetScheduledJobsAsync(string tenantId, string[] jobTypes, int limit = 10, CancellationToken cancellationToken = default) + { + lock (_gate) + { + return Task.FromResult>( + _jobs.Values + .Where(job => + string.Equals(job.TenantId, tenantId, StringComparison.Ordinal) && + jobTypes.Contains(job.JobType, StringComparer.Ordinal) && + job.Status is JobStatus.Pending or JobStatus.Scheduled) + .Take(limit) + .Select(job => job.ToEntity()) + .ToList()); + } + } + + public Task TryLeaseJobAsync(string tenantId, Guid jobId, string workerId, TimeSpan leaseDuration, CancellationToken cancellationToken = default) + { + lock (_gate) + { + if (!_jobs.TryGetValue((tenantId, jobId), out var stored) || + stored.Status is not (JobStatus.Pending or JobStatus.Scheduled)) + { + return Task.FromResult(null); + } + + stored.Status = JobStatus.Leased; + stored.LeaseId = Guid.Parse("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"); + stored.WorkerId = workerId; + stored.LeasedAt = DefaultTimestamp.AddMinutes(1); + stored.StartedAt = DefaultTimestamp.AddMinutes(1); + stored.LeaseUntil = stored.LeasedAt.Value.Add(leaseDuration); + + return Task.FromResult(stored.ToEntity()); + } + } + + public Task ExtendLeaseAsync(string tenantId, Guid jobId, Guid leaseId, TimeSpan extension, CancellationToken cancellationToken = default) + { + lock (_gate) + { + if (!_jobs.TryGetValue((tenantId, jobId), out var stored) || stored.LeaseId != leaseId) + { + return Task.FromResult(false); + } + + stored.LeaseUntil = (stored.LeaseUntil ?? DefaultTimestamp).Add(extension); + return Task.FromResult(true); + } + } + + public Task CompleteAsync(string tenantId, Guid jobId, Guid leaseId, string? result = null, CancellationToken cancellationToken = default) + { + lock (_gate) + { + if (!_jobs.TryGetValue((tenantId, jobId), out var stored) || stored.LeaseId != leaseId) + { + return Task.FromResult(false); + } + + stored.Status = JobStatus.Succeeded; + stored.Result = result; + stored.CompletedAt = DefaultTimestamp.AddMinutes(2); + return Task.FromResult(true); + } + } + + public Task FailAsync(string tenantId, Guid jobId, Guid leaseId, string reason, bool retry = true, CancellationToken cancellationToken = default) + { + lock (_gate) + { + if (!_jobs.TryGetValue((tenantId, jobId), out var stored) || stored.LeaseId != leaseId) + { + return Task.FromResult(false); + } + + stored.Status = retry ? JobStatus.Scheduled : JobStatus.Failed; + stored.Reason = reason; + stored.Attempt++; + return Task.FromResult(true); + } + } + + public Task CancelAsync(string tenantId, Guid jobId, string reason, CancellationToken cancellationToken = default) + { + lock (_gate) + { + if (!_jobs.TryGetValue((tenantId, jobId), out var stored)) + { + return Task.FromResult(false); + } + + stored.Status = JobStatus.Canceled; + stored.Reason = reason; + return Task.FromResult(true); + } + } + + public Task RecoverExpiredLeasesAsync(string tenantId, CancellationToken cancellationToken = default) + => Task.FromResult(0); + + public Task> GetByStatusAsync(string tenantId, JobStatus status, int limit = 100, int offset = 0, CancellationToken cancellationToken = default) + { + lock (_gate) + { + return Task.FromResult>( + _jobs.Values + .Where(job => string.Equals(job.TenantId, tenantId, StringComparison.Ordinal) && job.Status == status) + .Skip(offset) + .Take(limit) + .Select(job => job.ToEntity()) + .ToList()); + } + } + + private sealed class StoredJob + { + public required Guid Id { get; init; } + public required string TenantId { get; init; } + public string? ProjectId { get; init; } + public required string JobType { get; init; } + public JobStatus Status { get; set; } + public int Priority { get; init; } + public required string Payload { get; init; } + public required string PayloadDigest { get; init; } + public required string IdempotencyKey { get; init; } + public string? CorrelationId { get; init; } + public int Attempt { get; set; } + public int MaxAttempts { get; init; } + public Guid? LeaseId { get; set; } + public string? WorkerId { get; set; } + public DateTimeOffset? LeaseUntil { get; set; } + public DateTimeOffset? NotBefore { get; init; } + public string? Reason { get; set; } + public string? Result { get; set; } + public DateTimeOffset CreatedAt { get; set; } + public DateTimeOffset? ScheduledAt { get; init; } + public DateTimeOffset? LeasedAt { get; set; } + public DateTimeOffset? StartedAt { get; set; } + public DateTimeOffset? CompletedAt { get; set; } + public string? CreatedBy { get; init; } + + public static StoredJob From(JobEntity job) + { + return new StoredJob + { + Id = job.Id, + TenantId = job.TenantId, + ProjectId = job.ProjectId, + JobType = job.JobType, + Status = job.Status, + Priority = job.Priority, + Payload = job.Payload, + PayloadDigest = job.PayloadDigest, + IdempotencyKey = job.IdempotencyKey, + CorrelationId = job.CorrelationId, + Attempt = job.Attempt, + MaxAttempts = job.MaxAttempts, + LeaseId = job.LeaseId, + WorkerId = job.WorkerId, + LeaseUntil = job.LeaseUntil, + NotBefore = job.NotBefore, + Reason = job.Reason, + Result = job.Result, + CreatedAt = job.CreatedAt, + ScheduledAt = job.ScheduledAt, + LeasedAt = job.LeasedAt, + StartedAt = job.StartedAt, + CompletedAt = job.CompletedAt, + CreatedBy = job.CreatedBy + }; + } + + public JobEntity ToEntity() + { + return new JobEntity + { + Id = Id, + TenantId = TenantId, + ProjectId = ProjectId, + JobType = JobType, + Status = Status, + Priority = Priority, + Payload = Payload, + PayloadDigest = PayloadDigest, + IdempotencyKey = IdempotencyKey, + CorrelationId = CorrelationId, + Attempt = Attempt, + MaxAttempts = MaxAttempts, + LeaseId = LeaseId, + WorkerId = WorkerId, + LeaseUntil = LeaseUntil, + NotBefore = NotBefore, + Reason = Reason, + Result = Result, + CreatedAt = CreatedAt, + ScheduledAt = ScheduledAt, + LeasedAt = LeasedAt, + StartedAt = StartedAt, + CompletedAt = CompletedAt, + CreatedBy = CreatedBy + }; + } + } + } + + private sealed class FakeWorkerResultRepository : IWorkerResultRepository + { + private static readonly DateTimeOffset DefaultTimestamp = DateTimeOffset.Parse("2026-04-15T12:00:00Z"); + private readonly object _gate = new(); + private readonly Dictionary<(string TenantId, Guid Id), WorkerResultEntity> _results = []; + + public Task CreateAsync(WorkerResultEntity result, CancellationToken cancellationToken = default) + { + lock (_gate) + { + var stored = result with + { + CreatedAt = result.CreatedAt == default ? DefaultTimestamp : result.CreatedAt + }; + _results[(stored.TenantId, stored.Id)] = stored; + return Task.FromResult(stored); + } + } + + public Task GetByIdAsync(string tenantId, Guid id, CancellationToken cancellationToken = default) + { + lock (_gate) + { + return Task.FromResult(_results.TryGetValue((tenantId, id), out var result) ? result : null); + } + } + + public Task GetByJobAsync(string tenantId, string jobType, string jobId, CancellationToken cancellationToken = default) + { + lock (_gate) + { + var result = _results.Values.FirstOrDefault(item => + string.Equals(item.TenantId, tenantId, StringComparison.Ordinal) && + string.Equals(item.JobType, jobType, StringComparison.Ordinal) && + string.Equals(item.JobId, jobId, StringComparison.Ordinal)); + return Task.FromResult(result); + } + } + + public Task> GetByStatusAsync(string tenantId, string status, int limit = 100, CancellationToken cancellationToken = default) + { + lock (_gate) + { + return Task.FromResult>( + _results.Values + .Where(item => + string.Equals(item.TenantId, tenantId, StringComparison.Ordinal) && + string.Equals(item.Status, status, StringComparison.Ordinal)) + .Take(limit) + .ToList()); + } + } + + public Task> GetPendingAsync(string? jobType = null, int limit = 100, CancellationToken cancellationToken = default) + { + lock (_gate) + { + return Task.FromResult>( + _results.Values + .Where(item => + string.Equals(item.Status, "pending", StringComparison.Ordinal) && + (jobType is null || string.Equals(item.JobType, jobType, StringComparison.Ordinal))) + .OrderBy(item => item.CreatedAt) + .Take(limit) + .ToList()); + } + } + + public Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default) + { + lock (_gate) + { + IEnumerable query = _results.Values; + if (!string.IsNullOrWhiteSpace(tenantId)) + { + query = query.Where(item => string.Equals(item.TenantId, tenantId, StringComparison.Ordinal)); + } + + return Task.FromResult>( + query + .OrderBy(item => item.CreatedAt) + .ThenBy(item => item.JobId, StringComparer.Ordinal) + .ToList()); + } + } + + public Task UpdateProgressAsync(string tenantId, Guid id, string status, int progress, string? errorMessage = null, CancellationToken cancellationToken = default) + { + lock (_gate) + { + if (!_results.TryGetValue((tenantId, id), out var result)) + { + return Task.FromResult(false); + } + + _results[(tenantId, id)] = result with + { + Status = status, + Progress = progress, + ErrorMessage = errorMessage + }; + return Task.FromResult(true); + } + } + + public Task CompleteAsync(string tenantId, Guid id, string result, string? outputHash = null, CancellationToken cancellationToken = default) + { + lock (_gate) + { + if (!_results.TryGetValue((tenantId, id), out var existing)) + { + return Task.FromResult(false); + } + + _results[(tenantId, id)] = existing with + { + Status = "completed", + Progress = 100, + Result = result, + OutputHash = outputHash, + CompletedAt = DefaultTimestamp.AddMinutes(2) + }; + return Task.FromResult(true); + } + } + + public Task FailAsync(string tenantId, Guid id, string errorMessage, CancellationToken cancellationToken = default) + { + lock (_gate) + { + if (!_results.TryGetValue((tenantId, id), out var existing)) + { + return Task.FromResult(false); + } + + _results[(tenantId, id)] = existing with + { + Status = "failed", + ErrorMessage = errorMessage + }; + return Task.FromResult(true); + } + } + + public Task IncrementRetryAsync(string tenantId, Guid id, CancellationToken cancellationToken = default) + { + lock (_gate) + { + if (!_results.TryGetValue((tenantId, id), out var existing)) + { + return Task.FromResult(false); + } + + _results[(tenantId, id)] = existing with + { + RetryCount = existing.RetryCount + 1 + }; + return Task.FromResult(true); + } + } + } +} diff --git a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Ledger/PostgresLedgerExportStoreTests.cs b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Ledger/PostgresLedgerExportStoreTests.cs new file mode 100644 index 000000000..908127802 --- /dev/null +++ b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Ledger/PostgresLedgerExportStoreTests.cs @@ -0,0 +1,112 @@ +using StellaOps.Policy.Engine.Ledger; +using StellaOps.Policy.Persistence.Postgres.Models; +using StellaOps.Policy.Persistence.Postgres.Repositories; +using Xunit; + +namespace StellaOps.Policy.Engine.Tests.Ledger; + +public sealed class PostgresLedgerExportStoreTests +{ + [Fact] + public async Task SaveAndGet_RoundTripsPersistedExport() + { + var repository = new FakePolicyEngineLedgerExportRepository(); + var store = new PostgresLedgerExportStore(repository); + var export = CreateExport( + exportId: "exp-001", + tenantId: "tenant-a", + generatedAt: "2026-04-14T09:00:00.0000000Z"); + + await store.SaveAsync("tenant-a", export); + + var loaded = await store.GetAsync("exp-001"); + + Assert.NotNull(loaded); + Assert.Equal(export.Manifest.ExportId, loaded!.Manifest.ExportId); + Assert.Equal(export.Manifest.Sha256, loaded.Manifest.Sha256); + Assert.Equal(export.Records, loaded.Records); + Assert.Equal(export.Lines, loaded.Lines); + } + + [Fact] + public async Task ListAsync_FiltersByTenantAndPreservesOrdering() + { + var repository = new FakePolicyEngineLedgerExportRepository(); + var store = new PostgresLedgerExportStore(repository); + + await store.SaveAsync("tenant-a", CreateExport("exp-001", "tenant-a", "2026-04-14T09:00:00.0000000Z")); + await store.SaveAsync("tenant-b", CreateExport("exp-002", "tenant-b", "2026-04-14T08:00:00.0000000Z")); + await store.SaveAsync("tenant-a", CreateExport("exp-003", "tenant-a", "2026-04-14T10:00:00.0000000Z")); + + var tenantA = await store.ListAsync("tenant-a"); + + Assert.Collection( + tenantA, + item => Assert.Equal("exp-001", item.Manifest.ExportId), + item => Assert.Equal("exp-003", item.Manifest.ExportId)); + } + + private static LedgerExport CreateExport(string exportId, string tenantId, string generatedAt) + { + var manifest = new LedgerExportManifest( + ExportId: exportId, + SchemaVersion: "policy-ledger-export-v1", + GeneratedAt: generatedAt, + RecordCount: 1, + Sha256: $"sha256:{exportId}"); + + var records = new List + { + new( + TenantId: tenantId, + JobId: $"job-{exportId}", + ContextId: "ctx-001", + ComponentPurl: "pkg:oci/test/component@1.0.0", + AdvisoryId: "CVE-2026-0001", + Status: "blocked", + TraceRef: $"trace-{exportId}", + OccurredAt: generatedAt) + }; + + var lines = new List + { + $"manifest-{exportId}", + $"record-{exportId}" + }; + + return new LedgerExport(manifest, records, lines); + } + + private sealed class FakePolicyEngineLedgerExportRepository : IPolicyEngineLedgerExportRepository + { + private readonly Dictionary _documents = new(StringComparer.Ordinal); + + public Task SaveAsync(PolicyEngineLedgerExportDocument document, CancellationToken cancellationToken = default) + { + _documents[document.ExportId] = document with { CreatedAt = document.CreatedAt == default ? document.GeneratedAt : document.CreatedAt }; + return Task.CompletedTask; + } + + public Task GetAsync(string exportId, CancellationToken cancellationToken = default) + { + _documents.TryGetValue(exportId, out var document); + return Task.FromResult(document); + } + + public Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default) + { + IEnumerable items = _documents.Values; + if (!string.IsNullOrWhiteSpace(tenantId)) + { + items = items.Where(item => string.Equals(item.TenantId, tenantId, StringComparison.Ordinal)); + } + + var ordered = items + .OrderBy(item => item.GeneratedAt) + .ThenBy(item => item.ExportId, StringComparer.Ordinal) + .ToList(); + + return Task.FromResult>(ordered); + } + } +} diff --git a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/SnapshotServiceTests.cs b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/SnapshotServiceTests.cs index 959eb11ef..3d7d8a0c7 100644 --- a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/SnapshotServiceTests.cs +++ b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/SnapshotServiceTests.cs @@ -47,10 +47,18 @@ public sealed class SnapshotServiceTests await ledger.BuildAsync(new LedgerExportRequest("acme")); - var snapshot = await service.CreateAsync(new SnapshotRequest("acme", "overlay-1")); + var snapshot = await service.CreateAsync(new SnapshotRequest( + "acme", + "overlay-1", + ArtifactDigest: "sha256:test-artifact", + ArtifactRepository: "demo/api", + ArtifactTag: "1.2.3")); Assert.Equal("acme", snapshot.TenantId); Assert.Equal("overlay-1", snapshot.OverlayHash); + Assert.Equal("sha256:test-artifact", snapshot.ArtifactDigest); + Assert.Equal("demo/api", snapshot.ArtifactRepository); + Assert.Equal("1.2.3", snapshot.ArtifactTag); Assert.Single(snapshot.Records); Assert.Contains("violation", snapshot.StatusCounts.Keys); diff --git a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Snapshots/PostgresSnapshotStoreTests.cs b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Snapshots/PostgresSnapshotStoreTests.cs new file mode 100644 index 000000000..174dd680c --- /dev/null +++ b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Snapshots/PostgresSnapshotStoreTests.cs @@ -0,0 +1,118 @@ +using StellaOps.Policy.Engine.Ledger; +using StellaOps.Policy.Engine.Snapshots; +using StellaOps.Policy.Persistence.Postgres.Models; +using StellaOps.Policy.Persistence.Postgres.Repositories; +using Xunit; + +namespace StellaOps.Policy.Engine.Tests.Snapshots; + +public sealed class PostgresSnapshotStoreTests +{ + [Fact] + public async Task SaveAndGet_RoundTripsPersistedSnapshot() + { + var repository = new FakePolicyEngineSnapshotRepository(); + var store = new PostgresSnapshotStore(repository); + var snapshot = CreateSnapshot( + snapshotId: "snap-001", + tenantId: "tenant-a", + generatedAt: "2026-04-14T09:00:00.0000000Z"); + + await store.SaveAsync(snapshot); + + var loaded = await store.GetAsync("snap-001"); + + Assert.NotNull(loaded); + Assert.Equal(snapshot.SnapshotId, loaded.SnapshotId); + Assert.Equal(snapshot.TenantId, loaded.TenantId); + Assert.Equal(snapshot.LedgerExportId, loaded.LedgerExportId); + Assert.Equal(DateTimeOffset.Parse(snapshot.GeneratedAt), DateTimeOffset.Parse(loaded.GeneratedAt)); + Assert.Equal(snapshot.OverlayHash, loaded.OverlayHash); + Assert.Equal(snapshot.ArtifactDigest, loaded.ArtifactDigest); + Assert.Equal(snapshot.ArtifactRepository, loaded.ArtifactRepository); + Assert.Equal(snapshot.ArtifactTag, loaded.ArtifactTag); + Assert.Equal(snapshot.StatusCounts, loaded.StatusCounts); + Assert.Equal(snapshot.Records, loaded.Records); + } + + [Fact] + public async Task ListAsync_FiltersByTenantAndPreservesOrdering() + { + var repository = new FakePolicyEngineSnapshotRepository(); + var store = new PostgresSnapshotStore(repository); + + await store.SaveAsync(CreateSnapshot("snap-001", "tenant-a", "2026-04-14T09:00:00.0000000Z")); + await store.SaveAsync(CreateSnapshot("snap-002", "tenant-b", "2026-04-14T08:00:00.0000000Z")); + await store.SaveAsync(CreateSnapshot("snap-003", "tenant-a", "2026-04-14T10:00:00.0000000Z")); + + var tenantA = await store.ListAsync("tenant-a"); + + Assert.Collection( + tenantA, + item => Assert.Equal("snap-001", item.SnapshotId), + item => Assert.Equal("snap-003", item.SnapshotId)); + } + + private static SnapshotDetail CreateSnapshot(string snapshotId, string tenantId, string generatedAt) + { + return new SnapshotDetail( + SnapshotId: snapshotId, + TenantId: tenantId, + LedgerExportId: $"exp-{snapshotId}", + GeneratedAt: generatedAt, + OverlayHash: $"overlay-{snapshotId}", + ArtifactDigest: $"sha256:{snapshotId}", + ArtifactRepository: $"demo/{snapshotId}", + ArtifactTag: $"tag-{snapshotId}", + StatusCounts: new Dictionary(StringComparer.Ordinal) + { + ["blocked"] = 1, + ["warn"] = 2 + }, + Records: + [ + new LedgerExportRecord( + TenantId: tenantId, + JobId: $"job-{snapshotId}", + ContextId: "ctx-001", + ComponentPurl: "pkg:oci/test/component@1.0.0", + AdvisoryId: "CVE-2026-0001", + Status: "blocked", + TraceRef: $"trace-{snapshotId}", + OccurredAt: generatedAt) + ]); + } + + private sealed class FakePolicyEngineSnapshotRepository : IPolicyEngineSnapshotRepository + { + private readonly Dictionary _documents = new(StringComparer.Ordinal); + + public Task SaveAsync(PolicyEngineSnapshotDocument document, CancellationToken cancellationToken = default) + { + _documents[document.SnapshotId] = document with { CreatedAt = document.CreatedAt == default ? document.GeneratedAt : document.CreatedAt }; + return Task.CompletedTask; + } + + public Task GetAsync(string snapshotId, CancellationToken cancellationToken = default) + { + _documents.TryGetValue(snapshotId, out var document); + return Task.FromResult(document); + } + + public Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default) + { + IEnumerable items = _documents.Values; + if (!string.IsNullOrWhiteSpace(tenantId)) + { + items = items.Where(item => string.Equals(item.TenantId, tenantId, StringComparison.Ordinal)); + } + + var ordered = items + .OrderBy(item => item.GeneratedAt) + .ThenBy(item => item.SnapshotId, StringComparer.Ordinal) + .ToList(); + + return Task.FromResult>(ordered); + } + } +} diff --git a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/TASKS.md b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/TASKS.md index a3d8a92c0..6d90b0f58 100644 --- a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/TASKS.md +++ b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/TASKS.md @@ -8,3 +8,6 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | AUDIT-0442-M | DONE | Revalidated 2026-01-07; maintainability audit for StellaOps.Policy.Engine.Tests. | | AUDIT-0442-T | DONE | Revalidated 2026-01-07; test coverage audit for StellaOps.Policy.Engine.Tests. | | AUDIT-0442-A | DONE | Waived (test project; revalidated 2026-01-07). | +| NOMOCK-019-T | DONE | `SPRINT_20260410_001_Web_runtime_no_mocks_real_backend.md`: added `PolicyEngineRegistryWebhookRuntimeTests`, `PolicyEngineSchedulerWebhookRuntimeTests`, and `PolicyEngineGateTargetSnapshotRuntimeTests` to prove the merged-gateway async/runtime branches stay truthful and that gate evaluation now materializes persisted target snapshots before delta computation. | +| NOMOCK-020-T | DONE | `SPRINT_20260410_001_Web_runtime_no_mocks_real_backend.md`: extended Policy engine host/runtime tests to cover persisted orchestrator/worker store registrations plus first-run baseline bootstrap from completed persisted Policy result data, with the focused lanes passing `6/6` and `3/3`. | +| NOMOCK-021-T | DONE | `SPRINT_20260410_001_Web_runtime_no_mocks_real_backend.md`: added `PolicyEngineOrchestratorProducerRuntimeTests` to prove `/policy/orchestrator/jobs` now auto-executes queued work and materializes `policy.worker_results` without a manual `/policy/worker/run`, with the focused lane passing `1/1`. | diff --git a/src/Policy/__Tests/StellaOps.Policy.Gateway.Tests/PolicyGatewayPersistedDeltaRuntimeTests.cs b/src/Policy/__Tests/StellaOps.Policy.Gateway.Tests/PolicyGatewayPersistedDeltaRuntimeTests.cs new file mode 100644 index 000000000..391cd80b1 --- /dev/null +++ b/src/Policy/__Tests/StellaOps.Policy.Gateway.Tests/PolicyGatewayPersistedDeltaRuntimeTests.cs @@ -0,0 +1,220 @@ +using System.Text.Json; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; +using StellaOps.Auth.ServerIntegration.Tenancy; +using StellaOps.Policy.Deltas; +using StellaOps.Policy.Persistence.Postgres; +using StellaOps.Policy.Persistence.Postgres.Models; +using StellaOps.Policy.Persistence.Postgres.Repositories; +using Xunit; +using GatewayProgram = StellaOps.Policy.Gateway.Program; + +namespace StellaOps.Policy.Gateway.Tests; + +public sealed class PolicyGatewayPersistedDeltaRuntimeTests +{ + [Fact] + public async Task GatewayHost_ResolvesPostgresGateBypassAuditRepository_WithDefaultTenantFallback() + { + await using var factory = CreateFactory(); + using var scope = factory.Services.CreateScope(); + + var tenantAccessor = scope.ServiceProvider.GetRequiredService(); + tenantAccessor.TenantContext = null; + + var repository = scope.ServiceProvider.GetRequiredService(); + + var postgresRepository = Assert.IsType(repository); + Assert.Equal( + StellaOps.Policy.Engine.Tenancy.TenantContextConstants.DefaultTenantId, + ReadTenantId(postgresRepository)); + } + + [Fact] + public async Task GatewayHost_ResolvesPostgresGateBypassAuditRepository_WithCurrentTenant() + { + await using var factory = CreateFactory(); + using var scope = factory.Services.CreateScope(); + + var tenantAccessor = scope.ServiceProvider.GetRequiredService(); + tenantAccessor.TenantContext = new StellaOpsTenantContext + { + TenantId = "tenant-audit-001", + ActorId = "tester" + }; + + var repository = scope.ServiceProvider.GetRequiredService(); + + var postgresRepository = Assert.IsType(repository); + Assert.Equal("tenant-audit-001", ReadTenantId(postgresRepository)); + } + + [Fact] + public async Task GatewayHost_ResolvesPersistedSnapshotRuntime_AndProjectsRealDeltaInputs() + { + var document = CreateSnapshotDocument(); + + await using var factory = CreateFactory(document); + using var scope = factory.Services.CreateScope(); + + var manifestStore = scope.ServiceProvider.GetRequiredService(); + var deltaSnapshotService = scope.ServiceProvider.GetRequiredService(); + + Assert.Equal("PersistedKnowledgeSnapshotStore", manifestStore.GetType().Name); + + var snapshotData = await deltaSnapshotService.GetSnapshotAsync(document.SnapshotId); + + Assert.NotNull(snapshotData); + Assert.Equal(document.SnapshotId, snapshotData!.SnapshotId); + Assert.Equal("overlay-runtime", snapshotData.PolicyVersion); + Assert.Collection( + snapshotData.Packages, + package => + { + Assert.Equal("pkg:oci/demo/api@1.2.3", package.Purl); + Assert.Equal("1.2.3", package.Version); + }); + Assert.Contains(snapshotData.Reachability, item => item.CveId == "CVE-2026-2000" && item.IsReachable); + Assert.Contains(snapshotData.VexStatements, item => item.CveId == "CVE-2026-2000" && item.Status == "affected"); + Assert.Contains(snapshotData.PolicyViolations, item => item.RuleId.Contains("CVE-2026-2001", StringComparison.Ordinal)); + Assert.Contains(snapshotData.Unknowns, item => item.Id.Contains("CVE-2026-2002", StringComparison.Ordinal)); + } + + private static WebApplicationFactory CreateFactory(PolicyEngineSnapshotDocument document) + { + return new TestPolicyGatewayFactory() + .WithWebHostBuilder(builder => + { + builder.UseEnvironment("Development"); + builder.ConfigureTestServices(services => + { + services.RemoveAll(); + services.RemoveAll(); + services.AddSingleton(new FakePolicyEngineSnapshotRepository([document])); + }); + }); + } + + private static WebApplicationFactory CreateFactory() + { + return new TestPolicyGatewayFactory() + .WithWebHostBuilder(builder => + { + builder.UseEnvironment("Development"); + builder.ConfigureTestServices(services => + { + services.RemoveAll(); + }); + }); + } + + private static PolicyEngineSnapshotDocument CreateSnapshotDocument() + { + var records = JsonSerializer.Serialize(new object[] + { + new + { + tenant_id = "test-tenant", + job_id = "job-CVE-2026-2000", + context_id = "ctx-001", + component_purl = "pkg:oci/demo/api@1.2.3", + advisory_id = "CVE-2026-2000", + status = "affected", + trace_ref = "trace-CVE-2026-2000", + occurred_at = "2026-04-15T09:00:00.0000000Z" + }, + new + { + tenant_id = "test-tenant", + job_id = "job-CVE-2026-2001", + context_id = "ctx-001", + component_purl = "pkg:oci/demo/api@1.2.3", + advisory_id = "CVE-2026-2001", + status = "blocked", + trace_ref = "trace-CVE-2026-2001", + occurred_at = "2026-04-15T09:05:00.0000000Z" + }, + new + { + tenant_id = "test-tenant", + job_id = "job-CVE-2026-2002", + context_id = "ctx-001", + component_purl = "pkg:oci/demo/api@1.2.3", + advisory_id = "CVE-2026-2002", + status = "unknown", + trace_ref = "trace-CVE-2026-2002", + occurred_at = "2026-04-15T09:10:00.0000000Z" + } + }); + + var statusCounts = JsonSerializer.Serialize(new Dictionary(StringComparer.Ordinal) + { + ["affected"] = 1, + ["blocked"] = 1, + ["unknown"] = 1 + }); + + return new PolicyEngineSnapshotDocument + { + SnapshotId = "snap-delta-runtime-001", + TenantId = "test-tenant", + LedgerExportId = "exp-snap-delta-runtime-001", + GeneratedAt = DateTimeOffset.Parse("2026-04-15T09:15:00Z"), + OverlayHash = "overlay-runtime", + StatusCountsJson = statusCounts, + RecordsJson = records, + CreatedAt = DateTimeOffset.Parse("2026-04-15T09:15:30Z") + }; + } + + private sealed class FakePolicyEngineSnapshotRepository : IPolicyEngineSnapshotRepository + { + private readonly Dictionary _documents; + + public FakePolicyEngineSnapshotRepository(IEnumerable documents) + { + _documents = documents.ToDictionary(document => document.SnapshotId, StringComparer.Ordinal); + } + + public Task SaveAsync(PolicyEngineSnapshotDocument document, CancellationToken cancellationToken = default) + { + _documents[document.SnapshotId] = document; + return Task.CompletedTask; + } + + public Task GetAsync(string snapshotId, CancellationToken cancellationToken = default) + { + _documents.TryGetValue(snapshotId, out var document); + return Task.FromResult(document); + } + + public Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default) + { + IEnumerable documents = _documents.Values; + if (!string.IsNullOrWhiteSpace(tenantId)) + { + documents = documents.Where(document => string.Equals(document.TenantId, tenantId, StringComparison.Ordinal)); + } + + return Task.FromResult>( + documents + .OrderBy(document => document.GeneratedAt) + .ThenBy(document => document.SnapshotId, StringComparer.Ordinal) + .ToList()); + } + } + + private static string ReadTenantId(PostgresGateBypassAuditRepository repository) + { + var tenantIdField = typeof(PostgresGateBypassAuditRepository).GetField( + "_tenantId", + System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic); + + Assert.NotNull(tenantIdField); + return Assert.IsType(tenantIdField!.GetValue(repository)); + } +} diff --git a/src/Policy/__Tests/StellaOps.Policy.Gateway.Tests/RegistryWebhookQueueRuntimeTests.cs b/src/Policy/__Tests/StellaOps.Policy.Gateway.Tests/RegistryWebhookQueueRuntimeTests.cs new file mode 100644 index 000000000..d462deda0 --- /dev/null +++ b/src/Policy/__Tests/StellaOps.Policy.Gateway.Tests/RegistryWebhookQueueRuntimeTests.cs @@ -0,0 +1,567 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; +using StellaOps.Auth.ServerIntegration.Tenancy; +using StellaOps.Determinism; +using StellaOps.Policy.Engine.Services.Gateway; +using StellaOps.Policy.Persistence.Postgres.Models; +using StellaOps.Policy.Persistence.Postgres.Repositories; +using StellaOps.Scheduler.Persistence.Postgres.Models; +using StellaOps.Scheduler.Persistence.Postgres.Repositories; +using Xunit; +using GatewayProgram = StellaOps.Policy.Gateway.Program; + +namespace StellaOps.Policy.Gateway.Tests; + +public sealed class RegistryWebhookQueueRuntimeTests +{ + [Fact] + public async Task GateEvaluationQueue_ResolvesToUnsupportedRuntime() + { + await using var factory = CreateFactory(); + using var scope = factory.Services.CreateScope(); + + var queue = scope.ServiceProvider.GetRequiredService(); + + Assert.IsType(queue); + } + + [Fact] + public async Task GateEvaluationQueue_CanResolveToSchedulerBackedRuntime_InFocusedHost() + { + await using var factory = CreateSchedulerFactory(new FakeJobRepository(), new FakeWorkerResultRepository()); + using var scope = factory.Services.CreateScope(); + + var queue = scope.ServiceProvider.GetRequiredService(); + + Assert.IsType(queue); + } + + [Fact] + public async Task SchedulerBackedRuntime_CanEnqueuePendingJob_AndResolveStatus_InFocusedHost() + { + var jobs = new FakeJobRepository(); + var workerResults = new FakeWorkerResultRepository(); + + await using var factory = CreateSchedulerFactory(jobs, workerResults); + using var scope = factory.Services.CreateScope(); + var tenantAccessor = scope.ServiceProvider.GetRequiredService(); + tenantAccessor.TenantContext = new StellaOpsTenantContext + { + TenantId = TestPolicyGatewayFactory.DefaultTestTenant, + ActorId = "test-user", + Source = TenantSource.Unknown + }; + + var queue = scope.ServiceProvider.GetRequiredService(); + var jobId = await queue.EnqueueAsync(new GateEvaluationRequest + { + ImageDigest = "sha256:3333333333333333333333333333333333333333333333333333333333333333", + Repository = "demo/api", + Tag = "3.0.0", + BaselineRef = "baseline-003", + Source = "test", + Timestamp = DateTimeOffset.Parse("2026-04-15T12:00:00Z") + }); + + Assert.NotNull(await jobs.GetByIdAsync(TestPolicyGatewayFactory.DefaultTestTenant, Guid.Parse(jobId), CancellationToken.None)); + Assert.NotNull(await workerResults.GetByJobAsync(TestPolicyGatewayFactory.DefaultTestTenant, "policy.gate-evaluation", jobId, CancellationToken.None)); + + var statusService = scope.ServiceProvider.GetRequiredService(); + var status = await statusService.GetAsync(jobId); + + Assert.NotNull(status); + Assert.Equal("pending", status!.Status); + Assert.Equal("scheduled", status.SchedulerStatus); + Assert.Equal("sha256:3333333333333333333333333333333333333333333333333333333333333333", status.Request!.ImageDigest); + Assert.Equal("demo/api", status.Request.Repository); + } + + [Fact] + public async Task GenericRegistryWebhook_ReturnsNotImplemented_WhenAsyncQueueIsUnavailable() + { + await using var factory = CreateFactory(); + using var client = factory.CreateClient(); + client.DefaultRequestHeaders.Add("X-StellaOps-Tenant", TestPolicyGatewayFactory.DefaultTestTenant); + + var response = await client.PostAsJsonAsync( + "/api/v1/webhooks/registry/generic", + new + { + imageDigest = "sha256:1111111111111111111111111111111111111111111111111111111111111111", + repository = "demo/api", + tag = "1.2.3", + baselineRef = "baseline-001", + source = "test" + }); + + var body = await response.Content.ReadAsStringAsync(); + using var json = JsonDocument.Parse(body); + + Assert.Equal(HttpStatusCode.NotImplemented, response.StatusCode); + Assert.Equal("Async gate evaluation queue unavailable", json.RootElement.GetProperty("title").GetString()); + Assert.Contains( + "scheduler-backed async gate evaluation queue", + json.RootElement.GetProperty("detail").GetString(), + StringComparison.Ordinal); + } + + private static WebApplicationFactory CreateFactory() + { + return new TestPolicyGatewayFactory() + .WithWebHostBuilder(builder => + { + builder.UseEnvironment("Development"); + builder.ConfigureTestServices(services => + { + services.RemoveAll(); + }); + }); + } + + private static WebApplicationFactory CreateSchedulerFactory( + FakeJobRepository jobs, + FakeWorkerResultRepository workerResults) + { + return new TestPolicyGatewayFactory() + .WithWebHostBuilder(builder => + { + builder.UseEnvironment("Development"); + builder.ConfigureTestServices(services => + { + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + services.AddSingleton(new GateEvaluationQueueOptions + { + LeaseDurationSeconds = 30, + MaxAttempts = 2, + BatchSize = 10 + }); + services.AddSingleton(jobs); + services.AddSingleton(workerResults); + services.AddSingleton(new SequentialGuidProvider(Guid.Empty)); + services.AddSingleton(); + services.AddScoped(); + }); + }); + } + + private sealed class FakeJobRepository : IJobRepository + { + private static readonly DateTimeOffset DefaultTimestamp = DateTimeOffset.Parse("2026-04-15T12:00:00Z"); + private readonly object _gate = new(); + private readonly Dictionary<(string TenantId, Guid JobId), StoredJob> _jobs = []; + + public Task CreateAsync(JobEntity job, CancellationToken cancellationToken = default) + { + lock (_gate) + { + var stored = StoredJob.From(job); + if (stored.CreatedAt == default) + { + stored.CreatedAt = DefaultTimestamp; + } + + _jobs[(stored.TenantId, stored.Id)] = stored; + return Task.FromResult(stored.ToEntity()); + } + } + + public Task GetByIdAsync(string tenantId, Guid id, CancellationToken cancellationToken = default) + { + lock (_gate) + { + return Task.FromResult(_jobs.TryGetValue((tenantId, id), out var stored) ? stored.ToEntity() : null); + } + } + + public Task GetByIdempotencyKeyAsync(string tenantId, string idempotencyKey, CancellationToken cancellationToken = default) + { + lock (_gate) + { + var stored = _jobs.Values.FirstOrDefault(job => + string.Equals(job.TenantId, tenantId, StringComparison.Ordinal) && + string.Equals(job.IdempotencyKey, idempotencyKey, StringComparison.Ordinal)); + return Task.FromResult(stored?.ToEntity()); + } + } + + public Task> GetScheduledJobsAsync(string tenantId, string[] jobTypes, int limit = 10, CancellationToken cancellationToken = default) + { + lock (_gate) + { + return Task.FromResult>( + _jobs.Values + .Where(job => + string.Equals(job.TenantId, tenantId, StringComparison.Ordinal) && + jobTypes.Contains(job.JobType, StringComparer.Ordinal) && + job.Status is JobStatus.Pending or JobStatus.Scheduled) + .Take(limit) + .Select(job => job.ToEntity()) + .ToList()); + } + } + + public Task TryLeaseJobAsync(string tenantId, Guid jobId, string workerId, TimeSpan leaseDuration, CancellationToken cancellationToken = default) + { + lock (_gate) + { + if (!_jobs.TryGetValue((tenantId, jobId), out var stored) || + stored.Status is not (JobStatus.Pending or JobStatus.Scheduled)) + { + return Task.FromResult(null); + } + + stored.Status = JobStatus.Leased; + stored.LeaseId = Guid.Parse("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"); + stored.WorkerId = workerId; + stored.LeasedAt = DefaultTimestamp.AddMinutes(1); + stored.StartedAt = DefaultTimestamp.AddMinutes(1); + stored.LeaseUntil = stored.LeasedAt.Value.Add(leaseDuration); + + return Task.FromResult(stored.ToEntity()); + } + } + + public Task ExtendLeaseAsync(string tenantId, Guid jobId, Guid leaseId, TimeSpan extension, CancellationToken cancellationToken = default) + { + lock (_gate) + { + if (!_jobs.TryGetValue((tenantId, jobId), out var stored) || stored.LeaseId != leaseId) + { + return Task.FromResult(false); + } + + stored.LeaseUntil = (stored.LeaseUntil ?? DefaultTimestamp).Add(extension); + return Task.FromResult(true); + } + } + + public Task CompleteAsync(string tenantId, Guid jobId, Guid leaseId, string? result = null, CancellationToken cancellationToken = default) + { + lock (_gate) + { + if (!_jobs.TryGetValue((tenantId, jobId), out var stored) || stored.LeaseId != leaseId) + { + return Task.FromResult(false); + } + + stored.Status = JobStatus.Succeeded; + stored.Result = result; + stored.CompletedAt = DefaultTimestamp.AddMinutes(2); + return Task.FromResult(true); + } + } + + public Task FailAsync(string tenantId, Guid jobId, Guid leaseId, string reason, bool retry = true, CancellationToken cancellationToken = default) + { + lock (_gate) + { + if (!_jobs.TryGetValue((tenantId, jobId), out var stored) || stored.LeaseId != leaseId) + { + return Task.FromResult(false); + } + + stored.Status = retry ? JobStatus.Scheduled : JobStatus.Failed; + stored.Reason = reason; + stored.Attempt++; + return Task.FromResult(true); + } + } + + public Task CancelAsync(string tenantId, Guid jobId, string reason, CancellationToken cancellationToken = default) + { + lock (_gate) + { + if (!_jobs.TryGetValue((tenantId, jobId), out var stored)) + { + return Task.FromResult(false); + } + + stored.Status = JobStatus.Canceled; + stored.Reason = reason; + return Task.FromResult(true); + } + } + + public Task RecoverExpiredLeasesAsync(string tenantId, CancellationToken cancellationToken = default) + => Task.FromResult(0); + + public Task> GetByStatusAsync(string tenantId, JobStatus status, int limit = 100, int offset = 0, CancellationToken cancellationToken = default) + { + lock (_gate) + { + return Task.FromResult>( + _jobs.Values + .Where(job => string.Equals(job.TenantId, tenantId, StringComparison.Ordinal) && job.Status == status) + .Skip(offset) + .Take(limit) + .Select(job => job.ToEntity()) + .ToList()); + } + } + + private sealed class StoredJob + { + public required Guid Id { get; init; } + public required string TenantId { get; init; } + public string? ProjectId { get; init; } + public required string JobType { get; init; } + public JobStatus Status { get; set; } + public int Priority { get; init; } + public required string Payload { get; init; } + public required string PayloadDigest { get; init; } + public required string IdempotencyKey { get; init; } + public string? CorrelationId { get; init; } + public int Attempt { get; set; } + public int MaxAttempts { get; init; } + public Guid? LeaseId { get; set; } + public string? WorkerId { get; set; } + public DateTimeOffset? LeaseUntil { get; set; } + public DateTimeOffset? NotBefore { get; init; } + public string? Reason { get; set; } + public string? Result { get; set; } + public DateTimeOffset CreatedAt { get; set; } + public DateTimeOffset? ScheduledAt { get; init; } + public DateTimeOffset? LeasedAt { get; set; } + public DateTimeOffset? StartedAt { get; set; } + public DateTimeOffset? CompletedAt { get; set; } + public string? CreatedBy { get; init; } + + public static StoredJob From(JobEntity job) + { + return new StoredJob + { + Id = job.Id, + TenantId = job.TenantId, + ProjectId = job.ProjectId, + JobType = job.JobType, + Status = job.Status, + Priority = job.Priority, + Payload = job.Payload, + PayloadDigest = job.PayloadDigest, + IdempotencyKey = job.IdempotencyKey, + CorrelationId = job.CorrelationId, + Attempt = job.Attempt, + MaxAttempts = job.MaxAttempts, + LeaseId = job.LeaseId, + WorkerId = job.WorkerId, + LeaseUntil = job.LeaseUntil, + NotBefore = job.NotBefore, + Reason = job.Reason, + Result = job.Result, + CreatedAt = job.CreatedAt, + ScheduledAt = job.ScheduledAt, + LeasedAt = job.LeasedAt, + StartedAt = job.StartedAt, + CompletedAt = job.CompletedAt, + CreatedBy = job.CreatedBy + }; + } + + public JobEntity ToEntity() + { + return new JobEntity + { + Id = Id, + TenantId = TenantId, + ProjectId = ProjectId, + JobType = JobType, + Status = Status, + Priority = Priority, + Payload = Payload, + PayloadDigest = PayloadDigest, + IdempotencyKey = IdempotencyKey, + CorrelationId = CorrelationId, + Attempt = Attempt, + MaxAttempts = MaxAttempts, + LeaseId = LeaseId, + WorkerId = WorkerId, + LeaseUntil = LeaseUntil, + NotBefore = NotBefore, + Reason = Reason, + Result = Result, + CreatedAt = CreatedAt, + ScheduledAt = ScheduledAt, + LeasedAt = LeasedAt, + StartedAt = StartedAt, + CompletedAt = CompletedAt, + CreatedBy = CreatedBy + }; + } + } + } + + private sealed class FakeWorkerResultRepository : IWorkerResultRepository + { + private static readonly DateTimeOffset DefaultTimestamp = DateTimeOffset.Parse("2026-04-15T12:00:00Z"); + private readonly object _gate = new(); + private readonly Dictionary<(string TenantId, Guid Id), WorkerResultEntity> _results = []; + + public Task CreateAsync(WorkerResultEntity result, CancellationToken cancellationToken = default) + { + lock (_gate) + { + var stored = result with + { + CreatedAt = result.CreatedAt == default ? DefaultTimestamp : result.CreatedAt + }; + _results[(stored.TenantId, stored.Id)] = stored; + return Task.FromResult(stored); + } + } + + public Task GetByIdAsync(string tenantId, Guid id, CancellationToken cancellationToken = default) + { + lock (_gate) + { + return Task.FromResult(_results.TryGetValue((tenantId, id), out var result) ? result : null); + } + } + + public Task GetByJobAsync(string tenantId, string jobType, string jobId, CancellationToken cancellationToken = default) + { + lock (_gate) + { + var result = _results.Values.FirstOrDefault(item => + string.Equals(item.TenantId, tenantId, StringComparison.Ordinal) && + string.Equals(item.JobType, jobType, StringComparison.Ordinal) && + string.Equals(item.JobId, jobId, StringComparison.Ordinal)); + return Task.FromResult(result); + } + } + + public Task> GetByStatusAsync(string tenantId, string status, int limit = 100, CancellationToken cancellationToken = default) + { + lock (_gate) + { + return Task.FromResult>( + _results.Values + .Where(item => + string.Equals(item.TenantId, tenantId, StringComparison.Ordinal) && + string.Equals(item.Status, status, StringComparison.Ordinal)) + .Take(limit) + .ToList()); + } + } + + public Task> GetPendingAsync(string? jobType = null, int limit = 100, CancellationToken cancellationToken = default) + { + lock (_gate) + { + return Task.FromResult>( + _results.Values + .Where(item => + string.Equals(item.Status, "pending", StringComparison.Ordinal) && + (jobType is null || string.Equals(item.JobType, jobType, StringComparison.Ordinal))) + .OrderBy(item => item.CreatedAt) + .Take(limit) + .ToList()); + } + } + + public Task> ListAsync(string? tenantId = null, CancellationToken cancellationToken = default) + { + lock (_gate) + { + IEnumerable query = _results.Values; + if (!string.IsNullOrWhiteSpace(tenantId)) + { + query = query.Where(item => string.Equals(item.TenantId, tenantId, StringComparison.Ordinal)); + } + + return Task.FromResult>( + query + .OrderBy(item => item.CreatedAt) + .ThenBy(item => item.JobId, StringComparer.Ordinal) + .ToList()); + } + } + + public Task UpdateProgressAsync(string tenantId, Guid id, string status, int progress, string? errorMessage = null, CancellationToken cancellationToken = default) + { + lock (_gate) + { + if (!_results.TryGetValue((tenantId, id), out var result)) + { + return Task.FromResult(false); + } + + _results[(tenantId, id)] = result with + { + Status = status, + Progress = progress, + ErrorMessage = errorMessage + }; + return Task.FromResult(true); + } + } + + public Task CompleteAsync(string tenantId, Guid id, string result, string? outputHash = null, CancellationToken cancellationToken = default) + { + lock (_gate) + { + if (!_results.TryGetValue((tenantId, id), out var existing)) + { + return Task.FromResult(false); + } + + _results[(tenantId, id)] = existing with + { + Status = "completed", + Progress = 100, + Result = result, + OutputHash = outputHash, + CompletedAt = DefaultTimestamp.AddMinutes(2) + }; + return Task.FromResult(true); + } + } + + public Task FailAsync(string tenantId, Guid id, string errorMessage, CancellationToken cancellationToken = default) + { + lock (_gate) + { + if (!_results.TryGetValue((tenantId, id), out var existing)) + { + return Task.FromResult(false); + } + + _results[(tenantId, id)] = existing with + { + Status = "failed", + ErrorMessage = errorMessage + }; + return Task.FromResult(true); + } + } + + public Task IncrementRetryAsync(string tenantId, Guid id, CancellationToken cancellationToken = default) + { + lock (_gate) + { + if (!_results.TryGetValue((tenantId, id), out var existing)) + { + return Task.FromResult(false); + } + + _results[(tenantId, id)] = existing with + { + RetryCount = existing.RetryCount + 1 + }; + return Task.FromResult(true); + } + } + } +} diff --git a/src/Policy/__Tests/StellaOps.Policy.Gateway.Tests/TASKS.md b/src/Policy/__Tests/StellaOps.Policy.Gateway.Tests/TASKS.md index a24300a24..93eab1c70 100644 --- a/src/Policy/__Tests/StellaOps.Policy.Gateway.Tests/TASKS.md +++ b/src/Policy/__Tests/StellaOps.Policy.Gateway.Tests/TASKS.md @@ -9,3 +9,6 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | AUDIT-0446-T | DONE | Revalidated 2026-01-07; test coverage audit for StellaOps.Policy.Gateway.Tests. | | AUDIT-0446-A | DONE | Waived (test project; revalidated 2026-01-07). | | SPRINT-20260224-002-LOC-101-T | DONE | `SPRINT_20260224_002_Platform_translation_rollout_phase3_phase4.md`: added focused Policy Gateway locale-aware readiness test and validated German locale response text. | +| NOMOCK-017-T | DONE | `SPRINT_20260410_001_Web_runtime_no_mocks_real_backend.md`: added `PolicyGatewayPersistedDeltaRuntimeTests` to prove the standalone gateway resolves the persisted delta runtime projection instead of `InMemorySnapshotStore`. | +| NOMOCK-018-T | DONE | `SPRINT_20260410_001_Web_runtime_no_mocks_real_backend.md`: extended `PolicyGatewayPersistedDeltaRuntimeTests` to prove the standalone gateway resolves `PostgresGateBypassAuditRepository` for both current-tenant and default-tenant fallback cases. | +| NOMOCK-019-T | DONE | `SPRINT_20260410_001_Web_runtime_no_mocks_real_backend.md`: expanded `RegistryWebhookQueueRuntimeTests` to prove both standalone-gateway async branches: truthful `501` responses when the queue is unavailable, and scheduler-backed enqueue/status behavior when the real runtime is registered. | diff --git a/src/Policy/__Tests/StellaOps.Policy.Gateway.Tests/TestPolicyGatewayFactory.cs b/src/Policy/__Tests/StellaOps.Policy.Gateway.Tests/TestPolicyGatewayFactory.cs index a5ced3ea1..0c48ca747 100644 --- a/src/Policy/__Tests/StellaOps.Policy.Gateway.Tests/TestPolicyGatewayFactory.cs +++ b/src/Policy/__Tests/StellaOps.Policy.Gateway.Tests/TestPolicyGatewayFactory.cs @@ -38,6 +38,13 @@ namespace StellaOps.Policy.Gateway.Tests; /// public sealed class TestPolicyGatewayFactory : WebApplicationFactory { + private readonly IReadOnlyDictionary _extraSettings; + + public TestPolicyGatewayFactory(IReadOnlyDictionary? extraSettings = null) + { + _extraSettings = extraSettings ?? new Dictionary(StringComparer.Ordinal); + } + /// /// Symmetric signing key used for generating test JWTs. /// @@ -67,10 +74,15 @@ public sealed class TestPolicyGatewayFactory : WebApplicationFactory 0) + { + configurationBuilder.AddInMemoryCollection(_extraSettings); + } }); builder.ConfigureServices(services =>