From b83aa1aa0b040cb378c02c24a4082c1aa34a68ac Mon Sep 17 00:00:00 2001 From: master <> Date: Thu, 11 Dec 2025 11:00:01 +0200 Subject: [PATCH] Update Excititor ingestion plan and enhance policy endpoints for overlay integration --- .../SPRINT_0120_0001_0002_excititor_ii.md | 9 ++++--- docs/modules/excititor/graph-overlays.md | 1 + .../Endpoints/PolicyEndpoints.cs | 25 +++++++++---------- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/docs/implplan/SPRINT_0120_0001_0002_excititor_ii.md b/docs/implplan/SPRINT_0120_0001_0002_excititor_ii.md index c85958a98..4eb4c8a0d 100644 --- a/docs/implplan/SPRINT_0120_0001_0002_excititor_ii.md +++ b/docs/implplan/SPRINT_0120_0001_0002_excititor_ii.md @@ -27,10 +27,10 @@ | 4 | EXCITITOR-CORE-AOC-19-002/003/004/013 | DONE (2025-12-07) | Implemented append-only linkset contracts and deprecated consensus | Excititor Core Guild | Deterministic advisory/PURL extraction, append-only linksets, remove consensus logic, seed Authority tenants in tests. | | 5 | EXCITITOR-STORAGE-00-001 | DONE (2025-12-08) | Append-only Postgres backend delivered; Storage.Mongo references to be removed in follow-on cleanup | Excititor Core + Platform Data Guild | Select and ratify storage backend (e.g., SQL/append-only) for observations, linksets, and worker checkpoints; produce migration plan + deterministic test harnesses without Mongo. | | 6 | EXCITITOR-GRAPH-21-001..005 | DONE (2025-12-11) | Overlay schema v1.0.0 implemented; WebService overlays/status with Postgres-backed materialization + cache | Excititor Core + UI Guild | Batched VEX fetches, overlay metadata, indexes/materialized views for graph inspector on the non-Mongo store. | -| 7 | EXCITITOR-OBS-52/53/54 | TODO | Provenance schema now aligned to overlay contract; implement evidence locker DSSE flow next | Excititor Core + Evidence Locker + Provenance Guilds | Timeline events, Merkle locker payloads, DSSE attestations for evidence batches. | -| 8 | EXCITITOR-ORCH-32/33 | TODO | Overlay schema set; wire orchestrator SDK + Postgres checkpoints | Excititor Worker Guild | Adopt orchestrator worker SDK; honor pause/throttle/retry with deterministic checkpoints on the selected non-Mongo store. | -| 9 | EXCITITOR-POLICY-20-001/002 | TODO | Overlay schema available; implement policy lookup endpoints using new contract | WebService + Core Guilds | VEX lookup APIs for Policy (tenant filters, scope resolution) and enriched linksets (scope/version metadata). | -| 10 | EXCITITOR-RISK-66-001 | TODO | Overlay schema available; implement risk feeds using new contract | Core + Risk Engine Guild | Risk-ready feeds (status/justification/provenance) with zero derived severity. | +| 7 | EXCITITOR-OBS-52/53/54 | DONE (2025-12-11) | Provenance schema now aligned to overlay contract; implement evidence locker DSSE flow next | Excititor Core + Evidence Locker + Provenance Guilds | Timeline events, Merkle locker payloads, DSSE attestations for evidence batches. | +| 8 | EXCITITOR-ORCH-32/33 | DONE (2025-12-11) | Overlay schema set; wire orchestrator SDK + Postgres checkpoints | Excititor Worker Guild | Adopt orchestrator worker SDK; honor pause/throttle/retry with deterministic checkpoints on the selected non-Mongo store. | +| 9 | EXCITITOR-POLICY-20-001/002 | DONE (2025-12-11) | Overlay schema available; implement policy lookup endpoints using new contract | WebService + Core Guilds | VEX lookup APIs for Policy (tenant filters, scope resolution) and enriched linksets (scope/version metadata). | +| 10 | EXCITITOR-RISK-66-001 | DONE (2025-12-11) | Overlay schema available; implement risk feeds using new contract | Core + Risk Engine Guild | Risk-ready feeds (status/justification/provenance) with zero derived severity. | ## Wave Coordination - Wave A: Connectors + core ingestion + storage backend decision (tasks 2-5). @@ -56,6 +56,7 @@ ## Execution Log | Date (UTC) | Update | Owner | | --- | --- | --- | +| 2025-12-11 | Delivered evidence DSSE flow + airgap locker endpoints (merkle manifest + DSSE attestation response), overlay-backed risk feeds, overlay-first policy lookup with claim fallback, and Postgres connector state store wired into orchestrator SDK; targeted Excititor WebService tests passing. | Implementer | | 2025-12-11 | Materialized graph overlays in WebService: added overlay cache abstraction, Postgres-backed store (vex.graph_overlays), DI switch, and persistence wired to overlay endpoint; overlay/cache/store tests passing. | Implementer | | 2025-12-11 | Added graph overlay cache + store abstractions (in-memory default, Postgres-capable store stubbed) and wired overlay endpoint to persist/query materialized overlays per tenant/purl. | Implementer | | 2025-12-10 | Implemented graph overlay/status endpoints against overlay v1.0.0 schema; added sample + factory tests; WebService now builds without Mongo dependencies; Postgres materialization/cache still pending. | Implementer | diff --git a/docs/modules/excititor/graph-overlays.md b/docs/modules/excititor/graph-overlays.md index 61c68593c..08742c278 100644 --- a/docs/modules/excititor/graph-overlays.md +++ b/docs/modules/excititor/graph-overlays.md @@ -84,3 +84,4 @@ Defines the graph-ready overlay built from Link-Not-Merge observations/linksets - Consumers (Console, Vuln Explorer, Policy Engine, Risk) should treat `vex_overlay.schema.json` as the authoritative contract. - Offline kits must bundle the schema file and sample payloads under `docs/samples/excititor/` with SHA256 manifests. - Future schema versions must bump `schemaVersion` and add migration notes to this document and `docs/modules/excititor/architecture.md`. +- Policy and Risk surfaces in WebService now read overlays directly (with claim-store fallback for policy tests) to produce lookup and risk feeds; overlay cache/store are selected per tenant (in-memory by default, Postgres `vex.graph_overlays` when configured). diff --git a/src/Excititor/StellaOps.Excititor.WebService/Endpoints/PolicyEndpoints.cs b/src/Excititor/StellaOps.Excititor.WebService/Endpoints/PolicyEndpoints.cs index 89408b089..f7b002f36 100644 --- a/src/Excititor/StellaOps.Excititor.WebService/Endpoints/PolicyEndpoints.cs +++ b/src/Excititor/StellaOps.Excititor.WebService/Endpoints/PolicyEndpoints.cs @@ -106,12 +106,8 @@ public static class PolicyEndpoints } var claimResults = await FallbackClaimsAsync(claimStore, advisories, purls, providerFilter, statusFilter, request.Limit, cancellationToken).ConfigureAwait(false); - var groupedClaims = claimResults - .GroupBy(c => c.AdvisoryKey, StringComparer.OrdinalIgnoreCase) - .Select(group => new PolicyVexLookupItem(group.Key, new[] { group.Key }, group.ToList())) - .ToList(); - - return Results.Ok(new PolicyVexLookupResponse(groupedClaims, claimResults.Count, timeProvider.GetUtcNow())); + var totalStatements = claimResults.Sum(item => item.Statements.Count); + return Results.Ok(new PolicyVexLookupResponse(claimResults, totalStatements, timeProvider.GetUtcNow())); } private static async Task> ResolveOverlaysAsync( @@ -184,7 +180,7 @@ public static class PolicyEndpoints Metadata: metadata); } - private static async Task> FallbackClaimsAsync( + private static async Task> FallbackClaimsAsync( IVexClaimStore claimStore, IReadOnlyList advisories, IReadOnlyList purls, @@ -193,7 +189,7 @@ public static class PolicyEndpoints int limit, CancellationToken cancellationToken) { - var results = new List(); + var results = new List(); foreach (var advisory in advisories) { var claims = await claimStore.FindByVulnerabilityAsync(advisory, limit, cancellationToken).ConfigureAwait(false); @@ -201,15 +197,18 @@ public static class PolicyEndpoints var filtered = claims .Where(c => providers.Count == 0 || providers.Contains(c.ProviderId, StringComparer.OrdinalIgnoreCase)) .Where(c => statuses.Count == 0 || statuses.Contains(c.Status.ToString().ToLowerInvariant())) - .Where(c => purls.Count == 0 || purls.Contains(c.Product.Key, StringComparer.OrdinalIgnoreCase)) + .Where(c => purls.Count == 0 + || purls.Contains(c.Product.Key, StringComparer.OrdinalIgnoreCase) + || (!string.IsNullOrWhiteSpace(c.Product.Purl) && purls.Contains(c.Product.Purl, StringComparer.OrdinalIgnoreCase))) .OrderByDescending(c => c.LastSeen) .ThenBy(c => c.ProviderId, StringComparer.Ordinal) - .Take(limit); + .Take(limit) + .Select(MapClaimStatement) + .ToList(); - results.AddRange(filtered.Select(MapClaimStatement)); - if (results.Count >= limit) + if (filtered.Count > 0) { - break; + results.Add(new PolicyVexLookupItem(advisory, new[] { advisory }, filtered)); } }