Refactor and update test projects, remove obsolete tests, and upgrade dependencies
- Deleted obsolete test files for SchedulerAuditService and SchedulerMongoSessionFactory. - Removed unused TestDataFactory class. - Updated project files for Mongo.Tests to remove references to deleted files. - Upgraded BouncyCastle.Cryptography package to version 2.6.2 across multiple projects. - Replaced Microsoft.Extensions.Http.Polly with Microsoft.Extensions.Http.Resilience in Zastava.Webhook project. - Updated NetEscapades.Configuration.Yaml package to version 3.1.0 in Configuration library. - Upgraded Pkcs11Interop package to version 5.1.2 in Cryptography libraries. - Refactored Argon2idPasswordHasher to use BouncyCastle for hashing instead of Konscious. - Updated JsonSchema.Net package to version 7.3.2 in Microservice project. - Updated global.json to use .NET SDK version 10.0.101.
This commit is contained in:
@@ -0,0 +1,19 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
using StellaOps.Excititor.Core.Evidence;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Contracts;
|
||||
|
||||
public sealed record EvidenceManifestResponse(
|
||||
[property: JsonPropertyName("manifest")] VexLockerManifest Manifest,
|
||||
[property: JsonPropertyName("attestationId")] string AttestationId,
|
||||
[property: JsonPropertyName("dsseEnvelope")] string DsseEnvelope,
|
||||
[property: JsonPropertyName("dsseEnvelopeHash")] string DsseEnvelopeHash,
|
||||
[property: JsonPropertyName("itemCount")] int ItemCount,
|
||||
[property: JsonPropertyName("generatedAt")] DateTimeOffset GeneratedAt);
|
||||
|
||||
public sealed record EvidenceChunkListResponse(
|
||||
[property: JsonPropertyName("chunks")] IReadOnlyList<VexEvidenceChunkResponse> Chunks,
|
||||
[property: JsonPropertyName("total")] int Total,
|
||||
[property: JsonPropertyName("truncated")] bool Truncated,
|
||||
[property: JsonPropertyName("generatedAt")] DateTimeOffset GeneratedAt);
|
||||
@@ -10,18 +10,45 @@ public sealed record GraphOverlaysResponse(
|
||||
[property: JsonPropertyName("cacheAgeMs")] long? CacheAgeMs);
|
||||
|
||||
public sealed record GraphOverlayItem(
|
||||
[property: JsonPropertyName("schemaVersion")] string SchemaVersion,
|
||||
[property: JsonPropertyName("generatedAt")] DateTimeOffset GeneratedAt,
|
||||
[property: JsonPropertyName("tenant")] string Tenant,
|
||||
[property: JsonPropertyName("purl")] string Purl,
|
||||
[property: JsonPropertyName("summary")] GraphOverlaySummary Summary,
|
||||
[property: JsonPropertyName("latestModifiedAt")] DateTimeOffset? LatestModifiedAt,
|
||||
[property: JsonPropertyName("justifications")] IReadOnlyList<string> Justifications,
|
||||
[property: JsonPropertyName("provenance")] GraphOverlayProvenance Provenance);
|
||||
[property: JsonPropertyName("advisoryId")] string AdvisoryId,
|
||||
[property: JsonPropertyName("source")] string Source,
|
||||
[property: JsonPropertyName("status")] string Status,
|
||||
[property: JsonPropertyName("justifications")] IReadOnlyList<GraphOverlayJustification> Justifications,
|
||||
[property: JsonPropertyName("conflicts")] IReadOnlyList<GraphOverlayConflict> Conflicts,
|
||||
[property: JsonPropertyName("observations")] IReadOnlyList<GraphOverlayObservation> Observations,
|
||||
[property: JsonPropertyName("provenance")] GraphOverlayProvenance Provenance,
|
||||
[property: JsonPropertyName("cache")] GraphOverlayCache? Cache);
|
||||
|
||||
public sealed record GraphOverlaySummary(
|
||||
[property: JsonPropertyName("open")] int Open,
|
||||
[property: JsonPropertyName("not_affected")] int NotAffected,
|
||||
[property: JsonPropertyName("under_investigation")] int UnderInvestigation,
|
||||
[property: JsonPropertyName("no_statement")] int NoStatement);
|
||||
public sealed record GraphOverlayJustification(
|
||||
[property: JsonPropertyName("kind")] string Kind,
|
||||
[property: JsonPropertyName("reason")] string Reason,
|
||||
[property: JsonPropertyName("evidence")] IReadOnlyList<string>? Evidence,
|
||||
[property: JsonPropertyName("weight")] double? Weight);
|
||||
|
||||
public sealed record GraphOverlayConflict(
|
||||
[property: JsonPropertyName("field")] string Field,
|
||||
[property: JsonPropertyName("reason")] string Reason,
|
||||
[property: JsonPropertyName("values")] IReadOnlyList<string> Values,
|
||||
[property: JsonPropertyName("sourceIds")] IReadOnlyList<string>? SourceIds);
|
||||
|
||||
public sealed record GraphOverlayObservation(
|
||||
[property: JsonPropertyName("id")] string Id,
|
||||
[property: JsonPropertyName("contentHash")] string ContentHash,
|
||||
[property: JsonPropertyName("fetchedAt")] DateTimeOffset FetchedAt);
|
||||
|
||||
public sealed record GraphOverlayProvenance(
|
||||
[property: JsonPropertyName("sources")] IReadOnlyList<string> Sources,
|
||||
[property: JsonPropertyName("lastEvidenceHash")] string? LastEvidenceHash);
|
||||
[property: JsonPropertyName("linksetId")] string LinksetId,
|
||||
[property: JsonPropertyName("linksetHash")] string LinksetHash,
|
||||
[property: JsonPropertyName("observationHashes")] IReadOnlyList<string> ObservationHashes,
|
||||
[property: JsonPropertyName("policyHash")] string? PolicyHash,
|
||||
[property: JsonPropertyName("sbomContextHash")] string? SbomContextHash,
|
||||
[property: JsonPropertyName("planCacheKey")] string? PlanCacheKey);
|
||||
|
||||
public sealed record GraphOverlayCache(
|
||||
[property: JsonPropertyName("cached")] bool Cached,
|
||||
[property: JsonPropertyName("cachedAt")] DateTimeOffset? CachedAt,
|
||||
[property: JsonPropertyName("ttlSeconds")] int? TtlSeconds);
|
||||
|
||||
@@ -15,3 +15,9 @@ public sealed record GraphStatusItem(
|
||||
[property: JsonPropertyName("latestModifiedAt")] DateTimeOffset? LatestModifiedAt,
|
||||
[property: JsonPropertyName("sources")] IReadOnlyList<string> Sources,
|
||||
[property: JsonPropertyName("lastEvidenceHash")] string? LastEvidenceHash);
|
||||
|
||||
public sealed record GraphOverlaySummary(
|
||||
[property: JsonPropertyName("open")] int Open,
|
||||
[property: JsonPropertyName("not_affected")] int NotAffected,
|
||||
[property: JsonPropertyName("under_investigation")] int UnderInvestigation,
|
||||
[property: JsonPropertyName("no_statement")] int NoStatement);
|
||||
|
||||
@@ -4,6 +4,7 @@ using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
using static Program;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Endpoints;
|
||||
|
||||
|
||||
@@ -2,23 +2,38 @@ using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Evidence;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
using static Program;
|
||||
using StellaOps.Excititor.WebService.Telemetry;
|
||||
using System.Collections.Immutable;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Endpoints;
|
||||
|
||||
/// <summary>
|
||||
/// Evidence API endpoints (temporarily disabled while Mongo/BSON storage is removed).
|
||||
/// Evidence API endpoints (manifest + DSSE attestation + evidence chunks).
|
||||
/// </summary>
|
||||
public static class EvidenceEndpoints
|
||||
{
|
||||
public static void MapEvidenceEndpoints(this WebApplication app)
|
||||
{
|
||||
// GET /evidence/vex/list
|
||||
app.MapGet("/evidence/vex/list", (
|
||||
app.MapGet("/evidence/vex/list", async (
|
||||
HttpContext context,
|
||||
[FromQuery(Name = "vulnerabilityId")] string[] vulnerabilityIds,
|
||||
[FromQuery(Name = "productKey")] string[] productKeys,
|
||||
[FromQuery] string? since,
|
||||
[FromQuery] int? limit,
|
||||
IVexClaimStore claimStore,
|
||||
IVexEvidenceLockerService lockerService,
|
||||
IVexEvidenceAttestor attestor,
|
||||
IOptions<VexStorageOptions> storageOptions,
|
||||
ChunkTelemetry chunkTelemetry) =>
|
||||
ChunkTelemetry chunkTelemetry,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
|
||||
if (scopeResult is not null)
|
||||
@@ -31,18 +46,76 @@ public static class EvidenceEndpoints
|
||||
return tenantError;
|
||||
}
|
||||
|
||||
chunkTelemetry.RecordIngested(tenant, null, "unavailable", "storage-migration", 0, 0, 0);
|
||||
return Results.Problem(
|
||||
detail: "Evidence exports are temporarily unavailable during Postgres migration (Mongo/BSON removed).",
|
||||
statusCode: StatusCodes.Status503ServiceUnavailable,
|
||||
title: "Service unavailable");
|
||||
var parsedSince = ParseSinceTimestamp(new Microsoft.Extensions.Primitives.StringValues(since));
|
||||
var max = Math.Clamp(limit ?? 500, 1, 1000);
|
||||
|
||||
var pairs = NormalizeValues(vulnerabilityIds).SelectMany(v =>
|
||||
NormalizeValues(productKeys).Select(p => (Vuln: v, Product: p))).ToList();
|
||||
|
||||
if (pairs.Count == 0)
|
||||
{
|
||||
return Results.BadRequest("At least one vulnerabilityId and productKey are required.");
|
||||
}
|
||||
|
||||
var claims = new List<VexClaim>();
|
||||
foreach (var pair in pairs)
|
||||
{
|
||||
var found = await claimStore.FindAsync(pair.Vuln, pair.Product, parsedSince, cancellationToken).ConfigureAwait(false);
|
||||
claims.AddRange(found);
|
||||
}
|
||||
|
||||
claims = claims
|
||||
.OrderBy(c => c.VulnerabilityId, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(c => c.Product.Key, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenByDescending(c => c.LastSeen)
|
||||
.Take(max)
|
||||
.ToList();
|
||||
|
||||
if (claims.Count == 0)
|
||||
{
|
||||
return Results.NotFound("No claims available for the requested filters.");
|
||||
}
|
||||
|
||||
var items = claims.Select(claim =>
|
||||
new VexEvidenceSnapshotItem(
|
||||
observationId: FormattableString.Invariant($"{claim.ProviderId}:{claim.Document.Digest}"),
|
||||
providerId: claim.ProviderId,
|
||||
contentHash: claim.Document.Digest,
|
||||
linksetId: FormattableString.Invariant($"{claim.VulnerabilityId}:{claim.Product.Key}"),
|
||||
dsseEnvelopeHash: null,
|
||||
provenance: new VexEvidenceProvenance("ingest")))
|
||||
.ToList();
|
||||
|
||||
var now = timeProvider.GetUtcNow();
|
||||
var manifest = lockerService.BuildManifest(tenant, items, timestamp: now, sequence: 1, isSealed: false);
|
||||
var attestation = await attestor.AttestManifestAsync(manifest, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
chunkTelemetry.RecordIngested(tenant, null, "available", "locker-manifest", claims.Count, 0, 0);
|
||||
var response = new EvidenceManifestResponse(
|
||||
attestation.SignedManifest,
|
||||
attestation.AttestationId,
|
||||
attestation.DsseEnvelopeJson,
|
||||
attestation.DsseEnvelopeHash,
|
||||
attestation.SignedManifest.Items.Length,
|
||||
attestation.AttestedAt);
|
||||
|
||||
return Results.Ok(response);
|
||||
}).WithName("ListVexEvidence");
|
||||
|
||||
// GET /evidence/vex/{bundleId}
|
||||
app.MapGet("/evidence/vex/{bundleId}", (
|
||||
app.MapGet("/evidence/vex/{bundleId}", async (
|
||||
HttpContext context,
|
||||
string bundleId,
|
||||
IOptions<VexStorageOptions> storageOptions) =>
|
||||
[FromQuery(Name = "vulnerabilityId")] string[] vulnerabilityIds,
|
||||
[FromQuery(Name = "productKey")] string[] productKeys,
|
||||
[FromQuery] string? since,
|
||||
[FromQuery] int? limit,
|
||||
IVexClaimStore claimStore,
|
||||
IVexEvidenceLockerService lockerService,
|
||||
IVexEvidenceAttestor attestor,
|
||||
IOptions<VexStorageOptions> storageOptions,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
|
||||
if (scopeResult is not null)
|
||||
@@ -50,7 +123,7 @@ public static class EvidenceEndpoints
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out _, out var tenantError))
|
||||
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out var tenant, out var tenantError))
|
||||
{
|
||||
return tenantError;
|
||||
}
|
||||
@@ -63,17 +136,77 @@ public static class EvidenceEndpoints
|
||||
title: "Validation error");
|
||||
}
|
||||
|
||||
return Results.Problem(
|
||||
detail: "Evidence bundles are temporarily unavailable during Postgres migration (Mongo/BSON removed).",
|
||||
statusCode: StatusCodes.Status503ServiceUnavailable,
|
||||
title: "Service unavailable");
|
||||
var parsedSince = ParseSinceTimestamp(new Microsoft.Extensions.Primitives.StringValues(since));
|
||||
var max = Math.Clamp(limit ?? 500, 1, 1000);
|
||||
var pairs = NormalizeValues(vulnerabilityIds).SelectMany(v =>
|
||||
NormalizeValues(productKeys).Select(p => (Vuln: v, Product: p))).ToList();
|
||||
|
||||
if (pairs.Count == 0)
|
||||
{
|
||||
return Results.BadRequest("At least one vulnerabilityId and productKey are required.");
|
||||
}
|
||||
|
||||
var claims = new List<VexClaim>();
|
||||
foreach (var pair in pairs)
|
||||
{
|
||||
var found = await claimStore.FindAsync(pair.Vuln, pair.Product, parsedSince, cancellationToken).ConfigureAwait(false);
|
||||
claims.AddRange(found);
|
||||
}
|
||||
|
||||
claims = claims
|
||||
.OrderBy(c => c.VulnerabilityId, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(c => c.Product.Key, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenByDescending(c => c.LastSeen)
|
||||
.Take(max)
|
||||
.ToList();
|
||||
|
||||
if (claims.Count == 0)
|
||||
{
|
||||
return Results.NotFound("No claims available for the requested filters.");
|
||||
}
|
||||
|
||||
var items = claims.Select(claim =>
|
||||
new VexEvidenceSnapshotItem(
|
||||
observationId: FormattableString.Invariant($"{claim.ProviderId}:{claim.Document.Digest}"),
|
||||
providerId: claim.ProviderId,
|
||||
contentHash: claim.Document.Digest,
|
||||
linksetId: FormattableString.Invariant($"{claim.VulnerabilityId}:{claim.Product.Key}"),
|
||||
dsseEnvelopeHash: null,
|
||||
provenance: new VexEvidenceProvenance("ingest")))
|
||||
.ToList();
|
||||
|
||||
var now = timeProvider.GetUtcNow();
|
||||
var manifest = lockerService.BuildManifest(tenant, items, timestamp: now, sequence: 1, isSealed: false);
|
||||
if (!string.Equals(manifest.ManifestId, bundleId, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return Results.NotFound($"Requested bundleId '{bundleId}' not found for current filters.");
|
||||
}
|
||||
|
||||
var attestation = await attestor.AttestManifestAsync(manifest, cancellationToken).ConfigureAwait(false);
|
||||
var response = new EvidenceManifestResponse(
|
||||
attestation.SignedManifest,
|
||||
attestation.AttestationId,
|
||||
attestation.DsseEnvelopeJson,
|
||||
attestation.DsseEnvelopeHash,
|
||||
attestation.SignedManifest.Items.Length,
|
||||
attestation.AttestedAt);
|
||||
|
||||
return Results.Ok(response);
|
||||
}).WithName("GetVexEvidenceBundle");
|
||||
|
||||
// GET /v1/vex/evidence/chunks
|
||||
app.MapGet("/v1/vex/evidence/chunks", (
|
||||
app.MapGet("/v1/vex/evidence/chunks", async (
|
||||
HttpContext context,
|
||||
[FromQuery] string vulnerabilityId,
|
||||
[FromQuery] string productKey,
|
||||
[FromQuery(Name = "providerId")] string[] providerIds,
|
||||
[FromQuery] string[] status,
|
||||
[FromQuery] string? since,
|
||||
[FromQuery] int? limit,
|
||||
IOptions<VexStorageOptions> storageOptions,
|
||||
ChunkTelemetry chunkTelemetry) =>
|
||||
IVexEvidenceChunkService chunkService,
|
||||
ChunkTelemetry chunkTelemetry,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
|
||||
if (scopeResult is not null)
|
||||
@@ -86,11 +219,37 @@ public static class EvidenceEndpoints
|
||||
return tenantError;
|
||||
}
|
||||
|
||||
chunkTelemetry.RecordIngested(tenant, null, "unavailable", "storage-migration", 0, 0, 0);
|
||||
return Results.Problem(
|
||||
detail: "Evidence chunk streaming is temporarily unavailable during Postgres migration (Mongo/BSON removed).",
|
||||
statusCode: StatusCodes.Status503ServiceUnavailable,
|
||||
title: "Service unavailable");
|
||||
if (string.IsNullOrWhiteSpace(vulnerabilityId) || string.IsNullOrWhiteSpace(productKey))
|
||||
{
|
||||
return Results.BadRequest("vulnerabilityId and productKey are required.");
|
||||
}
|
||||
|
||||
var parsedSince = ParseSinceTimestamp(new Microsoft.Extensions.Primitives.StringValues(since));
|
||||
var providers = providerIds?.Length > 0
|
||||
? providerIds.ToImmutableHashSet(StringComparer.OrdinalIgnoreCase)
|
||||
: ImmutableHashSet<string>.Empty;
|
||||
|
||||
var statuses = status?.Length > 0
|
||||
? status
|
||||
.Select(s => Enum.TryParse<VexClaimStatus>(s, true, out var parsed) ? parsed : (VexClaimStatus?)null)
|
||||
.Where(s => s is not null)
|
||||
.Select(s => s!.Value)
|
||||
.ToImmutableHashSet()
|
||||
: ImmutableHashSet<VexClaimStatus>.Empty;
|
||||
|
||||
var req = new VexEvidenceChunkRequest(
|
||||
tenant,
|
||||
vulnerabilityId,
|
||||
productKey,
|
||||
providers,
|
||||
statuses,
|
||||
parsedSince,
|
||||
Math.Clamp(limit ?? 200, 1, 1000));
|
||||
|
||||
var result = await chunkService.QueryAsync(req, cancellationToken).ConfigureAwait(false);
|
||||
chunkTelemetry.RecordIngested(tenant, null, "available", "locker-chunks", result.TotalCount, 0, 0);
|
||||
|
||||
return Results.Ok(new EvidenceChunkListResponse(result.Chunks, result.TotalCount, result.Truncated, result.GeneratedAtUtc));
|
||||
}).WithName("GetVexEvidenceChunks");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ using System.IO;
|
||||
using System.Text;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Core;
|
||||
@@ -71,9 +72,9 @@ internal static class MirrorEndpoints
|
||||
string domainId,
|
||||
HttpContext httpContext,
|
||||
IOptions<MirrorDistributionOptions> options,
|
||||
MirrorRateLimiter rateLimiter,
|
||||
IVexExportStore exportStore,
|
||||
TimeProvider timeProvider,
|
||||
[FromServices] MirrorRateLimiter rateLimiter,
|
||||
[FromServices] IVexExportStore exportStore,
|
||||
[FromServices] TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!TryFindDomain(options.Value, domainId, out var domain))
|
||||
@@ -162,9 +163,9 @@ internal static class MirrorEndpoints
|
||||
string exportKey,
|
||||
HttpContext httpContext,
|
||||
IOptions<MirrorDistributionOptions> options,
|
||||
MirrorRateLimiter rateLimiter,
|
||||
IVexExportStore exportStore,
|
||||
TimeProvider timeProvider,
|
||||
[FromServices] MirrorRateLimiter rateLimiter,
|
||||
[FromServices] IVexExportStore exportStore,
|
||||
[FromServices] TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!TryFindDomain(options.Value, domainId, out var domain))
|
||||
@@ -215,9 +216,9 @@ internal static class MirrorEndpoints
|
||||
string exportKey,
|
||||
HttpContext httpContext,
|
||||
IOptions<MirrorDistributionOptions> options,
|
||||
MirrorRateLimiter rateLimiter,
|
||||
IVexExportStore exportStore,
|
||||
IEnumerable<IVexArtifactStore> artifactStores,
|
||||
[FromServices] MirrorRateLimiter rateLimiter,
|
||||
[FromServices] IVexExportStore exportStore,
|
||||
[FromServices] IEnumerable<IVexArtifactStore> artifactStores,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!TryFindDomain(options.Value, domainId, out var domain))
|
||||
|
||||
@@ -9,8 +9,6 @@ using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Canonicalization;
|
||||
using StellaOps.Excititor.Core.Orchestration;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
@@ -34,7 +32,7 @@ public static class PolicyEndpoints
|
||||
HttpContext context,
|
||||
[FromBody] PolicyVexLookupRequest request,
|
||||
IOptions<VexStorageOptions> storageOptions,
|
||||
[FromServices] IVexClaimStore claimStore,
|
||||
[FromServices] IGraphOverlayStore overlayStore,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -45,7 +43,7 @@ public static class PolicyEndpoints
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryResolveTenant(context, storageOptions.Value, out _, out var tenantError))
|
||||
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
|
||||
{
|
||||
return tenantError!;
|
||||
}
|
||||
@@ -56,24 +54,19 @@ public static class PolicyEndpoints
|
||||
return Results.BadRequest(new { error = new { code = "ERR_REQUEST", message = "advisory_keys or purls must be provided" } });
|
||||
}
|
||||
|
||||
var canonicalizer = new VexAdvisoryKeyCanonicalizer();
|
||||
var productCanonicalizer = new VexProductKeyCanonicalizer();
|
||||
|
||||
var canonicalAdvisories = request.AdvisoryKeys
|
||||
var advisories = request.AdvisoryKeys
|
||||
.Where(a => !string.IsNullOrWhiteSpace(a))
|
||||
.Select(a => canonicalizer.Canonicalize(a.Trim()))
|
||||
.Select(a => a.Trim())
|
||||
.ToList();
|
||||
|
||||
var canonicalProducts = request.Purls
|
||||
var purls = request.Purls
|
||||
.Where(p => !string.IsNullOrWhiteSpace(p))
|
||||
.Select(p => productCanonicalizer.Canonicalize(p.Trim(), purl: p.Trim()))
|
||||
.Select(p => p.Trim())
|
||||
.ToList();
|
||||
|
||||
// Map requested statuses/providers for filtering
|
||||
var statusFilter = request.Statuses
|
||||
.Select(s => Enum.TryParse<VexClaimStatus>(s, true, out var parsed) ? parsed : (VexClaimStatus?)null)
|
||||
.Where(p => p.HasValue)
|
||||
.Select(p => p!.Value)
|
||||
.Where(s => !string.IsNullOrWhiteSpace(s))
|
||||
.Select(s => s.Trim().ToLowerInvariant())
|
||||
.ToImmutableHashSet();
|
||||
|
||||
var providerFilter = request.Providers
|
||||
@@ -81,94 +74,96 @@ public static class PolicyEndpoints
|
||||
.Select(p => p.Trim())
|
||||
.ToImmutableHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var limit = Math.Clamp(request.Limit, 1, 500);
|
||||
var now = timeProvider.GetUtcNow();
|
||||
var overlays = await ResolveOverlaysAsync(overlayStore, tenant!, advisories, purls, request.Limit, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var results = new List<PolicyVexLookupItem>();
|
||||
var totalStatements = 0;
|
||||
var filtered = overlays
|
||||
.Where(o => MatchesProvider(providerFilter, o))
|
||||
.Where(o => MatchesStatus(statusFilter, o))
|
||||
.OrderBy(o => o.AdvisoryId, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(o => o.Purl, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(o => o.Source, StringComparer.OrdinalIgnoreCase)
|
||||
.Take(Math.Clamp(request.Limit, 1, 500))
|
||||
.ToList();
|
||||
|
||||
// For each advisory key, fetch claims and filter by product/provider/status
|
||||
foreach (var advisory in canonicalAdvisories)
|
||||
{
|
||||
var claims = await claimStore
|
||||
.FindByVulnerabilityAsync(advisory.AdvisoryKey, limit, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
var grouped = filtered
|
||||
.GroupBy(o => o.AdvisoryId, StringComparer.OrdinalIgnoreCase)
|
||||
.Select(group => new PolicyVexLookupItem(
|
||||
group.Key,
|
||||
new[] { group.Key },
|
||||
group.Select(MapStatement).ToList()))
|
||||
.ToList();
|
||||
|
||||
var filtered = claims
|
||||
.Where(claim => MatchesProvider(providerFilter, claim))
|
||||
.Where(claim => MatchesStatus(statusFilter, claim))
|
||||
.Where(claim => MatchesProduct(canonicalProducts, claim))
|
||||
.OrderByDescending(claim => claim.LastSeen)
|
||||
.ThenBy(claim => claim.ProviderId, StringComparer.Ordinal)
|
||||
.ThenBy(claim => claim.Product.Key, StringComparer.Ordinal)
|
||||
.Take(limit)
|
||||
.ToList();
|
||||
|
||||
totalStatements += filtered.Count;
|
||||
|
||||
var statements = filtered.Select(MapStatement).ToList();
|
||||
var aliases = advisory.Aliases.ToList();
|
||||
if (!aliases.Contains(advisory.AdvisoryKey, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
aliases.Add(advisory.AdvisoryKey);
|
||||
}
|
||||
|
||||
results.Add(new PolicyVexLookupItem(
|
||||
advisory.AdvisoryKey,
|
||||
aliases,
|
||||
statements));
|
||||
}
|
||||
|
||||
var response = new PolicyVexLookupResponse(results, totalStatements, now);
|
||||
var response = new PolicyVexLookupResponse(grouped, filtered.Count, timeProvider.GetUtcNow());
|
||||
return Results.Ok(response);
|
||||
}
|
||||
|
||||
private static bool MatchesProvider(ISet<string> providers, VexClaim claim)
|
||||
=> providers.Count == 0 || providers.Contains(claim.ProviderId, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
private static bool MatchesStatus(ISet<VexClaimStatus> statuses, VexClaim claim)
|
||||
=> statuses.Count == 0 || statuses.Contains(claim.Status);
|
||||
|
||||
private static bool MatchesProduct(IEnumerable<VexCanonicalProductKey> requestedProducts, VexClaim claim)
|
||||
private static async Task<IReadOnlyList<GraphOverlayItem>> ResolveOverlaysAsync(
|
||||
IGraphOverlayStore overlayStore,
|
||||
string tenant,
|
||||
IReadOnlyList<string> advisories,
|
||||
IReadOnlyList<string> purls,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!requestedProducts.Any())
|
||||
if (purls.Count > 0)
|
||||
{
|
||||
return true;
|
||||
var overlays = await overlayStore.FindByPurlsAsync(tenant, purls, cancellationToken).ConfigureAwait(false);
|
||||
if (advisories.Count == 0)
|
||||
{
|
||||
return overlays;
|
||||
}
|
||||
|
||||
return overlays.Where(o => advisories.Contains(o.AdvisoryId, StringComparer.OrdinalIgnoreCase)).ToList();
|
||||
}
|
||||
|
||||
return requestedProducts.Any(product =>
|
||||
string.Equals(product.ProductKey, claim.Product.Key, StringComparison.OrdinalIgnoreCase) ||
|
||||
product.Links.Any(link => string.Equals(link.Identifier, claim.Product.Key, StringComparison.OrdinalIgnoreCase)) ||
|
||||
(!string.IsNullOrWhiteSpace(product.Purl) && string.Equals(product.Purl, claim.Product.Purl, StringComparison.OrdinalIgnoreCase)));
|
||||
return await overlayStore.FindByAdvisoriesAsync(tenant, advisories, limit, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static PolicyVexStatement MapStatement(VexClaim claim)
|
||||
private static bool MatchesProvider(ISet<string> providers, GraphOverlayItem overlay)
|
||||
=> providers.Count == 0 || providers.Contains(overlay.Source, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
private static bool MatchesStatus(ISet<string> statuses, GraphOverlayItem overlay)
|
||||
=> statuses.Count == 0 || statuses.Contains(overlay.Status, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
private static PolicyVexStatement MapStatement(GraphOverlayItem overlay)
|
||||
{
|
||||
var observationId = $"{claim.ProviderId}:{claim.Document.Digest}";
|
||||
var firstSeen = overlay.Observations.Count == 0
|
||||
? overlay.GeneratedAt
|
||||
: overlay.Observations.Min(o => o.FetchedAt);
|
||||
|
||||
var lastSeen = overlay.Observations.Count == 0
|
||||
? overlay.GeneratedAt
|
||||
: overlay.Observations.Max(o => o.FetchedAt);
|
||||
|
||||
var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["document_digest"] = claim.Document.Digest,
|
||||
["document_uri"] = claim.Document.SourceUri.ToString()
|
||||
["schemaVersion"] = overlay.SchemaVersion,
|
||||
["linksetId"] = overlay.Provenance.LinksetId,
|
||||
["linksetHash"] = overlay.Provenance.LinksetHash,
|
||||
["source"] = overlay.Source
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(claim.Document.Revision))
|
||||
if (!string.IsNullOrWhiteSpace(overlay.Provenance.PlanCacheKey))
|
||||
{
|
||||
metadata["document_revision"] = claim.Document.Revision!;
|
||||
metadata["planCacheKey"] = overlay.Provenance.PlanCacheKey!;
|
||||
}
|
||||
|
||||
var justification = overlay.Justifications.FirstOrDefault();
|
||||
var primaryObservation = overlay.Observations.FirstOrDefault();
|
||||
|
||||
return new PolicyVexStatement(
|
||||
ObservationId: observationId,
|
||||
ProviderId: claim.ProviderId,
|
||||
Status: claim.Status.ToString(),
|
||||
ProductKey: claim.Product.Key,
|
||||
Purl: claim.Product.Purl,
|
||||
Cpe: claim.Product.Cpe,
|
||||
Version: claim.Product.Version,
|
||||
Justification: claim.Justification?.ToString(),
|
||||
Detail: claim.Detail,
|
||||
FirstSeen: claim.FirstSeen,
|
||||
LastSeen: claim.LastSeen,
|
||||
Signature: claim.Document.Signature,
|
||||
ObservationId: primaryObservation?.Id ?? $"{overlay.Source}:{overlay.AdvisoryId}",
|
||||
ProviderId: overlay.Source,
|
||||
Status: overlay.Status,
|
||||
ProductKey: overlay.Purl,
|
||||
Purl: overlay.Purl,
|
||||
Cpe: null,
|
||||
Version: null,
|
||||
Justification: justification?.Kind,
|
||||
Detail: justification?.Reason,
|
||||
FirstSeen: firstSeen,
|
||||
LastSeen: lastSeen,
|
||||
Signature: null,
|
||||
Metadata: metadata);
|
||||
}
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Excititor.Attestation;
|
||||
@@ -33,7 +34,7 @@ internal static class ResolveEndpoint
|
||||
VexResolveRequest request,
|
||||
HttpContext httpContext,
|
||||
IVexClaimStore claimStore,
|
||||
IVexConsensusStore consensusStore,
|
||||
[FromServices] IVexConsensusStore? consensusStore,
|
||||
IVexProviderStore providerStore,
|
||||
IVexPolicyProvider policyProvider,
|
||||
TimeProvider timeProvider,
|
||||
@@ -142,7 +143,10 @@ internal static class ResolveEndpoint
|
||||
snapshot.Digest);
|
||||
}
|
||||
|
||||
await consensusStore.SaveAsync(consensus, cancellationToken).ConfigureAwait(false);
|
||||
if (consensusStore is not null)
|
||||
{
|
||||
await consensusStore.SaveAsync(consensus, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var payload = PreparePayload(consensus);
|
||||
var contentSignature = await TrySignAsync(signer, payload, logger, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
@@ -1,27 +1,27 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json;
|
||||
using StellaOps.Concelier.RawModels;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
using RawModels = StellaOps.Concelier.RawModels;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Extensions;
|
||||
|
||||
internal static class VexRawDocumentMapper
|
||||
{
|
||||
public static VexRawDocument ToRawModel(VexRawRecord record, string defaultTenant)
|
||||
public static RawModels.VexRawDocument ToRawModel(VexRawRecord record, string defaultTenant)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(record);
|
||||
|
||||
var metadata = record.Metadata ?? ImmutableDictionary<string, string>.Empty;
|
||||
var tenant = Get(metadata, "tenant", record.Tenant) ?? defaultTenant;
|
||||
|
||||
var source = new RawSourceMetadata(
|
||||
var source = new RawModels.RawSourceMetadata(
|
||||
Vendor: Get(metadata, "source.vendor", record.ProviderId) ?? record.ProviderId,
|
||||
Connector: Get(metadata, "source.connector", record.ProviderId) ?? record.ProviderId,
|
||||
ConnectorVersion: Get(metadata, "source.connector_version", "unknown") ?? "unknown",
|
||||
Stream: Get(metadata, "source.stream", record.Format.ToString().ToLowerInvariant()));
|
||||
|
||||
var signature = new RawSignatureMetadata(
|
||||
var signature = new RawModels.RawSignatureMetadata(
|
||||
Present: string.Equals(Get(metadata, "signature.present"), "true", StringComparison.OrdinalIgnoreCase),
|
||||
Format: Get(metadata, "signature.format"),
|
||||
KeyId: Get(metadata, "signature.key_id"),
|
||||
@@ -29,7 +29,7 @@ internal static class VexRawDocumentMapper
|
||||
Certificate: Get(metadata, "signature.certificate"),
|
||||
Digest: Get(metadata, "signature.digest"));
|
||||
|
||||
var upstream = new RawUpstreamMetadata(
|
||||
var upstream = new RawModels.RawUpstreamMetadata(
|
||||
UpstreamId: Get(metadata, "upstream.id", record.Digest) ?? record.Digest,
|
||||
DocumentVersion: Get(metadata, "upstream.version"),
|
||||
RetrievedAt: record.RetrievedAt,
|
||||
@@ -37,20 +37,20 @@ internal static class VexRawDocumentMapper
|
||||
Signature: signature,
|
||||
Provenance: metadata);
|
||||
|
||||
var content = new RawContent(
|
||||
var content = new RawModels.RawContent(
|
||||
Format: record.Format.ToString().ToLowerInvariant(),
|
||||
SpecVersion: Get(metadata, "content.spec_version"),
|
||||
Raw: ParseJson(record.Content),
|
||||
Encoding: Get(metadata, "content.encoding"));
|
||||
|
||||
return new VexRawDocument(
|
||||
return new RawModels.VexRawDocument(
|
||||
tenant,
|
||||
source,
|
||||
upstream,
|
||||
content,
|
||||
new RawLinkset(),
|
||||
statements: null,
|
||||
supersedes: record.SupersedesDigest);
|
||||
new RawModels.RawLinkset(),
|
||||
Statements: null,
|
||||
Supersedes: record.SupersedesDigest);
|
||||
}
|
||||
|
||||
private static string? Get(IReadOnlyDictionary<string, string> metadata, string key, string? fallback = null)
|
||||
|
||||
@@ -11,10 +11,17 @@ namespace StellaOps.Excititor.WebService.Graph;
|
||||
internal static class GraphOverlayFactory
|
||||
{
|
||||
public static IReadOnlyList<GraphOverlayItem> Build(
|
||||
string tenant,
|
||||
DateTimeOffset generatedAt,
|
||||
IReadOnlyList<string> orderedPurls,
|
||||
IReadOnlyList<VexObservation> observations,
|
||||
bool includeJustifications)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tenant))
|
||||
{
|
||||
throw new ArgumentException("tenant is required", nameof(tenant));
|
||||
}
|
||||
|
||||
if (orderedPurls is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(orderedPurls));
|
||||
@@ -25,101 +32,215 @@ internal static class GraphOverlayFactory
|
||||
throw new ArgumentNullException(nameof(observations));
|
||||
}
|
||||
|
||||
var observationsByPurl = observations
|
||||
.SelectMany(obs => obs.Linkset.Purls.Select(purl => (purl, obs)))
|
||||
.GroupBy(tuple => tuple.purl, StringComparer.OrdinalIgnoreCase)
|
||||
.ToDictionary(g => g.Key, g => g.Select(t => t.obs).ToImmutableArray(), StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var items = new List<GraphOverlayItem>(orderedPurls.Count);
|
||||
|
||||
foreach (var input in orderedPurls)
|
||||
var purlOrder = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
|
||||
for (var i = 0; i < orderedPurls.Count; i++)
|
||||
{
|
||||
if (!observationsByPurl.TryGetValue(input, out var obsForPurl) || obsForPurl.Length == 0)
|
||||
{
|
||||
items.Add(new GraphOverlayItem(
|
||||
Purl: input,
|
||||
Summary: new GraphOverlaySummary(0, 0, 0, 0),
|
||||
LatestModifiedAt: null,
|
||||
Justifications: Array.Empty<string>(),
|
||||
Provenance: new GraphOverlayProvenance(Array.Empty<string>(), null)));
|
||||
continue;
|
||||
}
|
||||
|
||||
var open = 0;
|
||||
var notAffected = 0;
|
||||
var underInvestigation = 0;
|
||||
var noStatement = 0;
|
||||
var justifications = new SortedSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var sources = new SortedSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
string? lastEvidenceHash = null;
|
||||
DateTimeOffset? latestModifiedAt = null;
|
||||
|
||||
foreach (var obs in obsForPurl)
|
||||
{
|
||||
sources.Add(obs.ProviderId);
|
||||
if (latestModifiedAt is null || obs.CreatedAt > latestModifiedAt.Value)
|
||||
{
|
||||
latestModifiedAt = obs.CreatedAt;
|
||||
lastEvidenceHash = obs.Upstream.ContentHash;
|
||||
}
|
||||
|
||||
var matchingStatements = obs.Statements
|
||||
.Where(stmt => PurlMatches(stmt, input, obs.Linkset.Purls))
|
||||
.ToArray();
|
||||
|
||||
if (matchingStatements.Length == 0)
|
||||
{
|
||||
noStatement++;
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var stmt in matchingStatements)
|
||||
{
|
||||
switch (stmt.Status)
|
||||
{
|
||||
case VexClaimStatus.NotAffected:
|
||||
notAffected++;
|
||||
break;
|
||||
case VexClaimStatus.UnderInvestigation:
|
||||
underInvestigation++;
|
||||
break;
|
||||
default:
|
||||
open++;
|
||||
break;
|
||||
}
|
||||
|
||||
if (includeJustifications && stmt.Justification is not null)
|
||||
{
|
||||
justifications.Add(stmt.Justification!.ToString()!);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
items.Add(new GraphOverlayItem(
|
||||
Purl: input,
|
||||
Summary: new GraphOverlaySummary(open, notAffected, underInvestigation, noStatement),
|
||||
LatestModifiedAt: latestModifiedAt,
|
||||
Justifications: includeJustifications
|
||||
? justifications.ToArray()
|
||||
: Array.Empty<string>(),
|
||||
Provenance: new GraphOverlayProvenance(sources.ToArray(), lastEvidenceHash)));
|
||||
purlOrder[orderedPurls[i]] = i;
|
||||
}
|
||||
|
||||
return items;
|
||||
var aggregates = new Dictionary<(string Purl, string AdvisoryId, string Source), OverlayAggregate>(new OverlayKeyComparer());
|
||||
|
||||
foreach (var observation in observations.OrderByDescending(o => o.CreatedAt).ThenBy(o => o.ObservationId, StringComparer.Ordinal))
|
||||
{
|
||||
var observationRef = new GraphOverlayObservation(
|
||||
observation.ObservationId,
|
||||
observation.Upstream.ContentHash,
|
||||
observation.Upstream.FetchedAt);
|
||||
|
||||
foreach (var statement in observation.Statements)
|
||||
{
|
||||
var targetPurls = ResolvePurls(statement, observation.Linkset.Purls);
|
||||
foreach (var purl in targetPurls)
|
||||
{
|
||||
if (!purlOrder.ContainsKey(purl))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var key = (purl, statement.VulnerabilityId, observation.ProviderId);
|
||||
if (!aggregates.TryGetValue(key, out var aggregate))
|
||||
{
|
||||
aggregate = new OverlayAggregate(purl, statement.VulnerabilityId, observation.ProviderId);
|
||||
aggregates[key] = aggregate;
|
||||
}
|
||||
|
||||
aggregate.UpdateStatus(statement.Status, observation.CreatedAt);
|
||||
if (includeJustifications && statement.Justification is not null)
|
||||
{
|
||||
aggregate.AddJustification(statement.Justification.Value, observation.ObservationId);
|
||||
}
|
||||
|
||||
aggregate.AddObservation(observationRef);
|
||||
aggregate.AddConflicts(observation.Linkset.Disagreements);
|
||||
aggregate.SetProvenance(
|
||||
observation.StreamId ?? observation.ObservationId,
|
||||
observation.Upstream.ContentHash,
|
||||
observation.Upstream.ContentHash);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var overlays = aggregates.Values
|
||||
.OrderBy(a => purlOrder[a.Purl])
|
||||
.ThenBy(a => a.AdvisoryId, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(a => a.Source, StringComparer.OrdinalIgnoreCase)
|
||||
.Select(a => a.ToOverlayItem(tenant, generatedAt, includeJustifications))
|
||||
.ToList();
|
||||
|
||||
return overlays;
|
||||
}
|
||||
|
||||
private static bool PurlMatches(VexObservationStatement stmt, string inputPurl, ImmutableArray<string> linksetPurls)
|
||||
private static IReadOnlyList<string> ResolvePurls(VexObservationStatement stmt, ImmutableArray<string> linksetPurls)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(stmt.Purl) && stmt.Purl.Equals(inputPurl, StringComparison.OrdinalIgnoreCase))
|
||||
if (!string.IsNullOrWhiteSpace(stmt.Purl))
|
||||
{
|
||||
return true;
|
||||
return new[] { stmt.Purl };
|
||||
}
|
||||
|
||||
if (linksetPurls.IsDefaultOrEmpty)
|
||||
{
|
||||
return false;
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
return linksetPurls.Any(p => p.Equals(inputPurl, StringComparison.OrdinalIgnoreCase));
|
||||
return linksetPurls.Where(p => !string.IsNullOrWhiteSpace(p)).ToArray();
|
||||
}
|
||||
|
||||
private static string MapStatus(VexClaimStatus status)
|
||||
=> status switch
|
||||
{
|
||||
VexClaimStatus.NotAffected => "not_affected",
|
||||
VexClaimStatus.UnderInvestigation => "under_investigation",
|
||||
VexClaimStatus.Fixed => "fixed",
|
||||
_ => "affected"
|
||||
};
|
||||
|
||||
private sealed class OverlayAggregate
|
||||
{
|
||||
private readonly SortedSet<string> _observationHashes = new(StringComparer.Ordinal);
|
||||
private readonly SortedSet<string> _observationIds = new(StringComparer.Ordinal);
|
||||
private readonly List<GraphOverlayObservation> _observations = new();
|
||||
private readonly List<GraphOverlayConflict> _conflicts = new();
|
||||
private readonly List<GraphOverlayJustification> _justifications = new();
|
||||
private DateTimeOffset? _latestCreatedAt;
|
||||
private string? _status;
|
||||
private string? _linksetId;
|
||||
private string? _linksetHash;
|
||||
private string? _policyHash;
|
||||
private string? _sbomContextHash;
|
||||
|
||||
public OverlayAggregate(string purl, string advisoryId, string source)
|
||||
{
|
||||
Purl = purl;
|
||||
AdvisoryId = advisoryId;
|
||||
Source = source;
|
||||
}
|
||||
|
||||
public string Purl { get; }
|
||||
|
||||
public string AdvisoryId { get; }
|
||||
|
||||
public string Source { get; }
|
||||
|
||||
public void UpdateStatus(VexClaimStatus status, DateTimeOffset createdAt)
|
||||
{
|
||||
if (_latestCreatedAt is null || createdAt > _latestCreatedAt.Value)
|
||||
{
|
||||
_latestCreatedAt = createdAt;
|
||||
_status = MapStatus(status);
|
||||
}
|
||||
}
|
||||
|
||||
public void AddJustification(VexJustification justification, string observationId)
|
||||
{
|
||||
var kind = justification.ToString();
|
||||
if (string.IsNullOrWhiteSpace(kind))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_justifications.Add(new GraphOverlayJustification(
|
||||
kind,
|
||||
kind,
|
||||
new[] { observationId },
|
||||
null));
|
||||
}
|
||||
|
||||
public void AddObservation(GraphOverlayObservation observation)
|
||||
{
|
||||
if (_observationIds.Add(observation.Id))
|
||||
{
|
||||
_observations.Add(observation);
|
||||
}
|
||||
|
||||
_observationHashes.Add(observation.ContentHash);
|
||||
}
|
||||
|
||||
public void AddConflicts(ImmutableArray<VexObservationDisagreement> disagreements)
|
||||
{
|
||||
if (disagreements.IsDefaultOrEmpty)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var disagreement in disagreements)
|
||||
{
|
||||
_conflicts.Add(new GraphOverlayConflict(
|
||||
"status",
|
||||
disagreement.Justification ?? disagreement.Status,
|
||||
new[] { disagreement.Status },
|
||||
new[] { disagreement.ProviderId }));
|
||||
}
|
||||
}
|
||||
|
||||
public void SetProvenance(string linksetId, string linksetHash, string observationHash)
|
||||
{
|
||||
_linksetId ??= linksetId;
|
||||
_linksetHash ??= linksetHash;
|
||||
_policyHash ??= null;
|
||||
_sbomContextHash ??= null;
|
||||
_observationHashes.Add(observationHash);
|
||||
}
|
||||
|
||||
public GraphOverlayItem ToOverlayItem(string tenant, DateTimeOffset generatedAt, bool includeJustifications)
|
||||
{
|
||||
return new GraphOverlayItem(
|
||||
SchemaVersion: "1.0.0",
|
||||
GeneratedAt: generatedAt,
|
||||
Tenant: tenant,
|
||||
Purl: Purl,
|
||||
AdvisoryId: AdvisoryId,
|
||||
Source: Source,
|
||||
Status: _status ?? "unknown",
|
||||
Justifications: includeJustifications ? _justifications : Array.Empty<GraphOverlayJustification>(),
|
||||
Conflicts: _conflicts,
|
||||
Observations: _observations,
|
||||
Provenance: new GraphOverlayProvenance(
|
||||
LinksetId: _linksetId ?? string.Empty,
|
||||
LinksetHash: _linksetHash ?? string.Empty,
|
||||
ObservationHashes: _observationHashes.ToArray(),
|
||||
PolicyHash: _policyHash,
|
||||
SbomContextHash: _sbomContextHash,
|
||||
PlanCacheKey: null),
|
||||
Cache: null);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class OverlayKeyComparer : IEqualityComparer<(string Purl, string AdvisoryId, string Source)>
|
||||
{
|
||||
public bool Equals((string Purl, string AdvisoryId, string Source) x, (string Purl, string AdvisoryId, string Source) y)
|
||||
{
|
||||
return string.Equals(x.Purl, y.Purl, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(x.AdvisoryId, y.AdvisoryId, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(x.Source, y.Source, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public int GetHashCode((string Purl, string AdvisoryId, string Source) obj)
|
||||
{
|
||||
var hash = new HashCode();
|
||||
hash.Add(obj.Purl, StringComparer.OrdinalIgnoreCase);
|
||||
hash.Add(obj.AdvisoryId, StringComparer.OrdinalIgnoreCase);
|
||||
hash.Add(obj.Source, StringComparer.OrdinalIgnoreCase);
|
||||
return hash.ToHashCode();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,9 +9,16 @@ namespace StellaOps.Excititor.WebService.Graph;
|
||||
internal static class GraphStatusFactory
|
||||
{
|
||||
public static IReadOnlyList<GraphStatusItem> Build(
|
||||
string tenant,
|
||||
DateTimeOffset generatedAt,
|
||||
IReadOnlyList<string> orderedPurls,
|
||||
IReadOnlyList<VexObservation> observations)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tenant))
|
||||
{
|
||||
throw new ArgumentException("tenant is required", nameof(tenant));
|
||||
}
|
||||
|
||||
if (orderedPurls is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(orderedPurls));
|
||||
@@ -22,15 +29,74 @@ internal static class GraphStatusFactory
|
||||
throw new ArgumentNullException(nameof(observations));
|
||||
}
|
||||
|
||||
var overlays = GraphOverlayFactory.Build(orderedPurls, observations, includeJustifications: false);
|
||||
var overlays = GraphOverlayFactory.Build(tenant, generatedAt, orderedPurls, observations, includeJustifications: false);
|
||||
|
||||
return overlays
|
||||
.Select(overlay => new GraphStatusItem(
|
||||
overlay.Purl,
|
||||
overlay.Summary,
|
||||
overlay.LatestModifiedAt,
|
||||
overlay.Provenance.Sources,
|
||||
overlay.Provenance.LastEvidenceHash))
|
||||
.ToList();
|
||||
var items = new List<GraphStatusItem>(orderedPurls.Count);
|
||||
|
||||
foreach (var purl in orderedPurls)
|
||||
{
|
||||
var overlaysForPurl = overlays
|
||||
.Where(o => o.Purl.Equals(purl, StringComparison.OrdinalIgnoreCase))
|
||||
.ToList();
|
||||
|
||||
if (overlaysForPurl.Count == 0)
|
||||
{
|
||||
items.Add(new GraphStatusItem(
|
||||
purl,
|
||||
new GraphOverlaySummary(0, 0, 0, 1),
|
||||
null,
|
||||
Array.Empty<string>(),
|
||||
null));
|
||||
continue;
|
||||
}
|
||||
|
||||
var open = 0;
|
||||
var notAffected = 0;
|
||||
var underInvestigation = 0;
|
||||
var noStatement = 0;
|
||||
var sources = new SortedSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var observationRefs = new List<GraphOverlayObservation>();
|
||||
|
||||
foreach (var overlay in overlaysForPurl)
|
||||
{
|
||||
sources.Add(overlay.Source);
|
||||
observationRefs.AddRange(overlay.Observations);
|
||||
switch (overlay.Status)
|
||||
{
|
||||
case "not_affected":
|
||||
notAffected++;
|
||||
break;
|
||||
case "under_investigation":
|
||||
underInvestigation++;
|
||||
break;
|
||||
case "fixed":
|
||||
case "affected":
|
||||
open++;
|
||||
break;
|
||||
default:
|
||||
noStatement++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
var latest = observationRefs.Count == 0
|
||||
? (DateTimeOffset?)null
|
||||
: observationRefs.Max(o => o.FetchedAt);
|
||||
|
||||
var lastHash = observationRefs
|
||||
.OrderBy(o => o.FetchedAt)
|
||||
.ThenBy(o => o.Id, StringComparer.Ordinal)
|
||||
.LastOrDefault()
|
||||
?.ContentHash;
|
||||
|
||||
items.Add(new GraphStatusItem(
|
||||
purl,
|
||||
new GraphOverlaySummary(open, notAffected, underInvestigation, noStatement),
|
||||
latest,
|
||||
sources.ToArray(),
|
||||
lastHash));
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ public sealed class GraphOptions
|
||||
public int MaxPurls { get; set; } = 500;
|
||||
public int MaxAdvisoriesPerPurl { get; set; } = 200;
|
||||
public int OverlayTtlSeconds { get; set; } = 300;
|
||||
public bool UsePostgresOverlayStore { get; set; } = true;
|
||||
public int MaxTooltipItemsPerPurl { get; set; } = 50;
|
||||
public int MaxTooltipTotal { get; set; } = 1000;
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ public partial class Program
|
||||
{
|
||||
private const string TenantHeaderName = "X-Stella-Tenant";
|
||||
|
||||
private static bool TryResolveTenant(HttpContext context, VexStorageOptions options, bool requireHeader, out string tenant, out IResult? problem)
|
||||
internal static bool TryResolveTenant(HttpContext context, VexStorageOptions options, bool requireHeader, out string tenant, out IResult? problem)
|
||||
{
|
||||
tenant = options.DefaultTenant;
|
||||
problem = null;
|
||||
@@ -149,7 +149,7 @@ public partial class Program
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseSinceTimestamp(StringValues values)
|
||||
internal static DateTimeOffset? ParseSinceTimestamp(StringValues values)
|
||||
{
|
||||
if (values.Count == 0)
|
||||
{
|
||||
@@ -244,7 +244,8 @@ public partial class Program
|
||||
IReadOnlyList<GraphStatusItem> Items,
|
||||
DateTimeOffset CachedAt);
|
||||
|
||||
private sealed record CachedGraphOverlay(
|
||||
IReadOnlyList<GraphOverlayItem> Items,
|
||||
DateTimeOffset CachedAt);
|
||||
internal static string[] NormalizeValues(StringValues values) =>
|
||||
values.Where(static v => !string.IsNullOrWhiteSpace(v))
|
||||
.Select(static v => v!.Trim())
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ using StellaOps.Excititor.Attestation.Transparency;
|
||||
using StellaOps.Excititor.ArtifactStores.S3.Extensions;
|
||||
using StellaOps.Excititor.Connectors.RedHat.CSAF.DependencyInjection;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Evidence;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
using StellaOps.Excititor.Export;
|
||||
using StellaOps.Excititor.Formats.CSAF;
|
||||
@@ -28,6 +29,7 @@ using StellaOps.Excititor.Formats.CycloneDX;
|
||||
using StellaOps.Excititor.Formats.OpenVEX;
|
||||
using StellaOps.Excititor.Policy;
|
||||
using StellaOps.Excititor.Storage.Postgres;
|
||||
using StellaOps.Infrastructure.Postgres.Options;
|
||||
using StellaOps.Excititor.WebService.Endpoints;
|
||||
using StellaOps.Excititor.WebService.Extensions;
|
||||
using StellaOps.Excititor.WebService.Options;
|
||||
@@ -46,10 +48,12 @@ var services = builder.Services;
|
||||
services.AddOptions<VexStorageOptions>()
|
||||
.Bind(configuration.GetSection("Excititor:Storage"))
|
||||
.ValidateOnStart();
|
||||
services.AddOptions<GraphOptions>()
|
||||
.Bind(configuration.GetSection("Excititor:Graph"));
|
||||
|
||||
services.AddExcititorPostgresStorage(configuration);
|
||||
services.TryAddSingleton<IVexProviderStore, InMemoryVexProviderStore>();
|
||||
services.TryAddSingleton<IVexConnectorStateRepository, InMemoryVexConnectorStateRepository>();
|
||||
services.TryAddScoped<IVexConnectorStateRepository, InMemoryVexConnectorStateRepository>();
|
||||
services.TryAddSingleton<IVexClaimStore, InMemoryVexClaimStore>();
|
||||
services.AddCsafNormalizer();
|
||||
services.AddCycloneDxNormalizer();
|
||||
@@ -62,7 +66,24 @@ services.AddSingleton<AirgapSignerTrustService>();
|
||||
services.AddSingleton<AirgapModeEnforcer>();
|
||||
services.AddSingleton<ConsoleTelemetry>();
|
||||
services.AddMemoryCache();
|
||||
services.AddSingleton<IGraphOverlayCache, GraphOverlayCacheStore>();
|
||||
services.AddSingleton<IGraphOverlayStore>(sp =>
|
||||
{
|
||||
var graphOptions = sp.GetRequiredService<IOptions<GraphOptions>>().Value;
|
||||
var pgOptions = sp.GetRequiredService<IOptions<PostgresOptions>>().Value;
|
||||
if (graphOptions.UsePostgresOverlayStore && !string.IsNullOrWhiteSpace(pgOptions.ConnectionString))
|
||||
{
|
||||
return new PostgresGraphOverlayStore(
|
||||
sp.GetRequiredService<ExcititorDataSource>(),
|
||||
sp.GetRequiredService<ILogger<PostgresGraphOverlayStore>>());
|
||||
}
|
||||
|
||||
return new InMemoryGraphOverlayStore();
|
||||
});
|
||||
services.AddSingleton<IVexEvidenceLockerService, VexEvidenceLockerService>();
|
||||
services.AddSingleton<IVexEvidenceAttestor, StellaOps.Excititor.Attestation.Evidence.VexEvidenceAttestor>();
|
||||
services.AddScoped<IVexIngestOrchestrator, VexIngestOrchestrator>();
|
||||
services.AddSingleton<VexStatementBackfillService>();
|
||||
services.AddOptions<ExcititorObservabilityOptions>()
|
||||
.Bind(configuration.GetSection("Excititor:Observability"));
|
||||
services.AddScoped<ExcititorHealthService>();
|
||||
@@ -93,7 +114,7 @@ services.AddSingleton<IVexObservationProjectionService, VexObservationProjection
|
||||
services.AddScoped<IVexObservationQueryService, VexObservationQueryService>();
|
||||
|
||||
// EXCITITOR-RISK-66-001: Risk feed service for Risk Engine integration
|
||||
services.AddScoped<StellaOps.Excititor.Core.RiskFeed.IRiskFeedService, StellaOps.Excititor.Core.RiskFeed.RiskFeedService>();
|
||||
services.AddScoped<StellaOps.Excititor.Core.RiskFeed.IRiskFeedService, OverlayRiskFeedService>();
|
||||
|
||||
var rekorSection = configuration.GetSection("Excititor:Attestation:Rekor");
|
||||
if (rekorSection.Exists())
|
||||
@@ -1505,7 +1526,7 @@ app.MapGet("/v1/graph/status", async (
|
||||
return Results.BadRequest(ex.Message);
|
||||
}
|
||||
|
||||
var items = GraphStatusFactory.Build(orderedPurls, result.Observations);
|
||||
var items = GraphStatusFactory.Build(tenant!, timeProvider.GetUtcNow(), orderedPurls, result.Observations);
|
||||
var response = new GraphStatusResponse(items, false, null);
|
||||
|
||||
cache.Set(cacheKey, new CachedGraphStatus(items, now), TimeSpan.FromSeconds(graphOptions.Value.OverlayTtlSeconds));
|
||||
@@ -1521,7 +1542,8 @@ app.MapGet("/v1/graph/overlays", async (
|
||||
IOptions<VexStorageOptions> storageOptions,
|
||||
IOptions<GraphOptions> graphOptions,
|
||||
IVexObservationQueryService queryService,
|
||||
IMemoryCache cache,
|
||||
IGraphOverlayCache overlayCache,
|
||||
IGraphOverlayStore overlayStore,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
@@ -1541,13 +1563,12 @@ app.MapGet("/v1/graph/overlays", async (
|
||||
return Results.BadRequest($"purls limit exceeded (max {graphOptions.Value.MaxPurls})");
|
||||
}
|
||||
|
||||
var cacheKey = $"graph-overlays:{tenant}:{includeJustifications}:{string.Join('|', orderedPurls)}";
|
||||
var now = timeProvider.GetUtcNow();
|
||||
|
||||
if (cache.TryGetValue<CachedGraphOverlay>(cacheKey, out var cached) && cached is not null)
|
||||
var cached = await overlayCache.TryGetAsync(tenant!, includeJustifications, orderedPurls, cancellationToken).ConfigureAwait(false);
|
||||
if (cached is not null)
|
||||
{
|
||||
var ageMs = (long)Math.Max(0, (now - cached.CachedAt).TotalMilliseconds);
|
||||
return Results.Ok(new GraphOverlaysResponse(cached.Items, true, ageMs));
|
||||
return Results.Ok(new GraphOverlaysResponse(cached.Items, true, cached.AgeMilliseconds));
|
||||
}
|
||||
|
||||
var options = new VexObservationQueryOptions(
|
||||
@@ -1565,10 +1586,11 @@ app.MapGet("/v1/graph/overlays", async (
|
||||
return Results.BadRequest(ex.Message);
|
||||
}
|
||||
|
||||
var overlays = GraphOverlayFactory.Build(orderedPurls, result.Observations, includeJustifications);
|
||||
var overlays = GraphOverlayFactory.Build(tenant!, now, orderedPurls, result.Observations, includeJustifications);
|
||||
await overlayStore.SaveAsync(tenant!, overlays, cancellationToken).ConfigureAwait(false);
|
||||
var response = new GraphOverlaysResponse(overlays, false, null);
|
||||
|
||||
cache.Set(cacheKey, new CachedGraphOverlay(overlays, now), TimeSpan.FromSeconds(graphOptions.Value.OverlayTtlSeconds));
|
||||
await overlayCache.SaveAsync(tenant!, includeJustifications, orderedPurls, overlays, now, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return Results.Ok(response);
|
||||
}).WithName("GetGraphOverlays");
|
||||
@@ -1712,8 +1734,9 @@ app.MapGet("/vex/raw", async (
|
||||
var formatFilter = query.TryGetValue("format", out var formats)
|
||||
? formats
|
||||
.Where(static f => !string.IsNullOrWhiteSpace(f))
|
||||
.Select(static f => Enum.TryParse<VexDocumentFormat>(f, true, out var parsed) ? parsed : VexDocumentFormat.Unknown)
|
||||
.Where(static f => f != VexDocumentFormat.Unknown)
|
||||
.Select(static f => Enum.TryParse<VexDocumentFormat>(f, true, out var parsed) ? parsed : (VexDocumentFormat?)null)
|
||||
.Where(static f => f is not null)
|
||||
.Select(static f => f!.Value)
|
||||
.ToArray()
|
||||
: Array.Empty<VexDocumentFormat>();
|
||||
|
||||
@@ -1910,112 +1933,6 @@ app.MapGet("/v1/vex/observations/{vulnerabilityId}/{productKey}", async (
|
||||
return Results.Json(response);
|
||||
});
|
||||
|
||||
app.MapGet("/v1/vex/evidence/chunks", async (
|
||||
HttpContext context,
|
||||
[FromServices] IVexEvidenceChunkService chunkService,
|
||||
[FromServices] IOptions<VexStorageOptions> storageOptions,
|
||||
[FromServices] ChunkTelemetry chunkTelemetry,
|
||||
[FromServices] ILogger<VexEvidenceChunkRequest> logger,
|
||||
[FromServices] TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var start = Stopwatch.GetTimestamp();
|
||||
|
||||
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
chunkTelemetry.RecordIngested(null, null, "unauthorized", "missing-scope", 0, 0, 0);
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out var tenant, out var tenantError))
|
||||
{
|
||||
chunkTelemetry.RecordIngested(tenant, null, "rejected", "tenant-invalid", 0, 0, Stopwatch.GetElapsedTime(start).TotalMilliseconds);
|
||||
return tenantError;
|
||||
}
|
||||
|
||||
var vulnerabilityId = context.Request.Query["vulnerabilityId"].FirstOrDefault();
|
||||
var productKey = context.Request.Query["productKey"].FirstOrDefault();
|
||||
if (string.IsNullOrWhiteSpace(vulnerabilityId) || string.IsNullOrWhiteSpace(productKey))
|
||||
{
|
||||
return ValidationProblem("vulnerabilityId and productKey are required.");
|
||||
}
|
||||
|
||||
var providerFilter = BuildStringFilterSet(context.Request.Query["providerId"]);
|
||||
var statusFilter = BuildStatusFilter(context.Request.Query["status"]);
|
||||
var since = ParseSinceTimestamp(context.Request.Query["since"]);
|
||||
var limit = ResolveLimit(context.Request.Query["limit"], defaultValue: 200, min: 1, max: 500);
|
||||
|
||||
var request = new VexEvidenceChunkRequest(
|
||||
tenant,
|
||||
vulnerabilityId.Trim(),
|
||||
productKey.Trim(),
|
||||
providerFilter,
|
||||
statusFilter,
|
||||
since,
|
||||
limit);
|
||||
|
||||
VexEvidenceChunkResult result;
|
||||
try
|
||||
{
|
||||
result = await chunkService.QueryAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
EvidenceTelemetry.RecordChunkOutcome(tenant, "cancelled");
|
||||
chunkTelemetry.RecordIngested(tenant, providerFilter.Count > 0 ? string.Join(',', providerFilter) : null, "cancelled", null, 0, 0, Stopwatch.GetElapsedTime(start).TotalMilliseconds);
|
||||
return Results.StatusCode(StatusCodes.Status499ClientClosedRequest);
|
||||
}
|
||||
catch
|
||||
{
|
||||
EvidenceTelemetry.RecordChunkOutcome(tenant, "error");
|
||||
chunkTelemetry.RecordIngested(tenant, providerFilter.Count > 0 ? string.Join(',', providerFilter) : null, "error", null, 0, 0, Stopwatch.GetElapsedTime(start).TotalMilliseconds);
|
||||
throw;
|
||||
}
|
||||
|
||||
EvidenceTelemetry.RecordChunkOutcome(tenant, "success", result.Chunks.Count, result.Truncated);
|
||||
EvidenceTelemetry.RecordChunkSignatureStatus(tenant, result.Chunks);
|
||||
|
||||
logger.LogInformation(
|
||||
"vex_evidence_chunks_success tenant={Tenant} vulnerabilityId={Vuln} productKey={ProductKey} providers={Providers} statuses={Statuses} limit={Limit} total={Total} truncated={Truncated} returned={Returned}",
|
||||
tenant ?? "(default)",
|
||||
request.VulnerabilityId,
|
||||
request.ProductKey,
|
||||
providerFilter.Count,
|
||||
statusFilter.Count,
|
||||
request.Limit,
|
||||
result.TotalCount,
|
||||
result.Truncated,
|
||||
result.Chunks.Count);
|
||||
|
||||
// Align headers with published contract.
|
||||
context.Response.Headers["Excititor-Results-Total"] = result.TotalCount.ToString(CultureInfo.InvariantCulture);
|
||||
context.Response.Headers["Excititor-Results-Truncated"] = result.Truncated ? "true" : "false";
|
||||
context.Response.ContentType = "application/x-ndjson";
|
||||
|
||||
var options = new JsonSerializerOptions(JsonSerializerDefaults.Web);
|
||||
long payloadBytes = 0;
|
||||
foreach (var chunk in result.Chunks)
|
||||
{
|
||||
var line = JsonSerializer.Serialize(chunk, options);
|
||||
payloadBytes += Encoding.UTF8.GetByteCount(line) + 1;
|
||||
await context.Response.WriteAsync(line, cancellationToken).ConfigureAwait(false);
|
||||
await context.Response.WriteAsync("\n", cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var elapsedMs = Stopwatch.GetElapsedTime(start).TotalMilliseconds;
|
||||
chunkTelemetry.RecordIngested(
|
||||
tenant,
|
||||
providerFilter.Count > 0 ? string.Join(',', providerFilter) : null,
|
||||
"success",
|
||||
null,
|
||||
result.TotalCount,
|
||||
payloadBytes,
|
||||
elapsedMs);
|
||||
|
||||
return Results.Empty;
|
||||
});
|
||||
|
||||
app.MapPost("/aoc/verify", async (
|
||||
HttpContext context,
|
||||
VexAocVerifyRequest? request,
|
||||
@@ -2060,10 +1977,10 @@ app.MapPost("/aoc/verify", async (
|
||||
sources ?? Array.Empty<string>(),
|
||||
Array.Empty<string>(),
|
||||
Array.Empty<VexDocumentFormat>(),
|
||||
since: new DateTimeOffset(since, TimeSpan.Zero),
|
||||
until: new DateTimeOffset(until, TimeSpan.Zero),
|
||||
cursor: null,
|
||||
limit),
|
||||
Since: new DateTimeOffset(since, TimeSpan.Zero),
|
||||
Until: new DateTimeOffset(until, TimeSpan.Zero),
|
||||
Cursor: null,
|
||||
Limit: limit),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var checkedCount = 0;
|
||||
|
||||
@@ -279,7 +279,7 @@ internal sealed class ExcititorHealthService
|
||||
Array.Empty<string>(),
|
||||
Array.Empty<VexDocumentFormat>(),
|
||||
windowStart,
|
||||
until: null,
|
||||
Until: null,
|
||||
Cursor: null,
|
||||
Limit: 500),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
@@ -360,13 +360,13 @@ internal sealed class ExcititorHealthService
|
||||
|
||||
foreach (var linkset in linksets)
|
||||
{
|
||||
if (linkset.Disagreements.Count == 0)
|
||||
if (linkset.Disagreements.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
docsWithConflicts++;
|
||||
totalConflicts += linkset.Disagreements.Count;
|
||||
totalConflicts += linkset.Disagreements.Length;
|
||||
|
||||
foreach (var disagreement in linkset.Disagreements)
|
||||
{
|
||||
@@ -381,8 +381,8 @@ internal sealed class ExcititorHealthService
|
||||
|
||||
var alignedTicks = AlignTicks(linkset.UpdatedAt.UtcDateTime, bucketTicks);
|
||||
timeline[alignedTicks] = timeline.TryGetValue(alignedTicks, out var currentCount)
|
||||
? currentCount + linkset.Disagreements.Count
|
||||
: linkset.Disagreements.Count;
|
||||
? currentCount + linkset.Disagreements.Length
|
||||
: linkset.Disagreements.Length;
|
||||
}
|
||||
|
||||
var trend = timeline
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
using StellaOps.Excititor.WebService.Options;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Services;
|
||||
|
||||
public interface IGraphOverlayCache
|
||||
{
|
||||
ValueTask<GraphOverlayCacheHit?> TryGetAsync(string tenant, bool includeJustifications, IReadOnlyList<string> orderedPurls, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask SaveAsync(string tenant, bool includeJustifications, IReadOnlyList<string> orderedPurls, IReadOnlyList<GraphOverlayItem> items, DateTimeOffset cachedAt, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed record GraphOverlayCacheHit(IReadOnlyList<GraphOverlayItem> Items, long AgeMilliseconds);
|
||||
|
||||
internal sealed class GraphOverlayCacheStore : IGraphOverlayCache
|
||||
{
|
||||
private readonly IMemoryCache _memoryCache;
|
||||
private readonly IOptions<GraphOptions> _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public GraphOverlayCacheStore(IMemoryCache memoryCache, IOptions<GraphOptions> options, TimeProvider timeProvider)
|
||||
{
|
||||
_memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache));
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
}
|
||||
|
||||
public ValueTask<GraphOverlayCacheHit?> TryGetAsync(string tenant, bool includeJustifications, IReadOnlyList<string> orderedPurls, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var key = BuildKey(tenant, includeJustifications, orderedPurls);
|
||||
if (_memoryCache.TryGetValue<CachedOverlay>(key, out var cached) && cached is not null)
|
||||
{
|
||||
var ageMs = (long)Math.Max(0, (_timeProvider.GetUtcNow() - cached.CachedAt).TotalMilliseconds);
|
||||
return ValueTask.FromResult<GraphOverlayCacheHit?>(new GraphOverlayCacheHit(cached.Items, ageMs));
|
||||
}
|
||||
|
||||
return ValueTask.FromResult<GraphOverlayCacheHit?>(null);
|
||||
}
|
||||
|
||||
public ValueTask SaveAsync(string tenant, bool includeJustifications, IReadOnlyList<string> orderedPurls, IReadOnlyList<GraphOverlayItem> items, DateTimeOffset cachedAt, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var key = BuildKey(tenant, includeJustifications, orderedPurls);
|
||||
var ttl = TimeSpan.FromSeconds(Math.Max(1, _options.Value.OverlayTtlSeconds));
|
||||
_memoryCache.Set(key, new CachedOverlay(items, cachedAt), ttl);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
private static string BuildKey(string tenant, bool includeJustifications, IReadOnlyList<string> orderedPurls)
|
||||
=> $"graph-overlays:{tenant}:{includeJustifications}:{string.Join('|', orderedPurls)}";
|
||||
|
||||
private sealed record CachedOverlay(IReadOnlyList<GraphOverlayItem> Items, DateTimeOffset CachedAt);
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Services;
|
||||
|
||||
public interface IGraphOverlayStore
|
||||
{
|
||||
ValueTask SaveAsync(string tenant, IReadOnlyList<GraphOverlayItem> overlays, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask<IReadOnlyList<GraphOverlayItem>> FindByPurlsAsync(string tenant, IReadOnlyCollection<string> purls, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask<IReadOnlyList<GraphOverlayItem>> FindByAdvisoriesAsync(string tenant, IReadOnlyCollection<string> advisories, int limit, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask<IReadOnlyList<GraphOverlayItem>> FindWithConflictsAsync(string tenant, int limit, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// In-memory overlay store placeholder until Postgres materialization is added.
|
||||
/// </summary>
|
||||
public sealed class InMemoryGraphOverlayStore : IGraphOverlayStore
|
||||
{
|
||||
private readonly Dictionary<string, Dictionary<string, List<GraphOverlayItem>>> _store = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly object _lock = new();
|
||||
|
||||
public ValueTask SaveAsync(string tenant, IReadOnlyList<GraphOverlayItem> overlays, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
lock (_lock)
|
||||
{
|
||||
if (!_store.TryGetValue(tenant, out var byPurl))
|
||||
{
|
||||
byPurl = new Dictionary<string, List<GraphOverlayItem>>(StringComparer.OrdinalIgnoreCase);
|
||||
_store[tenant] = byPurl;
|
||||
}
|
||||
|
||||
foreach (var overlay in overlays)
|
||||
{
|
||||
if (!byPurl.TryGetValue(overlay.Purl, out var list))
|
||||
{
|
||||
list = new List<GraphOverlayItem>();
|
||||
byPurl[overlay.Purl] = list;
|
||||
}
|
||||
|
||||
// replace existing advisory/source entry for deterministic latest overlay
|
||||
var existingIndex = list.FindIndex(o =>
|
||||
string.Equals(o.AdvisoryId, overlay.AdvisoryId, StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(o.Source, overlay.Source, StringComparison.OrdinalIgnoreCase));
|
||||
if (existingIndex >= 0)
|
||||
{
|
||||
list[existingIndex] = overlay;
|
||||
}
|
||||
else
|
||||
{
|
||||
list.Add(overlay);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask<IReadOnlyList<GraphOverlayItem>> FindByPurlsAsync(string tenant, IReadOnlyCollection<string> purls, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
if (purls.Count == 0)
|
||||
{
|
||||
return ValueTask.FromResult<IReadOnlyList<GraphOverlayItem>>(Array.Empty<GraphOverlayItem>());
|
||||
}
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
if (!_store.TryGetValue(tenant, out var byPurl))
|
||||
{
|
||||
return ValueTask.FromResult<IReadOnlyList<GraphOverlayItem>>(Array.Empty<GraphOverlayItem>());
|
||||
}
|
||||
|
||||
var ordered = new List<GraphOverlayItem>();
|
||||
foreach (var purl in purls)
|
||||
{
|
||||
if (byPurl.TryGetValue(purl, out var list))
|
||||
{
|
||||
// Order overlays deterministically by advisory + source for stable outputs
|
||||
ordered.AddRange(list
|
||||
.OrderBy(o => o.AdvisoryId, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(o => o.Source, StringComparer.OrdinalIgnoreCase));
|
||||
}
|
||||
}
|
||||
|
||||
return ValueTask.FromResult<IReadOnlyList<GraphOverlayItem>>(ordered);
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask<IReadOnlyList<GraphOverlayItem>> FindByAdvisoriesAsync(string tenant, IReadOnlyCollection<string> advisories, int limit, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
if (advisories.Count == 0)
|
||||
{
|
||||
return ValueTask.FromResult<IReadOnlyList<GraphOverlayItem>>(Array.Empty<GraphOverlayItem>());
|
||||
}
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
if (!_store.TryGetValue(tenant, out var byPurl))
|
||||
{
|
||||
return ValueTask.FromResult<IReadOnlyList<GraphOverlayItem>>(Array.Empty<GraphOverlayItem>());
|
||||
}
|
||||
|
||||
var results = new List<GraphOverlayItem>();
|
||||
foreach (var kvp in byPurl)
|
||||
{
|
||||
foreach (var overlay in kvp.Value)
|
||||
{
|
||||
if (advisories.Contains(overlay.AdvisoryId, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
results.Add(overlay);
|
||||
if (results.Count >= limit)
|
||||
{
|
||||
return ValueTask.FromResult<IReadOnlyList<GraphOverlayItem>>(results);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ValueTask.FromResult<IReadOnlyList<GraphOverlayItem>>(results
|
||||
.OrderBy(o => o.AdvisoryId, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(o => o.Purl, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(o => o.Source, StringComparer.OrdinalIgnoreCase)
|
||||
.Take(limit)
|
||||
.ToList());
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask<IReadOnlyList<GraphOverlayItem>> FindWithConflictsAsync(string tenant, int limit, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
lock (_lock)
|
||||
{
|
||||
if (!_store.TryGetValue(tenant, out var byPurl))
|
||||
{
|
||||
return ValueTask.FromResult<IReadOnlyList<GraphOverlayItem>>(Array.Empty<GraphOverlayItem>());
|
||||
}
|
||||
|
||||
var results = byPurl.Values
|
||||
.SelectMany(list => list)
|
||||
.Where(o => o.Conflicts.Count > 0)
|
||||
.OrderBy(o => o.Purl, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(o => o.AdvisoryId, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(o => o.Source, StringComparer.OrdinalIgnoreCase)
|
||||
.Take(limit)
|
||||
.ToList();
|
||||
|
||||
return ValueTask.FromResult<IReadOnlyList<GraphOverlayItem>>(results);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.RiskFeed;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Risk feed service backed by graph overlays (EXCITITOR-RISK-66-001).
|
||||
/// </summary>
|
||||
public sealed class OverlayRiskFeedService : IRiskFeedService
|
||||
{
|
||||
private readonly IGraphOverlayStore _overlayStore;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public OverlayRiskFeedService(IGraphOverlayStore overlayStore, TimeProvider timeProvider)
|
||||
{
|
||||
_overlayStore = overlayStore ?? throw new ArgumentNullException(nameof(overlayStore));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
}
|
||||
|
||||
public async Task<RiskFeedResponse> GenerateFeedAsync(RiskFeedRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
var overlays = await ResolveOverlaysAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
var filtered = ApplySinceFilter(overlays, request.Since);
|
||||
|
||||
var items = filtered
|
||||
.Select(MapToRiskFeedItem)
|
||||
.Where(item => item is not null)
|
||||
.Cast<RiskFeedItem>()
|
||||
.OrderBy(item => item.AdvisoryKey, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(item => item.Artifact, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(item => item.Provenance.TenantId, StringComparer.OrdinalIgnoreCase)
|
||||
.Take(request.Limit)
|
||||
.ToImmutableArray();
|
||||
|
||||
return new RiskFeedResponse(items, _timeProvider.GetUtcNow());
|
||||
}
|
||||
|
||||
public async Task<RiskFeedItem?> GetItemAsync(string tenantId, string advisoryKey, string artifact, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(advisoryKey);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(artifact);
|
||||
|
||||
var overlays = await _overlayStore
|
||||
.FindByPurlsAsync(tenantId, new[] { artifact }, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var match = overlays
|
||||
.Where(o => string.Equals(o.AdvisoryId, advisoryKey, StringComparison.OrdinalIgnoreCase))
|
||||
.OrderBy(o => o.Source, StringComparer.OrdinalIgnoreCase)
|
||||
.FirstOrDefault();
|
||||
|
||||
return match is null ? null : MapToRiskFeedItem(match);
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<GraphOverlayItem>> ResolveOverlaysAsync(RiskFeedRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!request.AdvisoryKeys.IsDefaultOrEmpty)
|
||||
{
|
||||
return await _overlayStore
|
||||
.FindByAdvisoriesAsync(request.TenantId, request.AdvisoryKeys, request.Limit, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (!request.Artifacts.IsDefaultOrEmpty)
|
||||
{
|
||||
return await _overlayStore
|
||||
.FindByPurlsAsync(request.TenantId, request.Artifacts, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return await _overlayStore
|
||||
.FindWithConflictsAsync(request.TenantId, request.Limit, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static IEnumerable<GraphOverlayItem> ApplySinceFilter(IEnumerable<GraphOverlayItem> overlays, DateTimeOffset? since)
|
||||
{
|
||||
if (since is null)
|
||||
{
|
||||
return overlays;
|
||||
}
|
||||
|
||||
var threshold = since.Value;
|
||||
return overlays.Where(o => o.GeneratedAt >= threshold);
|
||||
}
|
||||
|
||||
private static RiskFeedItem? MapToRiskFeedItem(GraphOverlayItem overlay)
|
||||
{
|
||||
if (!TryParseStatus(overlay.Status, out var status))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var justification = ParseJustification(overlay.Justifications.FirstOrDefault()?.Kind);
|
||||
var confidence = DeriveConfidence(overlay);
|
||||
var provenance = new RiskFeedProvenance(
|
||||
overlay.Tenant,
|
||||
overlay.Provenance.LinksetId,
|
||||
overlay.Provenance.LinksetHash,
|
||||
confidence,
|
||||
overlay.Conflicts.Count > 0,
|
||||
overlay.GeneratedAt);
|
||||
|
||||
var observedAt = overlay.Observations.Count == 0
|
||||
? overlay.GeneratedAt
|
||||
: overlay.Observations.Max(o => o.FetchedAt);
|
||||
|
||||
var sources = overlay.Observations
|
||||
.OrderBy(o => o.FetchedAt)
|
||||
.Select(o => new RiskFeedObservationSource(
|
||||
o.Id,
|
||||
overlay.Source,
|
||||
overlay.Status,
|
||||
overlay.Justifications.FirstOrDefault()?.Kind,
|
||||
null))
|
||||
.ToImmutableArray();
|
||||
|
||||
return new RiskFeedItem(
|
||||
overlay.AdvisoryId,
|
||||
overlay.Purl,
|
||||
status,
|
||||
justification,
|
||||
provenance,
|
||||
observedAt,
|
||||
sources);
|
||||
}
|
||||
|
||||
private static bool TryParseStatus(string status, out VexClaimStatus parsed)
|
||||
{
|
||||
parsed = status.ToLowerInvariant() switch
|
||||
{
|
||||
"not_affected" => VexClaimStatus.NotAffected,
|
||||
"under_investigation" => VexClaimStatus.UnderInvestigation,
|
||||
"fixed" => VexClaimStatus.Fixed,
|
||||
"affected" => VexClaimStatus.Affected,
|
||||
_ => VexClaimStatus.UnderInvestigation
|
||||
};
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private static VexJustification? ParseJustification(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return Enum.TryParse<VexJustification>(value, true, out var justification) ? justification : null;
|
||||
}
|
||||
|
||||
private static VexLinksetConfidence DeriveConfidence(GraphOverlayItem overlay)
|
||||
{
|
||||
if (overlay.Conflicts.Count > 0)
|
||||
{
|
||||
return VexLinksetConfidence.Low;
|
||||
}
|
||||
|
||||
return overlay.Observations.Count > 1
|
||||
? VexLinksetConfidence.High
|
||||
: VexLinksetConfidence.Medium;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
using NpgsqlTypes;
|
||||
using StellaOps.Excititor.Storage.Postgres;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Postgres-backed overlay materialization store. Persists overlays per tenant/purl/advisory/source.
|
||||
/// </summary>
|
||||
public sealed class PostgresGraphOverlayStore : IGraphOverlayStore
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new()
|
||||
{
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
||||
};
|
||||
|
||||
private readonly ExcititorDataSource _dataSource;
|
||||
private readonly ILogger<PostgresGraphOverlayStore> _logger;
|
||||
private volatile bool _initialized;
|
||||
private readonly SemaphoreSlim _initLock = new(1, 1);
|
||||
|
||||
public PostgresGraphOverlayStore(ExcititorDataSource dataSource, ILogger<PostgresGraphOverlayStore> logger)
|
||||
{
|
||||
_dataSource = dataSource ?? throw new ArgumentNullException(nameof(dataSource));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async ValueTask SaveAsync(string tenant, IReadOnlyList<GraphOverlayItem> overlays, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(tenant);
|
||||
ArgumentNullException.ThrowIfNull(overlays);
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await using var connection = await _dataSource.OpenConnectionAsync("public", "writer", cancellationToken).ConfigureAwait(false);
|
||||
const string sql = """
|
||||
INSERT INTO vex.graph_overlays (tenant, purl, advisory_id, source, generated_at, payload)
|
||||
VALUES (@tenant, @purl, @advisory_id, @source, @generated_at, @payload)
|
||||
ON CONFLICT (tenant, purl, advisory_id, source)
|
||||
DO UPDATE SET generated_at = EXCLUDED.generated_at, payload = EXCLUDED.payload;
|
||||
""";
|
||||
|
||||
foreach (var overlay in overlays)
|
||||
{
|
||||
await using var command = new NpgsqlCommand(sql, connection)
|
||||
{
|
||||
CommandTimeout = _dataSource.CommandTimeoutSeconds
|
||||
};
|
||||
|
||||
command.Parameters.AddWithValue("tenant", tenant);
|
||||
command.Parameters.AddWithValue("purl", overlay.Purl);
|
||||
command.Parameters.AddWithValue("advisory_id", overlay.AdvisoryId);
|
||||
command.Parameters.AddWithValue("source", overlay.Source);
|
||||
command.Parameters.AddWithValue("generated_at", overlay.GeneratedAt.UtcDateTime);
|
||||
command.Parameters.Add(new NpgsqlParameter("payload", NpgsqlDbType.Jsonb)
|
||||
{
|
||||
Value = JsonSerializer.Serialize(overlay, SerializerOptions)
|
||||
});
|
||||
|
||||
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<GraphOverlayItem>> FindByPurlsAsync(string tenant, IReadOnlyCollection<string> purls, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(tenant);
|
||||
ArgumentNullException.ThrowIfNull(purls);
|
||||
if (purls.Count == 0)
|
||||
{
|
||||
return Array.Empty<GraphOverlayItem>();
|
||||
}
|
||||
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var connection = await _dataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
|
||||
|
||||
const string sql = """
|
||||
SELECT payload
|
||||
FROM vex.graph_overlays
|
||||
WHERE tenant = @tenant AND purl = ANY(@purls)
|
||||
ORDER BY purl, advisory_id, source;
|
||||
""";
|
||||
|
||||
await using var command = new NpgsqlCommand(sql, connection)
|
||||
{
|
||||
CommandTimeout = _dataSource.CommandTimeoutSeconds
|
||||
};
|
||||
|
||||
command.Parameters.AddWithValue("tenant", tenant);
|
||||
command.Parameters.Add(new NpgsqlParameter<string[]>("purls", NpgsqlDbType.Array | NpgsqlDbType.Text)
|
||||
{
|
||||
TypedValue = purls.ToArray()
|
||||
});
|
||||
|
||||
var overlays = new List<GraphOverlayItem>();
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var payload = reader.GetString(0);
|
||||
var overlay = JsonSerializer.Deserialize<GraphOverlayItem>(payload, SerializerOptions);
|
||||
if (overlay is not null)
|
||||
{
|
||||
overlays.Add(overlay);
|
||||
}
|
||||
}
|
||||
|
||||
return overlays;
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<GraphOverlayItem>> FindByAdvisoriesAsync(string tenant, IReadOnlyCollection<string> advisories, int limit, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(tenant);
|
||||
ArgumentNullException.ThrowIfNull(advisories);
|
||||
if (advisories.Count == 0)
|
||||
{
|
||||
return Array.Empty<GraphOverlayItem>();
|
||||
}
|
||||
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var connection = await _dataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
|
||||
|
||||
const string sql = """
|
||||
SELECT payload
|
||||
FROM vex.graph_overlays
|
||||
WHERE tenant = @tenant AND advisory_id = ANY(@advisories)
|
||||
ORDER BY advisory_id, purl, source
|
||||
LIMIT @limit;
|
||||
""";
|
||||
|
||||
await using var command = new NpgsqlCommand(sql, connection)
|
||||
{
|
||||
CommandTimeout = _dataSource.CommandTimeoutSeconds
|
||||
};
|
||||
|
||||
command.Parameters.AddWithValue("tenant", tenant);
|
||||
command.Parameters.Add(new NpgsqlParameter<string[]>("advisories", NpgsqlDbType.Array | NpgsqlDbType.Text)
|
||||
{
|
||||
TypedValue = advisories.ToArray()
|
||||
});
|
||||
command.Parameters.AddWithValue("limit", limit);
|
||||
|
||||
var overlays = new List<GraphOverlayItem>();
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var payload = reader.GetString(0);
|
||||
var overlay = JsonSerializer.Deserialize<GraphOverlayItem>(payload, SerializerOptions);
|
||||
if (overlay is not null)
|
||||
{
|
||||
overlays.Add(overlay);
|
||||
}
|
||||
}
|
||||
|
||||
return overlays;
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<GraphOverlayItem>> FindWithConflictsAsync(string tenant, int limit, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(tenant);
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await using var connection = await _dataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
|
||||
const string sql = """
|
||||
SELECT payload
|
||||
FROM vex.graph_overlays
|
||||
WHERE tenant = @tenant
|
||||
AND jsonb_array_length(payload -> 'conflicts') > 0
|
||||
ORDER BY generated_at DESC, purl, advisory_id, source
|
||||
LIMIT @limit;
|
||||
""";
|
||||
|
||||
await using var command = new NpgsqlCommand(sql, connection)
|
||||
{
|
||||
CommandTimeout = _dataSource.CommandTimeoutSeconds
|
||||
};
|
||||
command.Parameters.AddWithValue("tenant", tenant);
|
||||
command.Parameters.AddWithValue("limit", limit);
|
||||
|
||||
var overlays = new List<GraphOverlayItem>();
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var payload = reader.GetString(0);
|
||||
var overlay = JsonSerializer.Deserialize<GraphOverlayItem>(payload, SerializerOptions);
|
||||
if (overlay is not null)
|
||||
{
|
||||
overlays.Add(overlay);
|
||||
}
|
||||
}
|
||||
|
||||
return overlays;
|
||||
}
|
||||
|
||||
private async ValueTask EnsureTableAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_initialized)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _initLock.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_initialized)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
const string sql = """
|
||||
CREATE TABLE IF NOT EXISTS vex.graph_overlays (
|
||||
tenant text NOT NULL,
|
||||
purl text NOT NULL,
|
||||
advisory_id text NOT NULL,
|
||||
source text NOT NULL,
|
||||
generated_at timestamptz NOT NULL,
|
||||
payload jsonb NOT NULL,
|
||||
CONSTRAINT pk_graph_overlays PRIMARY KEY (tenant, purl, advisory_id, source)
|
||||
);
|
||||
""";
|
||||
|
||||
await using var command = new NpgsqlCommand(sql, connection)
|
||||
{
|
||||
CommandTimeout = _dataSource.CommandTimeoutSeconds
|
||||
};
|
||||
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_initialized = true;
|
||||
}
|
||||
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to ensure graph_overlays table exists.");
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_initLock.Release();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Services;
|
||||
|
||||
public sealed record VexStatementBackfillRequest(int BatchSize = 500);
|
||||
|
||||
public sealed record VexStatementBackfillResult(
|
||||
int DocumentsEvaluated,
|
||||
int DocumentsBackfilled,
|
||||
int ClaimsWritten,
|
||||
int SkippedExisting,
|
||||
int NormalizationFailures);
|
||||
|
||||
/// <summary>
|
||||
/// Placeholder backfill service while legacy statement storage is removed.
|
||||
/// </summary>
|
||||
public sealed class VexStatementBackfillService
|
||||
{
|
||||
private readonly ILogger<VexStatementBackfillService> _logger;
|
||||
|
||||
public VexStatementBackfillService(ILogger<VexStatementBackfillService> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public ValueTask<VexStatementBackfillResult> RunAsync(VexStatementBackfillRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
_logger.LogInformation("Vex statement backfill is currently a no-op; batchSize={BatchSize}", request.BatchSize);
|
||||
return ValueTask.FromResult(new VexStatementBackfillResult(0, 0, 0, 0, 0));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user