Refactor and update test projects, remove obsolete tests, and upgrade dependencies
- Deleted obsolete test files for SchedulerAuditService and SchedulerMongoSessionFactory. - Removed unused TestDataFactory class. - Updated project files for Mongo.Tests to remove references to deleted files. - Upgraded BouncyCastle.Cryptography package to version 2.6.2 across multiple projects. - Replaced Microsoft.Extensions.Http.Polly with Microsoft.Extensions.Http.Resilience in Zastava.Webhook project. - Updated NetEscapades.Configuration.Yaml package to version 3.1.0 in Configuration library. - Upgraded Pkcs11Interop package to version 5.1.2 in Cryptography libraries. - Refactored Argon2idPasswordHasher to use BouncyCastle for hashing instead of Konscious. - Updated JsonSchema.Net package to version 7.3.2 in Microservice project. - Updated global.json to use .NET SDK version 10.0.101.
This commit is contained in:
master
@@ -0,0 +1,19 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
using StellaOps.Excititor.Core.Evidence;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Contracts;
|
||||
|
||||
public sealed record EvidenceManifestResponse(
|
||||
[property: JsonPropertyName("manifest")] VexLockerManifest Manifest,
|
||||
[property: JsonPropertyName("attestationId")] string AttestationId,
|
||||
[property: JsonPropertyName("dsseEnvelope")] string DsseEnvelope,
|
||||
[property: JsonPropertyName("dsseEnvelopeHash")] string DsseEnvelopeHash,
|
||||
[property: JsonPropertyName("itemCount")] int ItemCount,
|
||||
[property: JsonPropertyName("generatedAt")] DateTimeOffset GeneratedAt);
|
||||
|
||||
public sealed record EvidenceChunkListResponse(
|
||||
[property: JsonPropertyName("chunks")] IReadOnlyList<VexEvidenceChunkResponse> Chunks,
|
||||
[property: JsonPropertyName("total")] int Total,
|
||||
[property: JsonPropertyName("truncated")] bool Truncated,
|
||||
[property: JsonPropertyName("generatedAt")] DateTimeOffset GeneratedAt);
|
||||
@@ -10,18 +10,45 @@ public sealed record GraphOverlaysResponse(
|
||||
[property: JsonPropertyName("cacheAgeMs")] long? CacheAgeMs);
|
||||
|
||||
public sealed record GraphOverlayItem(
|
||||
[property: JsonPropertyName("schemaVersion")] string SchemaVersion,
|
||||
[property: JsonPropertyName("generatedAt")] DateTimeOffset GeneratedAt,
|
||||
[property: JsonPropertyName("tenant")] string Tenant,
|
||||
[property: JsonPropertyName("purl")] string Purl,
|
||||
[property: JsonPropertyName("summary")] GraphOverlaySummary Summary,
|
||||
[property: JsonPropertyName("latestModifiedAt")] DateTimeOffset? LatestModifiedAt,
|
||||
[property: JsonPropertyName("justifications")] IReadOnlyList<string> Justifications,
|
||||
[property: JsonPropertyName("provenance")] GraphOverlayProvenance Provenance);
|
||||
[property: JsonPropertyName("advisoryId")] string AdvisoryId,
|
||||
[property: JsonPropertyName("source")] string Source,
|
||||
[property: JsonPropertyName("status")] string Status,
|
||||
[property: JsonPropertyName("justifications")] IReadOnlyList<GraphOverlayJustification> Justifications,
|
||||
[property: JsonPropertyName("conflicts")] IReadOnlyList<GraphOverlayConflict> Conflicts,
|
||||
[property: JsonPropertyName("observations")] IReadOnlyList<GraphOverlayObservation> Observations,
|
||||
[property: JsonPropertyName("provenance")] GraphOverlayProvenance Provenance,
|
||||
[property: JsonPropertyName("cache")] GraphOverlayCache? Cache);
|
||||
|
||||
public sealed record GraphOverlaySummary(
|
||||
[property: JsonPropertyName("open")] int Open,
|
||||
[property: JsonPropertyName("not_affected")] int NotAffected,
|
||||
[property: JsonPropertyName("under_investigation")] int UnderInvestigation,
|
||||
[property: JsonPropertyName("no_statement")] int NoStatement);
|
||||
public sealed record GraphOverlayJustification(
|
||||
[property: JsonPropertyName("kind")] string Kind,
|
||||
[property: JsonPropertyName("reason")] string Reason,
|
||||
[property: JsonPropertyName("evidence")] IReadOnlyList<string>? Evidence,
|
||||
[property: JsonPropertyName("weight")] double? Weight);
|
||||
|
||||
public sealed record GraphOverlayConflict(
|
||||
[property: JsonPropertyName("field")] string Field,
|
||||
[property: JsonPropertyName("reason")] string Reason,
|
||||
[property: JsonPropertyName("values")] IReadOnlyList<string> Values,
|
||||
[property: JsonPropertyName("sourceIds")] IReadOnlyList<string>? SourceIds);
|
||||
|
||||
public sealed record GraphOverlayObservation(
|
||||
[property: JsonPropertyName("id")] string Id,
|
||||
[property: JsonPropertyName("contentHash")] string ContentHash,
|
||||
[property: JsonPropertyName("fetchedAt")] DateTimeOffset FetchedAt);
|
||||
|
||||
public sealed record GraphOverlayProvenance(
|
||||
[property: JsonPropertyName("sources")] IReadOnlyList<string> Sources,
|
||||
[property: JsonPropertyName("lastEvidenceHash")] string? LastEvidenceHash);
|
||||
[property: JsonPropertyName("linksetId")] string LinksetId,
|
||||
[property: JsonPropertyName("linksetHash")] string LinksetHash,
|
||||
[property: JsonPropertyName("observationHashes")] IReadOnlyList<string> ObservationHashes,
|
||||
[property: JsonPropertyName("policyHash")] string? PolicyHash,
|
||||
[property: JsonPropertyName("sbomContextHash")] string? SbomContextHash,
|
||||
[property: JsonPropertyName("planCacheKey")] string? PlanCacheKey);
|
||||
|
||||
public sealed record GraphOverlayCache(
|
||||
[property: JsonPropertyName("cached")] bool Cached,
|
||||
[property: JsonPropertyName("cachedAt")] DateTimeOffset? CachedAt,
|
||||
[property: JsonPropertyName("ttlSeconds")] int? TtlSeconds);
|
||||
|
||||
@@ -15,3 +15,9 @@ public sealed record GraphStatusItem(
|
||||
[property: JsonPropertyName("latestModifiedAt")] DateTimeOffset? LatestModifiedAt,
|
||||
[property: JsonPropertyName("sources")] IReadOnlyList<string> Sources,
|
||||
[property: JsonPropertyName("lastEvidenceHash")] string? LastEvidenceHash);
|
||||
|
||||
public sealed record GraphOverlaySummary(
|
||||
[property: JsonPropertyName("open")] int Open,
|
||||
[property: JsonPropertyName("not_affected")] int NotAffected,
|
||||
[property: JsonPropertyName("under_investigation")] int UnderInvestigation,
|
||||
[property: JsonPropertyName("no_statement")] int NoStatement);
|
||||
|
||||
@@ -4,6 +4,7 @@ using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
using static Program;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Endpoints;
|
||||
|
||||
|
||||
@@ -2,23 +2,38 @@ using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Evidence;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
using static Program;
|
||||
using StellaOps.Excititor.WebService.Telemetry;
|
||||
using System.Collections.Immutable;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Endpoints;
|
||||
|
||||
/// <summary>
|
||||
/// Evidence API endpoints (temporarily disabled while Mongo/BSON storage is removed).
|
||||
/// Evidence API endpoints (manifest + DSSE attestation + evidence chunks).
|
||||
/// </summary>
|
||||
public static class EvidenceEndpoints
|
||||
{
|
||||
public static void MapEvidenceEndpoints(this WebApplication app)
|
||||
{
|
||||
// GET /evidence/vex/list
|
||||
app.MapGet("/evidence/vex/list", (
|
||||
app.MapGet("/evidence/vex/list", async (
|
||||
HttpContext context,
|
||||
[FromQuery(Name = "vulnerabilityId")] string[] vulnerabilityIds,
|
||||
[FromQuery(Name = "productKey")] string[] productKeys,
|
||||
[FromQuery] string? since,
|
||||
[FromQuery] int? limit,
|
||||
IVexClaimStore claimStore,
|
||||
IVexEvidenceLockerService lockerService,
|
||||
IVexEvidenceAttestor attestor,
|
||||
IOptions<VexStorageOptions> storageOptions,
|
||||
ChunkTelemetry chunkTelemetry) =>
|
||||
ChunkTelemetry chunkTelemetry,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
|
||||
if (scopeResult is not null)
|
||||
@@ -31,18 +46,76 @@ public static class EvidenceEndpoints
|
||||
return tenantError;
|
||||
}
|
||||
|
||||
chunkTelemetry.RecordIngested(tenant, null, "unavailable", "storage-migration", 0, 0, 0);
|
||||
return Results.Problem(
|
||||
detail: "Evidence exports are temporarily unavailable during Postgres migration (Mongo/BSON removed).",
|
||||
statusCode: StatusCodes.Status503ServiceUnavailable,
|
||||
title: "Service unavailable");
|
||||
var parsedSince = ParseSinceTimestamp(new Microsoft.Extensions.Primitives.StringValues(since));
|
||||
var max = Math.Clamp(limit ?? 500, 1, 1000);
|
||||
|
||||
var pairs = NormalizeValues(vulnerabilityIds).SelectMany(v =>
|
||||
NormalizeValues(productKeys).Select(p => (Vuln: v, Product: p))).ToList();
|
||||
|
||||
if (pairs.Count == 0)
|
||||
{
|
||||
return Results.BadRequest("At least one vulnerabilityId and productKey are required.");
|
||||
}
|
||||
|
||||
var claims = new List<VexClaim>();
|
||||
foreach (var pair in pairs)
|
||||
{
|
||||
var found = await claimStore.FindAsync(pair.Vuln, pair.Product, parsedSince, cancellationToken).ConfigureAwait(false);
|
||||
claims.AddRange(found);
|
||||
}
|
||||
|
||||
claims = claims
|
||||
.OrderBy(c => c.VulnerabilityId, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(c => c.Product.Key, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenByDescending(c => c.LastSeen)
|
||||
.Take(max)
|
||||
.ToList();
|
||||
|
||||
if (claims.Count == 0)
|
||||
{
|
||||
return Results.NotFound("No claims available for the requested filters.");
|
||||
}
|
||||
|
||||
var items = claims.Select(claim =>
|
||||
new VexEvidenceSnapshotItem(
|
||||
observationId: FormattableString.Invariant($"{claim.ProviderId}:{claim.Document.Digest}"),
|
||||
providerId: claim.ProviderId,
|
||||
contentHash: claim.Document.Digest,
|
||||
linksetId: FormattableString.Invariant($"{claim.VulnerabilityId}:{claim.Product.Key}"),
|
||||
dsseEnvelopeHash: null,
|
||||
provenance: new VexEvidenceProvenance("ingest")))
|
||||
.ToList();
|
||||
|
||||
var now = timeProvider.GetUtcNow();
|
||||
var manifest = lockerService.BuildManifest(tenant, items, timestamp: now, sequence: 1, isSealed: false);
|
||||
var attestation = await attestor.AttestManifestAsync(manifest, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
chunkTelemetry.RecordIngested(tenant, null, "available", "locker-manifest", claims.Count, 0, 0);
|
||||
var response = new EvidenceManifestResponse(
|
||||
attestation.SignedManifest,
|
||||
attestation.AttestationId,
|
||||
attestation.DsseEnvelopeJson,
|
||||
attestation.DsseEnvelopeHash,
|
||||
attestation.SignedManifest.Items.Length,
|
||||
attestation.AttestedAt);
|
||||
|
||||
return Results.Ok(response);
|
||||
}).WithName("ListVexEvidence");
|
||||
|
||||
// GET /evidence/vex/{bundleId}
|
||||
app.MapGet("/evidence/vex/{bundleId}", (
|
||||
app.MapGet("/evidence/vex/{bundleId}", async (
|
||||
HttpContext context,
|
||||
string bundleId,
|
||||
IOptions<VexStorageOptions> storageOptions) =>
|
||||
[FromQuery(Name = "vulnerabilityId")] string[] vulnerabilityIds,
|
||||
[FromQuery(Name = "productKey")] string[] productKeys,
|
||||
[FromQuery] string? since,
|
||||
[FromQuery] int? limit,
|
||||
IVexClaimStore claimStore,
|
||||
IVexEvidenceLockerService lockerService,
|
||||
IVexEvidenceAttestor attestor,
|
||||
IOptions<VexStorageOptions> storageOptions,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
|
||||
if (scopeResult is not null)
|
||||
@@ -50,7 +123,7 @@ public static class EvidenceEndpoints
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out _, out var tenantError))
|
||||
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out var tenant, out var tenantError))
|
||||
{
|
||||
return tenantError;
|
||||
}
|
||||
@@ -63,17 +136,77 @@ public static class EvidenceEndpoints
|
||||
title: "Validation error");
|
||||
}
|
||||
|
||||
return Results.Problem(
|
||||
detail: "Evidence bundles are temporarily unavailable during Postgres migration (Mongo/BSON removed).",
|
||||
statusCode: StatusCodes.Status503ServiceUnavailable,
|
||||
title: "Service unavailable");
|
||||
var parsedSince = ParseSinceTimestamp(new Microsoft.Extensions.Primitives.StringValues(since));
|
||||
var max = Math.Clamp(limit ?? 500, 1, 1000);
|
||||
var pairs = NormalizeValues(vulnerabilityIds).SelectMany(v =>
|
||||
NormalizeValues(productKeys).Select(p => (Vuln: v, Product: p))).ToList();
|
||||
|
||||
if (pairs.Count == 0)
|
||||
{
|
||||
return Results.BadRequest("At least one vulnerabilityId and productKey are required.");
|
||||
}
|
||||
|
||||
var claims = new List<VexClaim>();
|
||||
foreach (var pair in pairs)
|
||||
{
|
||||
var found = await claimStore.FindAsync(pair.Vuln, pair.Product, parsedSince, cancellationToken).ConfigureAwait(false);
|
||||
claims.AddRange(found);
|
||||
}
|
||||
|
||||
claims = claims
|
||||
.OrderBy(c => c.VulnerabilityId, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(c => c.Product.Key, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenByDescending(c => c.LastSeen)
|
||||
.Take(max)
|
||||
.ToList();
|
||||
|
||||
if (claims.Count == 0)
|
||||
{
|
||||
return Results.NotFound("No claims available for the requested filters.");
|
||||
}
|
||||
|
||||
var items = claims.Select(claim =>
|
||||
new VexEvidenceSnapshotItem(
|
||||
observationId: FormattableString.Invariant($"{claim.ProviderId}:{claim.Document.Digest}"),
|
||||
providerId: claim.ProviderId,
|
||||
contentHash: claim.Document.Digest,
|
||||
linksetId: FormattableString.Invariant($"{claim.VulnerabilityId}:{claim.Product.Key}"),
|
||||
dsseEnvelopeHash: null,
|
||||
provenance: new VexEvidenceProvenance("ingest")))
|
||||
.ToList();
|
||||
|
||||
var now = timeProvider.GetUtcNow();
|
||||
var manifest = lockerService.BuildManifest(tenant, items, timestamp: now, sequence: 1, isSealed: false);
|
||||
if (!string.Equals(manifest.ManifestId, bundleId, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return Results.NotFound($"Requested bundleId '{bundleId}' not found for current filters.");
|
||||
}
|
||||
|
||||
var attestation = await attestor.AttestManifestAsync(manifest, cancellationToken).ConfigureAwait(false);
|
||||
var response = new EvidenceManifestResponse(
|
||||
attestation.SignedManifest,
|
||||
attestation.AttestationId,
|
||||
attestation.DsseEnvelopeJson,
|
||||
attestation.DsseEnvelopeHash,
|
||||
attestation.SignedManifest.Items.Length,
|
||||
attestation.AttestedAt);
|
||||
|
||||
return Results.Ok(response);
|
||||
}).WithName("GetVexEvidenceBundle");
|
||||
|
||||
// GET /v1/vex/evidence/chunks
|
||||
app.MapGet("/v1/vex/evidence/chunks", (
|
||||
app.MapGet("/v1/vex/evidence/chunks", async (
|
||||
HttpContext context,
|
||||
[FromQuery] string vulnerabilityId,
|
||||
[FromQuery] string productKey,
|
||||
[FromQuery(Name = "providerId")] string[] providerIds,
|
||||
[FromQuery] string[] status,
|
||||
[FromQuery] string? since,
|
||||
[FromQuery] int? limit,
|
||||
IOptions<VexStorageOptions> storageOptions,
|
||||
ChunkTelemetry chunkTelemetry) =>
|
||||
IVexEvidenceChunkService chunkService,
|
||||
ChunkTelemetry chunkTelemetry,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
|
||||
if (scopeResult is not null)
|
||||
@@ -86,11 +219,37 @@ public static class EvidenceEndpoints
|
||||
return tenantError;
|
||||
}
|
||||
|
||||
chunkTelemetry.RecordIngested(tenant, null, "unavailable", "storage-migration", 0, 0, 0);
|
||||
return Results.Problem(
|
||||
detail: "Evidence chunk streaming is temporarily unavailable during Postgres migration (Mongo/BSON removed).",
|
||||
statusCode: StatusCodes.Status503ServiceUnavailable,
|
||||
title: "Service unavailable");
|
||||
if (string.IsNullOrWhiteSpace(vulnerabilityId) || string.IsNullOrWhiteSpace(productKey))
|
||||
{
|
||||
return Results.BadRequest("vulnerabilityId and productKey are required.");
|
||||
}
|
||||
|
||||
var parsedSince = ParseSinceTimestamp(new Microsoft.Extensions.Primitives.StringValues(since));
|
||||
var providers = providerIds?.Length > 0
|
||||
? providerIds.ToImmutableHashSet(StringComparer.OrdinalIgnoreCase)
|
||||
: ImmutableHashSet<string>.Empty;
|
||||
|
||||
var statuses = status?.Length > 0
|
||||
? status
|
||||
.Select(s => Enum.TryParse<VexClaimStatus>(s, true, out var parsed) ? parsed : (VexClaimStatus?)null)
|
||||
.Where(s => s is not null)
|
||||
.Select(s => s!.Value)
|
||||
.ToImmutableHashSet()
|
||||
: ImmutableHashSet<VexClaimStatus>.Empty;
|
||||
|
||||
var req = new VexEvidenceChunkRequest(
|
||||
tenant,
|
||||
vulnerabilityId,
|
||||
productKey,
|
||||
providers,
|
||||
statuses,
|
||||
parsedSince,
|
||||
Math.Clamp(limit ?? 200, 1, 1000));
|
||||
|
||||
var result = await chunkService.QueryAsync(req, cancellationToken).ConfigureAwait(false);
|
||||
chunkTelemetry.RecordIngested(tenant, null, "available", "locker-chunks", result.TotalCount, 0, 0);
|
||||
|
||||
return Results.Ok(new EvidenceChunkListResponse(result.Chunks, result.TotalCount, result.Truncated, result.GeneratedAtUtc));
|
||||
}).WithName("GetVexEvidenceChunks");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ using System.IO;
|
||||
using System.Text;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Core;
|
||||
@@ -71,9 +72,9 @@ internal static class MirrorEndpoints
|
||||
string domainId,
|
||||
HttpContext httpContext,
|
||||
IOptions<MirrorDistributionOptions> options,
|
||||
MirrorRateLimiter rateLimiter,
|
||||
IVexExportStore exportStore,
|
||||
TimeProvider timeProvider,
|
||||
[FromServices] MirrorRateLimiter rateLimiter,
|
||||
[FromServices] IVexExportStore exportStore,
|
||||
[FromServices] TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!TryFindDomain(options.Value, domainId, out var domain))
|
||||
@@ -162,9 +163,9 @@ internal static class MirrorEndpoints
|
||||
string exportKey,
|
||||
HttpContext httpContext,
|
||||
IOptions<MirrorDistributionOptions> options,
|
||||
MirrorRateLimiter rateLimiter,
|
||||
IVexExportStore exportStore,
|
||||
TimeProvider timeProvider,
|
||||
[FromServices] MirrorRateLimiter rateLimiter,
|
||||
[FromServices] IVexExportStore exportStore,
|
||||
[FromServices] TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!TryFindDomain(options.Value, domainId, out var domain))
|
||||
@@ -215,9 +216,9 @@ internal static class MirrorEndpoints
|
||||
string exportKey,
|
||||
HttpContext httpContext,
|
||||
IOptions<MirrorDistributionOptions> options,
|
||||
MirrorRateLimiter rateLimiter,
|
||||
IVexExportStore exportStore,
|
||||
IEnumerable<IVexArtifactStore> artifactStores,
|
||||
[FromServices] MirrorRateLimiter rateLimiter,
|
||||
[FromServices] IVexExportStore exportStore,
|
||||
[FromServices] IEnumerable<IVexArtifactStore> artifactStores,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!TryFindDomain(options.Value, domainId, out var domain))
|
||||
|
||||
@@ -9,8 +9,6 @@ using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Canonicalization;
|
||||
using StellaOps.Excititor.Core.Orchestration;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
@@ -34,7 +32,7 @@ public static class PolicyEndpoints
|
||||
HttpContext context,
|
||||
[FromBody] PolicyVexLookupRequest request,
|
||||
IOptions<VexStorageOptions> storageOptions,
|
||||
[FromServices] IVexClaimStore claimStore,
|
||||
[FromServices] IGraphOverlayStore overlayStore,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -45,7 +43,7 @@ public static class PolicyEndpoints
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryResolveTenant(context, storageOptions.Value, out _, out var tenantError))
|
||||
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
|
||||
{
|
||||
return tenantError!;
|
||||
}
|
||||
@@ -56,24 +54,19 @@ public static class PolicyEndpoints
|
||||
return Results.BadRequest(new { error = new { code = "ERR_REQUEST", message = "advisory_keys or purls must be provided" } });
|
||||
}
|
||||
|
||||
var canonicalizer = new VexAdvisoryKeyCanonicalizer();
|
||||
var productCanonicalizer = new VexProductKeyCanonicalizer();
|
||||
|
||||
var canonicalAdvisories = request.AdvisoryKeys
|
||||
var advisories = request.AdvisoryKeys
|
||||
.Where(a => !string.IsNullOrWhiteSpace(a))
|
||||
.Select(a => canonicalizer.Canonicalize(a.Trim()))
|
||||
.Select(a => a.Trim())
|
||||
.ToList();
|
||||
|
||||
var canonicalProducts = request.Purls
|
||||
var purls = request.Purls
|
||||
.Where(p => !string.IsNullOrWhiteSpace(p))
|
||||
.Select(p => productCanonicalizer.Canonicalize(p.Trim(), purl: p.Trim()))
|
||||
.Select(p => p.Trim())
|
||||
.ToList();
|
||||
|
||||
// Map requested statuses/providers for filtering
|
||||
var statusFilter = request.Statuses
|
||||
.Select(s => Enum.TryParse<VexClaimStatus>(s, true, out var parsed) ? parsed : (VexClaimStatus?)null)
|
||||
.Where(p => p.HasValue)
|
||||
.Select(p => p!.Value)
|
||||
.Where(s => !string.IsNullOrWhiteSpace(s))
|
||||
.Select(s => s.Trim().ToLowerInvariant())
|
||||
.ToImmutableHashSet();
|
||||
|
||||
var providerFilter = request.Providers
|
||||
@@ -81,94 +74,96 @@ public static class PolicyEndpoints
|
||||
.Select(p => p.Trim())
|
||||
.ToImmutableHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var limit = Math.Clamp(request.Limit, 1, 500);
|
||||
var now = timeProvider.GetUtcNow();
|
||||
var overlays = await ResolveOverlaysAsync(overlayStore, tenant!, advisories, purls, request.Limit, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var results = new List<PolicyVexLookupItem>();
|
||||
var totalStatements = 0;
|
||||
var filtered = overlays
|
||||
.Where(o => MatchesProvider(providerFilter, o))
|
||||
.Where(o => MatchesStatus(statusFilter, o))
|
||||
.OrderBy(o => o.AdvisoryId, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(o => o.Purl, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(o => o.Source, StringComparer.OrdinalIgnoreCase)
|
||||
.Take(Math.Clamp(request.Limit, 1, 500))
|
||||
.ToList();
|
||||
|
||||
// For each advisory key, fetch claims and filter by product/provider/status
|
||||
foreach (var advisory in canonicalAdvisories)
|
||||
{
|
||||
var claims = await claimStore
|
||||
.FindByVulnerabilityAsync(advisory.AdvisoryKey, limit, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
var grouped = filtered
|
||||
.GroupBy(o => o.AdvisoryId, StringComparer.OrdinalIgnoreCase)
|
||||
.Select(group => new PolicyVexLookupItem(
|
||||
group.Key,
|
||||
new[] { group.Key },
|
||||
group.Select(MapStatement).ToList()))
|
||||
.ToList();
|
||||
|
||||
var filtered = claims
|
||||
.Where(claim => MatchesProvider(providerFilter, claim))
|
||||
.Where(claim => MatchesStatus(statusFilter, claim))
|
||||
.Where(claim => MatchesProduct(canonicalProducts, claim))
|
||||
.OrderByDescending(claim => claim.LastSeen)
|
||||
.ThenBy(claim => claim.ProviderId, StringComparer.Ordinal)
|
||||
.ThenBy(claim => claim.Product.Key, StringComparer.Ordinal)
|
||||
.Take(limit)
|
||||
.ToList();
|
||||
|
||||
totalStatements += filtered.Count;
|
||||
|
||||
var statements = filtered.Select(MapStatement).ToList();
|
||||
var aliases = advisory.Aliases.ToList();
|
||||
if (!aliases.Contains(advisory.AdvisoryKey, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
aliases.Add(advisory.AdvisoryKey);
|
||||
}
|
||||
|
||||
results.Add(new PolicyVexLookupItem(
|
||||
advisory.AdvisoryKey,
|
||||
aliases,
|
||||
statements));
|
||||
}
|
||||
|
||||
var response = new PolicyVexLookupResponse(results, totalStatements, now);
|
||||
var response = new PolicyVexLookupResponse(grouped, filtered.Count, timeProvider.GetUtcNow());
|
||||
return Results.Ok(response);
|
||||
}
|
||||
|
||||
private static bool MatchesProvider(ISet<string> providers, VexClaim claim)
|
||||
=> providers.Count == 0 || providers.Contains(claim.ProviderId, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
private static bool MatchesStatus(ISet<VexClaimStatus> statuses, VexClaim claim)
|
||||
=> statuses.Count == 0 || statuses.Contains(claim.Status);
|
||||
|
||||
private static bool MatchesProduct(IEnumerable<VexCanonicalProductKey> requestedProducts, VexClaim claim)
|
||||
private static async Task<IReadOnlyList<GraphOverlayItem>> ResolveOverlaysAsync(
|
||||
IGraphOverlayStore overlayStore,
|
||||
string tenant,
|
||||
IReadOnlyList<string> advisories,
|
||||
IReadOnlyList<string> purls,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!requestedProducts.Any())
|
||||
if (purls.Count > 0)
|
||||
{
|
||||
return true;
|
||||
var overlays = await overlayStore.FindByPurlsAsync(tenant, purls, cancellationToken).ConfigureAwait(false);
|
||||
if (advisories.Count == 0)
|
||||
{
|
||||
return overlays;
|
||||
}
|
||||
|
||||
return overlays.Where(o => advisories.Contains(o.AdvisoryId, StringComparer.OrdinalIgnoreCase)).ToList();
|
||||
}
|
||||
|
||||
return requestedProducts.Any(product =>
|
||||
string.Equals(product.ProductKey, claim.Product.Key, StringComparison.OrdinalIgnoreCase) ||
|
||||
product.Links.Any(link => string.Equals(link.Identifier, claim.Product.Key, StringComparison.OrdinalIgnoreCase)) ||
|
||||
(!string.IsNullOrWhiteSpace(product.Purl) && string.Equals(product.Purl, claim.Product.Purl, StringComparison.OrdinalIgnoreCase)));
|
||||
return await overlayStore.FindByAdvisoriesAsync(tenant, advisories, limit, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static PolicyVexStatement MapStatement(VexClaim claim)
|
||||
private static bool MatchesProvider(ISet<string> providers, GraphOverlayItem overlay)
|
||||
=> providers.Count == 0 || providers.Contains(overlay.Source, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
private static bool MatchesStatus(ISet<string> statuses, GraphOverlayItem overlay)
|
||||
=> statuses.Count == 0 || statuses.Contains(overlay.Status, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
private static PolicyVexStatement MapStatement(GraphOverlayItem overlay)
|
||||
{
|
||||
var observationId = $"{claim.ProviderId}:{claim.Document.Digest}";
|
||||
var firstSeen = overlay.Observations.Count == 0
|
||||
? overlay.GeneratedAt
|
||||
: overlay.Observations.Min(o => o.FetchedAt);
|
||||
|
||||
var lastSeen = overlay.Observations.Count == 0
|
||||
? overlay.GeneratedAt
|
||||
: overlay.Observations.Max(o => o.FetchedAt);
|
||||
|
||||
var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["document_digest"] = claim.Document.Digest,
|
||||
["document_uri"] = claim.Document.SourceUri.ToString()
|
||||
["schemaVersion"] = overlay.SchemaVersion,
|
||||
["linksetId"] = overlay.Provenance.LinksetId,
|
||||
["linksetHash"] = overlay.Provenance.LinksetHash,
|
||||
["source"] = overlay.Source
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(claim.Document.Revision))
|
||||
if (!string.IsNullOrWhiteSpace(overlay.Provenance.PlanCacheKey))
|
||||
{
|
||||
metadata["document_revision"] = claim.Document.Revision!;
|
||||
metadata["planCacheKey"] = overlay.Provenance.PlanCacheKey!;
|
||||
}
|
||||
|
||||
var justification = overlay.Justifications.FirstOrDefault();
|
||||
var primaryObservation = overlay.Observations.FirstOrDefault();
|
||||
|
||||
return new PolicyVexStatement(
|
||||
ObservationId: observationId,
|
||||
ProviderId: claim.ProviderId,
|
||||
Status: claim.Status.ToString(),
|
||||
ProductKey: claim.Product.Key,
|
||||
Purl: claim.Product.Purl,
|
||||
Cpe: claim.Product.Cpe,
|
||||
Version: claim.Product.Version,
|
||||
Justification: claim.Justification?.ToString(),
|
||||
Detail: claim.Detail,
|
||||
FirstSeen: claim.FirstSeen,
|
||||
LastSeen: claim.LastSeen,
|
||||
Signature: claim.Document.Signature,
|
||||
ObservationId: primaryObservation?.Id ?? $"{overlay.Source}:{overlay.AdvisoryId}",
|
||||
ProviderId: overlay.Source,
|
||||
Status: overlay.Status,
|
||||
ProductKey: overlay.Purl,
|
||||
Purl: overlay.Purl,
|
||||
Cpe: null,
|
||||
Version: null,
|
||||
Justification: justification?.Kind,
|
||||
Detail: justification?.Reason,
|
||||
FirstSeen: firstSeen,
|
||||
LastSeen: lastSeen,
|
||||
Signature: null,
|
||||
Metadata: metadata);
|
||||
}
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Excititor.Attestation;
|
||||
@@ -33,7 +34,7 @@ internal static class ResolveEndpoint
|
||||
VexResolveRequest request,
|
||||
HttpContext httpContext,
|
||||
IVexClaimStore claimStore,
|
||||
IVexConsensusStore consensusStore,
|
||||
[FromServices] IVexConsensusStore? consensusStore,
|
||||
IVexProviderStore providerStore,
|
||||
IVexPolicyProvider policyProvider,
|
||||
TimeProvider timeProvider,
|
||||
@@ -142,7 +143,10 @@ internal static class ResolveEndpoint
|
||||
snapshot.Digest);
|
||||
}
|
||||
|
||||
await consensusStore.SaveAsync(consensus, cancellationToken).ConfigureAwait(false);
|
||||
if (consensusStore is not null)
|
||||
{
|
||||
await consensusStore.SaveAsync(consensus, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var payload = PreparePayload(consensus);
|
||||
var contentSignature = await TrySignAsync(signer, payload, logger, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
@@ -1,27 +1,27 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json;
|
||||
using StellaOps.Concelier.RawModels;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
using RawModels = StellaOps.Concelier.RawModels;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Extensions;
|
||||
|
||||
internal static class VexRawDocumentMapper
|
||||
{
|
||||
public static VexRawDocument ToRawModel(VexRawRecord record, string defaultTenant)
|
||||
public static RawModels.VexRawDocument ToRawModel(VexRawRecord record, string defaultTenant)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(record);
|
||||
|
||||
var metadata = record.Metadata ?? ImmutableDictionary<string, string>.Empty;
|
||||
var tenant = Get(metadata, "tenant", record.Tenant) ?? defaultTenant;
|
||||
|
||||
var source = new RawSourceMetadata(
|
||||
var source = new RawModels.RawSourceMetadata(
|
||||
Vendor: Get(metadata, "source.vendor", record.ProviderId) ?? record.ProviderId,
|
||||
Connector: Get(metadata, "source.connector", record.ProviderId) ?? record.ProviderId,
|
||||
ConnectorVersion: Get(metadata, "source.connector_version", "unknown") ?? "unknown",
|
||||
Stream: Get(metadata, "source.stream", record.Format.ToString().ToLowerInvariant()));
|
||||
|
||||
var signature = new RawSignatureMetadata(
|
||||
var signature = new RawModels.RawSignatureMetadata(
|
||||
Present: string.Equals(Get(metadata, "signature.present"), "true", StringComparison.OrdinalIgnoreCase),
|
||||
Format: Get(metadata, "signature.format"),
|
||||
KeyId: Get(metadata, "signature.key_id"),
|
||||
@@ -29,7 +29,7 @@ internal static class VexRawDocumentMapper
|
||||
Certificate: Get(metadata, "signature.certificate"),
|
||||
Digest: Get(metadata, "signature.digest"));
|
||||
|
||||
var upstream = new RawUpstreamMetadata(
|
||||
var upstream = new RawModels.RawUpstreamMetadata(
|
||||
UpstreamId: Get(metadata, "upstream.id", record.Digest) ?? record.Digest,
|
||||
DocumentVersion: Get(metadata, "upstream.version"),
|
||||
RetrievedAt: record.RetrievedAt,
|
||||
@@ -37,20 +37,20 @@ internal static class VexRawDocumentMapper
|
||||
Signature: signature,
|
||||
Provenance: metadata);
|
||||
|
||||
var content = new RawContent(
|
||||
var content = new RawModels.RawContent(
|
||||
Format: record.Format.ToString().ToLowerInvariant(),
|
||||
SpecVersion: Get(metadata, "content.spec_version"),
|
||||
Raw: ParseJson(record.Content),
|
||||
Encoding: Get(metadata, "content.encoding"));
|
||||
|
||||
return new VexRawDocument(
|
||||
return new RawModels.VexRawDocument(
|
||||
tenant,
|
||||
source,
|
||||
upstream,
|
||||
content,
|
||||
new RawLinkset(),
|
||||
statements: null,
|
||||
supersedes: record.SupersedesDigest);
|
||||
new RawModels.RawLinkset(),
|
||||
Statements: null,
|
||||
Supersedes: record.SupersedesDigest);
|
||||
}
|
||||
|
||||
private static string? Get(IReadOnlyDictionary<string, string> metadata, string key, string? fallback = null)
|
||||
|
||||
@@ -11,10 +11,17 @@ namespace StellaOps.Excititor.WebService.Graph;
|
||||
internal static class GraphOverlayFactory
|
||||
{
|
||||
public static IReadOnlyList<GraphOverlayItem> Build(
|
||||
string tenant,
|
||||
DateTimeOffset generatedAt,
|
||||
IReadOnlyList<string> orderedPurls,
|
||||
IReadOnlyList<VexObservation> observations,
|
||||
bool includeJustifications)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tenant))
|
||||
{
|
||||
throw new ArgumentException("tenant is required", nameof(tenant));
|
||||
}
|
||||
|
||||
if (orderedPurls is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(orderedPurls));
|
||||
@@ -25,101 +32,215 @@ internal static class GraphOverlayFactory
|
||||
throw new ArgumentNullException(nameof(observations));
|
||||
}
|
||||
|
||||
var observationsByPurl = observations
|
||||
.SelectMany(obs => obs.Linkset.Purls.Select(purl => (purl, obs)))
|
||||
.GroupBy(tuple => tuple.purl, StringComparer.OrdinalIgnoreCase)
|
||||
.ToDictionary(g => g.Key, g => g.Select(t => t.obs).ToImmutableArray(), StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var items = new List<GraphOverlayItem>(orderedPurls.Count);
|
||||
|
||||
foreach (var input in orderedPurls)
|
||||
var purlOrder = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
|
||||
for (var i = 0; i < orderedPurls.Count; i++)
|
||||
{
|
||||
if (!observationsByPurl.TryGetValue(input, out var obsForPurl) || obsForPurl.Length == 0)
|
||||
{
|
||||
items.Add(new GraphOverlayItem(
|
||||
Purl: input,
|
||||
Summary: new GraphOverlaySummary(0, 0, 0, 0),
|
||||
LatestModifiedAt: null,
|
||||
Justifications: Array.Empty<string>(),
|
||||
Provenance: new GraphOverlayProvenance(Array.Empty<string>(), null)));
|
||||
continue;
|
||||
}
|
||||
|
||||
var open = 0;
|
||||
var notAffected = 0;
|
||||
var underInvestigation = 0;
|
||||
var noStatement = 0;
|
||||
var justifications = new SortedSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var sources = new SortedSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
string? lastEvidenceHash = null;
|
||||
DateTimeOffset? latestModifiedAt = null;
|
||||
|
||||
foreach (var obs in obsForPurl)
|
||||
{
|
||||
sources.Add(obs.ProviderId);
|
||||
if (latestModifiedAt is null || obs.CreatedAt > latestModifiedAt.Value)
|
||||
{
|
||||
latestModifiedAt = obs.CreatedAt;
|
||||
lastEvidenceHash = obs.Upstream.ContentHash;
|
||||
}
|
||||
|
||||
var matchingStatements = obs.Statements
|
||||
.Where(stmt => PurlMatches(stmt, input, obs.Linkset.Purls))
|
||||
.ToArray();
|
||||
|
||||
if (matchingStatements.Length == 0)
|
||||
{
|
||||
noStatement++;
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var stmt in matchingStatements)
|
||||
{
|
||||
switch (stmt.Status)
|
||||
{
|
||||
case VexClaimStatus.NotAffected:
|
||||
notAffected++;
|
||||
break;
|
||||
case VexClaimStatus.UnderInvestigation:
|
||||
underInvestigation++;
|
||||
break;
|
||||
default:
|
||||
open++;
|
||||
break;
|
||||
}
|
||||
|
||||
if (includeJustifications && stmt.Justification is not null)
|
||||
{
|
||||
justifications.Add(stmt.Justification!.ToString()!);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
items.Add(new GraphOverlayItem(
|
||||
Purl: input,
|
||||
Summary: new GraphOverlaySummary(open, notAffected, underInvestigation, noStatement),
|
||||
LatestModifiedAt: latestModifiedAt,
|
||||
Justifications: includeJustifications
|
||||
? justifications.ToArray()
|
||||
: Array.Empty<string>(),
|
||||
Provenance: new GraphOverlayProvenance(sources.ToArray(), lastEvidenceHash)));
|
||||
purlOrder[orderedPurls[i]] = i;
|
||||
}
|
||||
|
||||
return items;
|
||||
var aggregates = new Dictionary<(string Purl, string AdvisoryId, string Source), OverlayAggregate>(new OverlayKeyComparer());
|
||||
|
||||
foreach (var observation in observations.OrderByDescending(o => o.CreatedAt).ThenBy(o => o.ObservationId, StringComparer.Ordinal))
|
||||
{
|
||||
var observationRef = new GraphOverlayObservation(
|
||||
observation.ObservationId,
|
||||
observation.Upstream.ContentHash,
|
||||
observation.Upstream.FetchedAt);
|
||||
|
||||
foreach (var statement in observation.Statements)
|
||||
{
|
||||
var targetPurls = ResolvePurls(statement, observation.Linkset.Purls);
|
||||
foreach (var purl in targetPurls)
|
||||
{
|
||||
if (!purlOrder.ContainsKey(purl))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var key = (purl, statement.VulnerabilityId, observation.ProviderId);
|
||||
if (!aggregates.TryGetValue(key, out var aggregate))
|
||||
{
|
||||
aggregate = new OverlayAggregate(purl, statement.VulnerabilityId, observation.ProviderId);
|
||||
aggregates[key] = aggregate;
|
||||
}
|
||||
|
||||
aggregate.UpdateStatus(statement.Status, observation.CreatedAt);
|
||||
if (includeJustifications && statement.Justification is not null)
|
||||
{
|
||||
aggregate.AddJustification(statement.Justification.Value, observation.ObservationId);
|
||||
}
|
||||
|
||||
aggregate.AddObservation(observationRef);
|
||||
aggregate.AddConflicts(observation.Linkset.Disagreements);
|
||||
aggregate.SetProvenance(
|
||||
observation.StreamId ?? observation.ObservationId,
|
||||
observation.Upstream.ContentHash,
|
||||
observation.Upstream.ContentHash);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var overlays = aggregates.Values
|
||||
.OrderBy(a => purlOrder[a.Purl])
|
||||
.ThenBy(a => a.AdvisoryId, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(a => a.Source, StringComparer.OrdinalIgnoreCase)
|
||||
.Select(a => a.ToOverlayItem(tenant, generatedAt, includeJustifications))
|
||||
.ToList();
|
||||
|
||||
return overlays;
|
||||
}
|
||||
|
||||
private static bool PurlMatches(VexObservationStatement stmt, string inputPurl, ImmutableArray<string> linksetPurls)
|
||||
private static IReadOnlyList<string> ResolvePurls(VexObservationStatement stmt, ImmutableArray<string> linksetPurls)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(stmt.Purl) && stmt.Purl.Equals(inputPurl, StringComparison.OrdinalIgnoreCase))
|
||||
if (!string.IsNullOrWhiteSpace(stmt.Purl))
|
||||
{
|
||||
return true;
|
||||
return new[] { stmt.Purl };
|
||||
}
|
||||
|
||||
if (linksetPurls.IsDefaultOrEmpty)
|
||||
{
|
||||
return false;
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
return linksetPurls.Any(p => p.Equals(inputPurl, StringComparison.OrdinalIgnoreCase));
|
||||
return linksetPurls.Where(p => !string.IsNullOrWhiteSpace(p)).ToArray();
|
||||
}
|
||||
|
||||
private static string MapStatus(VexClaimStatus status)
|
||||
=> status switch
|
||||
{
|
||||
VexClaimStatus.NotAffected => "not_affected",
|
||||
VexClaimStatus.UnderInvestigation => "under_investigation",
|
||||
VexClaimStatus.Fixed => "fixed",
|
||||
_ => "affected"
|
||||
};
|
||||
|
||||
private sealed class OverlayAggregate
|
||||
{
|
||||
private readonly SortedSet<string> _observationHashes = new(StringComparer.Ordinal);
|
||||
private readonly SortedSet<string> _observationIds = new(StringComparer.Ordinal);
|
||||
private readonly List<GraphOverlayObservation> _observations = new();
|
||||
private readonly List<GraphOverlayConflict> _conflicts = new();
|
||||
private readonly List<GraphOverlayJustification> _justifications = new();
|
||||
private DateTimeOffset? _latestCreatedAt;
|
||||
private string? _status;
|
||||
private string? _linksetId;
|
||||
private string? _linksetHash;
|
||||
private string? _policyHash;
|
||||
private string? _sbomContextHash;
|
||||
|
||||
public OverlayAggregate(string purl, string advisoryId, string source)
|
||||
{
|
||||
Purl = purl;
|
||||
AdvisoryId = advisoryId;
|
||||
Source = source;
|
||||
}
|
||||
|
||||
public string Purl { get; }
|
||||
|
||||
public string AdvisoryId { get; }
|
||||
|
||||
public string Source { get; }
|
||||
|
||||
public void UpdateStatus(VexClaimStatus status, DateTimeOffset createdAt)
|
||||
{
|
||||
if (_latestCreatedAt is null || createdAt > _latestCreatedAt.Value)
|
||||
{
|
||||
_latestCreatedAt = createdAt;
|
||||
_status = MapStatus(status);
|
||||
}
|
||||
}
|
||||
|
||||
public void AddJustification(VexJustification justification, string observationId)
|
||||
{
|
||||
var kind = justification.ToString();
|
||||
if (string.IsNullOrWhiteSpace(kind))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_justifications.Add(new GraphOverlayJustification(
|
||||
kind,
|
||||
kind,
|
||||
new[] { observationId },
|
||||
null));
|
||||
}
|
||||
|
||||
public void AddObservation(GraphOverlayObservation observation)
|
||||
{
|
||||
if (_observationIds.Add(observation.Id))
|
||||
{
|
||||
_observations.Add(observation);
|
||||
}
|
||||
|
||||
_observationHashes.Add(observation.ContentHash);
|
||||
}
|
||||
|
||||
public void AddConflicts(ImmutableArray<VexObservationDisagreement> disagreements)
|
||||
{
|
||||
if (disagreements.IsDefaultOrEmpty)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var disagreement in disagreements)
|
||||
{
|
||||
_conflicts.Add(new GraphOverlayConflict(
|
||||
"status",
|
||||
disagreement.Justification ?? disagreement.Status,
|
||||
new[] { disagreement.Status },
|
||||
new[] { disagreement.ProviderId }));
|
||||
}
|
||||
}
|
||||
|
||||
public void SetProvenance(string linksetId, string linksetHash, string observationHash)
|
||||
{
|
||||
_linksetId ??= linksetId;
|
||||
_linksetHash ??= linksetHash;
|
||||
_policyHash ??= null;
|
||||
_sbomContextHash ??= null;
|
||||
_observationHashes.Add(observationHash);
|
||||
}
|
||||
|
||||
public GraphOverlayItem ToOverlayItem(string tenant, DateTimeOffset generatedAt, bool includeJustifications)
|
||||
{
|
||||
return new GraphOverlayItem(
|
||||
SchemaVersion: "1.0.0",
|
||||
GeneratedAt: generatedAt,
|
||||
Tenant: tenant,
|
||||
Purl: Purl,
|
||||
AdvisoryId: AdvisoryId,
|
||||
Source: Source,
|
||||
Status: _status ?? "unknown",
|
||||
Justifications: includeJustifications ? _justifications : Array.Empty<GraphOverlayJustification>(),
|
||||
Conflicts: _conflicts,
|
||||
Observations: _observations,
|
||||
Provenance: new GraphOverlayProvenance(
|
||||
LinksetId: _linksetId ?? string.Empty,
|
||||
LinksetHash: _linksetHash ?? string.Empty,
|
||||
ObservationHashes: _observationHashes.ToArray(),
|
||||
PolicyHash: _policyHash,
|
||||
SbomContextHash: _sbomContextHash,
|
||||
PlanCacheKey: null),
|
||||
Cache: null);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class OverlayKeyComparer : IEqualityComparer<(string Purl, string AdvisoryId, string Source)>
|
||||
{
|
||||
public bool Equals((string Purl, string AdvisoryId, string Source) x, (string Purl, string AdvisoryId, string Source) y)
|
||||
{
|
||||
return string.Equals(x.Purl, y.Purl, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(x.AdvisoryId, y.AdvisoryId, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(x.Source, y.Source, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public int GetHashCode((string Purl, string AdvisoryId, string Source) obj)
|
||||
{
|
||||
var hash = new HashCode();
|
||||
hash.Add(obj.Purl, StringComparer.OrdinalIgnoreCase);
|
||||
hash.Add(obj.AdvisoryId, StringComparer.OrdinalIgnoreCase);
|
||||
hash.Add(obj.Source, StringComparer.OrdinalIgnoreCase);
|
||||
return hash.ToHashCode();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,9 +9,16 @@ namespace StellaOps.Excititor.WebService.Graph;
|
||||
internal static class GraphStatusFactory
|
||||
{
|
||||
public static IReadOnlyList<GraphStatusItem> Build(
|
||||
string tenant,
|
||||
DateTimeOffset generatedAt,
|
||||
IReadOnlyList<string> orderedPurls,
|
||||
IReadOnlyList<VexObservation> observations)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tenant))
|
||||
{
|
||||
throw new ArgumentException("tenant is required", nameof(tenant));
|
||||
}
|
||||
|
||||
if (orderedPurls is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(orderedPurls));
|
||||
@@ -22,15 +29,74 @@ internal static class GraphStatusFactory
|
||||
throw new ArgumentNullException(nameof(observations));
|
||||
}
|
||||
|
||||
var overlays = GraphOverlayFactory.Build(orderedPurls, observations, includeJustifications: false);
|
||||
var overlays = GraphOverlayFactory.Build(tenant, generatedAt, orderedPurls, observations, includeJustifications: false);
|
||||
|
||||
return overlays
|
||||
.Select(overlay => new GraphStatusItem(
|
||||
overlay.Purl,
|
||||
overlay.Summary,
|
||||
overlay.LatestModifiedAt,
|
||||
overlay.Provenance.Sources,
|
||||
overlay.Provenance.LastEvidenceHash))
|
||||
.ToList();
|
||||
var items = new List<GraphStatusItem>(orderedPurls.Count);
|
||||
|
||||
foreach (var purl in orderedPurls)
|
||||
{
|
||||
var overlaysForPurl = overlays
|
||||
.Where(o => o.Purl.Equals(purl, StringComparison.OrdinalIgnoreCase))
|
||||
.ToList();
|
||||
|
||||
if (overlaysForPurl.Count == 0)
|
||||
{
|
||||
items.Add(new GraphStatusItem(
|
||||
purl,
|
||||
new GraphOverlaySummary(0, 0, 0, 1),
|
||||
null,
|
||||
Array.Empty<string>(),
|
||||
null));
|
||||
continue;
|
||||
}
|
||||
|
||||
var open = 0;
|
||||
var notAffected = 0;
|
||||
var underInvestigation = 0;
|
||||
var noStatement = 0;
|
||||
var sources = new SortedSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var observationRefs = new List<GraphOverlayObservation>();
|
||||
|
||||
foreach (var overlay in overlaysForPurl)
|
||||
{
|
||||
sources.Add(overlay.Source);
|
||||
observationRefs.AddRange(overlay.Observations);
|
||||
switch (overlay.Status)
|
||||
{
|
||||
case "not_affected":
|
||||
notAffected++;
|
||||
break;
|
||||
case "under_investigation":
|
||||
underInvestigation++;
|
||||
break;
|
||||
case "fixed":
|
||||
case "affected":
|
||||
open++;
|
||||
break;
|
||||
default:
|
||||
noStatement++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
var latest = observationRefs.Count == 0
|
||||
? (DateTimeOffset?)null
|
||||
: observationRefs.Max(o => o.FetchedAt);
|
||||
|
||||
var lastHash = observationRefs
|
||||
.OrderBy(o => o.FetchedAt)
|
||||
.ThenBy(o => o.Id, StringComparer.Ordinal)
|
||||
.LastOrDefault()
|
||||
?.ContentHash;
|
||||
|
||||
items.Add(new GraphStatusItem(
|
||||
purl,
|
||||
new GraphOverlaySummary(open, notAffected, underInvestigation, noStatement),
|
||||
latest,
|
||||
sources.ToArray(),
|
||||
lastHash));
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ public sealed class GraphOptions
|
||||
public int MaxPurls { get; set; } = 500;
|
||||
public int MaxAdvisoriesPerPurl { get; set; } = 200;
|
||||
public int OverlayTtlSeconds { get; set; } = 300;
|
||||
public bool UsePostgresOverlayStore { get; set; } = true;
|
||||
public int MaxTooltipItemsPerPurl { get; set; } = 50;
|
||||
public int MaxTooltipTotal { get; set; } = 1000;
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ public partial class Program
|
||||
{
|
||||
private const string TenantHeaderName = "X-Stella-Tenant";
|
||||
|
||||
private static bool TryResolveTenant(HttpContext context, VexStorageOptions options, bool requireHeader, out string tenant, out IResult? problem)
|
||||
internal static bool TryResolveTenant(HttpContext context, VexStorageOptions options, bool requireHeader, out string tenant, out IResult? problem)
|
||||
{
|
||||
tenant = options.DefaultTenant;
|
||||
problem = null;
|
||||
@@ -149,7 +149,7 @@ public partial class Program
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseSinceTimestamp(StringValues values)
|
||||
internal static DateTimeOffset? ParseSinceTimestamp(StringValues values)
|
||||
{
|
||||
if (values.Count == 0)
|
||||
{
|
||||
@@ -244,7 +244,8 @@ public partial class Program
|
||||
IReadOnlyList<GraphStatusItem> Items,
|
||||
DateTimeOffset CachedAt);
|
||||
|
||||
private sealed record CachedGraphOverlay(
|
||||
IReadOnlyList<GraphOverlayItem> Items,
|
||||
DateTimeOffset CachedAt);
|
||||
internal static string[] NormalizeValues(StringValues values) =>
|
||||
values.Where(static v => !string.IsNullOrWhiteSpace(v))
|
||||
.Select(static v => v!.Trim())
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ using StellaOps.Excititor.Attestation.Transparency;
|
||||
using StellaOps.Excititor.ArtifactStores.S3.Extensions;
|
||||
using StellaOps.Excititor.Connectors.RedHat.CSAF.DependencyInjection;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Evidence;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
using StellaOps.Excititor.Export;
|
||||
using StellaOps.Excititor.Formats.CSAF;
|
||||
@@ -28,6 +29,7 @@ using StellaOps.Excititor.Formats.CycloneDX;
|
||||
using StellaOps.Excititor.Formats.OpenVEX;
|
||||
using StellaOps.Excititor.Policy;
|
||||
using StellaOps.Excititor.Storage.Postgres;
|
||||
using StellaOps.Infrastructure.Postgres.Options;
|
||||
using StellaOps.Excititor.WebService.Endpoints;
|
||||
using StellaOps.Excititor.WebService.Extensions;
|
||||
using StellaOps.Excititor.WebService.Options;
|
||||
@@ -46,10 +48,12 @@ var services = builder.Services;
|
||||
services.AddOptions<VexStorageOptions>()
|
||||
.Bind(configuration.GetSection("Excititor:Storage"))
|
||||
.ValidateOnStart();
|
||||
services.AddOptions<GraphOptions>()
|
||||
.Bind(configuration.GetSection("Excititor:Graph"));
|
||||
|
||||
services.AddExcititorPostgresStorage(configuration);
|
||||
services.TryAddSingleton<IVexProviderStore, InMemoryVexProviderStore>();
|
||||
services.TryAddSingleton<IVexConnectorStateRepository, InMemoryVexConnectorStateRepository>();
|
||||
services.TryAddScoped<IVexConnectorStateRepository, InMemoryVexConnectorStateRepository>();
|
||||
services.TryAddSingleton<IVexClaimStore, InMemoryVexClaimStore>();
|
||||
services.AddCsafNormalizer();
|
||||
services.AddCycloneDxNormalizer();
|
||||
@@ -62,7 +66,24 @@ services.AddSingleton<AirgapSignerTrustService>();
|
||||
services.AddSingleton<AirgapModeEnforcer>();
|
||||
services.AddSingleton<ConsoleTelemetry>();
|
||||
services.AddMemoryCache();
|
||||
services.AddSingleton<IGraphOverlayCache, GraphOverlayCacheStore>();
|
||||
services.AddSingleton<IGraphOverlayStore>(sp =>
|
||||
{
|
||||
var graphOptions = sp.GetRequiredService<IOptions<GraphOptions>>().Value;
|
||||
var pgOptions = sp.GetRequiredService<IOptions<PostgresOptions>>().Value;
|
||||
if (graphOptions.UsePostgresOverlayStore && !string.IsNullOrWhiteSpace(pgOptions.ConnectionString))
|
||||
{
|
||||
return new PostgresGraphOverlayStore(
|
||||
sp.GetRequiredService<ExcititorDataSource>(),
|
||||
sp.GetRequiredService<ILogger<PostgresGraphOverlayStore>>());
|
||||
}
|
||||
|
||||
return new InMemoryGraphOverlayStore();
|
||||
});
|
||||
services.AddSingleton<IVexEvidenceLockerService, VexEvidenceLockerService>();
|
||||
services.AddSingleton<IVexEvidenceAttestor, StellaOps.Excititor.Attestation.Evidence.VexEvidenceAttestor>();
|
||||
services.AddScoped<IVexIngestOrchestrator, VexIngestOrchestrator>();
|
||||
services.AddSingleton<VexStatementBackfillService>();
|
||||
services.AddOptions<ExcititorObservabilityOptions>()
|
||||
.Bind(configuration.GetSection("Excititor:Observability"));
|
||||
services.AddScoped<ExcititorHealthService>();
|
||||
@@ -93,7 +114,7 @@ services.AddSingleton<IVexObservationProjectionService, VexObservationProjection
|
||||
services.AddScoped<IVexObservationQueryService, VexObservationQueryService>();
|
||||
|
||||
// EXCITITOR-RISK-66-001: Risk feed service for Risk Engine integration
|
||||
services.AddScoped<StellaOps.Excititor.Core.RiskFeed.IRiskFeedService, StellaOps.Excititor.Core.RiskFeed.RiskFeedService>();
|
||||
services.AddScoped<StellaOps.Excititor.Core.RiskFeed.IRiskFeedService, OverlayRiskFeedService>();
|
||||
|
||||
var rekorSection = configuration.GetSection("Excititor:Attestation:Rekor");
|
||||
if (rekorSection.Exists())
|
||||
@@ -1505,7 +1526,7 @@ app.MapGet("/v1/graph/status", async (
|
||||
return Results.BadRequest(ex.Message);
|
||||
}
|
||||
|
||||
var items = GraphStatusFactory.Build(orderedPurls, result.Observations);
|
||||
var items = GraphStatusFactory.Build(tenant!, timeProvider.GetUtcNow(), orderedPurls, result.Observations);
|
||||
var response = new GraphStatusResponse(items, false, null);
|
||||
|
||||
cache.Set(cacheKey, new CachedGraphStatus(items, now), TimeSpan.FromSeconds(graphOptions.Value.OverlayTtlSeconds));
|
||||
@@ -1521,7 +1542,8 @@ app.MapGet("/v1/graph/overlays", async (
|
||||
IOptions<VexStorageOptions> storageOptions,
|
||||
IOptions<GraphOptions> graphOptions,
|
||||
IVexObservationQueryService queryService,
|
||||
IMemoryCache cache,
|
||||
IGraphOverlayCache overlayCache,
|
||||
IGraphOverlayStore overlayStore,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
@@ -1541,13 +1563,12 @@ app.MapGet("/v1/graph/overlays", async (
|
||||
return Results.BadRequest($"purls limit exceeded (max {graphOptions.Value.MaxPurls})");
|
||||
}
|
||||
|
||||
var cacheKey = $"graph-overlays:{tenant}:{includeJustifications}:{string.Join('|', orderedPurls)}";
|
||||
var now = timeProvider.GetUtcNow();
|
||||
|
||||
if (cache.TryGetValue<CachedGraphOverlay>(cacheKey, out var cached) && cached is not null)
|
||||
var cached = await overlayCache.TryGetAsync(tenant!, includeJustifications, orderedPurls, cancellationToken).ConfigureAwait(false);
|
||||
if (cached is not null)
|
||||
{
|
||||
var ageMs = (long)Math.Max(0, (now - cached.CachedAt).TotalMilliseconds);
|
||||
return Results.Ok(new GraphOverlaysResponse(cached.Items, true, ageMs));
|
||||
return Results.Ok(new GraphOverlaysResponse(cached.Items, true, cached.AgeMilliseconds));
|
||||
}
|
||||
|
||||
var options = new VexObservationQueryOptions(
|
||||
@@ -1565,10 +1586,11 @@ app.MapGet("/v1/graph/overlays", async (
|
||||
return Results.BadRequest(ex.Message);
|
||||
}
|
||||
|
||||
var overlays = GraphOverlayFactory.Build(orderedPurls, result.Observations, includeJustifications);
|
||||
var overlays = GraphOverlayFactory.Build(tenant!, now, orderedPurls, result.Observations, includeJustifications);
|
||||
await overlayStore.SaveAsync(tenant!, overlays, cancellationToken).ConfigureAwait(false);
|
||||
var response = new GraphOverlaysResponse(overlays, false, null);
|
||||
|
||||
cache.Set(cacheKey, new CachedGraphOverlay(overlays, now), TimeSpan.FromSeconds(graphOptions.Value.OverlayTtlSeconds));
|
||||
await overlayCache.SaveAsync(tenant!, includeJustifications, orderedPurls, overlays, now, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return Results.Ok(response);
|
||||
}).WithName("GetGraphOverlays");
|
||||
@@ -1712,8 +1734,9 @@ app.MapGet("/vex/raw", async (
|
||||
var formatFilter = query.TryGetValue("format", out var formats)
|
||||
? formats
|
||||
.Where(static f => !string.IsNullOrWhiteSpace(f))
|
||||
.Select(static f => Enum.TryParse<VexDocumentFormat>(f, true, out var parsed) ? parsed : VexDocumentFormat.Unknown)
|
||||
.Where(static f => f != VexDocumentFormat.Unknown)
|
||||
.Select(static f => Enum.TryParse<VexDocumentFormat>(f, true, out var parsed) ? parsed : (VexDocumentFormat?)null)
|
||||
.Where(static f => f is not null)
|
||||
.Select(static f => f!.Value)
|
||||
.ToArray()
|
||||
: Array.Empty<VexDocumentFormat>();
|
||||
|
||||
@@ -1910,112 +1933,6 @@ app.MapGet("/v1/vex/observations/{vulnerabilityId}/{productKey}", async (
|
||||
return Results.Json(response);
|
||||
});
|
||||
|
||||
app.MapGet("/v1/vex/evidence/chunks", async (
|
||||
HttpContext context,
|
||||
[FromServices] IVexEvidenceChunkService chunkService,
|
||||
[FromServices] IOptions<VexStorageOptions> storageOptions,
|
||||
[FromServices] ChunkTelemetry chunkTelemetry,
|
||||
[FromServices] ILogger<VexEvidenceChunkRequest> logger,
|
||||
[FromServices] TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var start = Stopwatch.GetTimestamp();
|
||||
|
||||
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
chunkTelemetry.RecordIngested(null, null, "unauthorized", "missing-scope", 0, 0, 0);
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out var tenant, out var tenantError))
|
||||
{
|
||||
chunkTelemetry.RecordIngested(tenant, null, "rejected", "tenant-invalid", 0, 0, Stopwatch.GetElapsedTime(start).TotalMilliseconds);
|
||||
return tenantError;
|
||||
}
|
||||
|
||||
var vulnerabilityId = context.Request.Query["vulnerabilityId"].FirstOrDefault();
|
||||
var productKey = context.Request.Query["productKey"].FirstOrDefault();
|
||||
if (string.IsNullOrWhiteSpace(vulnerabilityId) || string.IsNullOrWhiteSpace(productKey))
|
||||
{
|
||||
return ValidationProblem("vulnerabilityId and productKey are required.");
|
||||
}
|
||||
|
||||
var providerFilter = BuildStringFilterSet(context.Request.Query["providerId"]);
|
||||
var statusFilter = BuildStatusFilter(context.Request.Query["status"]);
|
||||
var since = ParseSinceTimestamp(context.Request.Query["since"]);
|
||||
var limit = ResolveLimit(context.Request.Query["limit"], defaultValue: 200, min: 1, max: 500);
|
||||
|
||||
var request = new VexEvidenceChunkRequest(
|
||||
tenant,
|
||||
vulnerabilityId.Trim(),
|
||||
productKey.Trim(),
|
||||
providerFilter,
|
||||
statusFilter,
|
||||
since,
|
||||
limit);
|
||||
|
||||
VexEvidenceChunkResult result;
|
||||
try
|
||||
{
|
||||
result = await chunkService.QueryAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
EvidenceTelemetry.RecordChunkOutcome(tenant, "cancelled");
|
||||
chunkTelemetry.RecordIngested(tenant, providerFilter.Count > 0 ? string.Join(',', providerFilter) : null, "cancelled", null, 0, 0, Stopwatch.GetElapsedTime(start).TotalMilliseconds);
|
||||
return Results.StatusCode(StatusCodes.Status499ClientClosedRequest);
|
||||
}
|
||||
catch
|
||||
{
|
||||
EvidenceTelemetry.RecordChunkOutcome(tenant, "error");
|
||||
chunkTelemetry.RecordIngested(tenant, providerFilter.Count > 0 ? string.Join(',', providerFilter) : null, "error", null, 0, 0, Stopwatch.GetElapsedTime(start).TotalMilliseconds);
|
||||
throw;
|
||||
}
|
||||
|
||||
EvidenceTelemetry.RecordChunkOutcome(tenant, "success", result.Chunks.Count, result.Truncated);
|
||||
EvidenceTelemetry.RecordChunkSignatureStatus(tenant, result.Chunks);
|
||||
|
||||
logger.LogInformation(
|
||||
"vex_evidence_chunks_success tenant={Tenant} vulnerabilityId={Vuln} productKey={ProductKey} providers={Providers} statuses={Statuses} limit={Limit} total={Total} truncated={Truncated} returned={Returned}",
|
||||
tenant ?? "(default)",
|
||||
request.VulnerabilityId,
|
||||
request.ProductKey,
|
||||
providerFilter.Count,
|
||||
statusFilter.Count,
|
||||
request.Limit,
|
||||
result.TotalCount,
|
||||
result.Truncated,
|
||||
result.Chunks.Count);
|
||||
|
||||
// Align headers with published contract.
|
||||
context.Response.Headers["Excititor-Results-Total"] = result.TotalCount.ToString(CultureInfo.InvariantCulture);
|
||||
context.Response.Headers["Excititor-Results-Truncated"] = result.Truncated ? "true" : "false";
|
||||
context.Response.ContentType = "application/x-ndjson";
|
||||
|
||||
var options = new JsonSerializerOptions(JsonSerializerDefaults.Web);
|
||||
long payloadBytes = 0;
|
||||
foreach (var chunk in result.Chunks)
|
||||
{
|
||||
var line = JsonSerializer.Serialize(chunk, options);
|
||||
payloadBytes += Encoding.UTF8.GetByteCount(line) + 1;
|
||||
await context.Response.WriteAsync(line, cancellationToken).ConfigureAwait(false);
|
||||
await context.Response.WriteAsync("\n", cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var elapsedMs = Stopwatch.GetElapsedTime(start).TotalMilliseconds;
|
||||
chunkTelemetry.RecordIngested(
|
||||
tenant,
|
||||
providerFilter.Count > 0 ? string.Join(',', providerFilter) : null,
|
||||
"success",
|
||||
null,
|
||||
result.TotalCount,
|
||||
payloadBytes,
|
||||
elapsedMs);
|
||||
|
||||
return Results.Empty;
|
||||
});
|
||||
|
||||
app.MapPost("/aoc/verify", async (
|
||||
HttpContext context,
|
||||
VexAocVerifyRequest? request,
|
||||
@@ -2060,10 +1977,10 @@ app.MapPost("/aoc/verify", async (
|
||||
sources ?? Array.Empty<string>(),
|
||||
Array.Empty<string>(),
|
||||
Array.Empty<VexDocumentFormat>(),
|
||||
since: new DateTimeOffset(since, TimeSpan.Zero),
|
||||
until: new DateTimeOffset(until, TimeSpan.Zero),
|
||||
cursor: null,
|
||||
limit),
|
||||
Since: new DateTimeOffset(since, TimeSpan.Zero),
|
||||
Until: new DateTimeOffset(until, TimeSpan.Zero),
|
||||
Cursor: null,
|
||||
Limit: limit),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var checkedCount = 0;
|
||||
|
||||
@@ -279,7 +279,7 @@ internal sealed class ExcititorHealthService
|
||||
Array.Empty<string>(),
|
||||
Array.Empty<VexDocumentFormat>(),
|
||||
windowStart,
|
||||
until: null,
|
||||
Until: null,
|
||||
Cursor: null,
|
||||
Limit: 500),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
@@ -360,13 +360,13 @@ internal sealed class ExcititorHealthService
|
||||
|
||||
foreach (var linkset in linksets)
|
||||
{
|
||||
if (linkset.Disagreements.Count == 0)
|
||||
if (linkset.Disagreements.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
docsWithConflicts++;
|
||||
totalConflicts += linkset.Disagreements.Count;
|
||||
totalConflicts += linkset.Disagreements.Length;
|
||||
|
||||
foreach (var disagreement in linkset.Disagreements)
|
||||
{
|
||||
@@ -381,8 +381,8 @@ internal sealed class ExcititorHealthService
|
||||
|
||||
var alignedTicks = AlignTicks(linkset.UpdatedAt.UtcDateTime, bucketTicks);
|
||||
timeline[alignedTicks] = timeline.TryGetValue(alignedTicks, out var currentCount)
|
||||
? currentCount + linkset.Disagreements.Count
|
||||
: linkset.Disagreements.Count;
|
||||
? currentCount + linkset.Disagreements.Length
|
||||
: linkset.Disagreements.Length;
|
||||
}
|
||||
|
||||
var trend = timeline
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
using StellaOps.Excititor.WebService.Options;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Services;
|
||||
|
||||
public interface IGraphOverlayCache
|
||||
{
|
||||
ValueTask<GraphOverlayCacheHit?> TryGetAsync(string tenant, bool includeJustifications, IReadOnlyList<string> orderedPurls, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask SaveAsync(string tenant, bool includeJustifications, IReadOnlyList<string> orderedPurls, IReadOnlyList<GraphOverlayItem> items, DateTimeOffset cachedAt, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed record GraphOverlayCacheHit(IReadOnlyList<GraphOverlayItem> Items, long AgeMilliseconds);
|
||||
|
||||
internal sealed class GraphOverlayCacheStore : IGraphOverlayCache
|
||||
{
|
||||
private readonly IMemoryCache _memoryCache;
|
||||
private readonly IOptions<GraphOptions> _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public GraphOverlayCacheStore(IMemoryCache memoryCache, IOptions<GraphOptions> options, TimeProvider timeProvider)
|
||||
{
|
||||
_memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache));
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
}
|
||||
|
||||
public ValueTask<GraphOverlayCacheHit?> TryGetAsync(string tenant, bool includeJustifications, IReadOnlyList<string> orderedPurls, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var key = BuildKey(tenant, includeJustifications, orderedPurls);
|
||||
if (_memoryCache.TryGetValue<CachedOverlay>(key, out var cached) && cached is not null)
|
||||
{
|
||||
var ageMs = (long)Math.Max(0, (_timeProvider.GetUtcNow() - cached.CachedAt).TotalMilliseconds);
|
||||
return ValueTask.FromResult<GraphOverlayCacheHit?>(new GraphOverlayCacheHit(cached.Items, ageMs));
|
||||
}
|
||||
|
||||
return ValueTask.FromResult<GraphOverlayCacheHit?>(null);
|
||||
}
|
||||
|
||||
public ValueTask SaveAsync(string tenant, bool includeJustifications, IReadOnlyList<string> orderedPurls, IReadOnlyList<GraphOverlayItem> items, DateTimeOffset cachedAt, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var key = BuildKey(tenant, includeJustifications, orderedPurls);
|
||||
var ttl = TimeSpan.FromSeconds(Math.Max(1, _options.Value.OverlayTtlSeconds));
|
||||
_memoryCache.Set(key, new CachedOverlay(items, cachedAt), ttl);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
private static string BuildKey(string tenant, bool includeJustifications, IReadOnlyList<string> orderedPurls)
|
||||
=> $"graph-overlays:{tenant}:{includeJustifications}:{string.Join('|', orderedPurls)}";
|
||||
|
||||
private sealed record CachedOverlay(IReadOnlyList<GraphOverlayItem> Items, DateTimeOffset CachedAt);
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Services;
|
||||
|
||||
public interface IGraphOverlayStore
|
||||
{
|
||||
ValueTask SaveAsync(string tenant, IReadOnlyList<GraphOverlayItem> overlays, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask<IReadOnlyList<GraphOverlayItem>> FindByPurlsAsync(string tenant, IReadOnlyCollection<string> purls, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask<IReadOnlyList<GraphOverlayItem>> FindByAdvisoriesAsync(string tenant, IReadOnlyCollection<string> advisories, int limit, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask<IReadOnlyList<GraphOverlayItem>> FindWithConflictsAsync(string tenant, int limit, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// In-memory overlay store placeholder until Postgres materialization is added.
|
||||
/// </summary>
|
||||
public sealed class InMemoryGraphOverlayStore : IGraphOverlayStore
|
||||
{
|
||||
private readonly Dictionary<string, Dictionary<string, List<GraphOverlayItem>>> _store = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly object _lock = new();
|
||||
|
||||
public ValueTask SaveAsync(string tenant, IReadOnlyList<GraphOverlayItem> overlays, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
lock (_lock)
|
||||
{
|
||||
if (!_store.TryGetValue(tenant, out var byPurl))
|
||||
{
|
||||
byPurl = new Dictionary<string, List<GraphOverlayItem>>(StringComparer.OrdinalIgnoreCase);
|
||||
_store[tenant] = byPurl;
|
||||
}
|
||||
|
||||
foreach (var overlay in overlays)
|
||||
{
|
||||
if (!byPurl.TryGetValue(overlay.Purl, out var list))
|
||||
{
|
||||
list = new List<GraphOverlayItem>();
|
||||
byPurl[overlay.Purl] = list;
|
||||
}
|
||||
|
||||
// replace existing advisory/source entry for deterministic latest overlay
|
||||
var existingIndex = list.FindIndex(o =>
|
||||
string.Equals(o.AdvisoryId, overlay.AdvisoryId, StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(o.Source, overlay.Source, StringComparison.OrdinalIgnoreCase));
|
||||
if (existingIndex >= 0)
|
||||
{
|
||||
list[existingIndex] = overlay;
|
||||
}
|
||||
else
|
||||
{
|
||||
list.Add(overlay);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask<IReadOnlyList<GraphOverlayItem>> FindByPurlsAsync(string tenant, IReadOnlyCollection<string> purls, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
if (purls.Count == 0)
|
||||
{
|
||||
return ValueTask.FromResult<IReadOnlyList<GraphOverlayItem>>(Array.Empty<GraphOverlayItem>());
|
||||
}
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
if (!_store.TryGetValue(tenant, out var byPurl))
|
||||
{
|
||||
return ValueTask.FromResult<IReadOnlyList<GraphOverlayItem>>(Array.Empty<GraphOverlayItem>());
|
||||
}
|
||||
|
||||
var ordered = new List<GraphOverlayItem>();
|
||||
foreach (var purl in purls)
|
||||
{
|
||||
if (byPurl.TryGetValue(purl, out var list))
|
||||
{
|
||||
// Order overlays deterministically by advisory + source for stable outputs
|
||||
ordered.AddRange(list
|
||||
.OrderBy(o => o.AdvisoryId, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(o => o.Source, StringComparer.OrdinalIgnoreCase));
|
||||
}
|
||||
}
|
||||
|
||||
return ValueTask.FromResult<IReadOnlyList<GraphOverlayItem>>(ordered);
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask<IReadOnlyList<GraphOverlayItem>> FindByAdvisoriesAsync(string tenant, IReadOnlyCollection<string> advisories, int limit, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
if (advisories.Count == 0)
|
||||
{
|
||||
return ValueTask.FromResult<IReadOnlyList<GraphOverlayItem>>(Array.Empty<GraphOverlayItem>());
|
||||
}
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
if (!_store.TryGetValue(tenant, out var byPurl))
|
||||
{
|
||||
return ValueTask.FromResult<IReadOnlyList<GraphOverlayItem>>(Array.Empty<GraphOverlayItem>());
|
||||
}
|
||||
|
||||
var results = new List<GraphOverlayItem>();
|
||||
foreach (var kvp in byPurl)
|
||||
{
|
||||
foreach (var overlay in kvp.Value)
|
||||
{
|
||||
if (advisories.Contains(overlay.AdvisoryId, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
results.Add(overlay);
|
||||
if (results.Count >= limit)
|
||||
{
|
||||
return ValueTask.FromResult<IReadOnlyList<GraphOverlayItem>>(results);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ValueTask.FromResult<IReadOnlyList<GraphOverlayItem>>(results
|
||||
.OrderBy(o => o.AdvisoryId, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(o => o.Purl, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(o => o.Source, StringComparer.OrdinalIgnoreCase)
|
||||
.Take(limit)
|
||||
.ToList());
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask<IReadOnlyList<GraphOverlayItem>> FindWithConflictsAsync(string tenant, int limit, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
lock (_lock)
|
||||
{
|
||||
if (!_store.TryGetValue(tenant, out var byPurl))
|
||||
{
|
||||
return ValueTask.FromResult<IReadOnlyList<GraphOverlayItem>>(Array.Empty<GraphOverlayItem>());
|
||||
}
|
||||
|
||||
var results = byPurl.Values
|
||||
.SelectMany(list => list)
|
||||
.Where(o => o.Conflicts.Count > 0)
|
||||
.OrderBy(o => o.Purl, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(o => o.AdvisoryId, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(o => o.Source, StringComparer.OrdinalIgnoreCase)
|
||||
.Take(limit)
|
||||
.ToList();
|
||||
|
||||
return ValueTask.FromResult<IReadOnlyList<GraphOverlayItem>>(results);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.RiskFeed;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Risk feed service backed by graph overlays (EXCITITOR-RISK-66-001).
|
||||
/// </summary>
|
||||
public sealed class OverlayRiskFeedService : IRiskFeedService
|
||||
{
|
||||
private readonly IGraphOverlayStore _overlayStore;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public OverlayRiskFeedService(IGraphOverlayStore overlayStore, TimeProvider timeProvider)
|
||||
{
|
||||
_overlayStore = overlayStore ?? throw new ArgumentNullException(nameof(overlayStore));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
}
|
||||
|
||||
public async Task<RiskFeedResponse> GenerateFeedAsync(RiskFeedRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
var overlays = await ResolveOverlaysAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
var filtered = ApplySinceFilter(overlays, request.Since);
|
||||
|
||||
var items = filtered
|
||||
.Select(MapToRiskFeedItem)
|
||||
.Where(item => item is not null)
|
||||
.Cast<RiskFeedItem>()
|
||||
.OrderBy(item => item.AdvisoryKey, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(item => item.Artifact, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(item => item.Provenance.TenantId, StringComparer.OrdinalIgnoreCase)
|
||||
.Take(request.Limit)
|
||||
.ToImmutableArray();
|
||||
|
||||
return new RiskFeedResponse(items, _timeProvider.GetUtcNow());
|
||||
}
|
||||
|
||||
public async Task<RiskFeedItem?> GetItemAsync(string tenantId, string advisoryKey, string artifact, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(advisoryKey);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(artifact);
|
||||
|
||||
var overlays = await _overlayStore
|
||||
.FindByPurlsAsync(tenantId, new[] { artifact }, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var match = overlays
|
||||
.Where(o => string.Equals(o.AdvisoryId, advisoryKey, StringComparison.OrdinalIgnoreCase))
|
||||
.OrderBy(o => o.Source, StringComparer.OrdinalIgnoreCase)
|
||||
.FirstOrDefault();
|
||||
|
||||
return match is null ? null : MapToRiskFeedItem(match);
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<GraphOverlayItem>> ResolveOverlaysAsync(RiskFeedRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!request.AdvisoryKeys.IsDefaultOrEmpty)
|
||||
{
|
||||
return await _overlayStore
|
||||
.FindByAdvisoriesAsync(request.TenantId, request.AdvisoryKeys, request.Limit, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (!request.Artifacts.IsDefaultOrEmpty)
|
||||
{
|
||||
return await _overlayStore
|
||||
.FindByPurlsAsync(request.TenantId, request.Artifacts, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return await _overlayStore
|
||||
.FindWithConflictsAsync(request.TenantId, request.Limit, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static IEnumerable<GraphOverlayItem> ApplySinceFilter(IEnumerable<GraphOverlayItem> overlays, DateTimeOffset? since)
|
||||
{
|
||||
if (since is null)
|
||||
{
|
||||
return overlays;
|
||||
}
|
||||
|
||||
var threshold = since.Value;
|
||||
return overlays.Where(o => o.GeneratedAt >= threshold);
|
||||
}
|
||||
|
||||
private static RiskFeedItem? MapToRiskFeedItem(GraphOverlayItem overlay)
|
||||
{
|
||||
if (!TryParseStatus(overlay.Status, out var status))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var justification = ParseJustification(overlay.Justifications.FirstOrDefault()?.Kind);
|
||||
var confidence = DeriveConfidence(overlay);
|
||||
var provenance = new RiskFeedProvenance(
|
||||
overlay.Tenant,
|
||||
overlay.Provenance.LinksetId,
|
||||
overlay.Provenance.LinksetHash,
|
||||
confidence,
|
||||
overlay.Conflicts.Count > 0,
|
||||
overlay.GeneratedAt);
|
||||
|
||||
var observedAt = overlay.Observations.Count == 0
|
||||
? overlay.GeneratedAt
|
||||
: overlay.Observations.Max(o => o.FetchedAt);
|
||||
|
||||
var sources = overlay.Observations
|
||||
.OrderBy(o => o.FetchedAt)
|
||||
.Select(o => new RiskFeedObservationSource(
|
||||
o.Id,
|
||||
overlay.Source,
|
||||
overlay.Status,
|
||||
overlay.Justifications.FirstOrDefault()?.Kind,
|
||||
null))
|
||||
.ToImmutableArray();
|
||||
|
||||
return new RiskFeedItem(
|
||||
overlay.AdvisoryId,
|
||||
overlay.Purl,
|
||||
status,
|
||||
justification,
|
||||
provenance,
|
||||
observedAt,
|
||||
sources);
|
||||
}
|
||||
|
||||
private static bool TryParseStatus(string status, out VexClaimStatus parsed)
|
||||
{
|
||||
parsed = status.ToLowerInvariant() switch
|
||||
{
|
||||
"not_affected" => VexClaimStatus.NotAffected,
|
||||
"under_investigation" => VexClaimStatus.UnderInvestigation,
|
||||
"fixed" => VexClaimStatus.Fixed,
|
||||
"affected" => VexClaimStatus.Affected,
|
||||
_ => VexClaimStatus.UnderInvestigation
|
||||
};
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private static VexJustification? ParseJustification(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return Enum.TryParse<VexJustification>(value, true, out var justification) ? justification : null;
|
||||
}
|
||||
|
||||
private static VexLinksetConfidence DeriveConfidence(GraphOverlayItem overlay)
|
||||
{
|
||||
if (overlay.Conflicts.Count > 0)
|
||||
{
|
||||
return VexLinksetConfidence.Low;
|
||||
}
|
||||
|
||||
return overlay.Observations.Count > 1
|
||||
? VexLinksetConfidence.High
|
||||
: VexLinksetConfidence.Medium;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
using NpgsqlTypes;
|
||||
using StellaOps.Excititor.Storage.Postgres;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Postgres-backed overlay materialization store. Persists overlays per tenant/purl/advisory/source.
|
||||
/// </summary>
|
||||
public sealed class PostgresGraphOverlayStore : IGraphOverlayStore
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new()
|
||||
{
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
||||
};
|
||||
|
||||
private readonly ExcititorDataSource _dataSource;
|
||||
private readonly ILogger<PostgresGraphOverlayStore> _logger;
|
||||
private volatile bool _initialized;
|
||||
private readonly SemaphoreSlim _initLock = new(1, 1);
|
||||
|
||||
public PostgresGraphOverlayStore(ExcititorDataSource dataSource, ILogger<PostgresGraphOverlayStore> logger)
|
||||
{
|
||||
_dataSource = dataSource ?? throw new ArgumentNullException(nameof(dataSource));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async ValueTask SaveAsync(string tenant, IReadOnlyList<GraphOverlayItem> overlays, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(tenant);
|
||||
ArgumentNullException.ThrowIfNull(overlays);
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await using var connection = await _dataSource.OpenConnectionAsync("public", "writer", cancellationToken).ConfigureAwait(false);
|
||||
const string sql = """
|
||||
INSERT INTO vex.graph_overlays (tenant, purl, advisory_id, source, generated_at, payload)
|
||||
VALUES (@tenant, @purl, @advisory_id, @source, @generated_at, @payload)
|
||||
ON CONFLICT (tenant, purl, advisory_id, source)
|
||||
DO UPDATE SET generated_at = EXCLUDED.generated_at, payload = EXCLUDED.payload;
|
||||
""";
|
||||
|
||||
foreach (var overlay in overlays)
|
||||
{
|
||||
await using var command = new NpgsqlCommand(sql, connection)
|
||||
{
|
||||
CommandTimeout = _dataSource.CommandTimeoutSeconds
|
||||
};
|
||||
|
||||
command.Parameters.AddWithValue("tenant", tenant);
|
||||
command.Parameters.AddWithValue("purl", overlay.Purl);
|
||||
command.Parameters.AddWithValue("advisory_id", overlay.AdvisoryId);
|
||||
command.Parameters.AddWithValue("source", overlay.Source);
|
||||
command.Parameters.AddWithValue("generated_at", overlay.GeneratedAt.UtcDateTime);
|
||||
command.Parameters.Add(new NpgsqlParameter("payload", NpgsqlDbType.Jsonb)
|
||||
{
|
||||
Value = JsonSerializer.Serialize(overlay, SerializerOptions)
|
||||
});
|
||||
|
||||
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<GraphOverlayItem>> FindByPurlsAsync(string tenant, IReadOnlyCollection<string> purls, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(tenant);
|
||||
ArgumentNullException.ThrowIfNull(purls);
|
||||
if (purls.Count == 0)
|
||||
{
|
||||
return Array.Empty<GraphOverlayItem>();
|
||||
}
|
||||
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var connection = await _dataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
|
||||
|
||||
const string sql = """
|
||||
SELECT payload
|
||||
FROM vex.graph_overlays
|
||||
WHERE tenant = @tenant AND purl = ANY(@purls)
|
||||
ORDER BY purl, advisory_id, source;
|
||||
""";
|
||||
|
||||
await using var command = new NpgsqlCommand(sql, connection)
|
||||
{
|
||||
CommandTimeout = _dataSource.CommandTimeoutSeconds
|
||||
};
|
||||
|
||||
command.Parameters.AddWithValue("tenant", tenant);
|
||||
command.Parameters.Add(new NpgsqlParameter<string[]>("purls", NpgsqlDbType.Array | NpgsqlDbType.Text)
|
||||
{
|
||||
TypedValue = purls.ToArray()
|
||||
});
|
||||
|
||||
var overlays = new List<GraphOverlayItem>();
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var payload = reader.GetString(0);
|
||||
var overlay = JsonSerializer.Deserialize<GraphOverlayItem>(payload, SerializerOptions);
|
||||
if (overlay is not null)
|
||||
{
|
||||
overlays.Add(overlay);
|
||||
}
|
||||
}
|
||||
|
||||
return overlays;
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<GraphOverlayItem>> FindByAdvisoriesAsync(string tenant, IReadOnlyCollection<string> advisories, int limit, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(tenant);
|
||||
ArgumentNullException.ThrowIfNull(advisories);
|
||||
if (advisories.Count == 0)
|
||||
{
|
||||
return Array.Empty<GraphOverlayItem>();
|
||||
}
|
||||
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var connection = await _dataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
|
||||
|
||||
const string sql = """
|
||||
SELECT payload
|
||||
FROM vex.graph_overlays
|
||||
WHERE tenant = @tenant AND advisory_id = ANY(@advisories)
|
||||
ORDER BY advisory_id, purl, source
|
||||
LIMIT @limit;
|
||||
""";
|
||||
|
||||
await using var command = new NpgsqlCommand(sql, connection)
|
||||
{
|
||||
CommandTimeout = _dataSource.CommandTimeoutSeconds
|
||||
};
|
||||
|
||||
command.Parameters.AddWithValue("tenant", tenant);
|
||||
command.Parameters.Add(new NpgsqlParameter<string[]>("advisories", NpgsqlDbType.Array | NpgsqlDbType.Text)
|
||||
{
|
||||
TypedValue = advisories.ToArray()
|
||||
});
|
||||
command.Parameters.AddWithValue("limit", limit);
|
||||
|
||||
var overlays = new List<GraphOverlayItem>();
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var payload = reader.GetString(0);
|
||||
var overlay = JsonSerializer.Deserialize<GraphOverlayItem>(payload, SerializerOptions);
|
||||
if (overlay is not null)
|
||||
{
|
||||
overlays.Add(overlay);
|
||||
}
|
||||
}
|
||||
|
||||
return overlays;
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<GraphOverlayItem>> FindWithConflictsAsync(string tenant, int limit, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(tenant);
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await using var connection = await _dataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
|
||||
const string sql = """
|
||||
SELECT payload
|
||||
FROM vex.graph_overlays
|
||||
WHERE tenant = @tenant
|
||||
AND jsonb_array_length(payload -> 'conflicts') > 0
|
||||
ORDER BY generated_at DESC, purl, advisory_id, source
|
||||
LIMIT @limit;
|
||||
""";
|
||||
|
||||
await using var command = new NpgsqlCommand(sql, connection)
|
||||
{
|
||||
CommandTimeout = _dataSource.CommandTimeoutSeconds
|
||||
};
|
||||
command.Parameters.AddWithValue("tenant", tenant);
|
||||
command.Parameters.AddWithValue("limit", limit);
|
||||
|
||||
var overlays = new List<GraphOverlayItem>();
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var payload = reader.GetString(0);
|
||||
var overlay = JsonSerializer.Deserialize<GraphOverlayItem>(payload, SerializerOptions);
|
||||
if (overlay is not null)
|
||||
{
|
||||
overlays.Add(overlay);
|
||||
}
|
||||
}
|
||||
|
||||
return overlays;
|
||||
}
|
||||
|
||||
private async ValueTask EnsureTableAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_initialized)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _initLock.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_initialized)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
const string sql = """
|
||||
CREATE TABLE IF NOT EXISTS vex.graph_overlays (
|
||||
tenant text NOT NULL,
|
||||
purl text NOT NULL,
|
||||
advisory_id text NOT NULL,
|
||||
source text NOT NULL,
|
||||
generated_at timestamptz NOT NULL,
|
||||
payload jsonb NOT NULL,
|
||||
CONSTRAINT pk_graph_overlays PRIMARY KEY (tenant, purl, advisory_id, source)
|
||||
);
|
||||
""";
|
||||
|
||||
await using var command = new NpgsqlCommand(sql, connection)
|
||||
{
|
||||
CommandTimeout = _dataSource.CommandTimeoutSeconds
|
||||
};
|
||||
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_initialized = true;
|
||||
}
|
||||
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to ensure graph_overlays table exists.");
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_initLock.Release();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Services;
|
||||
|
||||
public sealed record VexStatementBackfillRequest(int BatchSize = 500);
|
||||
|
||||
public sealed record VexStatementBackfillResult(
|
||||
int DocumentsEvaluated,
|
||||
int DocumentsBackfilled,
|
||||
int ClaimsWritten,
|
||||
int SkippedExisting,
|
||||
int NormalizationFailures);
|
||||
|
||||
/// <summary>
|
||||
/// Placeholder backfill service while legacy statement storage is removed.
|
||||
/// </summary>
|
||||
public sealed class VexStatementBackfillService
|
||||
{
|
||||
private readonly ILogger<VexStatementBackfillService> _logger;
|
||||
|
||||
public VexStatementBackfillService(ILogger<VexStatementBackfillService> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public ValueTask<VexStatementBackfillResult> RunAsync(VexStatementBackfillRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
_logger.LogInformation("Vex statement backfill is currently a no-op; batchSize={BatchSize}", request.BatchSize);
|
||||
return ValueTask.FromResult(new VexStatementBackfillResult(0, 0, 0, 0, 0));
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
@@ -50,7 +51,7 @@ services.AddOptions<VexStorageOptions>()
|
||||
|
||||
services.AddExcititorPostgresStorage(configuration);
|
||||
services.AddSingleton<IVexProviderStore, InMemoryVexProviderStore>();
|
||||
services.AddSingleton<IVexConnectorStateRepository, InMemoryVexConnectorStateRepository>();
|
||||
services.TryAddScoped<IVexConnectorStateRepository, InMemoryVexConnectorStateRepository>();
|
||||
services.AddSingleton<IVexClaimStore, InMemoryVexClaimStore>();
|
||||
services.AddCsafNormalizer();
|
||||
services.AddCycloneDxNormalizer();
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
using System.Collections.Immutable;
|
||||
|
||||
namespace StellaOps.Excititor.Core.Storage;
|
||||
|
||||
public sealed class DuplicateAirgapImportException : Exception
|
||||
{
|
||||
public DuplicateAirgapImportException(string message)
|
||||
: base(message)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Timeline entry for an imported airgap bundle.
|
||||
/// </summary>
|
||||
public sealed record AirgapTimelineEntry
|
||||
{
|
||||
public string EventType { get; init; } = string.Empty;
|
||||
|
||||
public DateTimeOffset CreatedAt { get; init; }
|
||||
|
||||
public string TenantId { get; init; } = "default";
|
||||
|
||||
public string BundleId { get; init; } = string.Empty;
|
||||
|
||||
public string MirrorGeneration { get; init; } = string.Empty;
|
||||
|
||||
public int? StalenessSeconds { get; init; }
|
||||
|
||||
public string? ErrorCode { get; init; }
|
||||
|
||||
public string? Message { get; init; }
|
||||
|
||||
public string? Remediation { get; init; }
|
||||
|
||||
public string? Actor { get; init; }
|
||||
|
||||
public string? Scopes { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Persisted airgap import record describing a mirror bundle and associated metadata.
|
||||
/// </summary>
|
||||
public sealed record AirgapImportRecord
|
||||
{
|
||||
public string Id { get; init; } = string.Empty;
|
||||
|
||||
public string TenantId { get; init; } = "default";
|
||||
|
||||
public string BundleId { get; init; } = string.Empty;
|
||||
|
||||
public string MirrorGeneration { get; init; } = "0";
|
||||
|
||||
public string Publisher { get; init; } = string.Empty;
|
||||
|
||||
public DateTimeOffset SignedAt { get; init; }
|
||||
|
||||
public DateTimeOffset ImportedAt { get; init; }
|
||||
|
||||
public string PayloadHash { get; init; } = string.Empty;
|
||||
|
||||
public string? PayloadUrl { get; init; }
|
||||
|
||||
public string Signature { get; init; } = string.Empty;
|
||||
|
||||
public string? TransparencyLog { get; init; }
|
||||
|
||||
public string? PortableManifestPath { get; init; }
|
||||
|
||||
public string? PortableManifestHash { get; init; }
|
||||
|
||||
public string? EvidenceLockerPath { get; init; }
|
||||
|
||||
public IReadOnlyList<AirgapTimelineEntry> Timeline { get; init; } = Array.Empty<AirgapTimelineEntry>();
|
||||
|
||||
public string? ImportActor { get; init; }
|
||||
|
||||
public string? ImportScopes { get; init; }
|
||||
}
|
||||
|
||||
public interface IAirgapImportStore
|
||||
{
|
||||
Task SaveAsync(AirgapImportRecord record, CancellationToken cancellationToken);
|
||||
|
||||
Task<AirgapImportRecord?> FindByBundleIdAsync(string tenantId, string bundleId, string? mirrorGeneration, CancellationToken cancellationToken);
|
||||
|
||||
Task<IReadOnlyList<AirgapImportRecord>> ListAsync(string tenantId, string? publisherFilter, DateTimeOffset? importedAfter, int limit, int offset, CancellationToken cancellationToken);
|
||||
|
||||
Task<int> CountAsync(string tenantId, string? publisherFilter, DateTimeOffset? importedAfter, CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -11,16 +11,24 @@ public sealed record VexConnectorState(
|
||||
string ConnectorId,
|
||||
DateTimeOffset? LastUpdated,
|
||||
ImmutableArray<string> DocumentDigests,
|
||||
ImmutableDictionary<string, string> ResumeTokens = default,
|
||||
ImmutableDictionary<string, string>? ResumeTokens = null,
|
||||
DateTimeOffset? LastSuccessAt = null,
|
||||
int FailureCount = 0,
|
||||
DateTimeOffset? NextEligibleRun = null,
|
||||
string? LastFailureReason = null,
|
||||
DateTimeOffset? LastCheckpoint = null)
|
||||
DateTimeOffset? LastCheckpoint = null,
|
||||
DateTimeOffset? LastHeartbeatAt = null,
|
||||
string? LastHeartbeatStatus = null,
|
||||
string? LastArtifactHash = null,
|
||||
string? LastArtifactKind = null)
|
||||
{
|
||||
public ImmutableDictionary<string, string> ResumeTokens { get; init; } = ResumeTokens.IsDefault
|
||||
? ImmutableDictionary<string, string>.Empty
|
||||
: ResumeTokens;
|
||||
public ImmutableArray<string> DocumentDigests { get; init; } =
|
||||
DocumentDigests.IsDefault ? ImmutableArray<string>.Empty : DocumentDigests;
|
||||
|
||||
public ImmutableDictionary<string, string> ResumeTokens { get; init; } =
|
||||
ResumeTokens is null || ResumeTokens.Count == 0
|
||||
? ImmutableDictionary<string, string>.Empty
|
||||
: ResumeTokens;
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -212,7 +212,7 @@ public sealed class InMemoryVexRawStore : IVexRawStore
|
||||
private static byte[] CanonicalizeJson(ReadOnlyMemory<byte> content)
|
||||
{
|
||||
using var jsonDocument = JsonDocument.Parse(content);
|
||||
using var buffer = new ArrayBufferWriter<byte>();
|
||||
var buffer = new ArrayBufferWriter<byte>();
|
||||
using (var writer = new Utf8JsonWriter(buffer, new JsonWriterOptions { Indented = false }))
|
||||
{
|
||||
WriteCanonical(writer, jsonDocument.RootElement);
|
||||
@@ -396,7 +396,7 @@ public sealed class InMemoryAppendOnlyLinksetStore : IAppendOnlyLinksetStore, IV
|
||||
tenant,
|
||||
vulnerabilityId,
|
||||
productKey,
|
||||
new VexProductScope(productKey, null, null, productKey, null, Array.Empty<string>()),
|
||||
new VexProductScope(productKey, "unknown", null, productKey, null, ImmutableArray<string>.Empty),
|
||||
Enumerable.Empty<VexLinksetObservationRefModel>(),
|
||||
Enumerable.Empty<VexObservationDisagreement>(),
|
||||
DateTimeOffset.UtcNow,
|
||||
@@ -554,7 +554,7 @@ public sealed class InMemoryAppendOnlyLinksetStore : IAppendOnlyLinksetStore, IV
|
||||
return ValueTask.FromResult(existing);
|
||||
}
|
||||
|
||||
var scope = new VexProductScope(productKey, null, null, productKey, null, Array.Empty<string>());
|
||||
var scope = new VexProductScope(productKey, "unknown", null, productKey, null, ImmutableArray<string>.Empty);
|
||||
var linkset = new VexLinkset(linksetId, tenant, vulnerabilityId, productKey, scope, Enumerable.Empty<VexLinksetObservationRefModel>());
|
||||
_linksets[key] = linkset;
|
||||
AddMutation(key, LinksetMutationEvent.MutationTypes.LinksetCreated, null, null, null, null);
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
namespace StellaOps.Excititor.Core.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Persistence abstraction for resolved VEX consensus documents.
|
||||
/// </summary>
|
||||
public interface IVexConsensusStore
|
||||
{
|
||||
ValueTask SaveAsync(VexConsensus consensus, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask<VexConsensus?> FindAsync(string vulnerabilityId, string productKey, CancellationToken cancellationToken);
|
||||
|
||||
IAsyncEnumerable<VexConsensus> FindCalculatedBeforeAsync(DateTimeOffset cutoff, int limit, CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
using StellaOps.Excititor.Core;
|
||||
|
||||
namespace StellaOps.Excititor.Export;
|
||||
|
||||
/// <summary>
|
||||
/// Persisted manifest store for export runs keyed by query signature and format.
|
||||
/// </summary>
|
||||
public interface IVexExportStore
|
||||
{
|
||||
ValueTask<VexExportManifest?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask SaveAsync(VexExportManifest manifest, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cache index used to track export cache entries by signature and format.
|
||||
/// </summary>
|
||||
public interface IVexCacheIndex
|
||||
{
|
||||
ValueTask<VexCacheEntry?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Maintenance operations for keeping the export cache consistent.
|
||||
/// </summary>
|
||||
public interface IVexCacheMaintenance
|
||||
{
|
||||
ValueTask<int> RemoveExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask<int> RemoveMissingManifestReferencesAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
using NpgsqlTypes;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
using StellaOps.Infrastructure.Postgres.Repositories;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Postgres.Repositories;
|
||||
|
||||
/// <summary>
|
||||
/// PostgreSQL-backed connector state repository for orchestrator checkpoints and heartbeats.
|
||||
/// </summary>
|
||||
public sealed class PostgresConnectorStateRepository : RepositoryBase<ExcititorDataSource>, IVexConnectorStateRepository
|
||||
{
|
||||
private volatile bool _initialized;
|
||||
private readonly SemaphoreSlim _initLock = new(1, 1);
|
||||
|
||||
public PostgresConnectorStateRepository(ExcititorDataSource dataSource, ILogger<PostgresConnectorStateRepository> logger)
|
||||
: base(dataSource, logger)
|
||||
{
|
||||
}
|
||||
|
||||
public async ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(connectorId);
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await using var connection = await DataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
|
||||
const string sql = """
|
||||
SELECT connector_id, last_updated, document_digests, resume_tokens, last_success_at, failure_count,
|
||||
next_eligible_run, last_failure_reason, last_checkpoint, last_heartbeat_at, last_heartbeat_status,
|
||||
last_artifact_hash, last_artifact_kind
|
||||
FROM vex.connector_states
|
||||
WHERE connector_id = @connector_id;
|
||||
""";
|
||||
|
||||
await using var command = CreateCommand(sql, connection);
|
||||
AddParameter(command, "connector_id", connectorId);
|
||||
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return Map(reader);
|
||||
}
|
||||
|
||||
public async ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(state);
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var lastUpdated = state.LastUpdated ?? DateTimeOffset.UtcNow;
|
||||
|
||||
await using var connection = await DataSource.OpenConnectionAsync("public", "writer", cancellationToken).ConfigureAwait(false);
|
||||
const string sql = """
|
||||
INSERT INTO vex.connector_states (
|
||||
connector_id, last_updated, document_digests, resume_tokens, last_success_at, failure_count,
|
||||
next_eligible_run, last_failure_reason, last_checkpoint, last_heartbeat_at, last_heartbeat_status,
|
||||
last_artifact_hash, last_artifact_kind)
|
||||
VALUES (
|
||||
@connector_id, @last_updated, @document_digests, @resume_tokens, @last_success_at, @failure_count,
|
||||
@next_eligible_run, @last_failure_reason, @last_checkpoint, @last_heartbeat_at, @last_heartbeat_status,
|
||||
@last_artifact_hash, @last_artifact_kind)
|
||||
ON CONFLICT (connector_id) DO UPDATE SET
|
||||
last_updated = EXCLUDED.last_updated,
|
||||
document_digests = EXCLUDED.document_digests,
|
||||
resume_tokens = EXCLUDED.resume_tokens,
|
||||
last_success_at = EXCLUDED.last_success_at,
|
||||
failure_count = EXCLUDED.failure_count,
|
||||
next_eligible_run = EXCLUDED.next_eligible_run,
|
||||
last_failure_reason = EXCLUDED.last_failure_reason,
|
||||
last_checkpoint = EXCLUDED.last_checkpoint,
|
||||
last_heartbeat_at = EXCLUDED.last_heartbeat_at,
|
||||
last_heartbeat_status = EXCLUDED.last_heartbeat_status,
|
||||
last_artifact_hash = EXCLUDED.last_artifact_hash,
|
||||
last_artifact_kind = EXCLUDED.last_artifact_kind;
|
||||
""";
|
||||
|
||||
await using var command = CreateCommand(sql, connection);
|
||||
AddParameter(command, "connector_id", state.ConnectorId);
|
||||
AddParameter(command, "last_updated", lastUpdated.UtcDateTime);
|
||||
AddParameter(command, "document_digests", state.DocumentDigests.IsDefault ? Array.Empty<string>() : state.DocumentDigests.ToArray());
|
||||
AddJsonbParameter(command, "resume_tokens", JsonSerializer.Serialize(state.ResumeTokens));
|
||||
AddParameter(command, "last_success_at", state.LastSuccessAt?.UtcDateTime);
|
||||
AddParameter(command, "failure_count", state.FailureCount);
|
||||
AddParameter(command, "next_eligible_run", state.NextEligibleRun?.UtcDateTime);
|
||||
AddParameter(command, "last_failure_reason", state.LastFailureReason);
|
||||
AddParameter(command, "last_checkpoint", state.LastCheckpoint?.UtcDateTime);
|
||||
AddParameter(command, "last_heartbeat_at", state.LastHeartbeatAt?.UtcDateTime);
|
||||
AddParameter(command, "last_heartbeat_status", state.LastHeartbeatStatus);
|
||||
AddParameter(command, "last_artifact_hash", state.LastArtifactHash);
|
||||
AddParameter(command, "last_artifact_kind", state.LastArtifactKind);
|
||||
|
||||
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyCollection<VexConnectorState>> ListAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var connection = await DataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
|
||||
|
||||
const string sql = """
|
||||
SELECT connector_id, last_updated, document_digests, resume_tokens, last_success_at, failure_count,
|
||||
next_eligible_run, last_failure_reason, last_checkpoint, last_heartbeat_at, last_heartbeat_status,
|
||||
last_artifact_hash, last_artifact_kind
|
||||
FROM vex.connector_states
|
||||
ORDER BY connector_id;
|
||||
""";
|
||||
|
||||
await using var command = CreateCommand(sql, connection);
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var results = new List<VexConnectorState>();
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
results.Add(Map(reader));
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private VexConnectorState Map(NpgsqlDataReader reader)
|
||||
{
|
||||
var connectorId = reader.GetString(0);
|
||||
var lastUpdated = reader.IsDBNull(1) ? (DateTimeOffset?)null : new DateTimeOffset(reader.GetDateTime(1), TimeSpan.Zero);
|
||||
var digests = reader.IsDBNull(2) ? ImmutableArray<string>.Empty : reader.GetFieldValue<string[]>(2).ToImmutableArray();
|
||||
var resumeTokens = reader.IsDBNull(3)
|
||||
? ImmutableDictionary<string, string>.Empty
|
||||
: JsonSerializer.Deserialize<ImmutableDictionary<string, string>>(reader.GetFieldValue<string>(3)) ?? ImmutableDictionary<string, string>.Empty;
|
||||
var lastSuccess = reader.IsDBNull(4) ? (DateTimeOffset?)null : new DateTimeOffset(reader.GetDateTime(4), TimeSpan.Zero);
|
||||
var failureCount = reader.IsDBNull(5) ? 0 : reader.GetInt32(5);
|
||||
var nextEligible = reader.IsDBNull(6) ? (DateTimeOffset?)null : new DateTimeOffset(reader.GetDateTime(6), TimeSpan.Zero);
|
||||
var lastFailureReason = reader.IsDBNull(7) ? null : reader.GetString(7);
|
||||
var lastCheckpoint = reader.IsDBNull(8) ? (DateTimeOffset?)null : new DateTimeOffset(reader.GetDateTime(8), TimeSpan.Zero);
|
||||
var lastHeartbeatAt = reader.IsDBNull(9) ? (DateTimeOffset?)null : new DateTimeOffset(reader.GetDateTime(9), TimeSpan.Zero);
|
||||
var lastHeartbeatStatus = reader.IsDBNull(10) ? null : reader.GetString(10);
|
||||
var lastArtifactHash = reader.IsDBNull(11) ? null : reader.GetString(11);
|
||||
var lastArtifactKind = reader.IsDBNull(12) ? null : reader.GetString(12);
|
||||
|
||||
return new VexConnectorState(
|
||||
connectorId,
|
||||
lastUpdated,
|
||||
digests,
|
||||
resumeTokens,
|
||||
lastSuccess,
|
||||
failureCount,
|
||||
nextEligible,
|
||||
lastFailureReason,
|
||||
lastCheckpoint,
|
||||
lastHeartbeatAt,
|
||||
lastHeartbeatStatus,
|
||||
lastArtifactHash,
|
||||
lastArtifactKind);
|
||||
}
|
||||
|
||||
private async ValueTask EnsureTableAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_initialized)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _initLock.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_initialized)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
const string sql = """
|
||||
CREATE TABLE IF NOT EXISTS vex.connector_states (
|
||||
connector_id text PRIMARY KEY,
|
||||
last_updated timestamptz NOT NULL,
|
||||
document_digests text[] NOT NULL,
|
||||
resume_tokens jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||
last_success_at timestamptz NULL,
|
||||
failure_count integer NOT NULL DEFAULT 0,
|
||||
next_eligible_run timestamptz NULL,
|
||||
last_failure_reason text NULL,
|
||||
last_checkpoint timestamptz NULL,
|
||||
last_heartbeat_at timestamptz NULL,
|
||||
last_heartbeat_status text NULL,
|
||||
last_artifact_hash text NULL,
|
||||
last_artifact_kind text NULL
|
||||
);
|
||||
""";
|
||||
|
||||
await using var command = CreateCommand(sql, connection);
|
||||
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
_initialized = true;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_initLock.Release();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -90,8 +90,9 @@ public sealed class PostgresVexRawStore : RepositoryBase<ExcititorDataSource>, I
|
||||
ON CONFLICT (digest) DO NOTHING;
|
||||
""";
|
||||
|
||||
await using (var command = CreateCommand(insertDocumentSql, connection, transaction))
|
||||
await using (var command = CreateCommand(insertDocumentSql, connection))
|
||||
{
|
||||
command.Transaction = transaction;
|
||||
AddParameter(command, "digest", digest);
|
||||
AddParameter(command, "tenant", tenant);
|
||||
AddParameter(command, "provider_id", providerId);
|
||||
@@ -117,7 +118,8 @@ public sealed class PostgresVexRawStore : RepositoryBase<ExcititorDataSource>, I
|
||||
ON CONFLICT (digest) DO NOTHING;
|
||||
""";
|
||||
|
||||
await using var blobCommand = CreateCommand(insertBlobSql, connection, transaction);
|
||||
await using var blobCommand = CreateCommand(insertBlobSql, connection);
|
||||
blobCommand.Transaction = transaction;
|
||||
AddParameter(blobCommand, "digest", digest);
|
||||
blobCommand.Parameters.Add(new NpgsqlParameter("payload", NpgsqlDbType.Bytea)
|
||||
{
|
||||
@@ -320,9 +322,15 @@ public sealed class PostgresVexRawStore : RepositoryBase<ExcititorDataSource>, I
|
||||
}
|
||||
|
||||
private static VexDocumentFormat ParseFormat(string value)
|
||||
=> Enum.TryParse<VexDocumentFormat>(value, ignoreCase: true, out var parsed)
|
||||
? parsed
|
||||
: VexDocumentFormat.Unknown;
|
||||
{
|
||||
if (Enum.TryParse<VexDocumentFormat>(value, ignoreCase: true, out var parsed))
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
|
||||
// Default to OpenVEX for unknown/legacy values to preserve compatibility with legacy rows.
|
||||
return VexDocumentFormat.OpenVex;
|
||||
}
|
||||
|
||||
private static ImmutableDictionary<string, string> ParseMetadata(string json)
|
||||
{
|
||||
|
||||
@@ -34,6 +34,7 @@ public static class ServiceCollectionExtensions
|
||||
services.AddScoped<IAppendOnlyLinksetStore, PostgresAppendOnlyLinksetStore>();
|
||||
services.AddScoped<IVexLinksetStore, PostgresAppendOnlyLinksetStore>();
|
||||
services.AddScoped<IVexRawStore, PostgresVexRawStore>();
|
||||
services.AddScoped<IVexConnectorStateRepository, PostgresConnectorStateRepository>();
|
||||
|
||||
return services;
|
||||
}
|
||||
@@ -56,6 +57,7 @@ public static class ServiceCollectionExtensions
|
||||
services.AddScoped<IAppendOnlyLinksetStore, PostgresAppendOnlyLinksetStore>();
|
||||
services.AddScoped<IVexLinksetStore, PostgresAppendOnlyLinksetStore>();
|
||||
services.AddScoped<IVexRawStore, PostgresVexRawStore>();
|
||||
services.AddScoped<IVexConnectorStateRepository, PostgresConnectorStateRepository>();
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
using Xunit;
|
||||
|
||||
@@ -8,6 +8,7 @@ using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
using StellaOps.Excititor.WebService.Options;
|
||||
using Xunit;
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
using StellaOps.Excititor.WebService.Options;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Tests;
|
||||
|
||||
public sealed class GraphOverlayCacheTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task SaveAndGet_RoundTripsOverlay()
|
||||
{
|
||||
var memoryCache = new MemoryCache(new MemoryCacheOptions());
|
||||
var options = Options.Create(new GraphOptions { OverlayTtlSeconds = 300 });
|
||||
var cache = new GraphOverlayCacheStore(memoryCache, options, TimeProvider.System);
|
||||
|
||||
var overlays = new[]
|
||||
{
|
||||
new GraphOverlayItem(
|
||||
SchemaVersion: "1.0.0",
|
||||
GeneratedAt: DateTimeOffset.UtcNow,
|
||||
Tenant: "tenant-a",
|
||||
Purl: "pkg:npm/example@1.0.0",
|
||||
AdvisoryId: "ADV-1",
|
||||
Source: "provider",
|
||||
Status: "not_affected",
|
||||
Summary: new GraphOverlaySummary(0, 1, 0, 0),
|
||||
Justifications: Array.Empty<GraphOverlayJustification>(),
|
||||
Conflicts: Array.Empty<GraphOverlayConflict>(),
|
||||
Observations: Array.Empty<GraphOverlayObservation>(),
|
||||
Provenance: new GraphOverlayProvenance("tenant-a", new[] { "provider" }, new[] { "CVE-1" }, new[] { "pkg:npm/example@1.0.0" }, Array.Empty<string>(), Array.Empty<string>()),
|
||||
Cache: null)
|
||||
};
|
||||
|
||||
await cache.SaveAsync("tenant-a", includeJustifications: false, overlays.Select(o => o.Purl).ToArray(), overlays, DateTimeOffset.UtcNow, CancellationToken.None);
|
||||
|
||||
var hit = await cache.TryGetAsync("tenant-a", includeJustifications: false, overlays.Select(o => o.Purl).ToArray(), CancellationToken.None);
|
||||
Assert.NotNull(hit);
|
||||
Assert.Equal(overlays, hit!.Items);
|
||||
Assert.True(hit.AgeMilliseconds >= 0);
|
||||
}
|
||||
}
|
||||
@@ -11,7 +11,7 @@ namespace StellaOps.Excititor.WebService.Tests;
|
||||
public sealed class GraphOverlayFactoryTests
|
||||
{
|
||||
[Fact]
|
||||
public void Build_ComputesSummariesAndProvenancePerPurl()
|
||||
public void Build_EmitsOverlayPerStatementWithProvenance()
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var observations = new[]
|
||||
@@ -55,20 +55,27 @@ public sealed class GraphOverlayFactoryTests
|
||||
};
|
||||
|
||||
var overlays = GraphOverlayFactory.Build(
|
||||
tenant: "tenant-a",
|
||||
generatedAt: now,
|
||||
orderedPurls: new[] { "pkg:rpm/redhat/openssl@1.1.1" },
|
||||
observations: observations,
|
||||
includeJustifications: true);
|
||||
|
||||
var overlay = Assert.Single(overlays);
|
||||
Assert.Equal("pkg:rpm/redhat/openssl@1.1.1", overlay.Purl);
|
||||
Assert.Equal(0, overlay.Summary.Open);
|
||||
Assert.Equal(1, overlay.Summary.NotAffected);
|
||||
Assert.Equal(1, overlay.Summary.UnderInvestigation);
|
||||
Assert.Equal(1, overlay.Summary.NoStatement);
|
||||
Assert.Equal(now, overlay.LatestModifiedAt);
|
||||
Assert.Equal(new[] { "ComponentNotPresent" }, overlay.Justifications);
|
||||
Assert.Equal("hash-new", overlay.Provenance.LastEvidenceHash);
|
||||
Assert.Equal(new[] { "oracle", "redhat", "ubuntu" }, overlay.Provenance.Sources);
|
||||
Assert.Equal(2, overlays.Count);
|
||||
|
||||
var notAffected = Assert.Single(overlays.Where(o => o.Status == "not_affected"));
|
||||
Assert.Equal("pkg:rpm/redhat/openssl@1.1.1", notAffected.Purl);
|
||||
Assert.Equal("CVE-2025-1000", notAffected.AdvisoryId);
|
||||
Assert.Equal("redhat", notAffected.Source);
|
||||
Assert.Single(notAffected.Justifications);
|
||||
Assert.Contains(notAffected.Observations, o => o.ContentHash == "hash-old");
|
||||
Assert.Contains("hash-old", notAffected.Provenance.ObservationHashes);
|
||||
|
||||
var underInvestigation = Assert.Single(overlays.Where(o => o.Status == "under_investigation"));
|
||||
Assert.Equal("CVE-2025-1001", underInvestigation.AdvisoryId);
|
||||
Assert.Equal("ubuntu", underInvestigation.Source);
|
||||
Assert.Empty(underInvestigation.Justifications);
|
||||
Assert.Contains("hash-new", underInvestigation.Provenance.ObservationHashes);
|
||||
}
|
||||
|
||||
private static VexObservation CreateObservation(
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Tests;
|
||||
|
||||
public sealed class GraphOverlayStoreTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task SaveAndFindByPurls_ReturnsLatestPerSourceAdvisory()
|
||||
{
|
||||
var store = new InMemoryGraphOverlayStore();
|
||||
var overlays = new[]
|
||||
{
|
||||
new GraphOverlayItem(
|
||||
SchemaVersion: "1.0.0",
|
||||
GeneratedAt: DateTimeOffset.UtcNow.AddMinutes(-1),
|
||||
Tenant: "tenant-a",
|
||||
Purl: "pkg:npm/example@1.0.0",
|
||||
AdvisoryId: "ADV-1",
|
||||
Source: "provider-a",
|
||||
Status: "not_affected",
|
||||
Summary: new GraphOverlaySummary(0, 1, 0, 0),
|
||||
Justifications: Array.Empty<GraphOverlayJustification>(),
|
||||
Conflicts: Array.Empty<GraphOverlayConflict>(),
|
||||
Observations: Array.Empty<GraphOverlayObservation>(),
|
||||
Provenance: new GraphOverlayProvenance("tenant-a", new[] { "provider-a" }, new[] { "ADV-1" }, new[] { "pkg:npm/example@1.0.0" }, Array.Empty<string>(), Array.Empty<string>()),
|
||||
Cache: null),
|
||||
new GraphOverlayItem(
|
||||
SchemaVersion: "1.0.0",
|
||||
GeneratedAt: DateTimeOffset.UtcNow,
|
||||
Tenant: "tenant-a",
|
||||
Purl: "pkg:npm/example@1.0.0",
|
||||
AdvisoryId: "ADV-1",
|
||||
Source: "provider-a",
|
||||
Status: "under_investigation",
|
||||
Summary: new GraphOverlaySummary(0, 0, 1, 0),
|
||||
Justifications: Array.Empty<GraphOverlayJustification>(),
|
||||
Conflicts: Array.Empty<GraphOverlayConflict>(),
|
||||
Observations: Array.Empty<GraphOverlayObservation>(),
|
||||
Provenance: new GraphOverlayProvenance("tenant-a", new[] { "provider-a" }, new[] { "ADV-1" }, new[] { "pkg:npm/example@1.0.0" }, Array.Empty<string>(), Array.Empty<string>()),
|
||||
Cache: null)
|
||||
};
|
||||
|
||||
await store.SaveAsync("tenant-a", overlays, CancellationToken.None);
|
||||
var results = await store.FindByPurlsAsync("tenant-a", new[] { "pkg:npm/example@1.0.0" }, CancellationToken.None);
|
||||
|
||||
var single = Assert.Single(results);
|
||||
Assert.Equal("under_investigation", single.Status);
|
||||
}
|
||||
}
|
||||
@@ -10,7 +10,7 @@ namespace StellaOps.Excititor.WebService.Tests;
|
||||
public sealed class GraphStatusFactoryTests
|
||||
{
|
||||
[Fact]
|
||||
public void Build_ProjectsOverlaySummariesAndProvenance()
|
||||
public void Build_ProjectsStatusCountsPerPurl()
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var observations = new[]
|
||||
@@ -39,6 +39,8 @@ public sealed class GraphStatusFactoryTests
|
||||
};
|
||||
|
||||
var items = GraphStatusFactory.Build(
|
||||
tenant: "tenant-a",
|
||||
generatedAt: now,
|
||||
orderedPurls: new[] { "pkg:rpm/redhat/openssl@1.1.1" },
|
||||
observations: observations);
|
||||
|
||||
@@ -47,10 +49,10 @@ public sealed class GraphStatusFactoryTests
|
||||
Assert.Equal(0, item.Summary.Open);
|
||||
Assert.Equal(1, item.Summary.NotAffected);
|
||||
Assert.Equal(0, item.Summary.UnderInvestigation);
|
||||
Assert.Equal(1, item.Summary.NoStatement);
|
||||
Assert.Equal(0, item.Summary.NoStatement);
|
||||
Assert.Equal(now, item.LatestModifiedAt);
|
||||
Assert.Equal("hash-new", item.LastEvidenceHash);
|
||||
Assert.Equal(new[] { "oracle", "ubuntu" }, item.Sources);
|
||||
Assert.Equal(new[] { "ubuntu" }, item.Sources);
|
||||
}
|
||||
|
||||
private static VexObservation CreateObservation(
|
||||
|
||||
@@ -2,6 +2,7 @@ using System.Net.Http.Json;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Tests;
|
||||
|
||||
Reference in New Issue
Block a user