nuget reorganization

This commit is contained in:
master
2025-11-18 23:45:25 +02:00
parent 77cee6a209
commit d3ecd7f8e6
7712 changed files with 13963 additions and 10007504 deletions

View File

@@ -0,0 +1,40 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Excititor.WebService.Contracts;
public sealed record VexLinksetListResponse(
[property: JsonPropertyName("items")] IReadOnlyList<VexLinksetListItem> Items,
[property: JsonPropertyName("nextCursor")] string? NextCursor);
public sealed record VexLinksetListItem(
[property: JsonPropertyName("linksetId")] string LinksetId,
[property: JsonPropertyName("tenant")] string Tenant,
[property: JsonPropertyName("vulnerabilityId")] string VulnerabilityId,
[property: JsonPropertyName("productKey")] string ProductKey,
[property: JsonPropertyName("providerIds")] IReadOnlyList<string> ProviderIds,
[property: JsonPropertyName("statuses")] IReadOnlyList<string> Statuses,
[property: JsonPropertyName("aliases")] IReadOnlyList<string> Aliases,
[property: JsonPropertyName("purls")] IReadOnlyList<string> Purls,
[property: JsonPropertyName("cpes")] IReadOnlyList<string> Cpes,
[property: JsonPropertyName("references")] IReadOnlyList<VexLinksetReference> References,
[property: JsonPropertyName("disagreements")] IReadOnlyList<VexLinksetDisagreement> Disagreements,
[property: JsonPropertyName("observations")] IReadOnlyList<VexLinksetObservationRef> Observations,
[property: JsonPropertyName("createdAt")] DateTimeOffset CreatedAt);
public sealed record VexLinksetReference(
[property: JsonPropertyName("type")] string Type,
[property: JsonPropertyName("url")] string Url);
public sealed record VexLinksetDisagreement(
[property: JsonPropertyName("providerId")] string ProviderId,
[property: JsonPropertyName("status")] string Status,
[property: JsonPropertyName("justification")] string? Justification,
[property: JsonPropertyName("confidence")] double? Confidence);
public sealed record VexLinksetObservationRef(
[property: JsonPropertyName("observationId")] string ObservationId,
[property: JsonPropertyName("providerId")] string ProviderId,
[property: JsonPropertyName("status")] string Status,
[property: JsonPropertyName("confidence")] double? Confidence);

View File

@@ -0,0 +1,20 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Excititor.WebService.Contracts;
public sealed record VexObservationListResponse(
[property: JsonPropertyName("items")] IReadOnlyList<VexObservationListItem> Items,
[property: JsonPropertyName("nextCursor")] string? NextCursor);
public sealed record VexObservationListItem(
[property: JsonPropertyName("observationId")] string ObservationId,
[property: JsonPropertyName("tenant")] string Tenant,
[property: JsonPropertyName("providerId")] string ProviderId,
[property: JsonPropertyName("vulnerabilityId")] string VulnerabilityId,
[property: JsonPropertyName("productKey")] string ProductKey,
[property: JsonPropertyName("status")] string Status,
[property: JsonPropertyName("createdAt")] DateTimeOffset CreatedAt,
[property: JsonPropertyName("lastObserved")] DateTimeOffset? LastObserved,
[property: JsonPropertyName("purls")] IReadOnlyList<string> Purls);

View File

@@ -10,6 +10,7 @@ using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Primitives;
using StellaOps.Excititor.Attestation.Verification;
using StellaOps.Excititor.Attestation.Extensions;
@@ -47,6 +48,7 @@ services.AddCycloneDxNormalizer();
services.AddOpenVexNormalizer();
services.AddSingleton<IVexSignatureVerifier, NoopVexSignatureVerifier>();
services.AddScoped<IVexIngestOrchestrator, VexIngestOrchestrator>();
services.AddScoped<IVexObservationLookup, MongoVexObservationLookup>();
services.AddOptions<ExcititorObservabilityOptions>()
.Bind(configuration.GetSection("Excititor:Observability"));
services.AddScoped<ExcititorHealthService>();
@@ -63,6 +65,8 @@ services.Configure<MirrorDistributionOptions>(configuration.GetSection(MirrorDis
services.AddSingleton<MirrorRateLimiter>();
services.TryAddSingleton(TimeProvider.System);
services.AddSingleton<IVexObservationProjectionService, VexObservationProjectionService>();
services.AddScoped<IVexObservationLookup, MongoVexObservationLookup>();
services.AddScoped<IVexObservationLookup, MongoVexObservationLookup>();
var rekorSection = configuration.GetSection("Excititor:Attestation:Rekor");
if (rekorSection.Exists())
@@ -522,10 +526,223 @@ app.MapGet("/v1/vex/observations/{vulnerabilityId}/{productKey}", async (
return Results.Json(response);
});
app.MapGet("/v1/vex/observations", async (
HttpContext context,
[FromServices] IVexObservationLookup observationLookup,
[FromServices] IOptions<VexMongoStorageOptions> storageOptions,
CancellationToken cancellationToken) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
{
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out var tenant, out var tenantError))
{
return tenantError;
}
var observationIds = BuildStringFilterSet(context.Request.Query["observationId"]);
var vulnerabilityIds = BuildStringFilterSet(context.Request.Query["vulnerabilityId"], toLower: true);
var productKeys = BuildStringFilterSet(context.Request.Query["productKey"], toLower: true);
var purls = BuildStringFilterSet(context.Request.Query["purl"], toLower: true);
var cpes = BuildStringFilterSet(context.Request.Query["cpe"], toLower: true);
var providerIds = BuildStringFilterSet(context.Request.Query["providerId"], toLower: true);
var statuses = BuildStatusFilter(context.Request.Query["status"]);
var limit = ResolveLimit(context.Request.Query["limit"], defaultValue: 500, min: 1, max: 2000);
var cursorRaw = context.Request.Query["cursor"].FirstOrDefault();
VexObservationCursor? cursor = null;
if (!string.IsNullOrWhiteSpace(cursorRaw))
{
try
{
cursor = VexObservationCursor.Parse(cursorRaw!);
}
catch
{
return Results.BadRequest("Cursor is malformed.");
}
}
IReadOnlyList<VexObservation> observations;
try
{
observations = await observationLookup.FindByFiltersAsync(
tenant,
observationIds,
vulnerabilityIds,
productKeys,
purls,
cpes,
providerIds,
statuses,
cursor,
limit,
cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
return Results.StatusCode(StatusCodes.Status499ClientClosedRequest);
}
var items = observations.Select(obs => new VexObservationListItem(
obs.ObservationId,
obs.Tenant,
obs.ProviderId,
obs.Statements.FirstOrDefault()?.VulnerabilityId ?? string.Empty,
obs.Statements.FirstOrDefault()?.ProductKey ?? string.Empty,
obs.Statements.FirstOrDefault()?.Status.ToString().ToLowerInvariant() ?? string.Empty,
obs.CreatedAt,
obs.Statements.FirstOrDefault()?.LastObserved,
obs.Linkset.Purls)).ToList();
var nextCursor = observations.Count == limit
? VexObservationCursor.FromObservation(observations.Last()).ToString()
: null;
var response = new VexObservationListResponse(items, nextCursor);
context.Response.Headers["Excititor-Results-Count"] = items.Count.ToString(CultureInfo.InvariantCulture);
if (nextCursor is not null)
{
context.Response.Headers["Excititor-Results-Cursor"] = nextCursor;
}
return Results.Json(response);
});
app.MapGet("/v1/vex/linksets", async (
HttpContext context,
[FromServices] IVexObservationLookup observationLookup,
[FromServices] IOptions<VexMongoStorageOptions> storageOptions,
CancellationToken cancellationToken) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
{
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out var tenant, out var tenantError))
{
return tenantError;
}
var vulnerabilityIds = BuildStringFilterSet(context.Request.Query["vulnerabilityId"], toLower: true);
var productKeys = BuildStringFilterSet(context.Request.Query["productKey"], toLower: true);
var providerIds = BuildStringFilterSet(context.Request.Query["providerId"], toLower: true);
var statuses = BuildStatusFilter(context.Request.Query["status"]);
var limit = ResolveLimit(context.Request.Query["limit"], defaultValue: 200, min: 1, max: 500);
var cursorRaw = context.Request.Query["cursor"].FirstOrDefault();
VexObservationCursor? cursor = null;
if (!string.IsNullOrWhiteSpace(cursorRaw))
{
try
{
cursor = VexObservationCursor.Parse(cursorRaw!);
}
catch
{
return Results.BadRequest("Cursor is malformed.");
}
}
IReadOnlyList<VexObservation> observations;
try
{
observations = await observationLookup.FindByFiltersAsync(
tenant,
observationIds: Array.Empty<string>(),
vulnerabilityIds,
productKeys,
purls: Array.Empty<string>(),
cpes: Array.Empty<string>(),
providerIds,
statuses,
cursor,
limit,
cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
return Results.StatusCode(StatusCodes.Status499ClientClosedRequest);
}
var grouped = observations
.GroupBy(obs => (VulnerabilityId: obs.Statements.FirstOrDefault()?.VulnerabilityId ?? string.Empty,
ProductKey: obs.Statements.FirstOrDefault()?.ProductKey ?? string.Empty))
.Select(group =>
{
var sample = group.FirstOrDefault();
var linkset = sample?.Linkset ?? new VexObservationLinkset(null, null, null, null);
var vulnerabilityId = group.Key.VulnerabilityId;
var productKey = group.Key.ProductKey;
var providerSet = new SortedSet<string>(StringComparer.OrdinalIgnoreCase);
var statusSet = new SortedSet<string>(StringComparer.OrdinalIgnoreCase);
var obsRefs = new List<VexLinksetObservationRef>();
foreach (var obs in group)
{
var stmt = obs.Statements.FirstOrDefault();
if (stmt is null)
{
continue;
}
providerSet.Add(obs.ProviderId);
statusSet.Add(stmt.Status.ToString().ToLowerInvariant());
obsRefs.Add(new VexLinksetObservationRef(
obs.ObservationId,
obs.ProviderId,
stmt.Status.ToString().ToLowerInvariant(),
stmt.Signals?.Severity?.Score));
}
var item = new VexLinksetListItem(
linksetId: string.Create(CultureInfo.InvariantCulture, $"{vulnerabilityId}:{productKey}"),
tenant,
vulnerabilityId,
productKey,
providerSet.ToList(),
statusSet.ToList(),
linkset.Aliases,
linkset.Purls,
linkset.Cpes,
linkset.References.Select(r => new VexLinksetReference(r.Type, r.Url)).ToList(),
linkset.Disagreements.Select(d => new VexLinksetDisagreement(d.ProviderId, d.Status, d.Justification, d.Confidence)).ToList(),
obsRefs,
createdAt: group.Min(o => o.CreatedAt));
return item;
})
.OrderBy(item => item.VulnerabilityId, StringComparer.Ordinal)
.ThenBy(item => item.ProductKey, StringComparer.Ordinal)
.Take(limit)
.ToList();
var nextCursor = grouped.Count == limit && observations.Count > 0
? VexObservationCursor.FromObservation(observations.Last()).ToString()
: null;
var response = new VexLinksetListResponse(grouped, nextCursor);
context.Response.Headers["Excititor-Results-Count"] = grouped.Count.ToString(CultureInfo.InvariantCulture);
if (nextCursor is not null)
{
context.Response.Headers["Excititor-Results-Cursor"] = nextCursor;
}
return Results.Json(response);
});
app.MapGet("/v1/vex/evidence/chunks", async (
HttpContext context,
[FromServices] IVexEvidenceChunkService chunkService,
[FromServices] IOptions<VexMongoStorageOptions> storageOptions,
[FromServices] ILogger<VexEvidenceChunkRequest> logger,
CancellationToken cancellationToken) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
@@ -579,6 +796,18 @@ app.MapGet("/v1/vex/evidence/chunks", async (
EvidenceTelemetry.RecordChunkOutcome(tenant, "success", result.Chunks.Count, result.Truncated);
EvidenceTelemetry.RecordChunkSignatureStatus(tenant, result.Chunks);
logger.LogInformation(
"vex_evidence_chunks_success tenant={Tenant} vulnerabilityId={Vuln} productKey={ProductKey} providers={Providers} statuses={Statuses} limit={Limit} total={Total} truncated={Truncated} returned={Returned}",
tenant ?? "(default)",
request.VulnerabilityId,
request.ProductKey,
providerFilter.Count,
statusFilter.Count,
request.Limit,
result.TotalCount,
result.Truncated,
result.Chunks.Count);
// Align headers with published contract.
context.Response.Headers["Excititor-Results-Total"] = result.TotalCount.ToString(CultureInfo.InvariantCulture);
context.Response.Headers["Excititor-Results-Truncated"] = result.Truncated ? "true" : "false";
@@ -841,3 +1070,90 @@ internal sealed record VexSeveritySignalRequest(string Scheme, double? Score, st
{
public VexSeveritySignal ToDomain() => new(Scheme, Score, Label, Vector);
}
app.MapGet(
"/v1/vex/observations",
async (
HttpContext context,
[FromServices] IVexObservationLookup observationLookup,
[FromServices] IOptions<VexMongoStorageOptions> storageOptions,
CancellationToken cancellationToken) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
{
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out var tenant, out var tenantError))
{
return tenantError;
}
var observationIds = BuildStringFilterSet(context.Request.Query["observationId"]);
var vulnerabilityIds = BuildStringFilterSet(context.Request.Query["vulnerabilityId"], toLower: true);
var productKeys = BuildStringFilterSet(context.Request.Query["productKey"], toLower: true);
var purls = BuildStringFilterSet(context.Request.Query["purl"], toLower: true);
var cpes = BuildStringFilterSet(context.Request.Query["cpe"], toLower: true);
var providerIds = BuildStringFilterSet(context.Request.Query["providerId"], toLower: true);
var statuses = BuildStatusFilter(context.Request.Query["status"]);
var limit = ResolveLimit(context.Request.Query["limit"], defaultValue: 200, min: 1, max: 500);
var cursorRaw = context.Request.Query["cursor"].FirstOrDefault();
VexObservationCursor? cursor = null;
if (!string.IsNullOrWhiteSpace(cursorRaw))
{
try
{
cursor = VexObservationCursor.Parse(cursorRaw!);
}
catch
{
return Results.BadRequest("Cursor is malformed.");
}
}
IReadOnlyList<VexObservation> observations;
try
{
observations = await observationLookup.FindByFiltersAsync(
tenant,
observationIds,
vulnerabilityIds,
productKeys,
purls,
cpes,
providerIds,
statuses,
cursor,
limit,
cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
return Results.StatusCode(StatusCodes.Status499ClientClosedRequest);
}
var items = observations.Select(obs => new VexObservationListItem(
obs.ObservationId,
obs.Tenant,
obs.ProviderId,
obs.Statements.FirstOrDefault()?.VulnerabilityId ?? string.Empty,
obs.Statements.FirstOrDefault()?.ProductKey ?? string.Empty,
obs.Statements.FirstOrDefault()?.Status.ToString().ToLowerInvariant() ?? string.Empty,
obs.CreatedAt,
obs.Statements.FirstOrDefault()?.LastObserved,
obs.Linkset.Purls)).ToList();
var nextCursor = observations.Count == limit
? VexObservationCursor.FromObservation(observations.Last()).ToString()
: null;
var response = new VexObservationListResponse(items, nextCursor);
context.Response.Headers["X-Count"] = items.Count.ToString(CultureInfo.InvariantCulture);
if (nextCursor is not null)
{
context.Response.Headers["X-Cursor"] = nextCursor;
}
return Results.Json(response);
});

View File

@@ -25,7 +25,7 @@ public static class VexLinksetUpdatedEventFactory
var observationRefs = (observations ?? Enumerable.Empty<VexObservation>())
.Where(obs => obs is not null)
.SelectMany(obs => obs.Statements.Select(statement => new VexLinksetObservationRef(
.SelectMany(obs => obs.Statements.Select(statement => new VexLinksetObservationRefCore(
observationId: obs.ObservationId,
providerId: obs.ProviderId,
status: statement.Status.ToString().ToLowerInvariant(),
@@ -70,11 +70,11 @@ public static class VexLinksetUpdatedEventFactory
return value.Trim();
}
private sealed class VexLinksetObservationRefComparer : IEqualityComparer<VexLinksetObservationRef>
private sealed class VexLinksetObservationRefComparer : IEqualityComparer<VexLinksetObservationRefCore>
{
public static readonly VexLinksetObservationRefComparer Instance = new();
public bool Equals(VexLinksetObservationRef? x, VexLinksetObservationRef? y)
public bool Equals(VexLinksetObservationRefCore? x, VexLinksetObservationRefCore? y)
{
if (ReferenceEquals(x, y))
{
@@ -92,7 +92,7 @@ public static class VexLinksetUpdatedEventFactory
&& Nullable.Equals(x.Confidence, y.Confidence);
}
public int GetHashCode(VexLinksetObservationRef obj)
public int GetHashCode(VexLinksetObservationRefCore obj)
{
var hash = new HashCode();
hash.Add(obj.ObservationId, StringComparer.Ordinal);
@@ -137,18 +137,12 @@ public static class VexLinksetUpdatedEventFactory
}
}
public sealed record VexLinksetObservationRef(
string ObservationId,
string ProviderId,
string Status,
double? Confidence);
public sealed record VexLinksetUpdatedEvent(
string EventType,
string Tenant,
string LinksetId,
string VulnerabilityId,
string ProductKey,
ImmutableArray<VexLinksetObservationRef> Observations,
ImmutableArray<VexLinksetObservationRefCore> Observations,
ImmutableArray<VexObservationDisagreement> Disagreements,
DateTimeOffset CreatedAtUtc);

View File

@@ -1,5 +1,6 @@
using System.Collections.Immutable;
using System.Text.Json.Nodes;
using System.Collections.Immutable;
using System.Collections.Generic;
using System.Text.Json.Nodes;
namespace StellaOps.Excititor.Core.Observations;
@@ -360,7 +361,8 @@ public sealed record VexObservationLinkset
IEnumerable<string>? cpes,
IEnumerable<VexObservationReference>? references,
IEnumerable<string>? reconciledFrom = null,
IEnumerable<VexObservationDisagreement>? disagreements = null)
IEnumerable<VexObservationDisagreement>? disagreements = null,
IEnumerable<VexLinksetObservationRefModel>? observationRefs = null)
{
Aliases = NormalizeSet(aliases, toLower: true);
Purls = NormalizeSet(purls, toLower: false);
@@ -368,6 +370,7 @@ public sealed record VexObservationLinkset
References = NormalizeReferences(references);
ReconciledFrom = NormalizeSet(reconciledFrom, toLower: false);
Disagreements = NormalizeDisagreements(disagreements);
Observations = NormalizeObservationRefs(observationRefs);
}
public ImmutableArray<string> Aliases { get; }
@@ -381,6 +384,8 @@ public sealed record VexObservationLinkset
public ImmutableArray<string> ReconciledFrom { get; }
public ImmutableArray<VexObservationDisagreement> Disagreements { get; }
public ImmutableArray<VexLinksetObservationRefModel> Observations { get; }
private static ImmutableArray<string> NormalizeSet(IEnumerable<string>? values, bool toLower)
{
@@ -499,8 +504,39 @@ public sealed record VexObservationLinkset
return set.Count == 0 ? ImmutableArray<VexObservationDisagreement>.Empty : set.ToImmutableArray();
}
private static ImmutableArray<VexLinksetObservationRefModel> NormalizeObservationRefs(
IEnumerable<VexLinksetObservationRefModel>? refs)
{
if (refs is null)
{
return ImmutableArray<VexLinksetObservationRefModel>.Empty;
}
var set = new SortedSet<VexLinksetObservationRefModel>(VexLinksetObservationRefComparer.Instance);
foreach (var item in refs)
{
if (item is null)
{
continue;
}
var obsId = VexObservation.TrimToNull(item.ObservationId);
var provider = VexObservation.TrimToNull(item.ProviderId);
var status = VexObservation.TrimToNull(item.Status);
if (obsId is null || provider is null || status is null)
{
continue;
}
var clamped = item.Confidence is null ? null : Math.Clamp(item.Confidence.Value, 0.0, 1.0);
set.Add(new VexLinksetObservationRefModel(obsId, provider, status, clamped));
}
return set.Count == 0 ? ImmutableArray<VexLinksetObservationRefModel>.Empty : set.ToImmutableArray();
}
}
public sealed record VexObservationReference
{
public VexObservationReference(string type, string url)
@@ -536,3 +572,52 @@ public sealed record VexObservationDisagreement
public double? Confidence { get; }
}
public sealed record VexLinksetObservationRefModel(
string ObservationId,
string ProviderId,
string Status,
double? Confidence);
internal sealed class VexLinksetObservationRefComparer : IComparer<VexLinksetObservationRefModel>
{
public static readonly VexLinksetObservationRefComparer Instance = new();
public int Compare(VexLinksetObservationRefModel? x, VexLinksetObservationRefModel? y)
{
if (ReferenceEquals(x, y))
{
return 0;
}
if (x is null)
{
return -1;
}
if (y is null)
{
return 1;
}
var providerCompare = StringComparer.OrdinalIgnoreCase.Compare(x.ProviderId, y.ProviderId);
if (providerCompare != 0)
{
return providerCompare;
}
var statusCompare = StringComparer.OrdinalIgnoreCase.Compare(x.Status, y.Status);
if (statusCompare != 0)
{
return statusCompare;
}
var obsCompare = StringComparer.Ordinal.Compare(x.ObservationId, y.ObservationId);
if (obsCompare != 0)
{
return obsCompare;
}
return Nullable.Compare(x.Confidence, y.Confidence);
}
}

View File

@@ -65,6 +65,10 @@ internal sealed class VexObservationCollectionsMigration : IVexMongoMigration
.Ascending(x => x.Tenant)
.Ascending(x => x.ProductKey);
var tenantProviderIndex = Builders<VexLinksetRecord>.IndexKeys
.Ascending(x => x.Tenant)
.Ascending("ProviderIds");
var tenantDisagreementProviderIndex = Builders<VexLinksetRecord>.IndexKeys
.Ascending(x => x.Tenant)
.Ascending("Disagreements.ProviderId")
@@ -74,6 +78,7 @@ internal sealed class VexObservationCollectionsMigration : IVexMongoMigration
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexLinksetRecord>(tenantLinksetIndex, new CreateIndexOptions { Unique = true }), cancellationToken: cancellationToken),
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexLinksetRecord>(tenantVulnIndex), cancellationToken: cancellationToken),
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexLinksetRecord>(tenantProductIndex), cancellationToken: cancellationToken),
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexLinksetRecord>(tenantProviderIndex), cancellationToken: cancellationToken),
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexLinksetRecord>(tenantDisagreementProviderIndex), cancellationToken: cancellationToken));
}
}

View File

@@ -0,0 +1,229 @@
using System.Collections.Immutable;
using System.Text.Json.Nodes;
using MongoDB.Driver;
using MongoDB.Bson;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Observations;
namespace StellaOps.Excititor.Storage.Mongo;
internal sealed class MongoVexObservationLookup : IVexObservationLookup
{
private readonly IMongoCollection<VexObservationRecord> _collection;
public MongoVexObservationLookup(IMongoDatabase database)
{
ArgumentNullException.ThrowIfNull(database);
_collection = database.GetCollection<VexObservationRecord>(VexMongoCollectionNames.Observations);
}
public async ValueTask<IReadOnlyList<VexObservation>> ListByTenantAsync(
string tenant,
CancellationToken cancellationToken)
{
var normalizedTenant = NormalizeTenant(tenant);
var filter = Builders<VexObservationRecord>.Filter.Eq(record => record.Tenant, normalizedTenant);
var records = await _collection
.Find(filter)
.Sort(Builders<VexObservationRecord>.Sort.Descending(r => r.CreatedAt).Descending(r => r.ObservationId))
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return records.Select(Map).ToList();
}
public async ValueTask<IReadOnlyList<VexObservation>> FindByFiltersAsync(
string tenant,
IReadOnlyCollection<string> observationIds,
IReadOnlyCollection<string> vulnerabilityIds,
IReadOnlyCollection<string> productKeys,
IReadOnlyCollection<string> purls,
IReadOnlyCollection<string> cpes,
IReadOnlyCollection<string> providerIds,
IReadOnlyCollection<VexClaimStatus> statuses,
VexObservationCursor? cursor,
int limit,
CancellationToken cancellationToken)
{
var normalizedTenant = NormalizeTenant(tenant);
var filters = new List<FilterDefinition<VexObservationRecord>>(capacity: 8)
{
Builders<VexObservationRecord>.Filter.Eq(r => r.Tenant, normalizedTenant)
};
AddInFilter(filters, r => r.ObservationId, observationIds);
AddInFilter(filters, r => r.VulnerabilityId, vulnerabilityIds.Select(v => v.ToLowerInvariant()));
AddInFilter(filters, r => r.ProductKey, productKeys.Select(p => p.ToLowerInvariant()));
AddInFilter(filters, r => r.ProviderId, providerIds.Select(p => p.ToLowerInvariant()));
if (statuses.Count > 0)
{
var statusStrings = statuses.Select(status => status.ToString().ToLowerInvariant()).ToArray();
filters.Add(Builders<VexObservationRecord>.Filter.In(r => r.Status, statusStrings));
}
if (cursor is not null)
{
var cursorFilter = Builders<VexObservationRecord>.Filter.Or(
Builders<VexObservationRecord>.Filter.Lt(r => r.CreatedAt, cursor.CreatedAtUtc.UtcDateTime),
Builders<VexObservationRecord>.Filter.And(
Builders<VexObservationRecord>.Filter.Eq(r => r.CreatedAt, cursor.CreatedAtUtc.UtcDateTime),
Builders<VexObservationRecord>.Filter.Lt(r => r.ObservationId, cursor.ObservationId)));
filters.Add(cursorFilter);
}
var combinedFilter = filters.Count == 1
? filters[0]
: Builders<VexObservationRecord>.Filter.And(filters);
var records = await _collection
.Find(combinedFilter)
.Sort(Builders<VexObservationRecord>.Sort.Descending(r => r.CreatedAt).Descending(r => r.ObservationId))
.Limit(limit)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return records.Select(Map).ToList();
}
private static void AddInFilter(
ICollection<FilterDefinition<VexObservationRecord>> filters,
System.Linq.Expressions.Expression<Func<VexObservationRecord, string>> field,
IEnumerable<string> values)
{
var normalized = values
.Where(value => !string.IsNullOrWhiteSpace(value))
.Select(value => value.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray();
if (normalized.Length > 0)
{
filters.Add(Builders<VexObservationRecord>.Filter.In(field, normalized));
}
}
private static VexObservation Map(VexObservationRecord record)
{
var statements = record.Statements.Select(MapStatement).ToImmutableArray();
var linkset = MapLinkset(record.Linkset);
var upstreamSignature = record.Upstream?.Signature is null
? new VexObservationSignature(false, null, null, null)
: new VexObservationSignature(
record.Upstream.Signature.Present,
record.Upstream.Signature.Subject,
record.Upstream.Signature.Issuer,
record.Upstream.Signature.VerifiedAt);
var upstream = record.Upstream is null
? new VexObservationUpstream(
upstreamId: record.ObservationId,
documentVersion: null,
fetchedAt: record.CreatedAt,
receivedAt: record.CreatedAt,
contentHash: record.Document.Digest,
signature: upstreamSignature)
: new VexObservationUpstream(
record.Upstream.UpstreamId,
record.Upstream.DocumentVersion,
record.Upstream.FetchedAt,
record.Upstream.ReceivedAt,
record.Upstream.ContentHash,
upstreamSignature);
var documentSignature = record.Document.Signature is null
? null
: new VexObservationSignature(
record.Document.Signature.Present,
record.Document.Signature.Subject,
record.Document.Signature.Issuer,
record.Document.Signature.VerifiedAt);
var content = record.Content is null
? new VexObservationContent("unknown", null, new JsonObject())
: new VexObservationContent(
record.Content.Format ?? "unknown",
record.Content.SpecVersion,
JsonNode.Parse(record.Content.Raw.ToJson()) ?? new JsonObject(),
metadata: ImmutableDictionary<string, string>.Empty);
var observation = new VexObservation(
observationId: record.ObservationId,
tenant: record.Tenant,
providerId: record.ProviderId,
streamId: string.IsNullOrWhiteSpace(record.StreamId) ? record.ProviderId : record.StreamId,
upstream: upstream,
statements: statements,
content: content,
linkset: linkset,
createdAt: new DateTimeOffset(record.CreatedAt, TimeSpan.Zero),
supersedes: ImmutableArray<string>.Empty,
attributes: ImmutableDictionary<string, string>.Empty);
return observation;
}
private static VexObservationStatement MapStatement(VexObservationStatementRecord record)
{
var justification = string.IsNullOrWhiteSpace(record.Justification)
? (VexJustification?)null
: Enum.Parse<VexJustification>(record.Justification, ignoreCase: true);
return new VexObservationStatement(
record.VulnerabilityId,
record.ProductKey,
Enum.Parse<VexClaimStatus>(record.Status, ignoreCase: true),
record.LastObserved,
locator: record.Locator,
justification: justification,
introducedVersion: record.IntroducedVersion,
fixedVersion: record.FixedVersion,
detail: record.Detail,
signals: new VexSignalSnapshot(
severity: record.ScopeScore.HasValue ? new VexSeveritySignal("scope", record.ScopeScore, "n/a", null) : null,
Kev: record.Kev,
Epss: record.Epss),
confidence: null,
metadata: ImmutableDictionary<string, string>.Empty,
evidence: null,
anchors: VexObservationAnchors.Empty,
additionalMetadata: ImmutableDictionary<string, string>.Empty,
signature: null);
}
private static VexObservationDisagreement MapDisagreement(VexLinksetDisagreementRecord record)
=> new(record.ProviderId, record.Status, record.Justification, record.Confidence);
private static VexObservationLinkset MapLinkset(VexObservationLinksetRecord record)
{
var aliases = record?.Aliases?.Where(NotNullOrWhiteSpace).Select(a => a.Trim()).ToImmutableArray() ?? ImmutableArray<string>.Empty;
var purls = record?.Purls?.Where(NotNullOrWhiteSpace).Select(p => p.Trim()).ToImmutableArray() ?? ImmutableArray<string>.Empty;
var cpes = record?.Cpes?.Where(NotNullOrWhiteSpace).Select(c => c.Trim()).ToImmutableArray() ?? ImmutableArray<string>.Empty;
var references = record?.References?.Select(r => new VexObservationReference(r.Type, r.Url)).ToImmutableArray() ?? ImmutableArray<VexObservationReference>.Empty;
var reconciledFrom = record?.ReconciledFrom?.Where(NotNullOrWhiteSpace).Select(r => r.Trim()).ToImmutableArray() ?? ImmutableArray<string>.Empty;
var disagreements = record?.Disagreements?.Select(MapDisagreement).ToImmutableArray() ?? ImmutableArray<VexObservationDisagreement>.Empty;
var observationRefs = record?.Observations?.Select(o => new VexLinksetObservationRef(
o.ObservationId,
o.ProviderId,
o.Status,
o.Confidence)).ToImmutableArray() ?? ImmutableArray<VexLinksetObservationRef>.Empty;
return new VexObservationLinkset(aliases, purls, cpes, references, reconciledFrom, disagreements, observationRefs);
}
private static bool NotNullOrWhiteSpace(string? value) => !string.IsNullOrWhiteSpace(value);
private static string NormalizeTenant(string tenant)
{
if (string.IsNullOrWhiteSpace(tenant))
{
throw new ArgumentException("tenant is required", nameof(tenant));
}
return tenant.Trim().ToLowerInvariant();
}
}

View File

@@ -57,6 +57,7 @@ public static class VexMongoServiceCollectionExtensions
services.AddScoped<IVexCacheMaintenance, MongoVexCacheMaintenance>();
services.AddScoped<IVexConnectorStateRepository, MongoVexConnectorStateRepository>();
services.AddScoped<VexStatementBackfillService>();
services.AddScoped<IVexObservationLookup, MongoVexObservationLookup>();
services.AddSingleton<IVexMongoMigration, VexInitialIndexMigration>();
services.AddSingleton<IVexMongoMigration, VexConsensusSignalsMigration>();
services.AddSingleton<IVexMongoMigration, VexConsensusHoldMigration>();

View File

@@ -292,13 +292,21 @@ internal sealed class VexObservationRecord
public string ProviderId { get; set; } = default!;
public string StreamId { get; set; } = string.Empty;
public string Status { get; set; } = default!;
public VexObservationDocumentRecord Document { get; set; } = new();
public DateTime CreatedAt { get; set; } = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc);
public VexObservationUpstreamRecord Upstream { get; set; } = new();
public List<VexLinksetDisagreementRecord> Disagreements { get; set; } = new();
public VexObservationContentRecord Content { get; set; } = new();
public List<VexObservationStatementRecord> Statements { get; set; } = new();
public VexObservationLinksetRecord Linkset { get; set; } = new();
public DateTime CreatedAt { get; set; } = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc);
}
[BsonIgnoreExtraElements]
@@ -308,6 +316,122 @@ internal sealed class VexObservationDocumentRecord
public string? SourceUri { get; set; }
= null;
public string? Format { get; set; }
= null;
public string? Revision { get; set; }
= null;
public VexObservationSignatureRecord? Signature { get; set; }
= null;
}
[BsonIgnoreExtraElements]
internal sealed class VexObservationSignatureRecord
{
public bool Present { get; set; }
= false;
public string? Subject { get; set; }
= null;
public string? Issuer { get; set; }
= null;
public DateTimeOffset? VerifiedAt { get; set; }
= null;
}
[BsonIgnoreExtraElements]
internal sealed class VexObservationUpstreamRecord
{
public string UpstreamId { get; set; } = default!;
public string? DocumentVersion { get; set; }
= null;
public DateTimeOffset FetchedAt { get; set; } = DateTimeOffset.UtcNow;
public DateTimeOffset ReceivedAt { get; set; } = DateTimeOffset.UtcNow;
public string ContentHash { get; set; } = default!;
public VexObservationSignatureRecord Signature { get; set; } = new();
}
[BsonIgnoreExtraElements]
internal sealed class VexObservationContentRecord
{
public string Format { get; set; } = "unknown";
public string? SpecVersion { get; set; }
= null;
public BsonDocument Raw { get; set; } = new();
}
[BsonIgnoreExtraElements]
internal sealed class VexObservationStatementRecord
{
public string VulnerabilityId { get; set; } = default!;
public string ProductKey { get; set; } = default!;
public string Status { get; set; } = default!;
public DateTimeOffset? LastObserved { get; set; }
= null;
public string? Locator { get; set; }
= null;
public string? Justification { get; set; }
= null;
public string? IntroducedVersion { get; set; }
= null;
public string? FixedVersion { get; set; }
= null;
public string? Detail { get; set; }
= null;
public double? ScopeScore { get; set; }
= null;
public double? Epss { get; set; }
= null;
public double? Kev { get; set; }
= null;
}
[BsonIgnoreExtraElements]
internal sealed class VexObservationLinksetRecord
{
public List<string> Aliases { get; set; } = new();
public List<string> Purls { get; set; } = new();
public List<string> Cpes { get; set; } = new();
public List<VexObservationReferenceRecord> References { get; set; } = new();
public List<string> ReconciledFrom { get; set; } = new();
public List<VexLinksetDisagreementRecord> Disagreements { get; set; } = new();
public List<VexObservationLinksetObservationRecord> Observations { get; set; } = new();
}
[BsonIgnoreExtraElements]
internal sealed class VexObservationReferenceRecord
{
public string Type { get; set; } = default!;
public string Url { get; set; } = default!;
}
[BsonIgnoreExtraElements]
@@ -324,6 +448,10 @@ internal sealed class VexLinksetRecord
public string ProductKey { get; set; } = default!;
public List<string> ProviderIds { get; set; } = new();
public List<string> Statuses { get; set; } = new();
public DateTime CreatedAt { get; set; } = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc);
public List<VexLinksetDisagreementRecord> Disagreements { get; set; } = new();
@@ -343,6 +471,19 @@ internal sealed class VexLinksetDisagreementRecord
= null;
}
[BsonIgnoreExtraElements]
internal sealed class VexObservationLinksetObservationRecord
{
public string ObservationId { get; set; } = default!;
public string ProviderId { get; set; } = default!;
public string Status { get; set; } = default!;
public double? Confidence { get; set; }
= null;
}
[BsonIgnoreExtraElements]
internal sealed class VexProviderRecord
{

View File

@@ -70,6 +70,26 @@ public sealed class VexEvidenceChunksEndpointTests : IDisposable
Assert.Equal("CVE-2025-0001", chunk.VulnerabilityId);
}
[Fact]
public async Task ChunksEndpoint_Sets_Results_Headers()
{
using var client = _factory.CreateClient(new WebApplicationFactoryClientOptions { AllowAutoRedirect = false });
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "vex.read");
client.DefaultRequestHeaders.Add("X-Stella-Tenant", "tests");
// No provider filter; limit forces truncation so headers should reflect total > limit.
var response = await client.GetAsync("/v1/vex/evidence/chunks?vulnerabilityId=CVE-2025-0001&productKey=pkg:docker/demo&limit=1");
response.EnsureSuccessStatusCode();
Assert.Equal("application/x-ndjson", response.Content.Headers.ContentType?.MediaType);
Assert.True(response.Headers.TryGetValues("Excititor-Results-Total", out var totalValues));
Assert.Equal("3", totalValues.Single());
Assert.True(response.Headers.TryGetValues("Excititor-Results-Truncated", out var truncatedValues));
Assert.Equal("true", truncatedValues.Single(), ignoreCase: true);
}
private void SeedStatements()
{
var client = new MongoClient(_runner.ConnectionString);

View File

@@ -0,0 +1,110 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http.Json;
using EphemeralMongo;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.Extensions.Configuration;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.WebService.Contracts;
using Xunit;
namespace StellaOps.Excititor.WebService.Tests;
public sealed class VexObservationListEndpointTests : IDisposable
{
private readonly IMongoRunner _runner;
private readonly TestWebApplicationFactory _factory;
public VexObservationListEndpointTests()
{
_runner = MongoRunner.Run(new MongoRunnerOptions { UseSingleNodeReplicaSet = true });
_factory = new TestWebApplicationFactory(
configureConfiguration: configuration =>
{
configuration.AddInMemoryCollection(new Dictionary<string, string?>
{
["Excititor:Storage:Mongo:ConnectionString"] = _runner.ConnectionString,
["Excititor:Storage:Mongo:DatabaseName"] = "observations_tests",
["Excititor:Storage:Mongo:DefaultTenant"] = "tests",
});
},
configureServices: services =>
{
TestServiceOverrides.Apply(services);
services.AddTestAuthentication();
});
SeedObservation();
}
[Fact]
public async Task ObservationsEndpoint_ReturnsFilteredResults()
{
using var client = _factory.CreateClient(new WebApplicationFactoryClientOptions { AllowAutoRedirect = false });
client.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", "vex.read");
client.DefaultRequestHeaders.Add("X-Stella-Tenant", "tests");
var response = await client.GetAsync("/v1/vex/observations?providerId=provider-a&status=affected&limit=10");
response.EnsureSuccessStatusCode();
var payload = await response.Content.ReadFromJsonAsync<VexObservationListResponse>();
Assert.NotNull(payload);
Assert.Single(payload!.Items);
var item = payload.Items[0];
Assert.Equal("obs-1", item.ObservationId);
Assert.Equal("provider-a", item.ProviderId);
Assert.Equal("cve-2025-0001", item.VulnerabilityId);
Assert.Equal("pkg:demo/app", item.ProductKey);
Assert.Equal("affected", item.Status);
Assert.Equal(new[] { "pkg:demo/app" }, item.Purls);
}
private void SeedObservation()
{
var client = new MongoClient(_runner.ConnectionString);
var database = client.GetDatabase("observations_tests");
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Observations);
var record = new BsonDocument
{
{ "_id", "obs-1" },
{ "Tenant", "tests" },
{ "ObservationId", "obs-1" },
{ "VulnerabilityId", "cve-2025-0001" },
{ "ProductKey", "pkg:demo/app" },
{ "ProviderId", "provider-a" },
{ "Status", "affected" },
{ "StreamId", "stream" },
{ "CreatedAt", DateTime.UtcNow },
{ "Document", new BsonDocument { { "Digest", "digest-1" }, { "Format", "csaf" }, { "SourceUri", "https://example.test/vex.json" } } },
{ "Upstream", new BsonDocument { { "UpstreamId", "up-1" }, { "ContentHash", "sha256:digest-1" }, { "Signature", new BsonDocument { { "Present", true }, { "Subject", "sub" }, { "Issuer", "iss" }, { "VerifiedAt", DateTime.UtcNow } } } } },
{ "Content", new BsonDocument { { "Format", "csaf" }, { "Raw", new BsonDocument { { "document", "payload" } } } } },
{ "Statements", new BsonArray
{
new BsonDocument
{
{ "VulnerabilityId", "cve-2025-0001" },
{ "ProductKey", "pkg:demo/app" },
{ "Status", "affected" },
{ "LastObserved", DateTime.UtcNow },
{ "Purl", "pkg:demo/app" }
}
}
},
{ "Linkset", new BsonDocument { { "Purls", new BsonArray { "pkg:demo/app" } } } }
};
collection.InsertOne(record);
}
public void Dispose()
{
_factory.Dispose();
_runner.Dispose();
}
}