up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Policy Simulation / policy-simulate (push) Has been cancelled
SDK Publish & Sign / sdk-publish (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
devportal-offline / build-offline (push) Has been cancelled

This commit is contained in:
StellaOps Bot
2025-11-25 22:09:44 +02:00
parent 6bee1fdcf5
commit 9f6e6f7fb3
116 changed files with 4495 additions and 730 deletions

View File

@@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Excititor.WebService.Contracts;
public sealed record GraphStatusResponse(
[property: JsonPropertyName("items")] IReadOnlyList<GraphStatusItem> Items,
[property: JsonPropertyName("cached")] bool Cached,
[property: JsonPropertyName("cacheAgeMs")] long? CacheAgeMs);
public sealed record GraphStatusItem(
[property: JsonPropertyName("purl")] string Purl,
[property: JsonPropertyName("summary")] GraphOverlaySummary Summary,
[property: JsonPropertyName("latestModifiedAt")] DateTimeOffset? LatestModifiedAt,
[property: JsonPropertyName("sources")] IReadOnlyList<string> Sources,
[property: JsonPropertyName("lastEvidenceHash")] string? LastEvidenceHash);

View File

@@ -0,0 +1,25 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Excititor.WebService.Contracts;
public sealed record GraphTooltipResponse(
[property: JsonPropertyName("items")] IReadOnlyList<GraphTooltipItem> Items,
[property: JsonPropertyName("nextCursor")] string? NextCursor,
[property: JsonPropertyName("hasMore")] bool HasMore);
public sealed record GraphTooltipItem(
[property: JsonPropertyName("purl")] string Purl,
[property: JsonPropertyName("observations")] IReadOnlyList<GraphTooltipObservation> Observations,
[property: JsonPropertyName("truncated")] bool Truncated);
public sealed record GraphTooltipObservation(
[property: JsonPropertyName("observationId")] string ObservationId,
[property: JsonPropertyName("advisoryId")] string AdvisoryId,
[property: JsonPropertyName("status")] string Status,
[property: JsonPropertyName("justification")] string? Justification,
[property: JsonPropertyName("providerId")] string ProviderId,
[property: JsonPropertyName("modifiedAt")] DateTimeOffset ModifiedAt,
[property: JsonPropertyName("evidenceHash")] string EvidenceHash,
[property: JsonPropertyName("dsseEnvelopeHash")] string? DsseEnvelopeHash);

View File

@@ -0,0 +1,36 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Excititor.Core.Observations;
using StellaOps.Excititor.WebService.Contracts;
namespace StellaOps.Excititor.WebService.Graph;
internal static class GraphStatusFactory
{
public static IReadOnlyList<GraphStatusItem> Build(
IReadOnlyList<string> orderedPurls,
IReadOnlyList<VexObservation> observations)
{
if (orderedPurls is null)
{
throw new ArgumentNullException(nameof(orderedPurls));
}
if (observations is null)
{
throw new ArgumentNullException(nameof(observations));
}
var overlays = GraphOverlayFactory.Build(orderedPurls, observations, includeJustifications: false);
return overlays
.Select(overlay => new GraphStatusItem(
overlay.Purl,
overlay.Summary,
overlay.LatestModifiedAt,
overlay.Provenance.Sources,
overlay.Provenance.LastEvidenceHash))
.ToList();
}
}

View File

@@ -0,0 +1,127 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using StellaOps.Excititor.Core.Observations;
using StellaOps.Excititor.WebService.Contracts;
namespace StellaOps.Excititor.WebService.Graph;
internal static class GraphTooltipFactory
{
public static IReadOnlyList<GraphTooltipItem> Build(
IReadOnlyList<string> orderedPurls,
IReadOnlyList<VexObservation> observations,
bool includeJustifications,
int maxItemsPerPurl)
{
if (orderedPurls is null)
{
throw new ArgumentNullException(nameof(orderedPurls));
}
if (observations is null)
{
throw new ArgumentNullException(nameof(observations));
}
if (maxItemsPerPurl <= 0)
{
throw new ArgumentOutOfRangeException(nameof(maxItemsPerPurl));
}
var requested = new HashSet<string>(orderedPurls, StringComparer.OrdinalIgnoreCase);
var byPurl = orderedPurls.ToDictionary(
keySelector: static purl => purl,
elementSelector: static _ => new List<GraphTooltipObservation>(),
comparer: StringComparer.OrdinalIgnoreCase);
foreach (var observation in observations)
{
var linksetPurls = observation.Linkset.Purls;
foreach (var statement in observation.Statements)
{
var targets = CollectTargets(statement, linksetPurls, requested);
if (targets.Count == 0)
{
continue;
}
var payload = new GraphTooltipObservation(
observation.ObservationId,
statement.VulnerabilityId,
statement.Status.ToString().ToLowerInvariant(),
includeJustifications ? statement.Justification?.ToString() : null,
observation.ProviderId,
observation.CreatedAt,
observation.Upstream.ContentHash,
observation.Upstream.Signature.Signature);
foreach (var target in targets)
{
byPurl[target].Add(payload);
}
}
}
var items = new List<GraphTooltipItem>(orderedPurls.Count);
foreach (var purl in orderedPurls)
{
if (!byPurl.TryGetValue(purl, out var observationsForPurl))
{
items.Add(new GraphTooltipItem(purl, Array.Empty<GraphTooltipObservation>(), false));
continue;
}
var ordered = observationsForPurl
.OrderByDescending(static o => o.ModifiedAt)
.ThenBy(static o => o.AdvisoryId, StringComparer.Ordinal)
.ThenBy(static o => o.ProviderId, StringComparer.OrdinalIgnoreCase)
.ThenBy(static o => o.ObservationId, StringComparer.Ordinal)
.ToList();
var truncated = ordered.Count > maxItemsPerPurl;
var limited = truncated ? ordered.Take(maxItemsPerPurl).ToList() : ordered;
items.Add(new GraphTooltipItem(purl, limited, truncated));
}
return items;
}
private static List<string> CollectTargets(
VexObservationStatement statement,
ImmutableArray<string> linksetPurls,
HashSet<string> requested)
{
var targets = new List<string>();
if (!string.IsNullOrWhiteSpace(statement.Purl))
{
var normalized = statement.Purl.ToLowerInvariant();
if (requested.Contains(normalized))
{
targets.Add(normalized);
}
}
if (targets.Count > 0)
{
return targets;
}
if (!linksetPurls.IsDefaultOrEmpty)
{
foreach (var purl in linksetPurls)
{
var normalized = purl?.ToLowerInvariant();
if (normalized is not null && requested.Contains(normalized))
{
targets.Add(normalized);
}
}
}
return targets;
}
}

View File

@@ -8,4 +8,6 @@ public sealed class GraphOptions
public int MaxPurls { get; set; } = 500;
public int MaxAdvisoriesPerPurl { get; set; } = 200;
public int OverlayTtlSeconds { get; set; } = 300;
public int MaxTooltipItemsPerPurl { get; set; } = 50;
public int MaxTooltipTotal { get; set; } = 1000;
}

View File

@@ -262,6 +262,10 @@ public partial class Program
signature.VerifiedAt));
}
private sealed record CachedGraphStatus(
IReadOnlyList<GraphStatusItem> Items,
DateTimeOffset CachedAt);
private sealed record CachedGraphOverlay(
IReadOnlyList<GraphOverlayItem> Items,
DateTimeOffset CachedAt);

View File

@@ -896,6 +896,64 @@ var response = new GraphLinkoutsResponse(items, notFound);
return Results.Ok(response);
}).WithName("PostGraphLinkouts");
app.MapGet("/v1/graph/status", async (
HttpContext context,
[FromQuery(Name = "purl")] string[]? purls,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<GraphOptions> graphOptions,
IVexObservationQueryService queryService,
IMemoryCache cache,
TimeProvider timeProvider,
CancellationToken cancellationToken) =>
{
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: true, out var tenant, out var tenantError))
{
return tenantError;
}
var orderedPurls = NormalizePurls(purls);
if (orderedPurls.Count == 0)
{
return Results.BadRequest("purl query parameter is required");
}
if (orderedPurls.Count > graphOptions.Value.MaxPurls)
{
return Results.BadRequest($"purls limit exceeded (max {graphOptions.Value.MaxPurls})");
}
var cacheKey = $"graph-status:{tenant}:{string.Join('|', orderedPurls)}";
var now = timeProvider.GetUtcNow();
if (cache.TryGetValue<CachedGraphStatus>(cacheKey, out var cached) && cached is not null)
{
var ageMs = (long)Math.Max(0, (now - cached.CachedAt).TotalMilliseconds);
return Results.Ok(new GraphStatusResponse(cached.Items, true, ageMs));
}
var options = new VexObservationQueryOptions(
tenant: tenant,
purls: orderedPurls,
limit: graphOptions.Value.MaxAdvisoriesPerPurl * orderedPurls.Count);
VexObservationQueryResult result;
try
{
result = await queryService.QueryAsync(options, cancellationToken).ConfigureAwait(false);
}
catch (FormatException ex)
{
return Results.BadRequest(ex.Message);
}
var items = GraphStatusFactory.Build(orderedPurls, result.Observations);
var response = new GraphStatusResponse(items, false, null);
cache.Set(cacheKey, new CachedGraphStatus(items, now), TimeSpan.FromSeconds(graphOptions.Value.OverlayTtlSeconds));
return Results.Ok(response);
}).WithName("GetGraphStatus");
// Cartographer overlays
app.MapGet("/v1/graph/overlays", async (
HttpContext context,
@@ -956,6 +1014,66 @@ app.MapGet("/v1/graph/overlays", async (
return Results.Ok(response);
}).WithName("GetGraphOverlays");
app.MapGet("/v1/graph/observations", async (
HttpContext context,
[FromQuery(Name = "purl")] string[]? purls,
[FromQuery] bool includeJustifications,
[FromQuery] int? limitPerPurl,
[FromQuery] string? cursor,
IOptions<VexMongoStorageOptions> storageOptions,
IOptions<GraphOptions> graphOptions,
IVexObservationQueryService queryService,
CancellationToken cancellationToken) =>
{
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: true, out var tenant, out var tenantError))
{
return tenantError;
}
var orderedPurls = NormalizePurls(purls);
if (orderedPurls.Count == 0)
{
return Results.BadRequest("purl query parameter is required");
}
if (orderedPurls.Count > graphOptions.Value.MaxPurls)
{
return Results.BadRequest($"purls limit exceeded (max {graphOptions.Value.MaxPurls})");
}
var perPurlLimit = limitPerPurl.GetValueOrDefault(graphOptions.Value.MaxTooltipItemsPerPurl);
if (perPurlLimit <= 0)
{
return Results.BadRequest("limitPerPurl must be greater than zero when provided.");
}
var effectivePerPurlLimit = Math.Min(perPurlLimit, graphOptions.Value.MaxAdvisoriesPerPurl);
var totalLimit = Math.Min(
Math.Max(1, effectivePerPurlLimit * orderedPurls.Count),
graphOptions.Value.MaxTooltipTotal);
var options = new VexObservationQueryOptions(
tenant: tenant,
purls: orderedPurls,
limit: totalLimit,
cursor: cursor);
VexObservationQueryResult result;
try
{
result = await queryService.QueryAsync(options, cancellationToken).ConfigureAwait(false);
}
catch (FormatException ex)
{
return Results.BadRequest(ex.Message);
}
var items = GraphTooltipFactory.Build(orderedPurls, result.Observations, includeJustifications, effectivePerPurlLimit);
var response = new GraphTooltipResponse(items, result.NextCursor, result.HasMore);
return Results.Ok(response);
}).WithName("GetGraphObservations");
app.MapPost("/ingest/vex", async (
HttpContext context,
VexIngestRequest request,

View File

@@ -0,0 +1,115 @@
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Bson;
using MongoDB.Driver;
namespace StellaOps.Excititor.Storage.Mongo.Migrations;
/// <summary>
/// Adds a $jsonSchema validator to the raw VEX collection to enforce aggregation-only
/// shape (immutable content hash + provenance fields).
/// ValidationAction=warn keeps rollout safe while surfacing violations.
/// </summary>
internal sealed class VexRawSchemaMigration : IVexMongoMigration
{
public string Id => "20251125-vex-raw-json-schema";
public async ValueTask ExecuteAsync(IMongoDatabase database, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(database);
var exists = await CollectionExistsAsync(database, VexMongoCollectionNames.Raw, cancellationToken)
.ConfigureAwait(false);
var validator = BuildValidator();
if (!exists)
{
await database.CreateCollectionAsync(
VexMongoCollectionNames.Raw,
new CreateCollectionOptions
{
Validator = validator,
ValidationAction = DocumentValidationAction.Warn,
ValidationLevel = DocumentValidationLevel.Moderate,
},
cancellationToken).ConfigureAwait(false);
return;
}
var command = new BsonDocument
{
{ "collMod", VexMongoCollectionNames.Raw },
{ "validator", validator },
{ "validationAction", "warn" },
{ "validationLevel", "moderate" },
};
await database.RunCommandAsync<BsonDocument>(command, cancellationToken: cancellationToken)
.ConfigureAwait(false);
}
private static async Task<bool> CollectionExistsAsync(
IMongoDatabase database,
string name,
CancellationToken cancellationToken)
{
using var cursor = await database.ListCollectionNamesAsync(
new ListCollectionNamesOptions { Filter = new BsonDocument("name", name) },
cancellationToken).ConfigureAwait(false);
return await cursor.AnyAsync(cancellationToken).ConfigureAwait(false);
}
private static BsonDocument BuildValidator()
{
var properties = new BsonDocument
{
{ "_id", new BsonDocument { { "bsonType", "string" }, { "description", "digest" } } },
{ "providerId", new BsonDocument { { "bsonType", "string" }, { "minLength", 1 } } },
{ "format", new BsonDocument
{
{ "bsonType", "string" },
{ "enum", new BsonArray { "csaf", "cyclonedx", "openvex" } }
}
},
{ "sourceUri", new BsonDocument { { "bsonType", "string" }, { "minLength", 1 } } },
{ "retrievedAt", new BsonDocument { { "bsonType", "date" } } },
{ "digest", new BsonDocument { { "bsonType", "string" }, { "minLength", 32 } } },
{ "content", new BsonDocument
{
{ "bsonType", new BsonArray { "binData", "string" } }
}
},
{ "gridFsObjectId", new BsonDocument
{
{ "bsonType", new BsonArray { "objectId", "null", "string" } }
}
},
{ "metadata", new BsonDocument
{
{ "bsonType", "object" },
{ "additionalProperties", true },
{ "patternProperties", new BsonDocument
{
{ ".*", new BsonDocument { { "bsonType", "string" } } }
}
}
}
}
};
return new BsonDocument
{
{
"$jsonSchema",
new BsonDocument
{
{ "bsonType", "object" },
{ "required", new BsonArray { "_id", "providerId", "format", "sourceUri", "retrievedAt", "digest" } },
{ "properties", properties },
{ "additionalProperties", true }
}
}
};
}
}

View File

@@ -61,6 +61,7 @@ public static class VexMongoServiceCollectionExtensions
services.AddScoped<VexStatementBackfillService>();
services.AddScoped<IVexObservationLookup, MongoVexObservationLookup>();
services.AddSingleton<IVexMongoMigration, VexInitialIndexMigration>();
services.AddSingleton<IVexMongoMigration, VexRawSchemaMigration>();
services.AddSingleton<IVexMongoMigration, VexConsensusSignalsMigration>();
services.AddSingleton<IVexMongoMigration, VexConsensusHoldMigration>();
services.AddSingleton<IVexMongoMigration, VexObservationCollectionsMigration>();

View File

@@ -0,0 +1,90 @@
using System;
using System.Collections.Immutable;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Observations;
using StellaOps.Excititor.WebService.Graph;
using Xunit;
namespace StellaOps.Excititor.WebService.Tests;
public sealed class GraphStatusFactoryTests
{
[Fact]
public void Build_ProjectsOverlaySummariesAndProvenance()
{
var now = DateTimeOffset.UtcNow;
var observations = new[]
{
CreateObservation(
providerId: "ubuntu",
createdAt: now,
purls: new[] { "pkg:rpm/redhat/openssl@1.1.1" },
statements: new[]
{
new VexObservationStatement(
vulnerabilityId: "CVE-2025-1001",
productKey: "pkg:rpm/redhat/openssl@1.1.1",
status: VexClaimStatus.NotAffected,
lastObserved: now,
justification: VexJustification.ComponentNotPresent,
purl: "pkg:rpm/redhat/openssl@1.1.1")
},
contentHash: "hash-new"),
CreateObservation(
providerId: "oracle",
createdAt: now.AddMinutes(-2),
purls: new[] { "pkg:rpm/redhat/openssl@1.1.1" },
statements: Array.Empty<VexObservationStatement>(),
contentHash: "hash-old")
};
var items = GraphStatusFactory.Build(
orderedPurls: new[] { "pkg:rpm/redhat/openssl@1.1.1" },
observations: observations);
var item = Assert.Single(items);
Assert.Equal("pkg:rpm/redhat/openssl@1.1.1", item.Purl);
Assert.Equal(0, item.Summary.Open);
Assert.Equal(1, item.Summary.NotAffected);
Assert.Equal(0, item.Summary.UnderInvestigation);
Assert.Equal(1, item.Summary.NoStatement);
Assert.Equal(now, item.LatestModifiedAt);
Assert.Equal("hash-new", item.LastEvidenceHash);
Assert.Equal(new[] { "oracle", "ubuntu" }, item.Sources);
}
private static VexObservation CreateObservation(
string providerId,
DateTimeOffset createdAt,
string[] purls,
VexObservationStatement[] statements,
string contentHash)
{
return new VexObservation(
observationId: $"obs-{providerId}-{createdAt.ToUnixTimeMilliseconds()}",
tenant: "tenant-a",
providerId: providerId,
streamId: "csaf",
upstream: new VexObservationUpstream(
upstreamId: Guid.NewGuid().ToString("N"),
documentVersion: "1",
fetchedAt: createdAt,
receivedAt: createdAt,
contentHash: contentHash,
signature: new VexObservationSignature(present: true, format: "sig", keyId: null, signature: null)),
statements: statements.ToImmutableArray(),
content: new VexObservationContent(
format: "csaf",
specVersion: "1",
raw: System.Text.Json.Nodes.JsonValue.Create("raw")!,
metadata: ImmutableDictionary<string, string>.Empty),
linkset: new VexObservationLinkset(
aliases: Array.Empty<string>(),
purls: purls,
cpes: Array.Empty<string>(),
references: Array.Empty<VexObservationReference>()),
createdAt: createdAt,
supersedes: ImmutableArray<string>.Empty,
attributes: ImmutableDictionary<string, string>.Empty);
}
}

View File

@@ -0,0 +1,137 @@
using System;
using System.Collections.Immutable;
using System.Linq;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Observations;
using StellaOps.Excititor.WebService.Graph;
using Xunit;
namespace StellaOps.Excititor.WebService.Tests;
public sealed class GraphTooltipFactoryTests
{
[Fact]
public void Build_OrdersByNewestAndTruncatesPerPurl()
{
var now = DateTimeOffset.UtcNow;
var observations = new[]
{
CreateObservation(
providerId: "ubuntu",
createdAt: now,
purls: new[] { "pkg:rpm/openssl@1.1.1" },
statements: new[]
{
new VexObservationStatement(
vulnerabilityId: "CVE-2025-1000",
productKey: "pkg:rpm/openssl@1.1.1",
status: VexClaimStatus.NotAffected,
lastObserved: now,
justification: VexJustification.ComponentNotPresent,
purl: "pkg:rpm/openssl@1.1.1")
},
contentHash: "hash-ubuntu"),
CreateObservation(
providerId: "redhat",
createdAt: now.AddMinutes(-1),
purls: new[] { "pkg:rpm/openssl@1.1.1" },
statements: new[]
{
new VexObservationStatement(
vulnerabilityId: "CVE-2025-1000",
productKey: "pkg:rpm/openssl@1.1.1",
status: VexClaimStatus.UnderInvestigation,
lastObserved: now.AddMinutes(-1),
justification: null,
purl: "pkg:rpm/openssl@1.1.1")
},
contentHash: "hash-redhat")
};
var items = GraphTooltipFactory.Build(
orderedPurls: new[] { "pkg:rpm/openssl@1.1.1" },
observations: observations,
includeJustifications: true,
maxItemsPerPurl: 1);
var item = Assert.Single(items);
Assert.True(item.Truncated);
var obs = Assert.Single(item.Observations);
Assert.Equal("CVE-2025-1000", obs.AdvisoryId);
Assert.Equal("notaffected", obs.Status);
Assert.Equal("ComponentNotPresent", obs.Justification);
Assert.Equal("ubuntu", obs.ProviderId);
Assert.Equal("hash-ubuntu", obs.EvidenceHash);
}
[Fact]
public void Build_UsesLinksetPurlsWhenStatementMissing()
{
var now = DateTimeOffset.UtcNow;
var observations = new[]
{
CreateObservation(
providerId: "oracle",
createdAt: now,
purls: new[] { "pkg:rpm/httpd@2.4.0" },
statements: new[]
{
new VexObservationStatement(
vulnerabilityId: "CVE-2025-2000",
productKey: "pkg:rpm/httpd@2.4.0",
status: VexClaimStatus.Affected,
lastObserved: now,
justification: null,
purl: null)
},
contentHash: "hash-oracle")
};
var items = GraphTooltipFactory.Build(
orderedPurls: new[] { "pkg:rpm/httpd@2.4.0" },
observations: observations,
includeJustifications: false,
maxItemsPerPurl: 5);
var item = Assert.Single(items);
var observation = Assert.Single(item.Observations);
Assert.Null(observation.Justification);
Assert.Equal("oracle", observation.ProviderId);
Assert.Equal("CVE-2025-2000", observation.AdvisoryId);
}
private static VexObservation CreateObservation(
string providerId,
DateTimeOffset createdAt,
string[] purls,
VexObservationStatement[] statements,
string contentHash)
{
return new VexObservation(
observationId: $"obs-{providerId}-{createdAt.ToUnixTimeMilliseconds()}",
tenant: "tenant-a",
providerId: providerId,
streamId: "csaf",
upstream: new VexObservationUpstream(
upstreamId: Guid.NewGuid().ToString("N"),
documentVersion: "1",
fetchedAt: createdAt,
receivedAt: createdAt,
contentHash: contentHash,
signature: new VexObservationSignature(present: true, format: "sig", keyId: null, signature: null)),
statements: statements.ToImmutableArray(),
content: new VexObservationContent(
format: "csaf",
specVersion: "1",
raw: System.Text.Json.Nodes.JsonValue.Create("raw")!,
metadata: ImmutableDictionary<string, string>.Empty),
linkset: new VexObservationLinkset(
aliases: Array.Empty<string>(),
purls: purls,
cpes: Array.Empty<string>(),
references: Array.Empty<VexObservationReference>()),
createdAt: createdAt,
supersedes: ImmutableArray<string>.Empty,
attributes: ImmutableDictionary<string, string>.Empty);
}
}

View File

@@ -37,6 +37,8 @@
<Compile Include="TestServiceOverrides.cs" />
<Compile Include="TestWebApplicationFactory.cs" />
<Compile Include="GraphOverlayFactoryTests.cs" />
<Compile Include="GraphStatusFactoryTests.cs" />
<Compile Include="GraphTooltipFactoryTests.cs" />
<Compile Include="AttestationVerifyEndpointTests.cs" />
</ItemGroup>
</Project>