feat: Add initial implementation of Vulnerability Resolver Jobs
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Created project for StellaOps.Scanner.Analyzers.Native.Tests with necessary dependencies.
- Documented roles and guidelines in AGENTS.md for Scheduler module.
- Implemented IResolverJobService interface and InMemoryResolverJobService for handling resolver jobs.
- Added ResolverBacklogNotifier and ResolverBacklogService for monitoring job metrics.
- Developed API endpoints for managing resolver jobs and retrieving metrics.
- Defined models for resolver job requests and responses.
- Integrated dependency injection for resolver job services.
- Implemented ImpactIndexSnapshot for persisting impact index data.
- Introduced SignalsScoringOptions for configurable scoring weights in reachability scoring.
- Added unit tests for ReachabilityScoringService and RuntimeFactsIngestionService.
- Created dotnet-filter.sh script to handle command-line arguments for dotnet.
- Established nuget-prime project for managing package downloads.
This commit is contained in:
master
2025-11-18 07:52:15 +02:00
parent e69b57d467
commit 8355e2ff75
299 changed files with 13293 additions and 2444 deletions

View File

@@ -5,14 +5,13 @@ namespace StellaOps.Concelier.WebService.Contracts;
public sealed record AdvisoryStructuredFieldResponse(
string AdvisoryKey,
string Fingerprint,
int Total,
bool Truncated,
IReadOnlyList<AdvisoryStructuredFieldEntry> Entries);
public sealed record AdvisoryStructuredFieldEntry(
string Type,
string DocumentId,
string FieldPath,
string ChunkId,
AdvisoryStructuredFieldContent Content,
AdvisoryStructuredFieldProvenance Provenance);
@@ -65,6 +64,8 @@ public sealed record AdvisoryStructuredAffectedContent(
string? Status);
public sealed record AdvisoryStructuredFieldProvenance(
string DocumentId,
string ObservationPath,
string Source,
string Kind,
string? Value,

View File

@@ -1,16 +1,19 @@
using System.Collections.Immutable;
using StellaOps.Concelier.Models.Observations;
using StellaOps.Concelier.Models.Observations;
using StellaOps.Concelier.RawModels;
namespace StellaOps.Concelier.WebService.Contracts;
public sealed record AdvisoryObservationQueryResponse(
ImmutableArray<AdvisoryObservation> Observations,
AdvisoryObservationLinksetAggregateResponse Linkset,
string? NextCursor,
bool HasMore);
public sealed record AdvisoryObservationLinksetAggregateResponse(
ImmutableArray<string> Aliases,
ImmutableArray<string> Purls,
ImmutableArray<string> Cpes,
ImmutableArray<AdvisoryObservationReference> References);
public sealed record AdvisoryObservationQueryResponse(
ImmutableArray<AdvisoryObservation> Observations,
AdvisoryObservationLinksetAggregateResponse Linkset,
string? NextCursor,
bool HasMore);
public sealed record AdvisoryObservationLinksetAggregateResponse(
ImmutableArray<string> Aliases,
ImmutableArray<string> Purls,
ImmutableArray<string> Cpes,
ImmutableArray<AdvisoryObservationReference> References,
ImmutableArray<string> Scopes,
ImmutableArray<RawRelationship> Relationships);

View File

@@ -45,18 +45,26 @@ public sealed record AdvisoryIdentifiersRequest(
[property: JsonPropertyName("primary")] string Primary,
[property: JsonPropertyName("aliases")] IReadOnlyList<string>? Aliases);
public sealed record AdvisoryLinksetRequest(
[property: JsonPropertyName("aliases")] IReadOnlyList<string>? Aliases,
[property: JsonPropertyName("purls")] IReadOnlyList<string>? PackageUrls,
[property: JsonPropertyName("cpes")] IReadOnlyList<string>? Cpes,
[property: JsonPropertyName("references")] IReadOnlyList<AdvisoryLinksetReferenceRequest>? References,
[property: JsonPropertyName("reconciledFrom")] IReadOnlyList<string>? ReconciledFrom,
[property: JsonPropertyName("notes")] IDictionary<string, string>? Notes);
public sealed record AdvisoryLinksetReferenceRequest(
[property: JsonPropertyName("type")] string Type,
[property: JsonPropertyName("url")] string Url,
[property: JsonPropertyName("source")] string? Source);
public sealed record AdvisoryLinksetRequest(
[property: JsonPropertyName("aliases")] IReadOnlyList<string>? Aliases,
[property: JsonPropertyName("scopes")] IReadOnlyList<string>? Scopes,
[property: JsonPropertyName("relationships")] IReadOnlyList<AdvisoryLinksetRelationshipRequest>? Relationships,
[property: JsonPropertyName("purls")] IReadOnlyList<string>? PackageUrls,
[property: JsonPropertyName("cpes")] IReadOnlyList<string>? Cpes,
[property: JsonPropertyName("references")] IReadOnlyList<AdvisoryLinksetReferenceRequest>? References,
[property: JsonPropertyName("reconciledFrom")] IReadOnlyList<string>? ReconciledFrom,
[property: JsonPropertyName("notes")] IDictionary<string, string>? Notes);
public sealed record AdvisoryLinksetRelationshipRequest(
[property: JsonPropertyName("type")] string Type,
[property: JsonPropertyName("source")] string Source,
[property: JsonPropertyName("target")] string Target,
[property: JsonPropertyName("provenance")] string? Provenance);
public sealed record AdvisoryLinksetReferenceRequest(
[property: JsonPropertyName("type")] string Type,
[property: JsonPropertyName("url")] string Url,
[property: JsonPropertyName("source")] string? Source);
public sealed record AdvisoryIngestResponse(
[property: JsonPropertyName("id")] string Id,

View File

@@ -68,6 +68,8 @@ internal static class AdvisoryRawRequestMapper
var linkset = new RawLinkset
{
Aliases = NormalizeStrings(linksetRequest?.Aliases),
Scopes = NormalizeStrings(linksetRequest?.Scopes),
Relationships = NormalizeRelationships(linksetRequest?.Relationships),
PackageUrls = NormalizeStrings(linksetRequest?.PackageUrls),
Cpes = NormalizeStrings(linksetRequest?.Cpes),
References = NormalizeReferences(linksetRequest?.References),
@@ -135,7 +137,7 @@ internal static class AdvisoryRawRequestMapper
if (references is null)
{
return ImmutableArray<RawReference>.Empty;
}
}
var builder = ImmutableArray.CreateBuilder<RawReference>();
foreach (var reference in references)
@@ -151,10 +153,38 @@ internal static class AdvisoryRawRequestMapper
}
builder.Add(new RawReference(reference.Type.Trim(), reference.Url.Trim(), string.IsNullOrWhiteSpace(reference.Source) ? null : reference.Source.Trim()));
}
return builder.Count == 0 ? ImmutableArray<RawReference>.Empty : builder.ToImmutable();
}
}
return builder.Count == 0 ? ImmutableArray<RawReference>.Empty : builder.ToImmutable();
}
private static ImmutableArray<RawRelationship> NormalizeRelationships(IEnumerable<AdvisoryLinksetRelationshipRequest>? relationships)
{
if (relationships is null)
{
return ImmutableArray<RawRelationship>.Empty;
}
var builder = ImmutableArray.CreateBuilder<RawRelationship>();
foreach (var relationship in relationships)
{
if (relationship is null
|| string.IsNullOrWhiteSpace(relationship.Type)
|| string.IsNullOrWhiteSpace(relationship.Source)
|| string.IsNullOrWhiteSpace(relationship.Target))
{
continue;
}
builder.Add(new RawRelationship(
relationship.Type.Trim(),
relationship.Source.Trim(),
relationship.Target.Trim(),
string.IsNullOrWhiteSpace(relationship.Provenance) ? null : relationship.Provenance.Trim()));
}
return builder.Count == 0 ? ImmutableArray<RawRelationship>.Empty : builder.ToImmutable();
}
private static JsonElement NormalizeRawContent(JsonElement element)
{

View File

@@ -438,7 +438,9 @@ var observationsEndpoint = app.MapGet("/concelier/observations", async (
result.Linkset.Aliases,
result.Linkset.Purls,
result.Linkset.Cpes,
result.Linkset.References),
result.Linkset.References,
result.Linkset.Scopes,
result.Linkset.Relationships),
result.NextCursor,
result.HasMore);
@@ -861,6 +863,7 @@ var advisoryChunksEndpoint = app.MapGet("/advisories/{advisoryKey}/chunks", asyn
var formatFilter = BuildFilterSet(context.Request.Query["format"]);
var resolution = await ResolveAdvisoryAsync(
tenant,
normalizedKey,
advisoryStore,
aliasStore,
@@ -891,6 +894,7 @@ var advisoryChunksEndpoint = app.MapGet("/advisories/{advisoryKey}/chunks", asyn
var observations = observationResult.Observations.ToArray();
var buildOptions = new AdvisoryChunkBuildOptions(
advisory.AdvisoryKey,
fingerprint,
chunkLimit,
observationLimit,
sectionFilter,
@@ -1319,11 +1323,17 @@ IResult? EnsureTenantAuthorized(HttpContext context, string tenant)
}
async Task<(Advisory Advisory, ImmutableArray<string> Aliases, string Fingerprint)?> ResolveAdvisoryAsync(
string tenant,
string advisoryKey,
IAdvisoryStore advisoryStore,
IAliasStore aliasStore,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(tenant))
{
return null;
}
ArgumentNullException.ThrowIfNull(advisoryStore);
ArgumentNullException.ThrowIfNull(aliasStore);

View File

@@ -12,6 +12,7 @@ namespace StellaOps.Concelier.WebService.Services;
internal sealed record AdvisoryChunkBuildOptions(
string AdvisoryKey,
string Fingerprint,
int ChunkLimit,
int ObservationLimit,
ImmutableHashSet<string> SectionFilter,
@@ -56,9 +57,7 @@ internal sealed class AdvisoryChunkBuilder
var vendorIndex = new ObservationIndex(observations);
var chunkLimit = Math.Max(1, options.ChunkLimit);
var entries = new List<AdvisoryStructuredFieldEntry>(chunkLimit);
var total = 0;
var truncated = false;
var entries = new List<AdvisoryStructuredFieldEntry>();
var sectionFilter = options.SectionFilter ?? ImmutableHashSet<string>.Empty;
foreach (var section in SectionOrder)
@@ -82,31 +81,25 @@ internal sealed class AdvisoryChunkBuilder
continue;
}
total += bucket.Count;
if (entries.Count >= chunkLimit)
{
truncated = true;
continue;
}
var remaining = chunkLimit - entries.Count;
if (bucket.Count <= remaining)
{
entries.AddRange(bucket);
}
else
{
entries.AddRange(bucket.Take(remaining));
truncated = true;
}
entries.AddRange(bucket);
}
var ordered = entries
.OrderBy(static entry => entry.Type, StringComparer.Ordinal)
.ThenBy(static entry => entry.Provenance.ObservationPath, StringComparer.Ordinal)
.ThenBy(static entry => entry.Provenance.DocumentId, StringComparer.Ordinal)
.ToArray();
var total = ordered.Length;
var truncated = total > chunkLimit;
var limited = truncated ? ordered.Take(chunkLimit).ToArray() : ordered;
var response = new AdvisoryStructuredFieldResponse(
options.AdvisoryKey,
options.Fingerprint,
total,
truncated,
entries);
limited);
var telemetry = new AdvisoryChunkTelemetrySummary(
vendorIndex.SourceCount,
@@ -284,11 +277,11 @@ internal sealed class AdvisoryChunkBuilder
return new AdvisoryStructuredFieldEntry(
type,
documentId,
fieldPath,
chunkId,
content,
new AdvisoryStructuredFieldProvenance(
documentId,
fieldPath,
provenance.Source,
provenance.Kind,
provenance.Value,

View File

@@ -13,6 +13,8 @@ public sealed record AdvisoryLinkset(
ImmutableArray<string> ObservationIds,
AdvisoryLinksetNormalized? Normalized,
AdvisoryLinksetProvenance? Provenance,
double? Confidence,
IReadOnlyList<AdvisoryLinksetConflict>? Conflicts,
DateTimeOffset CreatedAt,
string? BuiltByJobId);
@@ -34,6 +36,11 @@ public sealed record AdvisoryLinksetProvenance(
string? ToolVersion,
string? PolicyHash);
public sealed record AdvisoryLinksetConflict(
string Field,
string Reason,
IReadOnlyList<string>? Values);
internal static class BsonDocumentHelper
{
public static BsonDocument FromDictionary(Dictionary<string, object?> dictionary)

View File

@@ -1,3 +1,4 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
@@ -55,6 +56,8 @@ internal sealed class AdvisoryLinksetBackfillService : IAdvisoryLinksetBackfillS
observationIds,
normalized,
null,
1.0,
Array.Empty<AdvisoryLinksetConflict>(),
createdAt,
null);

View File

@@ -24,6 +24,19 @@ internal static class AdvisoryLinksetNormalization
return Build(purls);
}
public static (AdvisoryLinksetNormalized? normalized, double? confidence, IReadOnlyList<AdvisoryLinksetConflict> conflicts) FromRawLinksetWithConfidence(
RawLinkset linkset,
double? providedConfidence = null)
{
ArgumentNullException.ThrowIfNull(linkset);
var normalized = Build(linkset.PackageUrls);
var confidence = CoerceConfidence(providedConfidence);
var conflicts = ExtractConflicts(linkset);
return (normalized, confidence, conflicts);
}
private static AdvisoryLinksetNormalized? Build(IEnumerable<string> purlValues)
{
var normalizedPurls = NormalizePurls(purlValues);
@@ -75,4 +88,44 @@ internal static class AdvisoryLinksetNormalization
return versions.ToList();
}
private static double? CoerceConfidence(double? confidence)
{
if (!confidence.HasValue)
{
return null;
}
if (double.IsNaN(confidence.Value) || double.IsInfinity(confidence.Value))
{
return null;
}
return Math.Clamp(confidence.Value, 0d, 1d);
}
private static IReadOnlyList<AdvisoryLinksetConflict> ExtractConflicts(RawLinkset linkset)
{
if (linkset.Notes is null || linkset.Notes.Count == 0)
{
return Array.Empty<AdvisoryLinksetConflict>();
}
var conflicts = new List<AdvisoryLinksetConflict>();
foreach (var note in linkset.Notes)
{
if (string.IsNullOrWhiteSpace(note.Key) || string.IsNullOrWhiteSpace(note.Value))
{
continue;
}
conflicts.Add(new AdvisoryLinksetConflict(
note.Key.Trim(),
note.Value.Trim(),
null));
}
return conflicts;
}
}

View File

@@ -0,0 +1,15 @@
using System.Collections.Immutable;
using StellaOps.Concelier.Models.Observations;
namespace StellaOps.Concelier.Core.Observations;
/// <summary>
/// Aggregated linkset facets (aliases, purls, cpes, references, scopes, relationships) built from a set of observations.
/// </summary>
public sealed record AdvisoryObservationLinksetAggregate(
ImmutableArray<string> Aliases,
ImmutableArray<string> Purls,
ImmutableArray<string> Cpes,
ImmutableArray<AdvisoryObservationReference> References,
ImmutableArray<string> Scopes,
ImmutableArray<RawRelationship> Relationships);

View File

@@ -1,6 +1,7 @@
using System.Collections.Immutable;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Models.Observations;
using System.Collections.Immutable;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Models.Observations;
using StellaOps.Concelier.RawModels;
namespace StellaOps.Concelier.Core.Observations;
@@ -66,17 +67,19 @@ public sealed record AdvisoryObservationQueryOptions
/// <summary>
/// Query result containing observations and their aggregated linkset hints.
/// </summary>
public sealed record AdvisoryObservationQueryResult(
ImmutableArray<AdvisoryObservation> Observations,
AdvisoryObservationLinksetAggregate Linkset,
string? NextCursor,
bool HasMore);
/// <summary>
/// Aggregated linkset built from the observations returned by a query.
/// </summary>
public sealed record AdvisoryObservationLinksetAggregate(
ImmutableArray<string> Aliases,
ImmutableArray<string> Purls,
ImmutableArray<string> Cpes,
ImmutableArray<AdvisoryObservationReference> References);
public sealed record AdvisoryObservationQueryResult(
ImmutableArray<AdvisoryObservation> Observations,
AdvisoryObservationLinksetAggregate Linkset,
string? NextCursor,
bool HasMore);
/// <summary>
/// Aggregated linkset built from the observations returned by a query.
/// </summary>
public sealed record AdvisoryObservationLinksetAggregate(
ImmutableArray<string> Aliases,
ImmutableArray<string> Purls,
ImmutableArray<string> Cpes,
ImmutableArray<AdvisoryObservationReference> References,
ImmutableArray<string> Scopes,
ImmutableArray<RawRelationship> Relationships);

View File

@@ -1,8 +1,9 @@
using System.Collections.Immutable;
using System.Globalization;
using System.Text;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Models.Observations;
using System.Collections.Immutable;
using System.Globalization;
using System.Text;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Models.Observations;
using StellaOps.Concelier.RawModels;
namespace StellaOps.Concelier.Core.Observations;
@@ -195,24 +196,28 @@ public sealed class AdvisoryObservationQueryService : IAdvisoryObservationQueryS
private static AdvisoryObservationLinksetAggregate BuildAggregateLinkset(ImmutableArray<AdvisoryObservation> observations)
{
if (observations.IsDefaultOrEmpty)
{
return new AdvisoryObservationLinksetAggregate(
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty,
ImmutableArray<AdvisoryObservationReference>.Empty);
}
if (observations.IsDefaultOrEmpty)
{
return new AdvisoryObservationLinksetAggregate(
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty,
ImmutableArray<AdvisoryObservationReference>.Empty,
ImmutableArray<string>.Empty,
ImmutableArray<RawRelationship>.Empty);
}
var aliasSet = new HashSet<string>(StringComparer.Ordinal);
var purlSet = new HashSet<string>(StringComparer.Ordinal);
var cpeSet = new HashSet<string>(StringComparer.Ordinal);
var referenceSet = new HashSet<AdvisoryObservationReference>();
foreach (var observation in observations)
{
foreach (var alias in observation.Linkset.Aliases)
{
var referenceSet = new HashSet<AdvisoryObservationReference>();
var scopeSet = new HashSet<string>(StringComparer.Ordinal);
var relationshipSet = new HashSet<RawRelationship>();
foreach (var observation in observations)
{
foreach (var alias in observation.Linkset.Aliases)
{
aliasSet.Add(alias);
}
@@ -227,18 +232,34 @@ public sealed class AdvisoryObservationQueryService : IAdvisoryObservationQueryS
}
foreach (var reference in observation.Linkset.References)
{
referenceSet.Add(reference);
}
}
return new AdvisoryObservationLinksetAggregate(
aliasSet.OrderBy(static alias => alias, StringComparer.Ordinal).ToImmutableArray(),
purlSet.OrderBy(static purl => purl, StringComparer.Ordinal).ToImmutableArray(),
cpeSet.OrderBy(static cpe => cpe, StringComparer.Ordinal).ToImmutableArray(),
referenceSet
.OrderBy(static reference => reference.Type, StringComparer.Ordinal)
.ThenBy(static reference => reference.Url, StringComparer.Ordinal)
.ToImmutableArray());
}
}
{
referenceSet.Add(reference);
}
foreach (var scope in observation.RawLinkset.Scopes)
{
scopeSet.Add(scope);
}
foreach (var relationship in observation.RawLinkset.Relationships)
{
relationshipSet.Add(relationship);
}
}
return new AdvisoryObservationLinksetAggregate(
aliasSet.OrderBy(static alias => alias, StringComparer.Ordinal).ToImmutableArray(),
purlSet.OrderBy(static purl => purl, StringComparer.Ordinal).ToImmutableArray(),
cpeSet.OrderBy(static cpe => cpe, StringComparer.Ordinal).ToImmutableArray(),
referenceSet
.OrderBy(static reference => reference.Type, StringComparer.Ordinal)
.ThenBy(static reference => reference.Url, StringComparer.Ordinal)
.ToImmutableArray(),
scopeSet.OrderBy(static scope => scope, StringComparer.Ordinal).ToImmutableArray(),
relationshipSet
.OrderBy(static rel => rel.Type, StringComparer.Ordinal)
.ThenBy(static rel => rel.Source, StringComparer.Ordinal)
.ThenBy(static rel => rel.Target, StringComparer.Ordinal)
.ToImmutableArray());
}
}

View File

@@ -116,7 +116,10 @@ internal sealed class AdvisoryRawService : IAdvisoryRawService
var observation = _observationFactory.Create(enriched, _timeProvider.GetUtcNow());
await _observationSink.UpsertAsync(observation, cancellationToken).ConfigureAwait(false);
var normalizedLinkset = AdvisoryLinksetNormalization.FromRawLinkset(enriched.Linkset);
var (normalizedLinkset, confidence, conflicts) = AdvisoryLinksetNormalization.FromRawLinksetWithConfidence(
enriched.Linkset,
providedConfidence: null);
var linkset = new AdvisoryLinkset(
tenant,
source,
@@ -124,6 +127,8 @@ internal sealed class AdvisoryRawService : IAdvisoryRawService
ImmutableArray.Create(observation.ObservationId),
normalizedLinkset,
null,
confidence ?? 1.0,
conflicts,
_timeProvider.GetUtcNow(),
null);

View File

@@ -95,6 +95,26 @@ public sealed record AdvisoryObservation
return references;
}
static ImmutableArray<RawRelationship> SanitizeRelationships(ImmutableArray<RawRelationship> relationships)
{
if (relationships.IsDefault)
{
return ImmutableArray<RawRelationship>.Empty;
}
return relationships;
}
static ImmutableArray<string> SanitizeScopes(ImmutableArray<string> scopes)
{
if (scopes.IsDefault)
{
return ImmutableArray<string>.Empty;
}
return scopes;
}
static ImmutableDictionary<string, string> SanitizeNotes(ImmutableDictionary<string, string>? notes)
{
if (notes is null || notes.Count == 0)
@@ -108,6 +128,8 @@ public sealed record AdvisoryObservation
return rawLinkset with
{
Aliases = SanitizeStrings(rawLinkset.Aliases),
Scopes = SanitizeScopes(rawLinkset.Scopes),
Relationships = SanitizeRelationships(rawLinkset.Relationships),
PackageUrls = SanitizeStrings(rawLinkset.PackageUrls),
Cpes = SanitizeStrings(rawLinkset.Cpes),
References = SanitizeReferences(rawLinkset.References),

View File

@@ -0,0 +1,240 @@
using System.Collections.Immutable;
namespace StellaOps.Concelier.Models.Observations;
/// <summary>
/// Version 1 Link-Not-Merge observation record: immutable, per-source payload with provenance and tenant guards.
/// </summary>
public sealed record AdvisoryObservationV1
{
public AdvisoryObservationV1(
string observationId,
string tenant,
string source,
string advisoryId,
string? title,
string? summary,
ImmutableArray<ObservationSeverity> severities,
ImmutableArray<ObservationAffected> affected,
ImmutableArray<string> references,
ImmutableArray<string> weaknesses,
DateTimeOffset? published,
DateTimeOffset? modified,
ObservationProvenance provenance,
DateTimeOffset ingestedAt,
string? supersedesObservationId = null)
{
ObservationId = Validation.EnsureNotNullOrWhiteSpace(observationId, nameof(observationId));
Tenant = Validation.EnsureNotNullOrWhiteSpace(tenant, nameof(tenant)).ToLowerInvariant();
Source = Validation.EnsureNotNullOrWhiteSpace(source, nameof(source)).ToLowerInvariant();
AdvisoryId = Validation.EnsureNotNullOrWhiteSpace(advisoryId, nameof(advisoryId));
Title = Validation.TrimToNull(title);
Summary = Validation.TrimToNull(summary);
Severities = Normalize(severities);
Affected = Normalize(affected);
References = NormalizeStrings(references);
Weaknesses = NormalizeStrings(weaknesses);
Published = published?.ToUniversalTime();
Modified = modified?.ToUniversalTime();
Provenance = provenance ?? throw new ArgumentNullException(nameof(provenance));
IngestedAt = ingestedAt.ToUniversalTime();
SupersedesObservationId = Validation.TrimToNull(supersedesObservationId);
}
public string ObservationId { get; }
public string Tenant { get; }
public string Source { get; }
public string AdvisoryId { get; }
public string? Title { get; }
public string? Summary { get; }
public ImmutableArray<ObservationSeverity> Severities { get; }
public ImmutableArray<ObservationAffected> Affected { get; }
public ImmutableArray<string> References { get; }
public ImmutableArray<string> Weaknesses { get; }
public DateTimeOffset? Published { get; }
public DateTimeOffset? Modified { get; }
public ObservationProvenance Provenance { get; }
public DateTimeOffset IngestedAt { get; }
public string? SupersedesObservationId { get; }
private static ImmutableArray<string> NormalizeStrings(ImmutableArray<string> values)
{
if (values.IsDefaultOrEmpty)
{
return ImmutableArray<string>.Empty;
}
var builder = ImmutableArray.CreateBuilder<string>();
foreach (var value in values)
{
if (string.IsNullOrWhiteSpace(value))
{
continue;
}
builder.Add(value.Trim());
}
return builder.Count == 0 ? ImmutableArray<string>.Empty : builder.ToImmutable();
}
private static ImmutableArray<T> Normalize<T>(ImmutableArray<T> values)
{
if (values.IsDefaultOrEmpty)
{
return ImmutableArray<T>.Empty;
}
return values;
}
}
public sealed class ObservationSeverity
{
public ObservationSeverity(string system, double score, string? vector)
{
System = Validation.EnsureNotNullOrWhiteSpace(system, nameof(system));
Score = score;
Vector = Validation.TrimToNull(vector);
}
public string System { get; }
public double Score { get; }
public string? Vector { get; }
}
public sealed class ObservationAffected
{
public ObservationAffected(
string purl,
string? package,
ImmutableArray<string> versions,
ImmutableArray<ObservationVersionRange> ranges,
string? ecosystem,
ImmutableArray<string> cpes)
{
Purl = Validation.EnsureNotNullOrWhiteSpace(purl, nameof(purl));
Package = Validation.TrimToNull(package);
Versions = NormalizeStrings(versions);
Ranges = ranges.IsDefault ? ImmutableArray<ObservationVersionRange>.Empty : ranges;
Ecosystem = Validation.TrimToNull(ecosystem);
Cpes = NormalizeStrings(cpes);
}
public string Purl { get; }
public string? Package { get; }
public ImmutableArray<string> Versions { get; }
public ImmutableArray<ObservationVersionRange> Ranges { get; }
public string? Ecosystem { get; }
public ImmutableArray<string> Cpes { get; }
private static ImmutableArray<string> NormalizeStrings(ImmutableArray<string> values)
{
if (values.IsDefaultOrEmpty)
{
return ImmutableArray<string>.Empty;
}
var builder = ImmutableArray.CreateBuilder<string>();
foreach (var value in values)
{
if (string.IsNullOrWhiteSpace(value))
{
continue;
}
builder.Add(value.Trim());
}
return builder.Count == 0 ? ImmutableArray<string>.Empty : builder.ToImmutable();
}
}
public sealed class ObservationVersionRange
{
public ObservationVersionRange(string type, ImmutableArray<ObservationRangeEvent> events)
{
Type = Validation.EnsureNotNullOrWhiteSpace(type, nameof(type));
Events = events.IsDefault ? ImmutableArray<ObservationRangeEvent>.Empty : events;
}
public string Type { get; }
public ImmutableArray<ObservationRangeEvent> Events { get; }
}
public sealed class ObservationRangeEvent
{
public ObservationRangeEvent(string @event, string value)
{
Event = Validation.EnsureNotNullOrWhiteSpace(@event, nameof(@event));
Value = Validation.EnsureNotNullOrWhiteSpace(value, nameof(value));
}
public string Event { get; }
public string Value { get; }
}
public sealed class ObservationProvenance
{
public ObservationProvenance(
string sourceArtifactSha,
DateTimeOffset fetchedAt,
string? ingestJobId,
ObservationSignature? signature)
{
SourceArtifactSha = Validation.EnsureNotNullOrWhiteSpace(sourceArtifactSha, nameof(sourceArtifactSha));
FetchedAt = fetchedAt.ToUniversalTime();
IngestJobId = Validation.TrimToNull(ingestJobId);
Signature = signature;
}
public string SourceArtifactSha { get; }
public DateTimeOffset FetchedAt { get; }
public string? IngestJobId { get; }
public ObservationSignature? Signature { get; }
}
public sealed class ObservationSignature
{
public ObservationSignature(bool present, string? format, string? keyId, string? signatureValue)
{
Present = present;
Format = Validation.TrimToNull(format);
KeyId = Validation.TrimToNull(keyId);
SignatureValue = Validation.TrimToNull(signatureValue);
}
public bool Present { get; }
public string? Format { get; }
public string? KeyId { get; }
public string? SignatureValue { get; }
}

View File

@@ -55,23 +55,35 @@ public sealed record RawLinkset
{
[JsonPropertyName("aliases")]
public ImmutableArray<string> Aliases { get; init; } = ImmutableArray<string>.Empty;
[JsonPropertyName("purls")]
public ImmutableArray<string> PackageUrls { get; init; } = ImmutableArray<string>.Empty;
[JsonPropertyName("cpes")]
public ImmutableArray<string> Cpes { get; init; } = ImmutableArray<string>.Empty;
[JsonPropertyName("scopes")]
public ImmutableArray<string> Scopes { get; init; } = ImmutableArray<string>.Empty;
[JsonPropertyName("relationships")]
public ImmutableArray<RawRelationship> Relationships { get; init; } = ImmutableArray<RawRelationship>.Empty;
[JsonPropertyName("purls")]
public ImmutableArray<string> PackageUrls { get; init; } = ImmutableArray<string>.Empty;
[JsonPropertyName("cpes")]
public ImmutableArray<string> Cpes { get; init; } = ImmutableArray<string>.Empty;
[JsonPropertyName("references")]
public ImmutableArray<RawReference> References { get; init; } = ImmutableArray<RawReference>.Empty;
[JsonPropertyName("reconciled_from")]
public ImmutableArray<string> ReconciledFrom { get; init; } = ImmutableArray<string>.Empty;
[JsonPropertyName("notes")]
public ImmutableDictionary<string, string> Notes { get; init; } = ImmutableDictionary<string, string>.Empty;
[JsonPropertyName("notes")]
public ImmutableDictionary<string, string> Notes { get; init; } = ImmutableDictionary<string, string>.Empty;
}
public sealed record RawRelationship(
[property: JsonPropertyName("type")] string Type,
[property: JsonPropertyName("source")] string Source,
[property: JsonPropertyName("target")] string Target,
[property: JsonPropertyName("provenance")] string? Provenance = null);
public sealed record RawReference(
[property: JsonPropertyName("type")] string Type,
[property: JsonPropertyName("url")] string Url,

View File

@@ -29,6 +29,16 @@ public sealed class AdvisoryLinksetDocument
public AdvisoryLinksetNormalizedDocument? Normalized { get; set; }
= null;
[BsonElement("confidence")]
[BsonIgnoreIfNull]
public double? Confidence { get; set; }
= null;
[BsonElement("conflicts")]
[BsonIgnoreIfNull]
public List<AdvisoryLinksetConflictDocument>? Conflicts { get; set; }
= null;
[BsonElement("createdAt")]
public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
@@ -85,3 +95,18 @@ public sealed class AdvisoryLinksetProvenanceDocument
public string? PolicyHash { get; set; }
= null;
}
[BsonIgnoreExtraElements]
public sealed class AdvisoryLinksetConflictDocument
{
[BsonElement("field")]
public string Field { get; set; } = string.Empty;
[BsonElement("reason")]
public string Reason { get; set; } = string.Empty;
[BsonElement("values")]
[BsonIgnoreIfNull]
public List<string>? Values { get; set; }
= null;
}

View File

@@ -5,11 +5,12 @@ using CoreLinksets = StellaOps.Concelier.Core.Linksets;
namespace StellaOps.Concelier.Storage.Mongo.Linksets;
// Backcompat sink name retained for compile includes; forwards to the Mongo-specific store.
internal sealed class AdvisoryLinksetSink : CoreLinksets.IAdvisoryLinksetSink
{
private readonly IAdvisoryLinksetStore _store;
private readonly IMongoAdvisoryLinksetStore _store;
public AdvisoryLinksetSink(IAdvisoryLinksetStore store)
public AdvisoryLinksetSink(IMongoAdvisoryLinksetStore store)
{
_store = store ?? throw new ArgumentNullException(nameof(store));
}

View File

@@ -0,0 +1,20 @@
using System;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Concelier.Storage.Mongo.Linksets;
internal sealed class ConcelierMongoLinksetSink : global::StellaOps.Concelier.Core.Linksets.IAdvisoryLinksetSink
{
private readonly IMongoAdvisoryLinksetStore _store;
public ConcelierMongoLinksetSink(IMongoAdvisoryLinksetStore store)
{
_store = store ?? throw new ArgumentNullException(nameof(store));
}
public Task UpsertAsync(global::StellaOps.Concelier.Core.Linksets.AdvisoryLinkset linkset, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(linkset);
return _store.UpsertAsync(linkset, cancellationToken);
}
}

View File

@@ -6,15 +6,14 @@ using System.Threading;
using System.Threading.Tasks;
using MongoDB.Driver;
using CoreLinksets = StellaOps.Concelier.Core.Linksets;
namespace StellaOps.Concelier.Storage.Mongo.Linksets;
// Internal type kept in storage namespace to avoid name clash with core interface
internal sealed class MongoAdvisoryLinksetStore : CoreLinksets.IAdvisoryLinksetStore, CoreLinksets.IAdvisoryLinksetLookup
// Storage implementation of advisory linkset persistence.
internal sealed class ConcelierMongoLinksetStore : IMongoAdvisoryLinksetStore
{
private readonly IMongoCollection<AdvisoryLinksetDocument> _collection;
public MongoAdvisoryLinksetStore(IMongoCollection<AdvisoryLinksetDocument> collection)
public ConcelierMongoLinksetStore(IMongoCollection<AdvisoryLinksetDocument> collection)
{
_collection = collection ?? throw new ArgumentNullException(nameof(collection));
}
@@ -24,8 +23,9 @@ internal sealed class MongoAdvisoryLinksetStore : CoreLinksets.IAdvisoryLinksetS
ArgumentNullException.ThrowIfNull(linkset);
var document = MapToDocument(linkset);
var tenant = linkset.TenantId.ToLowerInvariant();
var filter = Builders<AdvisoryLinksetDocument>.Filter.And(
Builders<AdvisoryLinksetDocument>.Filter.Eq(d => d.TenantId, linkset.TenantId),
Builders<AdvisoryLinksetDocument>.Filter.Eq(d => d.TenantId, tenant),
Builders<AdvisoryLinksetDocument>.Filter.Eq(d => d.Source, linkset.Source),
Builders<AdvisoryLinksetDocument>.Filter.Eq(d => d.AdvisoryId, linkset.AdvisoryId));
@@ -73,9 +73,6 @@ internal sealed class MongoAdvisoryLinksetStore : CoreLinksets.IAdvisoryLinksetS
var filter = builder.And(filters);
var sort = Builders<AdvisoryLinksetDocument>.Sort.Descending(d => d.CreatedAt).Ascending(d => d.AdvisoryId);
var findFilter = filter;
if (cursor is not null)
{
var cursorFilter = builder.Or(
@@ -84,10 +81,11 @@ internal sealed class MongoAdvisoryLinksetStore : CoreLinksets.IAdvisoryLinksetS
builder.Eq(d => d.CreatedAt, cursor.CreatedAt.UtcDateTime),
builder.Gt(d => d.AdvisoryId, cursor.AdvisoryId)));
findFilter = builder.And(findFilter, cursorFilter);
filter = builder.And(filter, cursorFilter);
}
var documents = await _collection.Find(findFilter)
var sort = Builders<AdvisoryLinksetDocument>.Sort.Descending(d => d.CreatedAt).Ascending(d => d.AdvisoryId);
var documents = await _collection.Find(filter)
.Sort(sort)
.Limit(limit)
.ToListAsync(cancellationToken)
@@ -98,14 +96,23 @@ internal sealed class MongoAdvisoryLinksetStore : CoreLinksets.IAdvisoryLinksetS
private static AdvisoryLinksetDocument MapToDocument(CoreLinksets.AdvisoryLinkset linkset)
{
var doc = new AdvisoryLinksetDocument
return new AdvisoryLinksetDocument
{
TenantId = linkset.TenantId,
TenantId = linkset.TenantId.ToLowerInvariant(),
Source = linkset.Source,
AdvisoryId = linkset.AdvisoryId,
Observations = new List<string>(linkset.ObservationIds),
CreatedAt = linkset.CreatedAt.UtcDateTime,
BuiltByJobId = linkset.BuiltByJobId,
Confidence = linkset.Confidence,
Conflicts = linkset.Conflicts is null
? null
: linkset.Conflicts.Select(conflict => new AdvisoryLinksetConflictDocument
{
Field = conflict.Field,
Reason = conflict.Reason,
Values = conflict.Values is null ? null : new List<string>(conflict.Values)
}).ToList(),
Provenance = linkset.Provenance is null ? null : new AdvisoryLinksetProvenanceDocument
{
ObservationHashes = linkset.Provenance.ObservationHashes is null
@@ -122,26 +129,31 @@ internal sealed class MongoAdvisoryLinksetStore : CoreLinksets.IAdvisoryLinksetS
Severities = linkset.Normalized.SeveritiesToBson(),
}
};
return doc;
}
private static CoreLinksets.AdvisoryLinkset FromDocument(AdvisoryLinksetDocument doc)
{
return new AdvisoryLinkset(
return new CoreLinksets.AdvisoryLinkset(
doc.TenantId,
doc.Source,
doc.AdvisoryId,
doc.Observations.ToImmutableArray(),
doc.Normalized is null ? null : new AdvisoryLinksetNormalized(
doc.Normalized is null ? null : new CoreLinksets.AdvisoryLinksetNormalized(
doc.Normalized.Purls,
doc.Normalized.Versions,
doc.Normalized.Ranges?.Select(ToDictionary).ToList(),
doc.Normalized.Severities?.Select(ToDictionary).ToList()),
doc.Provenance is null ? null : new AdvisoryLinksetProvenance(
doc.Provenance is null ? null : new CoreLinksets.AdvisoryLinksetProvenance(
doc.Provenance.ObservationHashes,
doc.Provenance.ToolVersion,
doc.Provenance.PolicyHash),
doc.Confidence,
doc.Conflicts is null
? null
: doc.Conflicts.Select(conflict => new CoreLinksets.AdvisoryLinksetConflict(
conflict.Field,
conflict.Reason,
conflict.Values)).ToList(),
DateTime.SpecifyKind(doc.CreatedAt, DateTimeKind.Utc),
doc.BuiltByJobId);
}

View File

@@ -0,0 +1,5 @@
namespace StellaOps.Concelier.Storage.Mongo.Linksets;
public interface IMongoAdvisoryLinksetStore : global::StellaOps.Concelier.Core.Linksets.IAdvisoryLinksetStore, global::StellaOps.Concelier.Core.Linksets.IAdvisoryLinksetLookup
{
}

View File

@@ -0,0 +1,66 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Bson;
using MongoDB.Driver;
namespace StellaOps.Concelier.Storage.Mongo.Migrations;
/// <summary>
/// Normalises advisory_linksets tenant ids to lowercase to keep lookups/write paths consistent.
/// </summary>
public sealed class EnsureAdvisoryLinksetsTenantLowerMigration : IMongoMigration
{
private const string MigrationId = "20251117_advisory_linksets_tenant_lower";
private const int BatchSize = 500;
public string Id => MigrationId;
public string Description => "Lowercase tenant ids in advisory_linksets to match query filters.";
public async Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(database);
var collection = database.GetCollection<BsonDocument>(MongoStorageDefaults.Collections.AdvisoryLinksets);
var filter = Builders<BsonDocument>.Filter.Where(doc =>
doc.Contains("TenantId") &&
doc["TenantId"].BsonType == BsonType.String &&
doc["TenantId"].AsString != doc["TenantId"].AsString.ToLowerInvariant());
using var cursor = await collection.Find(filter).ToCursorAsync(cancellationToken).ConfigureAwait(false);
var writes = new List<WriteModel<BsonDocument>>(BatchSize);
while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false))
{
foreach (var doc in cursor.Current)
{
var currentTenant = doc["TenantId"].AsString;
var lower = currentTenant.ToLowerInvariant();
if (lower == currentTenant)
{
continue;
}
var idFilter = Builders<BsonDocument>.Filter.Eq("_id", doc["_id"]);
var update = Builders<BsonDocument>.Update.Set("TenantId", lower);
writes.Add(new UpdateOneModel<BsonDocument>(idFilter, update));
if (writes.Count >= BatchSize)
{
await collection.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = false }, cancellationToken)
.ConfigureAwait(false);
writes.Clear();
}
}
}
if (writes.Count > 0)
{
await collection.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = false }, cancellationToken)
.ConfigureAwait(false);
}
}
}

View File

@@ -28,5 +28,6 @@ public static class MongoStorageDefaults
public const string AdvisoryStatements = "advisory_statements";
public const string AdvisoryConflicts = "advisory_conflicts";
public const string AdvisoryObservations = "advisory_observations";
public const string AdvisoryLinksets = "advisory_linksets";
}
}

View File

@@ -175,6 +175,16 @@ public sealed class AdvisoryObservationRawLinksetDocument
public List<string>? Aliases { get; set; }
= new();
[BsonElement("scopes")]
[BsonIgnoreIfNull]
public List<string>? Scopes { get; set; }
= new();
[BsonElement("relationships")]
[BsonIgnoreIfNull]
public List<AdvisoryObservationRawRelationshipDocument>? Relationships { get; set; }
= new();
[BsonElement("purls")]
[BsonIgnoreIfNull]
public List<string>? PackageUrls { get; set; }
@@ -217,3 +227,21 @@ public sealed class AdvisoryObservationRawReferenceDocument
public string? Source { get; set; }
= null;
}
[BsonIgnoreExtraElements]
public sealed class AdvisoryObservationRawRelationshipDocument
{
[BsonElement("type")]
public string Type { get; set; } = string.Empty;
[BsonElement("source")]
public string Source { get; set; } = string.Empty;
[BsonElement("target")]
public string Target { get; set; } = string.Empty;
[BsonElement("provenance")]
[BsonIgnoreIfNull]
public string? Provenance { get; set; }
= null;
}

View File

@@ -1,8 +1,9 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text.Json.Nodes;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Nodes;
using MongoDB.Bson;
using MongoDB.Bson.IO;
using StellaOps.Concelier.Models.Observations;
@@ -10,11 +11,11 @@ using StellaOps.Concelier.RawModels;
namespace StellaOps.Concelier.Storage.Mongo.Observations;
internal static class AdvisoryObservationDocumentFactory
{
private static readonly JsonWriterSettings JsonSettings = new() { OutputMode = JsonOutputMode.RelaxedExtendedJson };
public static AdvisoryObservation ToModel(AdvisoryObservationDocument document)
internal static class AdvisoryObservationDocumentFactory
{
private static readonly JsonWriterSettings JsonSettings = new() { OutputMode = JsonOutputMode.RelaxedExtendedJson };
public static AdvisoryObservation ToModel(AdvisoryObservationDocument document)
{
ArgumentNullException.ThrowIfNull(document);
@@ -61,7 +62,69 @@ internal static class AdvisoryObservationDocumentFactory
return observation;
}
public static AdvisoryObservationDocument ToDocument(AdvisoryObservation observation)
{
ArgumentNullException.ThrowIfNull(observation);
var contentRaw = observation.Content.Raw?.ToJsonString(new JsonSerializerOptions
{
Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping
}) ?? "{}";
var document = new AdvisoryObservationDocument
{
Id = observation.ObservationId,
Tenant = observation.Tenant,
Source = new AdvisoryObservationSourceDocument
{
Vendor = observation.Source.Vendor,
Stream = observation.Source.Stream,
Api = observation.Source.Api,
CollectorVersion = observation.Source.CollectorVersion
},
Upstream = new AdvisoryObservationUpstreamDocument
{
UpstreamId = observation.Upstream.UpstreamId,
DocumentVersion = observation.Upstream.DocumentVersion,
FetchedAt = observation.Upstream.FetchedAt.UtcDateTime,
ReceivedAt = observation.Upstream.ReceivedAt.UtcDateTime,
ContentHash = observation.Upstream.ContentHash,
Signature = new AdvisoryObservationSignatureDocument
{
Present = observation.Upstream.Signature.Present,
Format = observation.Upstream.Signature.Format,
KeyId = observation.Upstream.Signature.KeyId,
Signature = observation.Upstream.Signature.Signature
},
Metadata = observation.Upstream.Metadata.ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal)
},
Content = new AdvisoryObservationContentDocument
{
Format = observation.Content.Format,
SpecVersion = observation.Content.SpecVersion,
Raw = BsonDocument.Parse(contentRaw),
Metadata = observation.Content.Metadata.ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal)
},
Linkset = new AdvisoryObservationLinksetDocument
{
Aliases = observation.Linkset.Aliases.ToList(),
Purls = observation.Linkset.Purls.ToList(),
Cpes = observation.Linkset.Cpes.ToList(),
References = observation.Linkset.References.Select(reference => new AdvisoryObservationReferenceDocument
{
Type = reference.Type,
Url = reference.Url
}).ToList()
},
RawLinkset = ToRawLinksetDocument(observation.RawLinkset),
CreatedAt = observation.CreatedAt.UtcDateTime,
Attributes = observation.Attributes.ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal)
};
return document;
}
private static JsonNode ParseJsonNode(BsonDocument raw)
{
if (raw is null || raw.ElementCount == 0)
@@ -113,6 +176,22 @@ internal static class AdvisoryObservationDocumentFactory
.ToImmutableArray();
}
static ImmutableArray<RawRelationship> ToImmutableRelationships(List<AdvisoryObservationRawRelationshipDocument>? relationships)
{
if (relationships is null || relationships.Count == 0)
{
return ImmutableArray<RawRelationship>.Empty;
}
return relationships
.Select(static relationship => new RawRelationship(
relationship.Type ?? string.Empty,
relationship.Source ?? string.Empty,
relationship.Target ?? string.Empty,
relationship.Provenance))
.ToImmutableArray();
}
static ImmutableArray<RawReference> ToImmutableReferences(List<AdvisoryObservationRawReferenceDocument>? references)
{
if (references is null || references.Count == 0)
@@ -152,6 +231,8 @@ internal static class AdvisoryObservationDocumentFactory
return new RawLinkset
{
Aliases = ToImmutableStringArray(document.Aliases),
Scopes = ToImmutableStringArray(document.Scopes),
Relationships = ToImmutableRelationships(document.Relationships),
PackageUrls = ToImmutableStringArray(document.PackageUrls),
Cpes = ToImmutableStringArray(document.Cpes),
References = ToImmutableReferences(document.References),
@@ -159,4 +240,35 @@ internal static class AdvisoryObservationDocumentFactory
Notes = ToImmutableDictionary(document.Notes)
};
}
private static AdvisoryObservationRawLinksetDocument ToRawLinksetDocument(RawLinkset rawLinkset)
{
return new AdvisoryObservationRawLinksetDocument
{
Aliases = rawLinkset.Aliases.IsDefault ? new List<string>() : rawLinkset.Aliases.ToList(),
Scopes = rawLinkset.Scopes.IsDefault ? new List<string>() : rawLinkset.Scopes.ToList(),
Relationships = rawLinkset.Relationships.IsDefault
? new List<AdvisoryObservationRawRelationshipDocument>()
: rawLinkset.Relationships.Select(relationship => new AdvisoryObservationRawRelationshipDocument
{
Type = relationship.Type,
Source = relationship.Source,
Target = relationship.Target,
Provenance = relationship.Provenance
}).ToList(),
PackageUrls = rawLinkset.PackageUrls.IsDefault ? new List<string>() : rawLinkset.PackageUrls.ToList(),
Cpes = rawLinkset.Cpes.IsDefault ? new List<string>() : rawLinkset.Cpes.ToList(),
References = rawLinkset.References.IsDefault
? new List<AdvisoryObservationRawReferenceDocument>()
: rawLinkset.References.Select(reference => new AdvisoryObservationRawReferenceDocument
{
Type = reference.Type,
Url = reference.Url,
Source = reference.Source
}).ToList(),
ReconciledFrom = rawLinkset.ReconciledFrom.IsDefault ? new List<string>() : rawLinkset.ReconciledFrom.ToList(),
Notes = rawLinkset.Notes.Count == 0 ? new Dictionary<string, string>(StringComparer.Ordinal)
: rawLinkset.Notes.ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal)
};
}
}

View File

@@ -2,6 +2,8 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Concelier.Core.Observations;
@@ -9,21 +11,21 @@ using StellaOps.Concelier.Models.Observations;
namespace StellaOps.Concelier.Storage.Mongo.Observations;
internal sealed class AdvisoryObservationStore : IAdvisoryObservationStore
{
private readonly IMongoCollection<AdvisoryObservationDocument> collection;
public AdvisoryObservationStore(IMongoCollection<AdvisoryObservationDocument> collection)
internal sealed class AdvisoryObservationStore : IAdvisoryObservationStore
{
private readonly IMongoCollection<AdvisoryObservationDocument> collection;
public AdvisoryObservationStore(IMongoCollection<AdvisoryObservationDocument> collection)
{
this.collection = collection ?? throw new ArgumentNullException(nameof(collection));
}
public async Task<IReadOnlyList<AdvisoryObservation>> ListByTenantAsync(string tenant, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
var filter = Builders<AdvisoryObservationDocument>.Filter.Eq(document => document.Tenant, tenant.ToLowerInvariant());
var documents = await collection
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
var filter = Builders<AdvisoryObservationDocument>.Filter.Eq(document => document.Tenant, tenant.ToLowerInvariant());
var documents = await collection
.Find(filter)
.SortByDescending(document => document.CreatedAt)
.ThenBy(document => document.Id)
@@ -111,6 +113,18 @@ internal sealed class AdvisoryObservationStore : IAdvisoryObservationStore
return documents.Select(AdvisoryObservationDocumentFactory.ToModel).ToArray();
}
public async Task UpsertAsync(AdvisoryObservation observation, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(observation);
cancellationToken.ThrowIfCancellationRequested();
var document = AdvisoryObservationDocumentFactory.ToDocument(observation);
var filter = Builders<AdvisoryObservationDocument>.Filter.Eq(d => d.Id, document.Id);
var options = new ReplaceOptions { IsUpsert = true };
await collection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false);
}
private static string[] NormalizeValues(IEnumerable<string>? values, Func<string, string> projector)
{
if (values is null)

View File

@@ -1,20 +1,23 @@
using System.Collections.Generic;
using StellaOps.Concelier.Models.Observations;
using StellaOps.Concelier.Core.Observations;
using System.Collections.Generic;
using System.Threading;
using StellaOps.Concelier.Models.Observations;
using StellaOps.Concelier.Core.Observations;
namespace StellaOps.Concelier.Storage.Mongo.Observations;
public interface IAdvisoryObservationStore
{
Task<IReadOnlyList<AdvisoryObservation>> ListByTenantAsync(string tenant, CancellationToken cancellationToken);
Task<IReadOnlyList<AdvisoryObservation>> FindByFiltersAsync(
string tenant,
IEnumerable<string>? observationIds,
IEnumerable<string>? aliases,
IEnumerable<string>? purls,
IEnumerable<string>? cpes,
AdvisoryObservationCursor? cursor,
int limit,
CancellationToken cancellationToken);
}
public interface IAdvisoryObservationStore
{
Task<IReadOnlyList<AdvisoryObservation>> ListByTenantAsync(string tenant, CancellationToken cancellationToken);
Task<IReadOnlyList<AdvisoryObservation>> FindByFiltersAsync(
string tenant,
IEnumerable<string>? observationIds,
IEnumerable<string>? aliases,
IEnumerable<string>? purls,
IEnumerable<string>? cpes,
AdvisoryObservationCursor? cursor,
int limit,
CancellationToken cancellationToken);
Task UpsertAsync(AdvisoryObservation observation, CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,167 @@
using System;
using System.Collections.Generic;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
namespace StellaOps.Concelier.Storage.Mongo.Observations.V1;
[BsonIgnoreExtraElements]
public sealed class AdvisoryObservationV1Document
{
[BsonId]
public ObjectId Id { get; set; }
[BsonElement("tenantId")]
public string TenantId { get; set; } = string.Empty;
[BsonElement("source")]
public string Source { get; set; } = string.Empty;
[BsonElement("advisoryId")]
public string AdvisoryId { get; set; } = string.Empty;
[BsonElement("title")]
[BsonIgnoreIfNull]
public string? Title { get; set; }
[BsonElement("summary")]
[BsonIgnoreIfNull]
public string? Summary { get; set; }
[BsonElement("severities")]
[BsonIgnoreIfNull]
public List<ObservationSeverityDocument>? Severities { get; set; }
[BsonElement("affected")]
[BsonIgnoreIfNull]
public List<ObservationAffectedDocument>? Affected { get; set; }
[BsonElement("references")]
[BsonIgnoreIfNull]
public List<string>? References { get; set; }
[BsonElement("weaknesses")]
[BsonIgnoreIfNull]
public List<string>? Weaknesses { get; set; }
[BsonElement("published")]
[BsonDateTimeOptions(Kind = DateTimeKind.Utc)]
[BsonIgnoreIfNull]
public DateTime? Published { get; set; }
[BsonElement("modified")]
[BsonDateTimeOptions(Kind = DateTimeKind.Utc)]
[BsonIgnoreIfNull]
public DateTime? Modified { get; set; }
[BsonElement("provenance")]
public ObservationProvenanceDocument Provenance { get; set; } = new();
[BsonElement("ingestedAt")]
[BsonDateTimeOptions(Kind = DateTimeKind.Utc)]
public DateTime IngestedAt { get; set; }
[BsonElement("supersedesId")]
[BsonIgnoreIfNull]
public ObjectId? SupersedesId { get; set; }
}
[BsonIgnoreExtraElements]
public sealed class ObservationSeverityDocument
{
[BsonElement("system")]
public string System { get; set; } = string.Empty;
[BsonElement("score")]
public double Score { get; set; }
[BsonElement("vector")]
[BsonIgnoreIfNull]
public string? Vector { get; set; }
}
[BsonIgnoreExtraElements]
public sealed class ObservationAffectedDocument
{
[BsonElement("purl")]
public string Purl { get; set; } = string.Empty;
[BsonElement("package")]
[BsonIgnoreIfNull]
public string? Package { get; set; }
[BsonElement("versions")]
[BsonIgnoreIfNull]
public List<string>? Versions { get; set; }
[BsonElement("ranges")]
[BsonIgnoreIfNull]
public List<ObservationVersionRangeDocument>? Ranges { get; set; }
[BsonElement("ecosystem")]
[BsonIgnoreIfNull]
public string? Ecosystem { get; set; }
[BsonElement("cpes")]
[BsonIgnoreIfNull]
public List<string>? Cpes { get; set; }
}
[BsonIgnoreExtraElements]
public sealed class ObservationVersionRangeDocument
{
[BsonElement("type")]
public string Type { get; set; } = string.Empty;
[BsonElement("events")]
[BsonIgnoreIfNull]
public List<ObservationRangeEventDocument>? Events { get; set; }
}
[BsonIgnoreExtraElements]
public sealed class ObservationRangeEventDocument
{
[BsonElement("event")]
public string Event { get; set; } = string.Empty;
[BsonElement("value")]
public string Value { get; set; } = string.Empty;
}
[BsonIgnoreExtraElements]
public sealed class ObservationProvenanceDocument
{
[BsonElement("sourceArtifactSha")]
public string SourceArtifactSha { get; set; } = string.Empty;
[BsonElement("fetchedAt")]
[BsonDateTimeOptions(Kind = DateTimeKind.Utc)]
public DateTime FetchedAt { get; set; }
[BsonElement("ingestJobId")]
[BsonIgnoreIfNull]
public string? IngestJobId { get; set; }
[BsonElement("signature")]
[BsonIgnoreIfNull]
public ObservationSignatureDocument? Signature { get; set; }
}
[BsonIgnoreExtraElements]
public sealed class ObservationSignatureDocument
{
[BsonElement("present")]
public bool Present { get; set; }
[BsonElement("format")]
[BsonIgnoreIfNull]
public string? Format { get; set; }
[BsonElement("keyId")]
[BsonIgnoreIfNull]
public string? KeyId { get; set; }
[BsonElement("signature")]
[BsonIgnoreIfNull]
public string? Signature { get; set; }
}

View File

@@ -0,0 +1,178 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using MongoDB.Bson;
using StellaOps.Concelier.Models.Observations;
namespace StellaOps.Concelier.Storage.Mongo.Observations.V1;
internal static class AdvisoryObservationV1DocumentFactory
{
public static AdvisoryObservationV1 ToModel(AdvisoryObservationV1Document document)
{
ArgumentNullException.ThrowIfNull(document);
var severities = document.Severities is null
? ImmutableArray<ObservationSeverity>.Empty
: document.Severities.Select(s => new ObservationSeverity(s.System, s.Score, s.Vector)).ToImmutableArray();
var affected = document.Affected is null
? ImmutableArray<ObservationAffected>.Empty
: document.Affected.Select(ToAffected).ToImmutableArray();
var references = ToImmutableStrings(document.References);
var weaknesses = ToImmutableStrings(document.Weaknesses);
var provenanceDoc = document.Provenance ?? throw new ArgumentNullException(nameof(document.Provenance));
var signatureDoc = provenanceDoc.Signature;
var signature = signatureDoc is null
? null
: new ObservationSignature(signatureDoc.Present, signatureDoc.Format, signatureDoc.KeyId, signatureDoc.Signature);
var provenance = new ObservationProvenance(
provenanceDoc.SourceArtifactSha,
DateTime.SpecifyKind(provenanceDoc.FetchedAt, DateTimeKind.Utc),
provenanceDoc.IngestJobId,
signature);
return new AdvisoryObservationV1(
document.Id.ToString(),
document.TenantId,
document.Source,
document.AdvisoryId,
document.Title,
document.Summary,
severities,
affected,
references,
weaknesses,
document.Published.HasValue ? DateTime.SpecifyKind(document.Published.Value, DateTimeKind.Utc) : null,
document.Modified.HasValue ? DateTime.SpecifyKind(document.Modified.Value, DateTimeKind.Utc) : null,
provenance,
DateTime.SpecifyKind(document.IngestedAt, DateTimeKind.Utc),
document.SupersedesId?.ToString());
}
public static AdvisoryObservationV1Document FromModel(AdvisoryObservationV1 model)
{
ArgumentNullException.ThrowIfNull(model);
var document = new AdvisoryObservationV1Document
{
Id = ObjectId.Parse(model.ObservationId),
TenantId = model.Tenant,
Source = model.Source,
AdvisoryId = model.AdvisoryId,
Title = model.Title,
Summary = model.Summary,
Severities = model.Severities
.Select(severity => new ObservationSeverityDocument
{
System = severity.System,
Score = severity.Score,
Vector = severity.Vector
})
.ToList(),
Affected = model.Affected.Select(ToDocument).ToList(),
References = model.References.IsDefaultOrEmpty ? null : model.References.ToList(),
Weaknesses = model.Weaknesses.IsDefaultOrEmpty ? null : model.Weaknesses.ToList(),
Published = model.Published?.UtcDateTime,
Modified = model.Modified?.UtcDateTime,
SupersedesId = string.IsNullOrWhiteSpace(model.SupersedesObservationId)
? null
: ObjectId.Parse(model.SupersedesObservationId!),
IngestedAt = model.IngestedAt.UtcDateTime,
Provenance = new ObservationProvenanceDocument
{
SourceArtifactSha = model.Provenance.SourceArtifactSha,
FetchedAt = model.Provenance.FetchedAt.UtcDateTime,
IngestJobId = model.Provenance.IngestJobId,
Signature = model.Provenance.Signature is null
? null
: new ObservationSignatureDocument
{
Present = model.Provenance.Signature.Present,
Format = model.Provenance.Signature.Format,
KeyId = model.Provenance.Signature.KeyId,
Signature = model.Provenance.Signature.SignatureValue
}
}
};
return document;
}
private static ImmutableArray<string> ToImmutableStrings(IEnumerable<string>? values)
{
if (values is null)
{
return ImmutableArray<string>.Empty;
}
var builder = ImmutableArray.CreateBuilder<string>();
foreach (var value in values)
{
if (string.IsNullOrWhiteSpace(value))
{
continue;
}
builder.Add(value.Trim());
}
return builder.Count == 0 ? ImmutableArray<string>.Empty : builder.ToImmutable();
}
private static ObservationAffected ToAffected(ObservationAffectedDocument document)
{
var ranges = document.Ranges is null
? ImmutableArray<ObservationVersionRange>.Empty
: document.Ranges.Select(ToRange).ToImmutableArray();
return new ObservationAffected(
document.Purl,
document.Package,
ToImmutableStrings(document.Versions),
ranges,
document.Ecosystem,
ToImmutableStrings(document.Cpes));
}
private static ObservationVersionRange ToRange(ObservationVersionRangeDocument document)
{
var events = document.Events is null
? ImmutableArray<ObservationRangeEvent>.Empty
: document.Events.Select(evt => new ObservationRangeEvent(evt.Event, evt.Value)).ToImmutableArray();
return new ObservationVersionRange(document.Type, events);
}
private static ObservationAffectedDocument ToDocument(ObservationAffected model)
{
return new ObservationAffectedDocument
{
Purl = model.Purl,
Package = model.Package,
Versions = model.Versions.IsDefaultOrEmpty ? null : model.Versions.ToList(),
Ranges = model.Ranges.IsDefaultOrEmpty ? null : model.Ranges.Select(ToDocument).ToList(),
Ecosystem = model.Ecosystem,
Cpes = model.Cpes.IsDefaultOrEmpty ? null : model.Cpes.ToList()
};
}
private static ObservationVersionRangeDocument ToDocument(ObservationVersionRange model)
{
return new ObservationVersionRangeDocument
{
Type = model.Type,
Events = model.Events.IsDefaultOrEmpty
? null
: model.Events.Select(evt => new ObservationRangeEventDocument
{
Event = evt.Event,
Value = evt.Value
}).ToList()
};
}
}

View File

@@ -0,0 +1,26 @@
using System;
using System.Security.Cryptography;
using System.Text;
using MongoDB.Bson;
namespace StellaOps.Concelier.Storage.Mongo.Observations.V1;
internal static class ObservationIdBuilder
{
public static ObjectId Create(string tenant, string source, string advisoryId, string sourceArtifactSha)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
ArgumentException.ThrowIfNullOrWhiteSpace(source);
ArgumentException.ThrowIfNullOrWhiteSpace(advisoryId);
ArgumentException.ThrowIfNullOrWhiteSpace(sourceArtifactSha);
var material = $"{tenant.Trim().ToLowerInvariant()}|{source.Trim().ToLowerInvariant()}|{advisoryId.Trim()}|{sourceArtifactSha.Trim()}";
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(material));
Span<byte> objectIdBytes = stackalloc byte[12];
hash.AsSpan(0, 12).CopyTo(objectIdBytes);
// ObjectId requires a byte[]; copy the stackalloc span into a managed array.
return new ObjectId(objectIdBytes.ToArray());
}
}

View File

@@ -645,23 +645,42 @@ internal sealed class MongoAdvisoryRawRepository : IAdvisoryRawRepository
private static RawLinkset MapLinkset(BsonDocument linkset)
{
var aliases = linkset.TryGetValue("aliases", out var aliasesValue) && aliasesValue.IsBsonArray
? aliasesValue.AsBsonArray.Select(BsonValueToString).ToImmutableArray()
: ImmutableArray<string>.Empty;
var purls = linkset.TryGetValue("purls", out var purlsValue) && purlsValue.IsBsonArray
? purlsValue.AsBsonArray.Select(BsonValueToString).ToImmutableArray()
: ImmutableArray<string>.Empty;
var cpes = linkset.TryGetValue("cpes", out var cpesValue) && cpesValue.IsBsonArray
? cpesValue.AsBsonArray.Select(BsonValueToString).ToImmutableArray()
: ImmutableArray<string>.Empty;
var references = linkset.TryGetValue("references", out var referencesValue) && referencesValue.IsBsonArray
? referencesValue.AsBsonArray
.Where(static value => value.IsBsonDocument)
.Select(value =>
{
var aliases = linkset.TryGetValue("aliases", out var aliasesValue) && aliasesValue.IsBsonArray
? aliasesValue.AsBsonArray.Select(BsonValueToString).ToImmutableArray()
: ImmutableArray<string>.Empty;
var scopes = linkset.TryGetValue("scopes", out var scopesValue) && scopesValue.IsBsonArray
? scopesValue.AsBsonArray.Select(BsonValueToString).ToImmutableArray()
: ImmutableArray<string>.Empty;
var purls = linkset.TryGetValue("purls", out var purlsValue) && purlsValue.IsBsonArray
? purlsValue.AsBsonArray.Select(BsonValueToString).ToImmutableArray()
: ImmutableArray<string>.Empty;
var cpes = linkset.TryGetValue("cpes", out var cpesValue) && cpesValue.IsBsonArray
? cpesValue.AsBsonArray.Select(BsonValueToString).ToImmutableArray()
: ImmutableArray<string>.Empty;
var relationships = linkset.TryGetValue("relationships", out var relationshipsValue) && relationshipsValue.IsBsonArray
? relationshipsValue.AsBsonArray
.Where(static value => value.IsBsonDocument)
.Select(value =>
{
var doc = value.AsBsonDocument;
return new RawRelationship(
GetRequiredString(doc, "type"),
GetRequiredString(doc, "source"),
GetRequiredString(doc, "target"),
GetOptionalString(doc, "provenance"));
})
.ToImmutableArray()
: ImmutableArray<RawRelationship>.Empty;
var references = linkset.TryGetValue("references", out var referencesValue) && referencesValue.IsBsonArray
? referencesValue.AsBsonArray
.Where(static value => value.IsBsonDocument)
.Select(value =>
{
var doc = value.AsBsonDocument;
return new RawReference(
GetRequiredString(doc, "type"),
@@ -684,14 +703,16 @@ internal sealed class MongoAdvisoryRawRepository : IAdvisoryRawRepository
}
}
return new RawLinkset
{
Aliases = aliases,
PackageUrls = purls,
Cpes = cpes,
References = references,
ReconciledFrom = reconciledFrom,
Notes = notesBuilder.ToImmutable()
return new RawLinkset
{
Aliases = aliases,
Scopes = scopes,
Relationships = relationships,
PackageUrls = purls,
Cpes = cpes,
References = references,
ReconciledFrom = reconciledFrom,
Notes = notesBuilder.ToImmutable()
};
}

View File

@@ -80,13 +80,13 @@ public static class ServiceCollectionExtensions
services.AddSingleton<IAdvisoryEventRepository, MongoAdvisoryEventRepository>();
services.AddSingleton<IAdvisoryEventLog, AdvisoryEventLog>();
services.AddSingleton<IAdvisoryRawRepository, MongoAdvisoryRawRepository>();
services.AddSingleton<StellaOps.Concelier.Storage.Mongo.Linksets.MongoAdvisoryLinksetStore>();
services.AddSingleton<StellaOps.Concelier.Storage.Mongo.Linksets.IMongoAdvisoryLinksetStore, StellaOps.Concelier.Storage.Mongo.Linksets.ConcelierMongoLinksetStore>();
services.AddSingleton<StellaOps.Concelier.Core.Linksets.IAdvisoryLinksetStore>(sp =>
sp.GetRequiredService<StellaOps.Concelier.Storage.Mongo.Linksets.MongoAdvisoryLinksetStore>());
sp.GetRequiredService<StellaOps.Concelier.Storage.Mongo.Linksets.IMongoAdvisoryLinksetStore>());
services.AddSingleton<StellaOps.Concelier.Core.Linksets.IAdvisoryLinksetLookup>(sp =>
sp.GetRequiredService<StellaOps.Concelier.Storage.Mongo.Linksets.MongoAdvisoryLinksetStore>());
services.AddSingleton<StellaOps.Concelier.Core.Linksets.IAdvisoryObservationSink, StellaOps.Concelier.Storage.Mongo.Linksets.AdvisoryObservationSink>();
services.AddSingleton<StellaOps.Concelier.Core.Linksets.IAdvisoryLinksetSink, StellaOps.Concelier.Storage.Mongo.Linksets.AdvisoryLinksetSink>();
sp.GetRequiredService<StellaOps.Concelier.Storage.Mongo.Linksets.IMongoAdvisoryLinksetStore>());
services.AddSingleton<StellaOps.Concelier.Core.Observations.IAdvisoryObservationSink, StellaOps.Concelier.Storage.Mongo.Observations.AdvisoryObservationSink>();
services.AddSingleton<StellaOps.Concelier.Core.Linksets.IAdvisoryLinksetSink, StellaOps.Concelier.Storage.Mongo.Linksets.ConcelierMongoLinksetSink>();
services.AddSingleton<IExportStateStore, ExportStateStore>();
services.TryAddSingleton<ExportStateManager>();

View File

@@ -0,0 +1,47 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using StellaOps.Concelier.Core.Linksets;
using StellaOps.Concelier.RawModels;
using Xunit;
namespace StellaOps.Concelier.Core.Tests.Linksets;
public sealed class AdvisoryLinksetNormalizationTests
{
[Fact]
public void FromRawLinksetWithConfidence_ExtractsNotesAsConflicts()
{
var linkset = new RawLinkset
{
PackageUrls = ImmutableArray.Create("pkg:npm/foo@1.0.0"),
Notes = new Dictionary<string, string>
{
{ "severity", "disagree" }
}
};
var (normalized, confidence, conflicts) = AdvisoryLinksetNormalization.FromRawLinksetWithConfidence(linkset, 0.8);
Assert.NotNull(normalized);
Assert.Equal(0.8, confidence);
Assert.Single(conflicts);
Assert.Equal("severity", conflicts[0].Field);
Assert.Equal("disagree", conflicts[0].Reason);
}
[Theory]
[InlineData(-1, 0)]
[InlineData(2, 1)]
[InlineData(double.NaN, null)]
public void FromRawLinksetWithConfidence_ClampsConfidence(double input, double? expected)
{
var linkset = new RawLinkset
{
PackageUrls = ImmutableArray<string>.Empty
};
var (_, confidence, _) = AdvisoryLinksetNormalization.FromRawLinksetWithConfidence(linkset, input);
Assert.Equal(expected, confidence);
}
}

View File

@@ -29,20 +29,30 @@ public sealed class AdvisoryObservationQueryServiceTests
{
new AdvisoryObservationReference("advisory", "https://example.test/advisory-1")
},
createdAt: DateTimeOffset.UtcNow.AddMinutes(-5)),
CreateObservation(
observationId: "tenant-a:osv:beta:1",
tenant: "tenant-a",
aliases: new[] { "CVE-2025-0002", "GHSA-xyzz" },
purls: new[] { "pkg:pypi/package-b@2.0.0" },
cpes: Array.Empty<string>(),
references: new[]
{
new AdvisoryObservationReference("advisory", "https://example.test/advisory-2"),
new AdvisoryObservationReference("patch", "https://example.test/patch-1")
},
createdAt: DateTimeOffset.UtcNow)
};
scopes: new[] { "runtime" },
relationships: new[]
{
new RawRelationship("depends_on", "pkg:npm/package-a@1.0.0", "pkg:npm/lib@2.0.0", "sbom-a")
},
createdAt: DateTimeOffset.UtcNow.AddMinutes(-5)),
CreateObservation(
observationId: "tenant-a:osv:beta:1",
tenant: "tenant-a",
aliases: new[] { "CVE-2025-0002", "GHSA-xyzz" },
purls: new[] { "pkg:pypi/package-b@2.0.0" },
cpes: Array.Empty<string>(),
references: new[]
{
new AdvisoryObservationReference("advisory", "https://example.test/advisory-2"),
new AdvisoryObservationReference("patch", "https://example.test/patch-1")
},
scopes: new[] { "build" },
relationships: new[]
{
new RawRelationship("affects", "pkg:pypi/package-b@2.0.0", "component-x", "sbom-b")
},
createdAt: DateTimeOffset.UtcNow)
};
var lookup = new InMemoryLookup(observations);
var service = new AdvisoryObservationQueryService(lookup);
@@ -63,15 +73,22 @@ public sealed class AdvisoryObservationQueryServiceTests
Assert.Equal(new[] { "cpe:/a:vendor:product:1.0" }, result.Linkset.Cpes);
Assert.Equal(3, result.Linkset.References.Length);
Assert.Equal("advisory", result.Linkset.References[0].Type);
Assert.Equal("https://example.test/advisory-1", result.Linkset.References[0].Url);
Assert.Equal("https://example.test/advisory-2", result.Linkset.References[1].Url);
Assert.Equal("patch", result.Linkset.References[2].Type);
Assert.False(result.HasMore);
Assert.Null(result.NextCursor);
}
Assert.Equal(3, result.Linkset.References.Length);
Assert.Equal("advisory", result.Linkset.References[0].Type);
Assert.Equal("https://example.test/advisory-1", result.Linkset.References[0].Url);
Assert.Equal("https://example.test/advisory-2", result.Linkset.References[1].Url);
Assert.Equal("patch", result.Linkset.References[2].Type);
Assert.Equal(new[] { "build", "runtime" }, result.Linkset.Scopes);
Assert.Equal(2, result.Linkset.Relationships.Length);
Assert.Equal("affects", result.Linkset.Relationships[0].Type);
Assert.Equal("component-x", result.Linkset.Relationships[0].Target);
Assert.Equal("depends_on", result.Linkset.Relationships[1].Type);
Assert.Equal("pkg:npm/lib@2.0.0", result.Linkset.Relationships[1].Target);
Assert.False(result.HasMore);
Assert.Null(result.NextCursor);
}
[Fact]
public async Task QueryAsync_WithAliasFilter_UsesAliasLookupAndFilters()
@@ -218,9 +235,11 @@ public sealed class AdvisoryObservationQueryServiceTests
IEnumerable<string> purls,
IEnumerable<string> cpes,
IEnumerable<AdvisoryObservationReference> references,
DateTimeOffset createdAt)
{
var raw = JsonNode.Parse("""{"message":"payload"}""") ?? throw new InvalidOperationException("Raw payload must not be null.");
DateTimeOffset createdAt,
IEnumerable<string>? scopes = null,
IEnumerable<RawRelationship>? relationships = null)
{
var raw = JsonNode.Parse("""{"message":"payload"}""") ?? throw new InvalidOperationException("Raw payload must not be null.");
var upstream = new AdvisoryObservationUpstream(
upstreamId: observationId,
@@ -239,7 +258,9 @@ public sealed class AdvisoryObservationQueryServiceTests
Cpes = cpes.ToImmutableArray(),
References = references
.Select(static reference => new RawReference(reference.Type, reference.Url))
.ToImmutableArray()
.ToImmutableArray(),
Scopes = scopes?.ToImmutableArray() ?? ImmutableArray<string>.Empty,
Relationships = relationships?.ToImmutableArray() ?? ImmutableArray<RawRelationship>.Empty
};
return new AdvisoryObservation(

View File

@@ -0,0 +1,171 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Driver;
using StellaOps.Concelier.Core.Linksets;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Linksets;
using StellaOps.Concelier.Testing;
using Xunit;
namespace StellaOps.Concelier.Storage.Mongo.Tests.Linksets;
public sealed class ConcelierMongoLinksetStoreTests : IClassFixture<MongoIntegrationFixture>
{
private readonly MongoIntegrationFixture _fixture;
public ConcelierMongoLinksetStoreTests(MongoIntegrationFixture fixture)
{
_fixture = fixture;
}
[Fact]
public void MapToDocument_StoresConfidenceAndConflicts()
{
var linkset = new AdvisoryLinkset(
"tenant",
"ghsa",
"GHSA-1234",
ImmutableArray.Create("obs-1", "obs-2"),
null,
new AdvisoryLinksetProvenance(new[] { "h1", "h2" }, "tool", "policy"),
0.82,
new List<AdvisoryLinksetConflict>
{
new("severity", "disagree", new[] { "HIGH", "MEDIUM" })
},
DateTimeOffset.UtcNow,
"job-1");
var method = typeof(ConcelierMongoLinksetStore).GetMethod(
"MapToDocument",
BindingFlags.NonPublic | BindingFlags.Static);
Assert.NotNull(method);
var document = (AdvisoryLinksetDocument)method!.Invoke(null, new object?[] { linkset })!;
Assert.Equal(linkset.Confidence, document.Confidence);
Assert.NotNull(document.Conflicts);
Assert.Single(document.Conflicts!);
Assert.Equal("severity", document.Conflicts![0].Field);
Assert.Equal("disagree", document.Conflicts![0].Reason);
}
[Fact]
public void FromDocument_RestoresConfidenceAndConflicts()
{
var doc = new AdvisoryLinksetDocument
{
TenantId = "tenant",
Source = "ghsa",
AdvisoryId = "GHSA-1234",
Observations = new List<string> { "obs-1" },
Confidence = 0.5,
Conflicts = new List<AdvisoryLinksetConflictDocument>
{
new()
{
Field = "references",
Reason = "mismatch",
Values = new List<string> { "url1", "url2" }
}
},
CreatedAt = DateTime.UtcNow
};
var method = typeof(ConcelierMongoLinksetStore).GetMethod(
"FromDocument",
BindingFlags.NonPublic | BindingFlags.Static);
Assert.NotNull(method);
var model = (AdvisoryLinkset)method!.Invoke(null, new object?[] { doc })!;
Assert.Equal(0.5, model.Confidence);
Assert.NotNull(model.Conflicts);
Assert.Single(model.Conflicts!);
Assert.Equal("references", model.Conflicts![0].Field);
}
[Fact]
public async Task FindByTenantAsync_OrdersByCreatedAtThenAdvisoryId()
{
await _fixture.Database.DropCollectionAsync(MongoStorageDefaults.Collections.AdvisoryLinksets);
var collection = _fixture.Database.GetCollection<AdvisoryLinksetDocument>(MongoStorageDefaults.Collections.AdvisoryLinksets);
var store = new ConcelierMongoLinksetStore(collection);
var now = DateTimeOffset.UtcNow;
var linksets = new[]
{
new AdvisoryLinkset("Tenant-A", "src", "ADV-002", ImmutableArray.Create("obs-1"), null, null, null, null, now, "job-1"),
new AdvisoryLinkset("Tenant-A", "src", "ADV-001", ImmutableArray.Create("obs-2"), null, null, null, null, now, "job-2"),
new AdvisoryLinkset("Tenant-A", "src", "ADV-003", ImmutableArray.Create("obs-3"), null, null, null, null, now.AddMinutes(-5), "job-3")
};
foreach (var linkset in linksets)
{
await store.UpsertAsync(linkset, CancellationToken.None);
}
var results = await store.FindByTenantAsync("TENANT-A", null, null, cursor: null, limit: 10, cancellationToken: CancellationToken.None);
Assert.Equal(new[] { "ADV-001", "ADV-002", "ADV-003" }, results.Select(r => r.AdvisoryId));
}
[Fact]
public async Task FindByTenantAsync_AppliesCursorForDeterministicPaging()
{
await _fixture.Database.DropCollectionAsync(MongoStorageDefaults.Collections.AdvisoryLinksets);
var collection = _fixture.Database.GetCollection<AdvisoryLinksetDocument>(MongoStorageDefaults.Collections.AdvisoryLinksets);
var store = new ConcelierMongoLinksetStore(collection);
var now = DateTimeOffset.UtcNow;
var firstPage = new[]
{
new AdvisoryLinkset("tenant-a", "src", "ADV-010", ImmutableArray.Create("obs-1"), null, null, null, null, now, "job-1"),
new AdvisoryLinkset("tenant-a", "src", "ADV-020", ImmutableArray.Create("obs-2"), null, null, null, null, now, "job-2"),
new AdvisoryLinkset("tenant-a", "src", "ADV-030", ImmutableArray.Create("obs-3"), null, null, null, null, now.AddMinutes(-10), "job-3")
};
foreach (var linkset in firstPage)
{
await store.UpsertAsync(linkset, CancellationToken.None);
}
var initial = await store.FindByTenantAsync("tenant-a", null, null, cursor: null, limit: 10, cancellationToken: CancellationToken.None);
var cursor = new AdvisoryLinksetCursor(initial[1].CreatedAt, initial[1].AdvisoryId);
var paged = await store.FindByTenantAsync("tenant-a", null, null, cursor, limit: 10, cancellationToken: CancellationToken.None);
Assert.Single(paged);
Assert.Equal("ADV-030", paged[0].AdvisoryId);
}
[Fact]
public async Task Upsert_NormalizesTenantToLowerInvariant()
{
await _fixture.Database.DropCollectionAsync(MongoStorageDefaults.Collections.AdvisoryLinksets);
var collection = _fixture.Database.GetCollection<AdvisoryLinksetDocument>(MongoStorageDefaults.Collections.AdvisoryLinksets);
var store = new ConcelierMongoLinksetStore(collection);
var linkset = new AdvisoryLinkset("Tenant-A", "ghsa", "GHSA-1", ImmutableArray.Create("obs-1"), null, null, null, null, DateTimeOffset.UtcNow, "job-1");
await store.UpsertAsync(linkset, CancellationToken.None);
var fetched = await collection.Find(Builders<AdvisoryLinksetDocument>.Filter.Empty).FirstOrDefaultAsync();
Assert.NotNull(fetched);
Assert.Equal("tenant-a", fetched!.TenantId);
var results = await store.FindByTenantAsync("TENANT-A", null, null, cursor: null, limit: 10, cancellationToken: CancellationToken.None);
Assert.Single(results);
Assert.Equal("GHSA-1", results[0].AdvisoryId);
}
}

View File

@@ -0,0 +1,39 @@
using System.Threading.Tasks;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Concelier.Storage.Mongo.Migrations;
using StellaOps.Concelier.Testing;
using Xunit;
namespace StellaOps.Concelier.Storage.Mongo.Tests.Migrations;
[Collection("mongo-fixture")]
public sealed class EnsureAdvisoryLinksetsTenantLowerMigrationTests : IClassFixture<MongoIntegrationFixture>
{
private readonly MongoIntegrationFixture _fixture;
public EnsureAdvisoryLinksetsTenantLowerMigrationTests(MongoIntegrationFixture fixture)
{
_fixture = fixture;
}
[Fact]
public async Task ApplyAsync_LowersTenantIds()
{
var collection = _fixture.Database.GetCollection<BsonDocument>(MongoStorageDefaults.Collections.AdvisoryLinksets);
await _fixture.Database.DropCollectionAsync(MongoStorageDefaults.Collections.AdvisoryLinksets);
await collection.InsertManyAsync(new[]
{
new BsonDocument { { "TenantId", "Tenant-A" }, { "Source", "src" }, { "AdvisoryId", "ADV-1" }, { "Observations", new BsonArray() } },
new BsonDocument { { "TenantId", "tenant-b" }, { "Source", "src" }, { "AdvisoryId", "ADV-2" }, { "Observations", new BsonArray() } }
});
var migration = new EnsureAdvisoryLinksetsTenantLowerMigration();
await migration.ApplyAsync(_fixture.Database, default);
var all = await collection.Find(FilterDefinition<BsonDocument>.Empty).ToListAsync();
Assert.Contains(all, doc => doc["TenantId"] == "tenant-a");
Assert.Contains(all, doc => doc["TenantId"] == "tenant-b");
}
}

View File

@@ -56,6 +56,11 @@ public sealed class AdvisoryObservationDocumentFactoryTests
RawLinkset = new AdvisoryObservationRawLinksetDocument
{
Aliases = new List<string> { "CVE-2025-1234", "cve-2025-1234" },
Scopes = new List<string> { "runtime", "build" },
Relationships = new List<AdvisoryObservationRawRelationshipDocument>
{
new() { Type = "depends_on", Source = "componentA", Target = "componentB", Provenance = "sbom-manifest" }
},
PackageUrls = new List<string> { "pkg:generic/foo@1.0.0" },
Cpes = new List<string> { "cpe:/a:vendor:product:1" },
References = new List<AdvisoryObservationRawReferenceDocument>
@@ -78,6 +83,11 @@ public sealed class AdvisoryObservationDocumentFactoryTests
Assert.True(observation.Content.Raw?["example"]?.GetValue<bool>());
Assert.Equal(document.Linkset.References![0].Type, observation.Linkset.References[0].Type);
Assert.Equal(new[] { "CVE-2025-1234", "cve-2025-1234" }, observation.RawLinkset.Aliases);
Assert.Equal(new[] { "runtime", "build" }, observation.RawLinkset.Scopes);
Assert.Equal("depends_on", observation.RawLinkset.Relationships[0].Type);
Assert.Equal("componentA", observation.RawLinkset.Relationships[0].Source);
Assert.Equal("componentB", observation.RawLinkset.Relationships[0].Target);
Assert.Equal("sbom-manifest", observation.RawLinkset.Relationships[0].Provenance);
Assert.Equal("Advisory", observation.RawLinkset.References[0].Type);
Assert.Equal("vendor", observation.RawLinkset.References[0].Source);
Assert.Equal("note-value", observation.RawLinkset.Notes["note-key"]);

View File

@@ -0,0 +1,94 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using MongoDB.Bson;
using StellaOps.Concelier.Models.Observations;
using StellaOps.Concelier.Storage.Mongo.Observations.V1;
using Xunit;
namespace StellaOps.Concelier.Storage.Mongo.Tests.Observations;
public sealed class AdvisoryObservationV1DocumentFactoryTests
{
[Fact]
public void ObservationIdBuilder_IsDeterministic()
{
var id1 = ObservationIdBuilder.Create("TENANT", "Ghsa", "GHSA-1234", "sha256:abc");
var id2 = ObservationIdBuilder.Create("tenant", "ghsa", "GHSA-1234", "sha256:abc");
Assert.Equal(id1, id2);
}
[Fact]
public void ToModel_MapsAndNormalizes()
{
var document = new AdvisoryObservationV1Document
{
Id = new ObjectId("6710f1f1a1b2c3d4e5f60708"),
TenantId = "TENANT-01",
Source = "GHSA",
AdvisoryId = "GHSA-2025-0001",
Title = "Test title",
Summary = "Summary",
Severities = new List<ObservationSeverityDocument>
{
new() { System = "cvssv3.1", Score = 7.5, Vector = "AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N" }
},
Affected = new List<ObservationAffectedDocument>
{
new()
{
Purl = "pkg:nuget/foo@1.2.3",
Package = "foo",
Versions = new List<string>{ "1.2.3" },
Ranges = new List<ObservationVersionRangeDocument>
{
new()
{
Type = "ECOSYSTEM",
Events = new List<ObservationRangeEventDocument>
{
new(){ Event = "introduced", Value = "1.0.0" },
new(){ Event = "fixed", Value = "1.2.3" }
}
}
},
Ecosystem = "nuget",
Cpes = new List<string>{ "cpe:/a:foo:bar:1.2.3" }
}
},
References = new List<string>{ "https://example.test/advisory" },
Weaknesses = new List<string>{ "CWE-79" },
Published = new DateTime(2025, 11, 1, 0, 0, 0, DateTimeKind.Utc),
Modified = new DateTime(2025, 11, 10, 0, 0, 0, DateTimeKind.Utc),
IngestedAt = new DateTime(2025, 11, 12, 0, 0, 0, DateTimeKind.Utc),
Provenance = new ObservationProvenanceDocument
{
SourceArtifactSha = "sha256:abc",
FetchedAt = new DateTime(2025, 11, 12, 0, 0, 0, DateTimeKind.Utc),
IngestJobId = "job-1",
Signature = new ObservationSignatureDocument
{
Present = true,
Format = "dsse",
KeyId = "k1",
Signature = "sig"
}
}
};
var model = AdvisoryObservationV1DocumentFactory.ToModel(document);
Assert.Equal("6710f1f1a1b2c3d4e5f60708", model.ObservationId);
Assert.Equal("tenant-01", model.Tenant);
Assert.Equal("ghsa", model.Source);
Assert.Equal("GHSA-2025-0001", model.AdvisoryId);
Assert.Equal("Test title", model.Title);
Assert.Single(model.Severities);
Assert.Single(model.Affected);
Assert.Single(model.References);
Assert.Single(model.Weaknesses);
Assert.Equal(new DateTimeOffset(2025, 11, 12, 0, 0, 0, TimeSpan.Zero), model.IngestedAt);
Assert.NotNull(model.Provenance.Signature);
}
}

View File

@@ -0,0 +1,56 @@
using System;
using System.Collections.Generic;
using MongoDB.Bson.Serialization.Attributes;
namespace StellaOps.Concelier.WebService.Tests;
/// <summary>
/// Minimal linkset document used only for seeding the Mongo collection in WebService integration tests.
/// Matches the shape written by the linkset ingestion pipeline.
/// </summary>
internal sealed class AdvisoryLinksetDocument
{
[BsonElement("tenantId")]
public string TenantId { get; init; } = string.Empty;
[BsonElement("source")]
public string Source { get; init; } = string.Empty;
[BsonElement("advisoryId")]
public string AdvisoryId { get; init; } = string.Empty;
[BsonElement("observations")]
public IReadOnlyList<string> Observations { get; init; } = Array.Empty<string>();
[BsonElement("createdAt")]
public DateTime CreatedAt { get; init; }
[BsonElement("normalized")]
public AdvisoryLinksetNormalizedDocument Normalized { get; init; } = new();
}
internal sealed class AdvisoryLinksetNormalizedDocument
{
[BsonElement("purls")]
public IReadOnlyList<string> Purls { get; init; } = Array.Empty<string>();
[BsonElement("versions")]
public IReadOnlyList<string> Versions { get; init; } = Array.Empty<string>();
}
/// <summary>
/// Shape used when reading /linksets responses in WebService endpoint tests.
/// </summary>
internal sealed class AdvisoryLinksetQueryResponse
{
public AdvisoryLinksetResponse[] Linksets { get; init; } = Array.Empty<AdvisoryLinksetResponse>();
public bool HasMore { get; init; }
public string? NextCursor { get; init; }
}
internal sealed class AdvisoryLinksetResponse
{
public string AdvisoryId { get; init; } = string.Empty;
public IReadOnlyList<string> Purls { get; init; } = Array.Empty<string>();
public IReadOnlyList<string> Versions { get; init; } = Array.Empty<string>();
}

View File

@@ -33,6 +33,7 @@ using StellaOps.Concelier.Merge.Services;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Observations;
using StellaOps.Concelier.Storage.Mongo.Linksets;
using StellaOps.Concelier.Core.Raw;
using StellaOps.Concelier.WebService.Jobs;
using StellaOps.Concelier.WebService.Options;
@@ -376,13 +377,12 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
var root = document.RootElement;
Assert.Equal("CVE-2025-0001", root.GetProperty("advisoryKey").GetString());
Assert.False(string.IsNullOrWhiteSpace(root.GetProperty("fingerprint").GetString()));
Assert.Equal(1, root.GetProperty("total").GetInt32());
Assert.False(root.GetProperty("truncated").GetBoolean());
var entry = Assert.Single(root.GetProperty("entries").EnumerateArray());
Assert.Equal("workaround", entry.GetProperty("type").GetString());
Assert.Equal("tenant-a:chunk:newest", entry.GetProperty("documentId").GetString());
Assert.Equal("/references/0", entry.GetProperty("fieldPath").GetString());
Assert.False(string.IsNullOrWhiteSpace(entry.GetProperty("chunkId").GetString()));
var content = entry.GetProperty("content");
@@ -391,6 +391,8 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
Assert.Equal("https://vendor.example/workaround", content.GetProperty("url").GetString());
var provenance = entry.GetProperty("provenance");
Assert.Equal("tenant-a:chunk:newest", provenance.GetProperty("documentId").GetString());
Assert.Equal("/references/0", provenance.GetProperty("observationPath").GetString());
Assert.Equal("nvd", provenance.GetProperty("source").GetString());
Assert.Equal("workaround", provenance.GetProperty("kind").GetString());
Assert.Equal("tenant-a:chunk:newest", provenance.GetProperty("value").GetString());
@@ -638,6 +640,9 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
using var client = _factory.CreateClient();
long expectedSegments = 0;
string expectedTruncatedTag = "false";
var metrics = await CaptureMetricsAsync(
AdvisoryAiMetrics.MeterName,
new[]
@@ -654,6 +659,13 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
var first = await client.GetAsync(url);
first.EnsureSuccessStatusCode();
using (var firstDocument = await first.Content.ReadFromJsonAsync<JsonDocument>())
{
Assert.NotNull(firstDocument);
expectedSegments = firstDocument!.RootElement.GetProperty("entries").GetArrayLength();
expectedTruncatedTag = firstDocument.RootElement.GetProperty("truncated").GetBoolean() ? "true" : "false";
}
var second = await client.GetAsync(url);
second.EnsureSuccessStatusCode();
});
@@ -679,7 +691,11 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
Assert.True(metrics.TryGetValue("advisory_ai_chunk_segments", out var segmentMeasurements));
Assert.Equal(2, segmentMeasurements!.Count);
Assert.Contains(segmentMeasurements!, measurement => GetTagValue(measurement, "truncated") == "false");
Assert.All(segmentMeasurements!, measurement =>
{
Assert.Equal(expectedSegments, measurement.Value);
Assert.Equal(expectedTruncatedTag, GetTagValue(measurement, "truncated"));
});
Assert.True(metrics.TryGetValue("advisory_ai_chunk_sources", out var sourceMeasurements));
Assert.Equal(2, sourceMeasurements!.Count);
@@ -2522,6 +2538,7 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
Array.Empty<string>(),
references,
Array.Empty<string>(),
Array.Empty<string>(),
new Dictionary<string, string> { ["note"] = "ingest-test" }));
}