feat: Add initial implementation of Vulnerability Resolver Jobs
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Created project for StellaOps.Scanner.Analyzers.Native.Tests with necessary dependencies.
- Documented roles and guidelines in AGENTS.md for Scheduler module.
- Implemented IResolverJobService interface and InMemoryResolverJobService for handling resolver jobs.
- Added ResolverBacklogNotifier and ResolverBacklogService for monitoring job metrics.
- Developed API endpoints for managing resolver jobs and retrieving metrics.
- Defined models for resolver job requests and responses.
- Integrated dependency injection for resolver job services.
- Implemented ImpactIndexSnapshot for persisting impact index data.
- Introduced SignalsScoringOptions for configurable scoring weights in reachability scoring.
- Added unit tests for ReachabilityScoringService and RuntimeFactsIngestionService.
- Created dotnet-filter.sh script to handle command-line arguments for dotnet.
- Established nuget-prime project for managing package downloads.
This commit is contained in:
master
2025-11-18 07:52:15 +02:00
parent e69b57d467
commit 8355e2ff75
299 changed files with 13293 additions and 2444 deletions

View File

@@ -13,6 +13,8 @@ public sealed record AdvisoryLinkset(
ImmutableArray<string> ObservationIds,
AdvisoryLinksetNormalized? Normalized,
AdvisoryLinksetProvenance? Provenance,
double? Confidence,
IReadOnlyList<AdvisoryLinksetConflict>? Conflicts,
DateTimeOffset CreatedAt,
string? BuiltByJobId);
@@ -34,6 +36,11 @@ public sealed record AdvisoryLinksetProvenance(
string? ToolVersion,
string? PolicyHash);
public sealed record AdvisoryLinksetConflict(
string Field,
string Reason,
IReadOnlyList<string>? Values);
internal static class BsonDocumentHelper
{
public static BsonDocument FromDictionary(Dictionary<string, object?> dictionary)

View File

@@ -1,3 +1,4 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
@@ -55,6 +56,8 @@ internal sealed class AdvisoryLinksetBackfillService : IAdvisoryLinksetBackfillS
observationIds,
normalized,
null,
1.0,
Array.Empty<AdvisoryLinksetConflict>(),
createdAt,
null);

View File

@@ -24,6 +24,19 @@ internal static class AdvisoryLinksetNormalization
return Build(purls);
}
public static (AdvisoryLinksetNormalized? normalized, double? confidence, IReadOnlyList<AdvisoryLinksetConflict> conflicts) FromRawLinksetWithConfidence(
RawLinkset linkset,
double? providedConfidence = null)
{
ArgumentNullException.ThrowIfNull(linkset);
var normalized = Build(linkset.PackageUrls);
var confidence = CoerceConfidence(providedConfidence);
var conflicts = ExtractConflicts(linkset);
return (normalized, confidence, conflicts);
}
private static AdvisoryLinksetNormalized? Build(IEnumerable<string> purlValues)
{
var normalizedPurls = NormalizePurls(purlValues);
@@ -75,4 +88,44 @@ internal static class AdvisoryLinksetNormalization
return versions.ToList();
}
private static double? CoerceConfidence(double? confidence)
{
if (!confidence.HasValue)
{
return null;
}
if (double.IsNaN(confidence.Value) || double.IsInfinity(confidence.Value))
{
return null;
}
return Math.Clamp(confidence.Value, 0d, 1d);
}
private static IReadOnlyList<AdvisoryLinksetConflict> ExtractConflicts(RawLinkset linkset)
{
if (linkset.Notes is null || linkset.Notes.Count == 0)
{
return Array.Empty<AdvisoryLinksetConflict>();
}
var conflicts = new List<AdvisoryLinksetConflict>();
foreach (var note in linkset.Notes)
{
if (string.IsNullOrWhiteSpace(note.Key) || string.IsNullOrWhiteSpace(note.Value))
{
continue;
}
conflicts.Add(new AdvisoryLinksetConflict(
note.Key.Trim(),
note.Value.Trim(),
null));
}
return conflicts;
}
}

View File

@@ -0,0 +1,15 @@
using System.Collections.Immutable;
using StellaOps.Concelier.Models.Observations;
namespace StellaOps.Concelier.Core.Observations;
/// <summary>
/// Aggregated linkset facets (aliases, purls, cpes, references, scopes, relationships) built from a set of observations.
/// </summary>
public sealed record AdvisoryObservationLinksetAggregate(
ImmutableArray<string> Aliases,
ImmutableArray<string> Purls,
ImmutableArray<string> Cpes,
ImmutableArray<AdvisoryObservationReference> References,
ImmutableArray<string> Scopes,
ImmutableArray<RawRelationship> Relationships);

View File

@@ -1,6 +1,7 @@
using System.Collections.Immutable;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Models.Observations;
using System.Collections.Immutable;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Models.Observations;
using StellaOps.Concelier.RawModels;
namespace StellaOps.Concelier.Core.Observations;
@@ -66,17 +67,19 @@ public sealed record AdvisoryObservationQueryOptions
/// <summary>
/// Query result containing observations and their aggregated linkset hints.
/// </summary>
public sealed record AdvisoryObservationQueryResult(
ImmutableArray<AdvisoryObservation> Observations,
AdvisoryObservationLinksetAggregate Linkset,
string? NextCursor,
bool HasMore);
/// <summary>
/// Aggregated linkset built from the observations returned by a query.
/// </summary>
public sealed record AdvisoryObservationLinksetAggregate(
ImmutableArray<string> Aliases,
ImmutableArray<string> Purls,
ImmutableArray<string> Cpes,
ImmutableArray<AdvisoryObservationReference> References);
public sealed record AdvisoryObservationQueryResult(
ImmutableArray<AdvisoryObservation> Observations,
AdvisoryObservationLinksetAggregate Linkset,
string? NextCursor,
bool HasMore);
/// <summary>
/// Aggregated linkset built from the observations returned by a query.
/// </summary>
public sealed record AdvisoryObservationLinksetAggregate(
ImmutableArray<string> Aliases,
ImmutableArray<string> Purls,
ImmutableArray<string> Cpes,
ImmutableArray<AdvisoryObservationReference> References,
ImmutableArray<string> Scopes,
ImmutableArray<RawRelationship> Relationships);

View File

@@ -1,8 +1,9 @@
using System.Collections.Immutable;
using System.Globalization;
using System.Text;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Models.Observations;
using System.Collections.Immutable;
using System.Globalization;
using System.Text;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Models.Observations;
using StellaOps.Concelier.RawModels;
namespace StellaOps.Concelier.Core.Observations;
@@ -195,24 +196,28 @@ public sealed class AdvisoryObservationQueryService : IAdvisoryObservationQueryS
private static AdvisoryObservationLinksetAggregate BuildAggregateLinkset(ImmutableArray<AdvisoryObservation> observations)
{
if (observations.IsDefaultOrEmpty)
{
return new AdvisoryObservationLinksetAggregate(
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty,
ImmutableArray<AdvisoryObservationReference>.Empty);
}
if (observations.IsDefaultOrEmpty)
{
return new AdvisoryObservationLinksetAggregate(
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty,
ImmutableArray<AdvisoryObservationReference>.Empty,
ImmutableArray<string>.Empty,
ImmutableArray<RawRelationship>.Empty);
}
var aliasSet = new HashSet<string>(StringComparer.Ordinal);
var purlSet = new HashSet<string>(StringComparer.Ordinal);
var cpeSet = new HashSet<string>(StringComparer.Ordinal);
var referenceSet = new HashSet<AdvisoryObservationReference>();
foreach (var observation in observations)
{
foreach (var alias in observation.Linkset.Aliases)
{
var referenceSet = new HashSet<AdvisoryObservationReference>();
var scopeSet = new HashSet<string>(StringComparer.Ordinal);
var relationshipSet = new HashSet<RawRelationship>();
foreach (var observation in observations)
{
foreach (var alias in observation.Linkset.Aliases)
{
aliasSet.Add(alias);
}
@@ -227,18 +232,34 @@ public sealed class AdvisoryObservationQueryService : IAdvisoryObservationQueryS
}
foreach (var reference in observation.Linkset.References)
{
referenceSet.Add(reference);
}
}
return new AdvisoryObservationLinksetAggregate(
aliasSet.OrderBy(static alias => alias, StringComparer.Ordinal).ToImmutableArray(),
purlSet.OrderBy(static purl => purl, StringComparer.Ordinal).ToImmutableArray(),
cpeSet.OrderBy(static cpe => cpe, StringComparer.Ordinal).ToImmutableArray(),
referenceSet
.OrderBy(static reference => reference.Type, StringComparer.Ordinal)
.ThenBy(static reference => reference.Url, StringComparer.Ordinal)
.ToImmutableArray());
}
}
{
referenceSet.Add(reference);
}
foreach (var scope in observation.RawLinkset.Scopes)
{
scopeSet.Add(scope);
}
foreach (var relationship in observation.RawLinkset.Relationships)
{
relationshipSet.Add(relationship);
}
}
return new AdvisoryObservationLinksetAggregate(
aliasSet.OrderBy(static alias => alias, StringComparer.Ordinal).ToImmutableArray(),
purlSet.OrderBy(static purl => purl, StringComparer.Ordinal).ToImmutableArray(),
cpeSet.OrderBy(static cpe => cpe, StringComparer.Ordinal).ToImmutableArray(),
referenceSet
.OrderBy(static reference => reference.Type, StringComparer.Ordinal)
.ThenBy(static reference => reference.Url, StringComparer.Ordinal)
.ToImmutableArray(),
scopeSet.OrderBy(static scope => scope, StringComparer.Ordinal).ToImmutableArray(),
relationshipSet
.OrderBy(static rel => rel.Type, StringComparer.Ordinal)
.ThenBy(static rel => rel.Source, StringComparer.Ordinal)
.ThenBy(static rel => rel.Target, StringComparer.Ordinal)
.ToImmutableArray());
}
}

View File

@@ -116,7 +116,10 @@ internal sealed class AdvisoryRawService : IAdvisoryRawService
var observation = _observationFactory.Create(enriched, _timeProvider.GetUtcNow());
await _observationSink.UpsertAsync(observation, cancellationToken).ConfigureAwait(false);
var normalizedLinkset = AdvisoryLinksetNormalization.FromRawLinkset(enriched.Linkset);
var (normalizedLinkset, confidence, conflicts) = AdvisoryLinksetNormalization.FromRawLinksetWithConfidence(
enriched.Linkset,
providedConfidence: null);
var linkset = new AdvisoryLinkset(
tenant,
source,
@@ -124,6 +127,8 @@ internal sealed class AdvisoryRawService : IAdvisoryRawService
ImmutableArray.Create(observation.ObservationId),
normalizedLinkset,
null,
confidence ?? 1.0,
conflicts,
_timeProvider.GetUtcNow(),
null);

View File

@@ -95,6 +95,26 @@ public sealed record AdvisoryObservation
return references;
}
static ImmutableArray<RawRelationship> SanitizeRelationships(ImmutableArray<RawRelationship> relationships)
{
if (relationships.IsDefault)
{
return ImmutableArray<RawRelationship>.Empty;
}
return relationships;
}
static ImmutableArray<string> SanitizeScopes(ImmutableArray<string> scopes)
{
if (scopes.IsDefault)
{
return ImmutableArray<string>.Empty;
}
return scopes;
}
static ImmutableDictionary<string, string> SanitizeNotes(ImmutableDictionary<string, string>? notes)
{
if (notes is null || notes.Count == 0)
@@ -108,6 +128,8 @@ public sealed record AdvisoryObservation
return rawLinkset with
{
Aliases = SanitizeStrings(rawLinkset.Aliases),
Scopes = SanitizeScopes(rawLinkset.Scopes),
Relationships = SanitizeRelationships(rawLinkset.Relationships),
PackageUrls = SanitizeStrings(rawLinkset.PackageUrls),
Cpes = SanitizeStrings(rawLinkset.Cpes),
References = SanitizeReferences(rawLinkset.References),

View File

@@ -0,0 +1,240 @@
using System.Collections.Immutable;
namespace StellaOps.Concelier.Models.Observations;
/// <summary>
/// Version 1 Link-Not-Merge observation record: immutable, per-source payload with provenance and tenant guards.
/// </summary>
public sealed record AdvisoryObservationV1
{
public AdvisoryObservationV1(
string observationId,
string tenant,
string source,
string advisoryId,
string? title,
string? summary,
ImmutableArray<ObservationSeverity> severities,
ImmutableArray<ObservationAffected> affected,
ImmutableArray<string> references,
ImmutableArray<string> weaknesses,
DateTimeOffset? published,
DateTimeOffset? modified,
ObservationProvenance provenance,
DateTimeOffset ingestedAt,
string? supersedesObservationId = null)
{
ObservationId = Validation.EnsureNotNullOrWhiteSpace(observationId, nameof(observationId));
Tenant = Validation.EnsureNotNullOrWhiteSpace(tenant, nameof(tenant)).ToLowerInvariant();
Source = Validation.EnsureNotNullOrWhiteSpace(source, nameof(source)).ToLowerInvariant();
AdvisoryId = Validation.EnsureNotNullOrWhiteSpace(advisoryId, nameof(advisoryId));
Title = Validation.TrimToNull(title);
Summary = Validation.TrimToNull(summary);
Severities = Normalize(severities);
Affected = Normalize(affected);
References = NormalizeStrings(references);
Weaknesses = NormalizeStrings(weaknesses);
Published = published?.ToUniversalTime();
Modified = modified?.ToUniversalTime();
Provenance = provenance ?? throw new ArgumentNullException(nameof(provenance));
IngestedAt = ingestedAt.ToUniversalTime();
SupersedesObservationId = Validation.TrimToNull(supersedesObservationId);
}
public string ObservationId { get; }
public string Tenant { get; }
public string Source { get; }
public string AdvisoryId { get; }
public string? Title { get; }
public string? Summary { get; }
public ImmutableArray<ObservationSeverity> Severities { get; }
public ImmutableArray<ObservationAffected> Affected { get; }
public ImmutableArray<string> References { get; }
public ImmutableArray<string> Weaknesses { get; }
public DateTimeOffset? Published { get; }
public DateTimeOffset? Modified { get; }
public ObservationProvenance Provenance { get; }
public DateTimeOffset IngestedAt { get; }
public string? SupersedesObservationId { get; }
private static ImmutableArray<string> NormalizeStrings(ImmutableArray<string> values)
{
if (values.IsDefaultOrEmpty)
{
return ImmutableArray<string>.Empty;
}
var builder = ImmutableArray.CreateBuilder<string>();
foreach (var value in values)
{
if (string.IsNullOrWhiteSpace(value))
{
continue;
}
builder.Add(value.Trim());
}
return builder.Count == 0 ? ImmutableArray<string>.Empty : builder.ToImmutable();
}
private static ImmutableArray<T> Normalize<T>(ImmutableArray<T> values)
{
if (values.IsDefaultOrEmpty)
{
return ImmutableArray<T>.Empty;
}
return values;
}
}
public sealed class ObservationSeverity
{
public ObservationSeverity(string system, double score, string? vector)
{
System = Validation.EnsureNotNullOrWhiteSpace(system, nameof(system));
Score = score;
Vector = Validation.TrimToNull(vector);
}
public string System { get; }
public double Score { get; }
public string? Vector { get; }
}
public sealed class ObservationAffected
{
public ObservationAffected(
string purl,
string? package,
ImmutableArray<string> versions,
ImmutableArray<ObservationVersionRange> ranges,
string? ecosystem,
ImmutableArray<string> cpes)
{
Purl = Validation.EnsureNotNullOrWhiteSpace(purl, nameof(purl));
Package = Validation.TrimToNull(package);
Versions = NormalizeStrings(versions);
Ranges = ranges.IsDefault ? ImmutableArray<ObservationVersionRange>.Empty : ranges;
Ecosystem = Validation.TrimToNull(ecosystem);
Cpes = NormalizeStrings(cpes);
}
public string Purl { get; }
public string? Package { get; }
public ImmutableArray<string> Versions { get; }
public ImmutableArray<ObservationVersionRange> Ranges { get; }
public string? Ecosystem { get; }
public ImmutableArray<string> Cpes { get; }
private static ImmutableArray<string> NormalizeStrings(ImmutableArray<string> values)
{
if (values.IsDefaultOrEmpty)
{
return ImmutableArray<string>.Empty;
}
var builder = ImmutableArray.CreateBuilder<string>();
foreach (var value in values)
{
if (string.IsNullOrWhiteSpace(value))
{
continue;
}
builder.Add(value.Trim());
}
return builder.Count == 0 ? ImmutableArray<string>.Empty : builder.ToImmutable();
}
}
public sealed class ObservationVersionRange
{
public ObservationVersionRange(string type, ImmutableArray<ObservationRangeEvent> events)
{
Type = Validation.EnsureNotNullOrWhiteSpace(type, nameof(type));
Events = events.IsDefault ? ImmutableArray<ObservationRangeEvent>.Empty : events;
}
public string Type { get; }
public ImmutableArray<ObservationRangeEvent> Events { get; }
}
public sealed class ObservationRangeEvent
{
public ObservationRangeEvent(string @event, string value)
{
Event = Validation.EnsureNotNullOrWhiteSpace(@event, nameof(@event));
Value = Validation.EnsureNotNullOrWhiteSpace(value, nameof(value));
}
public string Event { get; }
public string Value { get; }
}
public sealed class ObservationProvenance
{
public ObservationProvenance(
string sourceArtifactSha,
DateTimeOffset fetchedAt,
string? ingestJobId,
ObservationSignature? signature)
{
SourceArtifactSha = Validation.EnsureNotNullOrWhiteSpace(sourceArtifactSha, nameof(sourceArtifactSha));
FetchedAt = fetchedAt.ToUniversalTime();
IngestJobId = Validation.TrimToNull(ingestJobId);
Signature = signature;
}
public string SourceArtifactSha { get; }
public DateTimeOffset FetchedAt { get; }
public string? IngestJobId { get; }
public ObservationSignature? Signature { get; }
}
public sealed class ObservationSignature
{
public ObservationSignature(bool present, string? format, string? keyId, string? signatureValue)
{
Present = present;
Format = Validation.TrimToNull(format);
KeyId = Validation.TrimToNull(keyId);
SignatureValue = Validation.TrimToNull(signatureValue);
}
public bool Present { get; }
public string? Format { get; }
public string? KeyId { get; }
public string? SignatureValue { get; }
}

View File

@@ -55,23 +55,35 @@ public sealed record RawLinkset
{
[JsonPropertyName("aliases")]
public ImmutableArray<string> Aliases { get; init; } = ImmutableArray<string>.Empty;
[JsonPropertyName("purls")]
public ImmutableArray<string> PackageUrls { get; init; } = ImmutableArray<string>.Empty;
[JsonPropertyName("cpes")]
public ImmutableArray<string> Cpes { get; init; } = ImmutableArray<string>.Empty;
[JsonPropertyName("scopes")]
public ImmutableArray<string> Scopes { get; init; } = ImmutableArray<string>.Empty;
[JsonPropertyName("relationships")]
public ImmutableArray<RawRelationship> Relationships { get; init; } = ImmutableArray<RawRelationship>.Empty;
[JsonPropertyName("purls")]
public ImmutableArray<string> PackageUrls { get; init; } = ImmutableArray<string>.Empty;
[JsonPropertyName("cpes")]
public ImmutableArray<string> Cpes { get; init; } = ImmutableArray<string>.Empty;
[JsonPropertyName("references")]
public ImmutableArray<RawReference> References { get; init; } = ImmutableArray<RawReference>.Empty;
[JsonPropertyName("reconciled_from")]
public ImmutableArray<string> ReconciledFrom { get; init; } = ImmutableArray<string>.Empty;
[JsonPropertyName("notes")]
public ImmutableDictionary<string, string> Notes { get; init; } = ImmutableDictionary<string, string>.Empty;
[JsonPropertyName("notes")]
public ImmutableDictionary<string, string> Notes { get; init; } = ImmutableDictionary<string, string>.Empty;
}
public sealed record RawRelationship(
[property: JsonPropertyName("type")] string Type,
[property: JsonPropertyName("source")] string Source,
[property: JsonPropertyName("target")] string Target,
[property: JsonPropertyName("provenance")] string? Provenance = null);
public sealed record RawReference(
[property: JsonPropertyName("type")] string Type,
[property: JsonPropertyName("url")] string Url,

View File

@@ -29,6 +29,16 @@ public sealed class AdvisoryLinksetDocument
public AdvisoryLinksetNormalizedDocument? Normalized { get; set; }
= null;
[BsonElement("confidence")]
[BsonIgnoreIfNull]
public double? Confidence { get; set; }
= null;
[BsonElement("conflicts")]
[BsonIgnoreIfNull]
public List<AdvisoryLinksetConflictDocument>? Conflicts { get; set; }
= null;
[BsonElement("createdAt")]
public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
@@ -85,3 +95,18 @@ public sealed class AdvisoryLinksetProvenanceDocument
public string? PolicyHash { get; set; }
= null;
}
[BsonIgnoreExtraElements]
public sealed class AdvisoryLinksetConflictDocument
{
[BsonElement("field")]
public string Field { get; set; } = string.Empty;
[BsonElement("reason")]
public string Reason { get; set; } = string.Empty;
[BsonElement("values")]
[BsonIgnoreIfNull]
public List<string>? Values { get; set; }
= null;
}

View File

@@ -5,11 +5,12 @@ using CoreLinksets = StellaOps.Concelier.Core.Linksets;
namespace StellaOps.Concelier.Storage.Mongo.Linksets;
// Backcompat sink name retained for compile includes; forwards to the Mongo-specific store.
internal sealed class AdvisoryLinksetSink : CoreLinksets.IAdvisoryLinksetSink
{
private readonly IAdvisoryLinksetStore _store;
private readonly IMongoAdvisoryLinksetStore _store;
public AdvisoryLinksetSink(IAdvisoryLinksetStore store)
public AdvisoryLinksetSink(IMongoAdvisoryLinksetStore store)
{
_store = store ?? throw new ArgumentNullException(nameof(store));
}

View File

@@ -0,0 +1,20 @@
using System;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Concelier.Storage.Mongo.Linksets;
internal sealed class ConcelierMongoLinksetSink : global::StellaOps.Concelier.Core.Linksets.IAdvisoryLinksetSink
{
private readonly IMongoAdvisoryLinksetStore _store;
public ConcelierMongoLinksetSink(IMongoAdvisoryLinksetStore store)
{
_store = store ?? throw new ArgumentNullException(nameof(store));
}
public Task UpsertAsync(global::StellaOps.Concelier.Core.Linksets.AdvisoryLinkset linkset, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(linkset);
return _store.UpsertAsync(linkset, cancellationToken);
}
}

View File

@@ -6,15 +6,14 @@ using System.Threading;
using System.Threading.Tasks;
using MongoDB.Driver;
using CoreLinksets = StellaOps.Concelier.Core.Linksets;
namespace StellaOps.Concelier.Storage.Mongo.Linksets;
// Internal type kept in storage namespace to avoid name clash with core interface
internal sealed class MongoAdvisoryLinksetStore : CoreLinksets.IAdvisoryLinksetStore, CoreLinksets.IAdvisoryLinksetLookup
// Storage implementation of advisory linkset persistence.
internal sealed class ConcelierMongoLinksetStore : IMongoAdvisoryLinksetStore
{
private readonly IMongoCollection<AdvisoryLinksetDocument> _collection;
public MongoAdvisoryLinksetStore(IMongoCollection<AdvisoryLinksetDocument> collection)
public ConcelierMongoLinksetStore(IMongoCollection<AdvisoryLinksetDocument> collection)
{
_collection = collection ?? throw new ArgumentNullException(nameof(collection));
}
@@ -24,8 +23,9 @@ internal sealed class MongoAdvisoryLinksetStore : CoreLinksets.IAdvisoryLinksetS
ArgumentNullException.ThrowIfNull(linkset);
var document = MapToDocument(linkset);
var tenant = linkset.TenantId.ToLowerInvariant();
var filter = Builders<AdvisoryLinksetDocument>.Filter.And(
Builders<AdvisoryLinksetDocument>.Filter.Eq(d => d.TenantId, linkset.TenantId),
Builders<AdvisoryLinksetDocument>.Filter.Eq(d => d.TenantId, tenant),
Builders<AdvisoryLinksetDocument>.Filter.Eq(d => d.Source, linkset.Source),
Builders<AdvisoryLinksetDocument>.Filter.Eq(d => d.AdvisoryId, linkset.AdvisoryId));
@@ -73,9 +73,6 @@ internal sealed class MongoAdvisoryLinksetStore : CoreLinksets.IAdvisoryLinksetS
var filter = builder.And(filters);
var sort = Builders<AdvisoryLinksetDocument>.Sort.Descending(d => d.CreatedAt).Ascending(d => d.AdvisoryId);
var findFilter = filter;
if (cursor is not null)
{
var cursorFilter = builder.Or(
@@ -84,10 +81,11 @@ internal sealed class MongoAdvisoryLinksetStore : CoreLinksets.IAdvisoryLinksetS
builder.Eq(d => d.CreatedAt, cursor.CreatedAt.UtcDateTime),
builder.Gt(d => d.AdvisoryId, cursor.AdvisoryId)));
findFilter = builder.And(findFilter, cursorFilter);
filter = builder.And(filter, cursorFilter);
}
var documents = await _collection.Find(findFilter)
var sort = Builders<AdvisoryLinksetDocument>.Sort.Descending(d => d.CreatedAt).Ascending(d => d.AdvisoryId);
var documents = await _collection.Find(filter)
.Sort(sort)
.Limit(limit)
.ToListAsync(cancellationToken)
@@ -98,14 +96,23 @@ internal sealed class MongoAdvisoryLinksetStore : CoreLinksets.IAdvisoryLinksetS
private static AdvisoryLinksetDocument MapToDocument(CoreLinksets.AdvisoryLinkset linkset)
{
var doc = new AdvisoryLinksetDocument
return new AdvisoryLinksetDocument
{
TenantId = linkset.TenantId,
TenantId = linkset.TenantId.ToLowerInvariant(),
Source = linkset.Source,
AdvisoryId = linkset.AdvisoryId,
Observations = new List<string>(linkset.ObservationIds),
CreatedAt = linkset.CreatedAt.UtcDateTime,
BuiltByJobId = linkset.BuiltByJobId,
Confidence = linkset.Confidence,
Conflicts = linkset.Conflicts is null
? null
: linkset.Conflicts.Select(conflict => new AdvisoryLinksetConflictDocument
{
Field = conflict.Field,
Reason = conflict.Reason,
Values = conflict.Values is null ? null : new List<string>(conflict.Values)
}).ToList(),
Provenance = linkset.Provenance is null ? null : new AdvisoryLinksetProvenanceDocument
{
ObservationHashes = linkset.Provenance.ObservationHashes is null
@@ -122,26 +129,31 @@ internal sealed class MongoAdvisoryLinksetStore : CoreLinksets.IAdvisoryLinksetS
Severities = linkset.Normalized.SeveritiesToBson(),
}
};
return doc;
}
private static CoreLinksets.AdvisoryLinkset FromDocument(AdvisoryLinksetDocument doc)
{
return new AdvisoryLinkset(
return new CoreLinksets.AdvisoryLinkset(
doc.TenantId,
doc.Source,
doc.AdvisoryId,
doc.Observations.ToImmutableArray(),
doc.Normalized is null ? null : new AdvisoryLinksetNormalized(
doc.Normalized is null ? null : new CoreLinksets.AdvisoryLinksetNormalized(
doc.Normalized.Purls,
doc.Normalized.Versions,
doc.Normalized.Ranges?.Select(ToDictionary).ToList(),
doc.Normalized.Severities?.Select(ToDictionary).ToList()),
doc.Provenance is null ? null : new AdvisoryLinksetProvenance(
doc.Provenance is null ? null : new CoreLinksets.AdvisoryLinksetProvenance(
doc.Provenance.ObservationHashes,
doc.Provenance.ToolVersion,
doc.Provenance.PolicyHash),
doc.Confidence,
doc.Conflicts is null
? null
: doc.Conflicts.Select(conflict => new CoreLinksets.AdvisoryLinksetConflict(
conflict.Field,
conflict.Reason,
conflict.Values)).ToList(),
DateTime.SpecifyKind(doc.CreatedAt, DateTimeKind.Utc),
doc.BuiltByJobId);
}

View File

@@ -0,0 +1,5 @@
namespace StellaOps.Concelier.Storage.Mongo.Linksets;
public interface IMongoAdvisoryLinksetStore : global::StellaOps.Concelier.Core.Linksets.IAdvisoryLinksetStore, global::StellaOps.Concelier.Core.Linksets.IAdvisoryLinksetLookup
{
}

View File

@@ -0,0 +1,66 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Bson;
using MongoDB.Driver;
namespace StellaOps.Concelier.Storage.Mongo.Migrations;
/// <summary>
/// Normalises advisory_linksets tenant ids to lowercase to keep lookups/write paths consistent.
/// </summary>
public sealed class EnsureAdvisoryLinksetsTenantLowerMigration : IMongoMigration
{
private const string MigrationId = "20251117_advisory_linksets_tenant_lower";
private const int BatchSize = 500;
public string Id => MigrationId;
public string Description => "Lowercase tenant ids in advisory_linksets to match query filters.";
public async Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(database);
var collection = database.GetCollection<BsonDocument>(MongoStorageDefaults.Collections.AdvisoryLinksets);
var filter = Builders<BsonDocument>.Filter.Where(doc =>
doc.Contains("TenantId") &&
doc["TenantId"].BsonType == BsonType.String &&
doc["TenantId"].AsString != doc["TenantId"].AsString.ToLowerInvariant());
using var cursor = await collection.Find(filter).ToCursorAsync(cancellationToken).ConfigureAwait(false);
var writes = new List<WriteModel<BsonDocument>>(BatchSize);
while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false))
{
foreach (var doc in cursor.Current)
{
var currentTenant = doc["TenantId"].AsString;
var lower = currentTenant.ToLowerInvariant();
if (lower == currentTenant)
{
continue;
}
var idFilter = Builders<BsonDocument>.Filter.Eq("_id", doc["_id"]);
var update = Builders<BsonDocument>.Update.Set("TenantId", lower);
writes.Add(new UpdateOneModel<BsonDocument>(idFilter, update));
if (writes.Count >= BatchSize)
{
await collection.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = false }, cancellationToken)
.ConfigureAwait(false);
writes.Clear();
}
}
}
if (writes.Count > 0)
{
await collection.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = false }, cancellationToken)
.ConfigureAwait(false);
}
}
}

View File

@@ -28,5 +28,6 @@ public static class MongoStorageDefaults
public const string AdvisoryStatements = "advisory_statements";
public const string AdvisoryConflicts = "advisory_conflicts";
public const string AdvisoryObservations = "advisory_observations";
public const string AdvisoryLinksets = "advisory_linksets";
}
}

View File

@@ -175,6 +175,16 @@ public sealed class AdvisoryObservationRawLinksetDocument
public List<string>? Aliases { get; set; }
= new();
[BsonElement("scopes")]
[BsonIgnoreIfNull]
public List<string>? Scopes { get; set; }
= new();
[BsonElement("relationships")]
[BsonIgnoreIfNull]
public List<AdvisoryObservationRawRelationshipDocument>? Relationships { get; set; }
= new();
[BsonElement("purls")]
[BsonIgnoreIfNull]
public List<string>? PackageUrls { get; set; }
@@ -217,3 +227,21 @@ public sealed class AdvisoryObservationRawReferenceDocument
public string? Source { get; set; }
= null;
}
[BsonIgnoreExtraElements]
public sealed class AdvisoryObservationRawRelationshipDocument
{
[BsonElement("type")]
public string Type { get; set; } = string.Empty;
[BsonElement("source")]
public string Source { get; set; } = string.Empty;
[BsonElement("target")]
public string Target { get; set; } = string.Empty;
[BsonElement("provenance")]
[BsonIgnoreIfNull]
public string? Provenance { get; set; }
= null;
}

View File

@@ -1,8 +1,9 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text.Json.Nodes;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Nodes;
using MongoDB.Bson;
using MongoDB.Bson.IO;
using StellaOps.Concelier.Models.Observations;
@@ -10,11 +11,11 @@ using StellaOps.Concelier.RawModels;
namespace StellaOps.Concelier.Storage.Mongo.Observations;
internal static class AdvisoryObservationDocumentFactory
{
private static readonly JsonWriterSettings JsonSettings = new() { OutputMode = JsonOutputMode.RelaxedExtendedJson };
public static AdvisoryObservation ToModel(AdvisoryObservationDocument document)
internal static class AdvisoryObservationDocumentFactory
{
private static readonly JsonWriterSettings JsonSettings = new() { OutputMode = JsonOutputMode.RelaxedExtendedJson };
public static AdvisoryObservation ToModel(AdvisoryObservationDocument document)
{
ArgumentNullException.ThrowIfNull(document);
@@ -61,7 +62,69 @@ internal static class AdvisoryObservationDocumentFactory
return observation;
}
public static AdvisoryObservationDocument ToDocument(AdvisoryObservation observation)
{
ArgumentNullException.ThrowIfNull(observation);
var contentRaw = observation.Content.Raw?.ToJsonString(new JsonSerializerOptions
{
Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping
}) ?? "{}";
var document = new AdvisoryObservationDocument
{
Id = observation.ObservationId,
Tenant = observation.Tenant,
Source = new AdvisoryObservationSourceDocument
{
Vendor = observation.Source.Vendor,
Stream = observation.Source.Stream,
Api = observation.Source.Api,
CollectorVersion = observation.Source.CollectorVersion
},
Upstream = new AdvisoryObservationUpstreamDocument
{
UpstreamId = observation.Upstream.UpstreamId,
DocumentVersion = observation.Upstream.DocumentVersion,
FetchedAt = observation.Upstream.FetchedAt.UtcDateTime,
ReceivedAt = observation.Upstream.ReceivedAt.UtcDateTime,
ContentHash = observation.Upstream.ContentHash,
Signature = new AdvisoryObservationSignatureDocument
{
Present = observation.Upstream.Signature.Present,
Format = observation.Upstream.Signature.Format,
KeyId = observation.Upstream.Signature.KeyId,
Signature = observation.Upstream.Signature.Signature
},
Metadata = observation.Upstream.Metadata.ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal)
},
Content = new AdvisoryObservationContentDocument
{
Format = observation.Content.Format,
SpecVersion = observation.Content.SpecVersion,
Raw = BsonDocument.Parse(contentRaw),
Metadata = observation.Content.Metadata.ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal)
},
Linkset = new AdvisoryObservationLinksetDocument
{
Aliases = observation.Linkset.Aliases.ToList(),
Purls = observation.Linkset.Purls.ToList(),
Cpes = observation.Linkset.Cpes.ToList(),
References = observation.Linkset.References.Select(reference => new AdvisoryObservationReferenceDocument
{
Type = reference.Type,
Url = reference.Url
}).ToList()
},
RawLinkset = ToRawLinksetDocument(observation.RawLinkset),
CreatedAt = observation.CreatedAt.UtcDateTime,
Attributes = observation.Attributes.ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal)
};
return document;
}
private static JsonNode ParseJsonNode(BsonDocument raw)
{
if (raw is null || raw.ElementCount == 0)
@@ -113,6 +176,22 @@ internal static class AdvisoryObservationDocumentFactory
.ToImmutableArray();
}
static ImmutableArray<RawRelationship> ToImmutableRelationships(List<AdvisoryObservationRawRelationshipDocument>? relationships)
{
if (relationships is null || relationships.Count == 0)
{
return ImmutableArray<RawRelationship>.Empty;
}
return relationships
.Select(static relationship => new RawRelationship(
relationship.Type ?? string.Empty,
relationship.Source ?? string.Empty,
relationship.Target ?? string.Empty,
relationship.Provenance))
.ToImmutableArray();
}
static ImmutableArray<RawReference> ToImmutableReferences(List<AdvisoryObservationRawReferenceDocument>? references)
{
if (references is null || references.Count == 0)
@@ -152,6 +231,8 @@ internal static class AdvisoryObservationDocumentFactory
return new RawLinkset
{
Aliases = ToImmutableStringArray(document.Aliases),
Scopes = ToImmutableStringArray(document.Scopes),
Relationships = ToImmutableRelationships(document.Relationships),
PackageUrls = ToImmutableStringArray(document.PackageUrls),
Cpes = ToImmutableStringArray(document.Cpes),
References = ToImmutableReferences(document.References),
@@ -159,4 +240,35 @@ internal static class AdvisoryObservationDocumentFactory
Notes = ToImmutableDictionary(document.Notes)
};
}
private static AdvisoryObservationRawLinksetDocument ToRawLinksetDocument(RawLinkset rawLinkset)
{
return new AdvisoryObservationRawLinksetDocument
{
Aliases = rawLinkset.Aliases.IsDefault ? new List<string>() : rawLinkset.Aliases.ToList(),
Scopes = rawLinkset.Scopes.IsDefault ? new List<string>() : rawLinkset.Scopes.ToList(),
Relationships = rawLinkset.Relationships.IsDefault
? new List<AdvisoryObservationRawRelationshipDocument>()
: rawLinkset.Relationships.Select(relationship => new AdvisoryObservationRawRelationshipDocument
{
Type = relationship.Type,
Source = relationship.Source,
Target = relationship.Target,
Provenance = relationship.Provenance
}).ToList(),
PackageUrls = rawLinkset.PackageUrls.IsDefault ? new List<string>() : rawLinkset.PackageUrls.ToList(),
Cpes = rawLinkset.Cpes.IsDefault ? new List<string>() : rawLinkset.Cpes.ToList(),
References = rawLinkset.References.IsDefault
? new List<AdvisoryObservationRawReferenceDocument>()
: rawLinkset.References.Select(reference => new AdvisoryObservationRawReferenceDocument
{
Type = reference.Type,
Url = reference.Url,
Source = reference.Source
}).ToList(),
ReconciledFrom = rawLinkset.ReconciledFrom.IsDefault ? new List<string>() : rawLinkset.ReconciledFrom.ToList(),
Notes = rawLinkset.Notes.Count == 0 ? new Dictionary<string, string>(StringComparer.Ordinal)
: rawLinkset.Notes.ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal)
};
}
}

View File

@@ -2,6 +2,8 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Concelier.Core.Observations;
@@ -9,21 +11,21 @@ using StellaOps.Concelier.Models.Observations;
namespace StellaOps.Concelier.Storage.Mongo.Observations;
internal sealed class AdvisoryObservationStore : IAdvisoryObservationStore
{
private readonly IMongoCollection<AdvisoryObservationDocument> collection;
public AdvisoryObservationStore(IMongoCollection<AdvisoryObservationDocument> collection)
internal sealed class AdvisoryObservationStore : IAdvisoryObservationStore
{
private readonly IMongoCollection<AdvisoryObservationDocument> collection;
public AdvisoryObservationStore(IMongoCollection<AdvisoryObservationDocument> collection)
{
this.collection = collection ?? throw new ArgumentNullException(nameof(collection));
}
public async Task<IReadOnlyList<AdvisoryObservation>> ListByTenantAsync(string tenant, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
var filter = Builders<AdvisoryObservationDocument>.Filter.Eq(document => document.Tenant, tenant.ToLowerInvariant());
var documents = await collection
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
var filter = Builders<AdvisoryObservationDocument>.Filter.Eq(document => document.Tenant, tenant.ToLowerInvariant());
var documents = await collection
.Find(filter)
.SortByDescending(document => document.CreatedAt)
.ThenBy(document => document.Id)
@@ -111,6 +113,18 @@ internal sealed class AdvisoryObservationStore : IAdvisoryObservationStore
return documents.Select(AdvisoryObservationDocumentFactory.ToModel).ToArray();
}
public async Task UpsertAsync(AdvisoryObservation observation, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(observation);
cancellationToken.ThrowIfCancellationRequested();
var document = AdvisoryObservationDocumentFactory.ToDocument(observation);
var filter = Builders<AdvisoryObservationDocument>.Filter.Eq(d => d.Id, document.Id);
var options = new ReplaceOptions { IsUpsert = true };
await collection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false);
}
private static string[] NormalizeValues(IEnumerable<string>? values, Func<string, string> projector)
{
if (values is null)

View File

@@ -1,20 +1,23 @@
using System.Collections.Generic;
using StellaOps.Concelier.Models.Observations;
using StellaOps.Concelier.Core.Observations;
using System.Collections.Generic;
using System.Threading;
using StellaOps.Concelier.Models.Observations;
using StellaOps.Concelier.Core.Observations;
namespace StellaOps.Concelier.Storage.Mongo.Observations;
public interface IAdvisoryObservationStore
{
Task<IReadOnlyList<AdvisoryObservation>> ListByTenantAsync(string tenant, CancellationToken cancellationToken);
Task<IReadOnlyList<AdvisoryObservation>> FindByFiltersAsync(
string tenant,
IEnumerable<string>? observationIds,
IEnumerable<string>? aliases,
IEnumerable<string>? purls,
IEnumerable<string>? cpes,
AdvisoryObservationCursor? cursor,
int limit,
CancellationToken cancellationToken);
}
public interface IAdvisoryObservationStore
{
Task<IReadOnlyList<AdvisoryObservation>> ListByTenantAsync(string tenant, CancellationToken cancellationToken);
Task<IReadOnlyList<AdvisoryObservation>> FindByFiltersAsync(
string tenant,
IEnumerable<string>? observationIds,
IEnumerable<string>? aliases,
IEnumerable<string>? purls,
IEnumerable<string>? cpes,
AdvisoryObservationCursor? cursor,
int limit,
CancellationToken cancellationToken);
Task UpsertAsync(AdvisoryObservation observation, CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,167 @@
using System;
using System.Collections.Generic;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
namespace StellaOps.Concelier.Storage.Mongo.Observations.V1;
[BsonIgnoreExtraElements]
public sealed class AdvisoryObservationV1Document
{
[BsonId]
public ObjectId Id { get; set; }
[BsonElement("tenantId")]
public string TenantId { get; set; } = string.Empty;
[BsonElement("source")]
public string Source { get; set; } = string.Empty;
[BsonElement("advisoryId")]
public string AdvisoryId { get; set; } = string.Empty;
[BsonElement("title")]
[BsonIgnoreIfNull]
public string? Title { get; set; }
[BsonElement("summary")]
[BsonIgnoreIfNull]
public string? Summary { get; set; }
[BsonElement("severities")]
[BsonIgnoreIfNull]
public List<ObservationSeverityDocument>? Severities { get; set; }
[BsonElement("affected")]
[BsonIgnoreIfNull]
public List<ObservationAffectedDocument>? Affected { get; set; }
[BsonElement("references")]
[BsonIgnoreIfNull]
public List<string>? References { get; set; }
[BsonElement("weaknesses")]
[BsonIgnoreIfNull]
public List<string>? Weaknesses { get; set; }
[BsonElement("published")]
[BsonDateTimeOptions(Kind = DateTimeKind.Utc)]
[BsonIgnoreIfNull]
public DateTime? Published { get; set; }
[BsonElement("modified")]
[BsonDateTimeOptions(Kind = DateTimeKind.Utc)]
[BsonIgnoreIfNull]
public DateTime? Modified { get; set; }
[BsonElement("provenance")]
public ObservationProvenanceDocument Provenance { get; set; } = new();
[BsonElement("ingestedAt")]
[BsonDateTimeOptions(Kind = DateTimeKind.Utc)]
public DateTime IngestedAt { get; set; }
[BsonElement("supersedesId")]
[BsonIgnoreIfNull]
public ObjectId? SupersedesId { get; set; }
}
[BsonIgnoreExtraElements]
public sealed class ObservationSeverityDocument
{
[BsonElement("system")]
public string System { get; set; } = string.Empty;
[BsonElement("score")]
public double Score { get; set; }
[BsonElement("vector")]
[BsonIgnoreIfNull]
public string? Vector { get; set; }
}
[BsonIgnoreExtraElements]
public sealed class ObservationAffectedDocument
{
[BsonElement("purl")]
public string Purl { get; set; } = string.Empty;
[BsonElement("package")]
[BsonIgnoreIfNull]
public string? Package { get; set; }
[BsonElement("versions")]
[BsonIgnoreIfNull]
public List<string>? Versions { get; set; }
[BsonElement("ranges")]
[BsonIgnoreIfNull]
public List<ObservationVersionRangeDocument>? Ranges { get; set; }
[BsonElement("ecosystem")]
[BsonIgnoreIfNull]
public string? Ecosystem { get; set; }
[BsonElement("cpes")]
[BsonIgnoreIfNull]
public List<string>? Cpes { get; set; }
}
[BsonIgnoreExtraElements]
public sealed class ObservationVersionRangeDocument
{
[BsonElement("type")]
public string Type { get; set; } = string.Empty;
[BsonElement("events")]
[BsonIgnoreIfNull]
public List<ObservationRangeEventDocument>? Events { get; set; }
}
[BsonIgnoreExtraElements]
public sealed class ObservationRangeEventDocument
{
[BsonElement("event")]
public string Event { get; set; } = string.Empty;
[BsonElement("value")]
public string Value { get; set; } = string.Empty;
}
[BsonIgnoreExtraElements]
public sealed class ObservationProvenanceDocument
{
[BsonElement("sourceArtifactSha")]
public string SourceArtifactSha { get; set; } = string.Empty;
[BsonElement("fetchedAt")]
[BsonDateTimeOptions(Kind = DateTimeKind.Utc)]
public DateTime FetchedAt { get; set; }
[BsonElement("ingestJobId")]
[BsonIgnoreIfNull]
public string? IngestJobId { get; set; }
[BsonElement("signature")]
[BsonIgnoreIfNull]
public ObservationSignatureDocument? Signature { get; set; }
}
[BsonIgnoreExtraElements]
public sealed class ObservationSignatureDocument
{
[BsonElement("present")]
public bool Present { get; set; }
[BsonElement("format")]
[BsonIgnoreIfNull]
public string? Format { get; set; }
[BsonElement("keyId")]
[BsonIgnoreIfNull]
public string? KeyId { get; set; }
[BsonElement("signature")]
[BsonIgnoreIfNull]
public string? Signature { get; set; }
}

View File

@@ -0,0 +1,178 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using MongoDB.Bson;
using StellaOps.Concelier.Models.Observations;
namespace StellaOps.Concelier.Storage.Mongo.Observations.V1;
internal static class AdvisoryObservationV1DocumentFactory
{
public static AdvisoryObservationV1 ToModel(AdvisoryObservationV1Document document)
{
ArgumentNullException.ThrowIfNull(document);
var severities = document.Severities is null
? ImmutableArray<ObservationSeverity>.Empty
: document.Severities.Select(s => new ObservationSeverity(s.System, s.Score, s.Vector)).ToImmutableArray();
var affected = document.Affected is null
? ImmutableArray<ObservationAffected>.Empty
: document.Affected.Select(ToAffected).ToImmutableArray();
var references = ToImmutableStrings(document.References);
var weaknesses = ToImmutableStrings(document.Weaknesses);
var provenanceDoc = document.Provenance ?? throw new ArgumentNullException(nameof(document.Provenance));
var signatureDoc = provenanceDoc.Signature;
var signature = signatureDoc is null
? null
: new ObservationSignature(signatureDoc.Present, signatureDoc.Format, signatureDoc.KeyId, signatureDoc.Signature);
var provenance = new ObservationProvenance(
provenanceDoc.SourceArtifactSha,
DateTime.SpecifyKind(provenanceDoc.FetchedAt, DateTimeKind.Utc),
provenanceDoc.IngestJobId,
signature);
return new AdvisoryObservationV1(
document.Id.ToString(),
document.TenantId,
document.Source,
document.AdvisoryId,
document.Title,
document.Summary,
severities,
affected,
references,
weaknesses,
document.Published.HasValue ? DateTime.SpecifyKind(document.Published.Value, DateTimeKind.Utc) : null,
document.Modified.HasValue ? DateTime.SpecifyKind(document.Modified.Value, DateTimeKind.Utc) : null,
provenance,
DateTime.SpecifyKind(document.IngestedAt, DateTimeKind.Utc),
document.SupersedesId?.ToString());
}
public static AdvisoryObservationV1Document FromModel(AdvisoryObservationV1 model)
{
ArgumentNullException.ThrowIfNull(model);
var document = new AdvisoryObservationV1Document
{
Id = ObjectId.Parse(model.ObservationId),
TenantId = model.Tenant,
Source = model.Source,
AdvisoryId = model.AdvisoryId,
Title = model.Title,
Summary = model.Summary,
Severities = model.Severities
.Select(severity => new ObservationSeverityDocument
{
System = severity.System,
Score = severity.Score,
Vector = severity.Vector
})
.ToList(),
Affected = model.Affected.Select(ToDocument).ToList(),
References = model.References.IsDefaultOrEmpty ? null : model.References.ToList(),
Weaknesses = model.Weaknesses.IsDefaultOrEmpty ? null : model.Weaknesses.ToList(),
Published = model.Published?.UtcDateTime,
Modified = model.Modified?.UtcDateTime,
SupersedesId = string.IsNullOrWhiteSpace(model.SupersedesObservationId)
? null
: ObjectId.Parse(model.SupersedesObservationId!),
IngestedAt = model.IngestedAt.UtcDateTime,
Provenance = new ObservationProvenanceDocument
{
SourceArtifactSha = model.Provenance.SourceArtifactSha,
FetchedAt = model.Provenance.FetchedAt.UtcDateTime,
IngestJobId = model.Provenance.IngestJobId,
Signature = model.Provenance.Signature is null
? null
: new ObservationSignatureDocument
{
Present = model.Provenance.Signature.Present,
Format = model.Provenance.Signature.Format,
KeyId = model.Provenance.Signature.KeyId,
Signature = model.Provenance.Signature.SignatureValue
}
}
};
return document;
}
private static ImmutableArray<string> ToImmutableStrings(IEnumerable<string>? values)
{
if (values is null)
{
return ImmutableArray<string>.Empty;
}
var builder = ImmutableArray.CreateBuilder<string>();
foreach (var value in values)
{
if (string.IsNullOrWhiteSpace(value))
{
continue;
}
builder.Add(value.Trim());
}
return builder.Count == 0 ? ImmutableArray<string>.Empty : builder.ToImmutable();
}
private static ObservationAffected ToAffected(ObservationAffectedDocument document)
{
var ranges = document.Ranges is null
? ImmutableArray<ObservationVersionRange>.Empty
: document.Ranges.Select(ToRange).ToImmutableArray();
return new ObservationAffected(
document.Purl,
document.Package,
ToImmutableStrings(document.Versions),
ranges,
document.Ecosystem,
ToImmutableStrings(document.Cpes));
}
private static ObservationVersionRange ToRange(ObservationVersionRangeDocument document)
{
var events = document.Events is null
? ImmutableArray<ObservationRangeEvent>.Empty
: document.Events.Select(evt => new ObservationRangeEvent(evt.Event, evt.Value)).ToImmutableArray();
return new ObservationVersionRange(document.Type, events);
}
private static ObservationAffectedDocument ToDocument(ObservationAffected model)
{
return new ObservationAffectedDocument
{
Purl = model.Purl,
Package = model.Package,
Versions = model.Versions.IsDefaultOrEmpty ? null : model.Versions.ToList(),
Ranges = model.Ranges.IsDefaultOrEmpty ? null : model.Ranges.Select(ToDocument).ToList(),
Ecosystem = model.Ecosystem,
Cpes = model.Cpes.IsDefaultOrEmpty ? null : model.Cpes.ToList()
};
}
private static ObservationVersionRangeDocument ToDocument(ObservationVersionRange model)
{
return new ObservationVersionRangeDocument
{
Type = model.Type,
Events = model.Events.IsDefaultOrEmpty
? null
: model.Events.Select(evt => new ObservationRangeEventDocument
{
Event = evt.Event,
Value = evt.Value
}).ToList()
};
}
}

View File

@@ -0,0 +1,26 @@
using System;
using System.Security.Cryptography;
using System.Text;
using MongoDB.Bson;
namespace StellaOps.Concelier.Storage.Mongo.Observations.V1;
internal static class ObservationIdBuilder
{
public static ObjectId Create(string tenant, string source, string advisoryId, string sourceArtifactSha)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
ArgumentException.ThrowIfNullOrWhiteSpace(source);
ArgumentException.ThrowIfNullOrWhiteSpace(advisoryId);
ArgumentException.ThrowIfNullOrWhiteSpace(sourceArtifactSha);
var material = $"{tenant.Trim().ToLowerInvariant()}|{source.Trim().ToLowerInvariant()}|{advisoryId.Trim()}|{sourceArtifactSha.Trim()}";
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(material));
Span<byte> objectIdBytes = stackalloc byte[12];
hash.AsSpan(0, 12).CopyTo(objectIdBytes);
// ObjectId requires a byte[]; copy the stackalloc span into a managed array.
return new ObjectId(objectIdBytes.ToArray());
}
}

View File

@@ -645,23 +645,42 @@ internal sealed class MongoAdvisoryRawRepository : IAdvisoryRawRepository
private static RawLinkset MapLinkset(BsonDocument linkset)
{
var aliases = linkset.TryGetValue("aliases", out var aliasesValue) && aliasesValue.IsBsonArray
? aliasesValue.AsBsonArray.Select(BsonValueToString).ToImmutableArray()
: ImmutableArray<string>.Empty;
var purls = linkset.TryGetValue("purls", out var purlsValue) && purlsValue.IsBsonArray
? purlsValue.AsBsonArray.Select(BsonValueToString).ToImmutableArray()
: ImmutableArray<string>.Empty;
var cpes = linkset.TryGetValue("cpes", out var cpesValue) && cpesValue.IsBsonArray
? cpesValue.AsBsonArray.Select(BsonValueToString).ToImmutableArray()
: ImmutableArray<string>.Empty;
var references = linkset.TryGetValue("references", out var referencesValue) && referencesValue.IsBsonArray
? referencesValue.AsBsonArray
.Where(static value => value.IsBsonDocument)
.Select(value =>
{
var aliases = linkset.TryGetValue("aliases", out var aliasesValue) && aliasesValue.IsBsonArray
? aliasesValue.AsBsonArray.Select(BsonValueToString).ToImmutableArray()
: ImmutableArray<string>.Empty;
var scopes = linkset.TryGetValue("scopes", out var scopesValue) && scopesValue.IsBsonArray
? scopesValue.AsBsonArray.Select(BsonValueToString).ToImmutableArray()
: ImmutableArray<string>.Empty;
var purls = linkset.TryGetValue("purls", out var purlsValue) && purlsValue.IsBsonArray
? purlsValue.AsBsonArray.Select(BsonValueToString).ToImmutableArray()
: ImmutableArray<string>.Empty;
var cpes = linkset.TryGetValue("cpes", out var cpesValue) && cpesValue.IsBsonArray
? cpesValue.AsBsonArray.Select(BsonValueToString).ToImmutableArray()
: ImmutableArray<string>.Empty;
var relationships = linkset.TryGetValue("relationships", out var relationshipsValue) && relationshipsValue.IsBsonArray
? relationshipsValue.AsBsonArray
.Where(static value => value.IsBsonDocument)
.Select(value =>
{
var doc = value.AsBsonDocument;
return new RawRelationship(
GetRequiredString(doc, "type"),
GetRequiredString(doc, "source"),
GetRequiredString(doc, "target"),
GetOptionalString(doc, "provenance"));
})
.ToImmutableArray()
: ImmutableArray<RawRelationship>.Empty;
var references = linkset.TryGetValue("references", out var referencesValue) && referencesValue.IsBsonArray
? referencesValue.AsBsonArray
.Where(static value => value.IsBsonDocument)
.Select(value =>
{
var doc = value.AsBsonDocument;
return new RawReference(
GetRequiredString(doc, "type"),
@@ -684,14 +703,16 @@ internal sealed class MongoAdvisoryRawRepository : IAdvisoryRawRepository
}
}
return new RawLinkset
{
Aliases = aliases,
PackageUrls = purls,
Cpes = cpes,
References = references,
ReconciledFrom = reconciledFrom,
Notes = notesBuilder.ToImmutable()
return new RawLinkset
{
Aliases = aliases,
Scopes = scopes,
Relationships = relationships,
PackageUrls = purls,
Cpes = cpes,
References = references,
ReconciledFrom = reconciledFrom,
Notes = notesBuilder.ToImmutable()
};
}

View File

@@ -80,13 +80,13 @@ public static class ServiceCollectionExtensions
services.AddSingleton<IAdvisoryEventRepository, MongoAdvisoryEventRepository>();
services.AddSingleton<IAdvisoryEventLog, AdvisoryEventLog>();
services.AddSingleton<IAdvisoryRawRepository, MongoAdvisoryRawRepository>();
services.AddSingleton<StellaOps.Concelier.Storage.Mongo.Linksets.MongoAdvisoryLinksetStore>();
services.AddSingleton<StellaOps.Concelier.Storage.Mongo.Linksets.IMongoAdvisoryLinksetStore, StellaOps.Concelier.Storage.Mongo.Linksets.ConcelierMongoLinksetStore>();
services.AddSingleton<StellaOps.Concelier.Core.Linksets.IAdvisoryLinksetStore>(sp =>
sp.GetRequiredService<StellaOps.Concelier.Storage.Mongo.Linksets.MongoAdvisoryLinksetStore>());
sp.GetRequiredService<StellaOps.Concelier.Storage.Mongo.Linksets.IMongoAdvisoryLinksetStore>());
services.AddSingleton<StellaOps.Concelier.Core.Linksets.IAdvisoryLinksetLookup>(sp =>
sp.GetRequiredService<StellaOps.Concelier.Storage.Mongo.Linksets.MongoAdvisoryLinksetStore>());
services.AddSingleton<StellaOps.Concelier.Core.Linksets.IAdvisoryObservationSink, StellaOps.Concelier.Storage.Mongo.Linksets.AdvisoryObservationSink>();
services.AddSingleton<StellaOps.Concelier.Core.Linksets.IAdvisoryLinksetSink, StellaOps.Concelier.Storage.Mongo.Linksets.AdvisoryLinksetSink>();
sp.GetRequiredService<StellaOps.Concelier.Storage.Mongo.Linksets.IMongoAdvisoryLinksetStore>());
services.AddSingleton<StellaOps.Concelier.Core.Observations.IAdvisoryObservationSink, StellaOps.Concelier.Storage.Mongo.Observations.AdvisoryObservationSink>();
services.AddSingleton<StellaOps.Concelier.Core.Linksets.IAdvisoryLinksetSink, StellaOps.Concelier.Storage.Mongo.Linksets.ConcelierMongoLinksetSink>();
services.AddSingleton<IExportStateStore, ExportStateStore>();
services.TryAddSingleton<ExportStateManager>();