stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 12 Releases Wiki Activity

Add unit tests and implementations for MongoDB index models and OpenAPI metadata

Browse Source 
- Implemented `MongoIndexModelTests` to verify index models for various stores.
- Created `OpenApiMetadataFactory` with methods to generate OpenAPI metadata.
- Added tests for `OpenApiMetadataFactory` to ensure expected defaults and URL overrides.
- Introduced `ObserverSurfaceSecrets` and `WebhookSurfaceSecrets` for managing secrets.
- Developed `RuntimeSurfaceFsClient` and `WebhookSurfaceFsClient` for manifest retrieval.
- Added dependency injection tests for `SurfaceEnvironmentRegistration` in both Observer and Webhook contexts.
- Implemented tests for secret resolution in `ObserverSurfaceSecretsTests` and `WebhookSurfaceSecretsTests`.
- Created `EnsureLinkNotMergeCollectionsMigrationTests` to validate MongoDB migration logic.
- Added project files for MongoDB tests and NuGet package mirroring.
This commit is contained in:
master
2025-11-17 21:21:56 +02:00
parent d3128aec24
commit 9075bad2d9
146 changed files with 152183 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
src/Concelier/__Libraries/StellaOps.Concelier.Core/Linksets/AdvisoryLinkset.cs Normal file
View File

@@ -0,0 +1,50 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using MongoDB.Bson;
using System.Linq;
namespace StellaOps.Concelier.Core.Linksets;
public sealed record AdvisoryLinkset(
string TenantId,
string Source,
string AdvisoryId,
ImmutableArray<string> ObservationIds,
AdvisoryLinksetNormalized? Normalized,
AdvisoryLinksetProvenance? Provenance,
DateTimeOffset CreatedAt,
string? BuiltByJobId);
public sealed record AdvisoryLinksetNormalized(
IReadOnlyList<string>? Purls,
IReadOnlyList<string>? Versions,
IReadOnlyList<Dictionary<string, object?>>? Ranges,
IReadOnlyList<Dictionary<string, object?>>? Severities)
{
public List<BsonDocument>? RangesToBson()
=> Ranges is null ? null : Ranges.Select(BsonDocumentHelper.FromDictionary).ToList();
public List<BsonDocument>? SeveritiesToBson()
=> Severities is null ? null : Severities.Select(BsonDocumentHelper.FromDictionary).ToList();
}
public sealed record AdvisoryLinksetProvenance(
IReadOnlyList<string>? ObservationHashes,
string? ToolVersion,
string? PolicyHash);
internal static class BsonDocumentHelper
{
public static BsonDocument FromDictionary(Dictionary<string, object?> dictionary)
{
ArgumentNullException.ThrowIfNull(dictionary);
var doc = new BsonDocument();
foreach (var kvp in dictionary)
{
doc[kvp.Key] = kvp.Value is null ? BsonNull.Value : BsonValue.Create(kvp.Value);
}
return doc;
}
}

82
src/Concelier/__Libraries/StellaOps.Concelier.Core/Linksets/AdvisoryLinksetBackfillService.cs Normal file
View File

@@ -0,0 +1,82 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Observations;
namespace StellaOps.Concelier.Core.Linksets;
internal sealed class AdvisoryLinksetBackfillService : IAdvisoryLinksetBackfillService
{
private readonly IAdvisoryObservationLookup _observations;
private readonly IAdvisoryLinksetSink _linksetSink;
private readonly TimeProvider _timeProvider;
public AdvisoryLinksetBackfillService(
IAdvisoryObservationLookup observations,
IAdvisoryLinksetSink linksetSink,
TimeProvider timeProvider)
{
_observations = observations ?? throw new ArgumentNullException(nameof(observations));
_linksetSink = linksetSink ?? throw new ArgumentNullException(nameof(linksetSink));
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
}
public async Task<int> BackfillTenantAsync(string tenant, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
cancellationToken.ThrowIfCancellationRequested();
var observations = await _observations.ListByTenantAsync(tenant, cancellationToken).ConfigureAwait(false);
if (observations.Count == 0)
{
return 0;
}
var groups = observations.GroupBy(
o => (o.Source.Vendor, o.Upstream.UpstreamId),
new VendorUpstreamComparer());
var count = 0;
var now = _timeProvider.GetUtcNow();
foreach (var group in groups)
{
cancellationToken.ThrowIfCancellationRequested();
var observationIds = group.Select(o => o.ObservationId).Distinct(StringComparer.Ordinal).ToImmutableArray();
var createdAt = group.Max(o => o.CreatedAt);
var normalized = AdvisoryLinksetNormalization.FromPurls(group.SelectMany(o => o.Linkset.Purls));
var linkset = new AdvisoryLinkset(
tenant,
group.Key.Vendor,
group.Key.UpstreamId,
observationIds,
normalized,
null,
createdAt,
null);
await _linksetSink.UpsertAsync(linkset, cancellationToken).ConfigureAwait(false);
count++;
}
return count;
}
}
internal sealed class VendorUpstreamComparer : IEqualityComparer<(string Vendor, string UpstreamId)>
{
public bool Equals((string Vendor, string UpstreamId) x, (string Vendor, string UpstreamId) y)
=> StringComparer.Ordinal.Equals(x.Vendor, y.Vendor)
&& StringComparer.Ordinal.Equals(x.UpstreamId, y.UpstreamId);
public int GetHashCode((string Vendor, string UpstreamId) obj)
{
var hash = new HashCode();
hash.Add(obj.Vendor, StringComparer.Ordinal);
hash.Add(obj.UpstreamId, StringComparer.Ordinal);
return hash.ToHashCode();
}
}

5
src/Concelier/__Libraries/StellaOps.Concelier.Core/Linksets/AdvisoryLinksetCursor.cs Normal file
View File

@@ -0,0 +1,5 @@
using System;
namespace StellaOps.Concelier.Core.Linksets;
public sealed record AdvisoryLinksetCursor(DateTimeOffset CreatedAt, string AdvisoryId);

78
src/Concelier/__Libraries/StellaOps.Concelier.Core/Linksets/AdvisoryLinksetNormalization.cs Normal file
View File

@@ -0,0 +1,78 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.RawModels;
using StellaOps.Concelier.Models;
namespace StellaOps.Concelier.Core.Linksets;
internal static class AdvisoryLinksetNormalization
{
public static AdvisoryLinksetNormalized? FromRawLinkset(RawLinkset linkset)
{
ArgumentNullException.ThrowIfNull(linkset);
return Build(linkset.PackageUrls);
}
public static AdvisoryLinksetNormalized? FromPurls(IEnumerable<string>? purls)
{
if (purls is null)
{
return null;
}
return Build(purls);
}
private static AdvisoryLinksetNormalized? Build(IEnumerable<string> purlValues)
{
var normalizedPurls = NormalizePurls(purlValues);
var versions = ExtractVersions(normalizedPurls);
if (normalizedPurls.Count == 0 && versions.Count == 0)
{
return null;
}
return new AdvisoryLinksetNormalized(normalizedPurls, versions, null, null);
}
private static List<string> NormalizePurls(IEnumerable<string> purls)
{
var distinct = new SortedSet<string>(StringComparer.Ordinal);
foreach (var purl in purls)
{
var normalized = Validation.TrimToNull(purl);
if (normalized is null)
{
continue;
}
distinct.Add(normalized);
}
return distinct.ToList();
}
private static List<string> ExtractVersions(IReadOnlyCollection<string> purls)
{
var versions = new SortedSet<string>(StringComparer.Ordinal);
foreach (var purl in purls)
{
var atIndex = purl.LastIndexOf('@');
if (atIndex < 0 || atIndex >= purl.Length - 1)
{
continue;
}
var version = purl[(atIndex + 1)..];
if (!string.IsNullOrWhiteSpace(version))
{
versions.Add(version);
}
}
return versions.ToList();
}
}

10
src/Concelier/__Libraries/StellaOps.Concelier.Core/Linksets/AdvisoryLinksetQueryOptions.cs Normal file
View File

@@ -0,0 +1,10 @@
using System.Collections.Generic;
namespace StellaOps.Concelier.Core.Linksets;
public sealed record AdvisoryLinksetQueryOptions(
string Tenant,
IEnumerable<string>? AdvisoryIds = null,
IEnumerable<string>? Sources = null,
int? Limit = null,
string? Cursor = null);

111
src/Concelier/__Libraries/StellaOps.Concelier.Core/Linksets/AdvisoryLinksetQueryService.cs Normal file
View File

@@ -0,0 +1,111 @@
using System.Collections.Immutable;
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Concelier.Core.Linksets;
public interface IAdvisoryLinksetQueryService
{
Task<AdvisoryLinksetQueryResult> QueryAsync(AdvisoryLinksetQueryOptions options, CancellationToken cancellationToken);
}
public sealed record AdvisoryLinksetQueryResult(ImmutableArray<AdvisoryLinkset> Linksets, string? NextCursor, bool HasMore);
public sealed record AdvisoryLinksetPage(ImmutableArray<AdvisoryLinkset> Linksets, string? NextCursor, bool HasMore);
public sealed class AdvisoryLinksetQueryService : IAdvisoryLinksetQueryService
{
private const int DefaultLimit = 100;
private const int MaxLimit = 500;
private readonly IAdvisoryLinksetLookup _store;
public AdvisoryLinksetQueryService(IAdvisoryLinksetLookup store)
{
_store = store ?? throw new ArgumentNullException(nameof(store));
}
public async Task<AdvisoryLinksetQueryResult> QueryAsync(AdvisoryLinksetQueryOptions options, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(options);
cancellationToken.ThrowIfCancellationRequested();
var tenant = string.IsNullOrWhiteSpace(options.Tenant)
? throw new ArgumentNullException(nameof(options.Tenant))
: options.Tenant.ToLowerInvariant();
var limit = NormalizeLimit(options.Limit);
var cursor = DecodeCursor(options.Cursor);
var linksets = await _store
.FindByTenantAsync(tenant, options.AdvisoryIds, options.Sources, cursor, limit + 1, cancellationToken)
.ConfigureAwait(false);
var ordered = linksets
.OrderByDescending(ls => ls.CreatedAt)
.ThenBy(ls => ls.AdvisoryId, StringComparer.Ordinal)
.ToImmutableArray();
var hasMore = ordered.Length > limit;
var page = hasMore ? ordered.Take(limit).ToImmutableArray() : ordered;
var nextCursor = hasMore ? EncodeCursor(page[^1]) : null;
return new AdvisoryLinksetQueryResult(page, nextCursor, hasMore);
}
private static int NormalizeLimit(int? requested)
{
if (!requested.HasValue || requested <= 0)
{
return DefaultLimit;
}
return requested.Value > MaxLimit ? MaxLimit : requested.Value;
}
private static AdvisoryLinksetCursor? DecodeCursor(string? cursor)
{
if (string.IsNullOrWhiteSpace(cursor))
{
return null;
}
try
{
var buffer = Convert.FromBase64String(cursor.Trim());
var payload = System.Text.Encoding.UTF8.GetString(buffer);
var separator = payload.IndexOf(':');
if (separator <= 0 || separator >= payload.Length - 1)
{
throw new FormatException("Cursor format invalid.");
}
var ticksText = payload[..separator];
if (!long.TryParse(ticksText, out var ticks))
{
throw new FormatException("Cursor timestamp invalid.");
}
var advisoryId = payload[(separator + 1)..];
if (string.IsNullOrWhiteSpace(advisoryId))
{
throw new FormatException("Cursor advisoryId missing.");
}
return new AdvisoryLinksetCursor(new DateTimeOffset(new DateTime(ticks, DateTimeKind.Utc)), advisoryId);
}
catch (FormatException)
{
throw;
}
catch (Exception ex)
{
throw new FormatException("Cursor is malformed.", ex);
}
}
private static string? EncodeCursor(AdvisoryLinkset linkset)
{
var payload = $"{linkset.CreatedAt.UtcTicks}:{linkset.AdvisoryId}";
return Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(payload));
}
}

9
src/Concelier/__Libraries/StellaOps.Concelier.Core/Linksets/IAdvisoryLinksetBackfillService.cs Normal file
View File

@@ -0,0 +1,9 @@
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Concelier.Core.Linksets;
public interface IAdvisoryLinksetBackfillService
{
Task<int> BackfillTenantAsync(string tenant, CancellationToken cancellationToken);
}

9
src/Concelier/__Libraries/StellaOps.Concelier.Core/Linksets/IAdvisoryLinksetSink.cs Normal file
View File

@@ -0,0 +1,9 @@
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Concelier.Core.Linksets;
public interface IAdvisoryLinksetSink
{
Task UpsertAsync(AdvisoryLinkset linkset, CancellationToken cancellationToken);
}

20
src/Concelier/__Libraries/StellaOps.Concelier.Core/Linksets/IAdvisoryLinksetStore.cs Normal file
View File

@@ -0,0 +1,20 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Concelier.Core.Linksets;
public interface IAdvisoryLinksetStore : IAdvisoryLinksetSink, IAdvisoryLinksetLookup
{
}
public interface IAdvisoryLinksetLookup
{
Task<IReadOnlyList<AdvisoryLinkset>> FindByTenantAsync(
string tenantId,
IEnumerable<string>? advisoryIds,
IEnumerable<string>? sources,
AdvisoryLinksetCursor? cursor,
int limit,
CancellationToken cancellationToken);
}

45
src/Concelier/__Libraries/StellaOps.Concelier.Core/Linksets/ObservationPipelineServiceCollectionExtensions.cs Normal file
View File

@@ -0,0 +1,45 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using StellaOps.Concelier.Core.Observations;
using StellaOps.Concelier.Core.Linksets;
namespace StellaOps.Concelier.Core.Linksets;
public static class ObservationPipelineServiceCollectionExtensions
{
public static IServiceCollection AddConcelierObservationPipeline(this IServiceCollection services)
{
ArgumentNullException.ThrowIfNull(services);
services.TryAddSingleton<IAdvisoryObservationSink, NullObservationSink>();
services.TryAddSingleton<IAdvisoryLinksetSink, NullLinksetSink>();
services.TryAddSingleton<IAdvisoryLinksetLookup, NullLinksetLookup>();
services.TryAddSingleton<IAdvisoryLinksetBackfillService, AdvisoryLinksetBackfillService>();
return services;
}
private sealed class NullObservationSink : IAdvisoryObservationSink
{
public Task UpsertAsync(Models.Observations.AdvisoryObservation observation, CancellationToken cancellationToken)
=> Task.CompletedTask;
}
private sealed class NullLinksetSink : IAdvisoryLinksetSink
{
public Task UpsertAsync(AdvisoryLinkset linkset, CancellationToken cancellationToken)
=> Task.CompletedTask;
}
private sealed class NullLinksetLookup : IAdvisoryLinksetLookup
{
public Task<IReadOnlyList<AdvisoryLinkset>> FindByTenantAsync(
string tenantId,
IEnumerable<string>? advisoryIds,
IEnumerable<string>? sources,
AdvisoryLinksetCursor? cursor,
int limit,
CancellationToken cancellationToken)
=> Task.FromResult<IReadOnlyList<AdvisoryLinkset>>(Array.Empty<AdvisoryLinkset>());
}
}

10
src/Concelier/__Libraries/StellaOps.Concelier.Core/Observations/IAdvisoryObservationSink.cs Normal file
View File

@@ -0,0 +1,10 @@
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Models.Observations;
namespace StellaOps.Concelier.Core.Observations;
public interface IAdvisoryObservationSink
{
Task UpsertAsync(AdvisoryObservation observation, CancellationToken cancellationToken);
}

69
src/Concelier/__Libraries/StellaOps.Concelier.Core/Raw/AdvisoryRawService.cs
View File

@@ -9,7 +9,8 @@ using Microsoft.Extensions.Logging;
using StellaOps.Aoc;
using StellaOps.Ingestion.Telemetry;
using StellaOps.Concelier.Core.Aoc;
using StellaOps.Concelier.Core.Linksets;
using StellaOps.Concelier.Core.Linksets;
using StellaOps.Concelier.Core.Observations;
using StellaOps.Concelier.RawModels;
using StellaOps.Concelier.Models;
@@ -19,28 +20,37 @@ internal sealed class AdvisoryRawService : IAdvisoryRawService
{
private static readonly ImmutableArray<string> EmptyArray = ImmutableArray<string>.Empty;
private readonly IAdvisoryRawRepository _repository;
private readonly IAdvisoryRawWriteGuard _writeGuard;
private readonly IAocGuard _aocGuard;
private readonly IAdvisoryLinksetMapper _linksetMapper;
private readonly TimeProvider _timeProvider;
private readonly ILogger<AdvisoryRawService> _logger;
private readonly IAdvisoryRawRepository _repository;
private readonly IAdvisoryRawWriteGuard _writeGuard;
private readonly IAocGuard _aocGuard;
private readonly IAdvisoryLinksetMapper _linksetMapper;
private readonly IAdvisoryObservationFactory _observationFactory;
private readonly IAdvisoryObservationSink _observationSink;
private readonly IAdvisoryLinksetSink _linksetSink;
private readonly TimeProvider _timeProvider;
private readonly ILogger<AdvisoryRawService> _logger;
public AdvisoryRawService(
IAdvisoryRawRepository repository,
IAdvisoryRawWriteGuard writeGuard,
IAocGuard aocGuard,
IAdvisoryLinksetMapper linksetMapper,
TimeProvider timeProvider,
ILogger<AdvisoryRawService> logger)
{
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
_writeGuard = writeGuard ?? throw new ArgumentNullException(nameof(writeGuard));
_aocGuard = aocGuard ?? throw new ArgumentNullException(nameof(aocGuard));
_linksetMapper = linksetMapper ?? throw new ArgumentNullException(nameof(linksetMapper));
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
IAdvisoryRawRepository repository,
IAdvisoryRawWriteGuard writeGuard,
IAocGuard aocGuard,
IAdvisoryLinksetMapper linksetMapper,
IAdvisoryObservationFactory observationFactory,
IAdvisoryObservationSink observationSink,
IAdvisoryLinksetSink linksetSink,
TimeProvider timeProvider,
ILogger<AdvisoryRawService> logger)
{
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
_writeGuard = writeGuard ?? throw new ArgumentNullException(nameof(writeGuard));
_aocGuard = aocGuard ?? throw new ArgumentNullException(nameof(aocGuard));
_linksetMapper = linksetMapper ?? throw new ArgumentNullException(nameof(linksetMapper));
_observationFactory = observationFactory ?? throw new ArgumentNullException(nameof(observationFactory));
_observationSink = observationSink ?? throw new ArgumentNullException(nameof(observationSink));
_linksetSink = linksetSink ?? throw new ArgumentNullException(nameof(linksetSink));
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<AdvisoryRawUpsertResult> IngestAsync(AdvisoryRawDocument document, CancellationToken cancellationToken)
{
@@ -102,6 +112,23 @@ internal sealed class AdvisoryRawService : IAdvisoryRawService
var result = await _repository.UpsertAsync(enriched, cancellationToken).ConfigureAwait(false);
IngestionTelemetry.RecordWriteAttempt(tenant, source, result.Inserted ? IngestionTelemetry.ResultOk : IngestionTelemetry.ResultNoop);
// Persist observation + linkset for Link-Not-Merge consumers (idempotent upserts).
var observation = _observationFactory.Create(enriched, _timeProvider.GetUtcNow());
await _observationSink.UpsertAsync(observation, cancellationToken).ConfigureAwait(false);
var normalizedLinkset = AdvisoryLinksetNormalization.FromRawLinkset(enriched.Linkset);
var linkset = new AdvisoryLinkset(
tenant,
source,
enriched.Upstream.UpstreamId,
ImmutableArray.Create(observation.ObservationId),
normalizedLinkset,
null,
_timeProvider.GetUtcNow(),
null);
await _linksetSink.UpsertAsync(linkset, cancellationToken).ConfigureAwait(false);
if (result.Inserted)
{
_logger.LogInformation(

87
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/Linksets/AdvisoryLinksetDocument.cs Normal file
View File

@@ -0,0 +1,87 @@
using System;
using System.Collections.Generic;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
namespace StellaOps.Concelier.Storage.Mongo.Linksets;
[BsonIgnoreExtraElements]
public sealed class AdvisoryLinksetDocument
{
[BsonId]
public ObjectId Id { get; set; }
= ObjectId.GenerateNewId();
[BsonElement("tenantId")]
public string TenantId { get; set; } = string.Empty;
[BsonElement("source")]
public string Source { get; set; } = string.Empty;
[BsonElement("advisoryId")]
public string AdvisoryId { get; set; } = string.Empty;
[BsonElement("observations")]
public List<string> Observations { get; set; } = new();
[BsonElement("normalized")]
[BsonIgnoreIfNull]
public AdvisoryLinksetNormalizedDocument? Normalized { get; set; }
= null;
[BsonElement("createdAt")]
public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
[BsonElement("builtByJobId")]
[BsonIgnoreIfNull]
public string? BuiltByJobId { get; set; }
= null;
[BsonElement("provenance")]
[BsonIgnoreIfNull]
public AdvisoryLinksetProvenanceDocument? Provenance { get; set; }
= null;
}
[BsonIgnoreExtraElements]
public sealed class AdvisoryLinksetNormalizedDocument
{
[BsonElement("purls")]
[BsonIgnoreIfNull]
public List<string>? Purls { get; set; }
= new();
[BsonElement("versions")]
[BsonIgnoreIfNull]
public List<string>? Versions { get; set; }
= new();
[BsonElement("ranges")]
[BsonIgnoreIfNull]
public List<BsonDocument>? Ranges { get; set; }
= new();
[BsonElement("severities")]
[BsonIgnoreIfNull]
public List<BsonDocument>? Severities { get; set; }
= new();
}
[BsonIgnoreExtraElements]
public sealed class AdvisoryLinksetProvenanceDocument
{
[BsonElement("observationHashes")]
[BsonIgnoreIfNull]
public List<string>? ObservationHashes { get; set; }
= new();
[BsonElement("toolVersion")]
[BsonIgnoreIfNull]
public string? ToolVersion { get; set; }
= null;
[BsonElement("policyHash")]
[BsonIgnoreIfNull]
public string? PolicyHash { get; set; }
= null;
}

22
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/Linksets/AdvisoryLinksetSink.cs Normal file
View File

@@ -0,0 +1,22 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using CoreLinksets = StellaOps.Concelier.Core.Linksets;
namespace StellaOps.Concelier.Storage.Mongo.Linksets;
internal sealed class AdvisoryLinksetSink : CoreLinksets.IAdvisoryLinksetSink
{
private readonly IAdvisoryLinksetStore _store;
public AdvisoryLinksetSink(IAdvisoryLinksetStore store)
{
_store = store ?? throw new ArgumentNullException(nameof(store));
}
public Task UpsertAsync(CoreLinksets.AdvisoryLinkset linkset, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(linkset);
return _store.UpsertAsync(linkset, cancellationToken);
}
}

170
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/Linksets/AdvisoryLinksetStore.cs Normal file
View File

@@ -0,0 +1,170 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Driver;
using CoreLinksets = StellaOps.Concelier.Core.Linksets;
namespace StellaOps.Concelier.Storage.Mongo.Linksets;
// Internal type kept in storage namespace to avoid name clash with core interface
internal sealed class MongoAdvisoryLinksetStore : CoreLinksets.IAdvisoryLinksetStore, CoreLinksets.IAdvisoryLinksetLookup
{
private readonly IMongoCollection<AdvisoryLinksetDocument> _collection;
public MongoAdvisoryLinksetStore(IMongoCollection<AdvisoryLinksetDocument> collection)
{
_collection = collection ?? throw new ArgumentNullException(nameof(collection));
}
public async Task UpsertAsync(CoreLinksets.AdvisoryLinkset linkset, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(linkset);
var document = MapToDocument(linkset);
var filter = Builders<AdvisoryLinksetDocument>.Filter.And(
Builders<AdvisoryLinksetDocument>.Filter.Eq(d => d.TenantId, linkset.TenantId),
Builders<AdvisoryLinksetDocument>.Filter.Eq(d => d.Source, linkset.Source),
Builders<AdvisoryLinksetDocument>.Filter.Eq(d => d.AdvisoryId, linkset.AdvisoryId));
var options = new ReplaceOptions { IsUpsert = true };
await _collection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false);
}
public async Task<IReadOnlyList<CoreLinksets.AdvisoryLinkset>> FindByTenantAsync(
string tenantId,
IEnumerable<string>? advisoryIds,
IEnumerable<string>? sources,
CoreLinksets.AdvisoryLinksetCursor? cursor,
int limit,
CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
if (limit <= 0)
{
throw new ArgumentOutOfRangeException(nameof(limit));
}
var builder = Builders<AdvisoryLinksetDocument>.Filter;
var filters = new List<FilterDefinition<AdvisoryLinksetDocument>>
{
builder.Eq(d => d.TenantId, tenantId.ToLowerInvariant())
};
if (advisoryIds is not null)
{
var ids = advisoryIds.Where(v => !string.IsNullOrWhiteSpace(v)).ToArray();
if (ids.Length > 0)
{
filters.Add(builder.In(d => d.AdvisoryId, ids));
}
}
if (sources is not null)
{
var srcs = sources.Where(v => !string.IsNullOrWhiteSpace(v)).ToArray();
if (srcs.Length > 0)
{
filters.Add(builder.In(d => d.Source, srcs));
}
}
var filter = builder.And(filters);
var sort = Builders<AdvisoryLinksetDocument>.Sort.Descending(d => d.CreatedAt).Ascending(d => d.AdvisoryId);
var findFilter = filter;
if (cursor is not null)
{
var cursorFilter = builder.Or(
builder.Lt(d => d.CreatedAt, cursor.CreatedAt.UtcDateTime),
builder.And(
builder.Eq(d => d.CreatedAt, cursor.CreatedAt.UtcDateTime),
builder.Gt(d => d.AdvisoryId, cursor.AdvisoryId)));
findFilter = builder.And(findFilter, cursorFilter);
}
var documents = await _collection.Find(findFilter)
.Sort(sort)
.Limit(limit)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return documents.Select(FromDocument).ToArray();
}
private static AdvisoryLinksetDocument MapToDocument(CoreLinksets.AdvisoryLinkset linkset)
{
var doc = new AdvisoryLinksetDocument
{
TenantId = linkset.TenantId,
Source = linkset.Source,
AdvisoryId = linkset.AdvisoryId,
Observations = new List<string>(linkset.ObservationIds),
CreatedAt = linkset.CreatedAt.UtcDateTime,
BuiltByJobId = linkset.BuiltByJobId,
Provenance = linkset.Provenance is null ? null : new AdvisoryLinksetProvenanceDocument
{
ObservationHashes = linkset.Provenance.ObservationHashes is null
? null
: new List<string>(linkset.Provenance.ObservationHashes),
ToolVersion = linkset.Provenance.ToolVersion,
PolicyHash = linkset.Provenance.PolicyHash,
},
Normalized = linkset.Normalized is null ? null : new AdvisoryLinksetNormalizedDocument
{
Purls = linkset.Normalized.Purls is null ? null : new List<string>(linkset.Normalized.Purls),
Versions = linkset.Normalized.Versions is null ? null : new List<string>(linkset.Normalized.Versions),
Ranges = linkset.Normalized.RangesToBson(),
Severities = linkset.Normalized.SeveritiesToBson(),
}
};
return doc;
}
private static CoreLinksets.AdvisoryLinkset FromDocument(AdvisoryLinksetDocument doc)
{
return new AdvisoryLinkset(
doc.TenantId,
doc.Source,
doc.AdvisoryId,
doc.Observations.ToImmutableArray(),
doc.Normalized is null ? null : new AdvisoryLinksetNormalized(
doc.Normalized.Purls,
doc.Normalized.Versions,
doc.Normalized.Ranges?.Select(ToDictionary).ToList(),
doc.Normalized.Severities?.Select(ToDictionary).ToList()),
doc.Provenance is null ? null : new AdvisoryLinksetProvenance(
doc.Provenance.ObservationHashes,
doc.Provenance.ToolVersion,
doc.Provenance.PolicyHash),
DateTime.SpecifyKind(doc.CreatedAt, DateTimeKind.Utc),
doc.BuiltByJobId);
}
private static Dictionary<string, object?> ToDictionary(MongoDB.Bson.BsonDocument bson)
{
var dict = new Dictionary<string, object?>(StringComparer.Ordinal);
foreach (var element in bson.Elements)
{
dict[element.Name] = element.Value switch
{
MongoDB.Bson.BsonString s => s.AsString,
MongoDB.Bson.BsonInt32 i => i.AsInt32,
MongoDB.Bson.BsonInt64 l => l.AsInt64,
MongoDB.Bson.BsonDouble d => d.AsDouble,
MongoDB.Bson.BsonDecimal128 dec => dec.ToDecimal(),
MongoDB.Bson.BsonBoolean b => b.AsBoolean,
MongoDB.Bson.BsonDateTime dt => dt.ToUniversalTime(),
MongoDB.Bson.BsonNull => (object?)null,
MongoDB.Bson.BsonArray arr => arr.Select(v => v.ToString()).ToArray(),
_ => element.Value.ToString()
};
}
return dict;
}
}

242
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/Migrations/EnsureLinkNotMergeCollectionsMigration.cs Normal file
View File

@@ -0,0 +1,242 @@
using System.Collections.Generic;
using MongoDB.Bson;
using MongoDB.Driver;
namespace StellaOps.Concelier.Storage.Mongo.Migrations;
internal sealed class EnsureLinkNotMergeCollectionsMigration : IMongoMigration
{
public string Id => "20251116_link_not_merge_collections";
public string Description => "Ensure advisory_observations and advisory_linksets collections exist with validators and indexes for Link-Not-Merge";
public async Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(database);
await EnsureObservationsAsync(database, cancellationToken).ConfigureAwait(false);
await EnsureLinksetsAsync(database, cancellationToken).ConfigureAwait(false);
}
private static async Task EnsureObservationsAsync(IMongoDatabase database, CancellationToken ct)
{
var collectionName = MongoStorageDefaults.Collections.AdvisoryObservations;
var validator = new BsonDocument("$jsonSchema", BuildObservationSchema());
await EnsureCollectionWithValidatorAsync(database, collectionName, validator, ct).ConfigureAwait(false);
var collection = database.GetCollection<BsonDocument>(collectionName);
var indexes = new List<CreateIndexModel<BsonDocument>>
{
new(new BsonDocument
{
{"tenant", 1},
{"source", 1},
{"advisoryId", 1},
{"upstream.fetchedAt", -1},
},
new CreateIndexOptions { Name = "obs_tenant_source_adv_fetchedAt" }),
new(new BsonDocument
{
{"provenance.sourceArtifactSha", 1},
},
new CreateIndexOptions { Name = "obs_prov_sourceArtifactSha_unique", Unique = true }),
};
await collection.Indexes.CreateManyAsync(indexes, cancellationToken: ct).ConfigureAwait(false);
}
private static async Task EnsureLinksetsAsync(IMongoDatabase database, CancellationToken ct)
{
var collectionName = MongoStorageDefaults.Collections.AdvisoryLinksets;
var validator = new BsonDocument("$jsonSchema", BuildLinksetSchema());
await EnsureCollectionWithValidatorAsync(database, collectionName, validator, ct).ConfigureAwait(false);
var collection = database.GetCollection<BsonDocument>(collectionName);
var indexes = new List<CreateIndexModel<BsonDocument>>
{
new(new BsonDocument
{
{"tenantId", 1},
{"advisoryId", 1},
{"source", 1},
},
new CreateIndexOptions { Name = "linkset_tenant_advisory_source", Unique = true }),
new(new BsonDocument { { "observations", 1 } }, new CreateIndexOptions { Name = "linkset_observations" })
};
await collection.Indexes.CreateManyAsync(indexes, cancellationToken: ct).ConfigureAwait(false);
}
private static async Task EnsureCollectionWithValidatorAsync(
IMongoDatabase database,
string collectionName,
BsonDocument validator,
CancellationToken ct)
{
var filter = new BsonDocument("name", collectionName);
var existing = await database.ListCollectionsAsync(new ListCollectionsOptions { Filter = filter }, ct)
.ConfigureAwait(false);
var exists = await existing.AnyAsync(ct).ConfigureAwait(false);
if (!exists)
{
var options = new CreateCollectionOptions<BsonDocument>
{
Validator = validator,
ValidationLevel = DocumentValidationLevel.Moderate,
ValidationAction = DocumentValidationAction.Error,
};
await database.CreateCollectionAsync(collectionName, options, ct).ConfigureAwait(false);
}
else
{
var command = new BsonDocument
{
{ "collMod", collectionName },
{ "validator", validator },
{ "validationLevel", "moderate" },
{ "validationAction", "error" },
};
await database.RunCommandAsync<BsonDocument>(command, cancellationToken: ct).ConfigureAwait(false);
}
}
private static BsonDocument BuildObservationSchema()
{
return new BsonDocument
{
{ "bsonType", "object" },
{ "required", new BsonArray { "_id", "tenantId", "source", "advisoryId", "affected", "provenance", "ingestedAt" } },
{ "properties", new BsonDocument
{
{ "_id", new BsonDocument("bsonType", "string") },
{ "tenantId", new BsonDocument("bsonType", "string") },
{ "source", new BsonDocument("bsonType", "string") },
{ "advisoryId", new BsonDocument("bsonType", "string") },
{ "title", new BsonDocument("bsonType", new BsonArray { "string", "null" }) },
{ "summary", new BsonDocument("bsonType", new BsonArray { "string", "null" }) },
{ "severities", new BsonDocument
{
{ "bsonType", "array" },
{ "items", new BsonDocument
{
{ "bsonType", "object" },
{ "required", new BsonArray { "system", "score" } },
{ "properties", new BsonDocument
{
{ "system", new BsonDocument("bsonType", "string") },
{ "score", new BsonDocument("bsonType", new BsonArray { "double", "int", "long", "decimal" }) },
{ "vector", new BsonDocument("bsonType", new BsonArray { "string", "null" }) }
}
}
}
}
}
},
{ "affected", new BsonDocument
{
{ "bsonType", "array" },
{ "items", new BsonDocument
{
{ "bsonType", "object" },
{ "required", new BsonArray { "purl" } },
{ "properties", new BsonDocument
{
{ "purl", new BsonDocument("bsonType", "string") },
{ "package", new BsonDocument("bsonType", new BsonArray { "string", "null" }) },
{ "versions", new BsonDocument("bsonType", new BsonArray { "array", "null" }) },
{ "ranges", new BsonDocument("bsonType", new BsonArray { "array", "null" }) },
{ "ecosystem", new BsonDocument("bsonType", new BsonArray { "string", "null" }) },
{ "cpe", new BsonDocument("bsonType", new BsonArray { "array", "null" }) },
{ "cpes", new BsonDocument("bsonType", new BsonArray { "array", "null" }) }
}
}
}
}
}
},
{ "references", new BsonDocument
{
{ "bsonType", new BsonArray { "array", "null" } },
{ "items", new BsonDocument("bsonType", "string") }
}
},
{ "weaknesses", new BsonDocument
{
{ "bsonType", new BsonArray { "array", "null" } },
{ "items", new BsonDocument("bsonType", "string") }
}
},
{ "published", new BsonDocument("bsonType", new BsonArray { "date", "null" }) },
{ "modified", new BsonDocument("bsonType", new BsonArray { "date", "null" }) },
{ "provenance", new BsonDocument
{
{ "bsonType", "object" },
{ "required", new BsonArray { "sourceArtifactSha", "fetchedAt" } },
{ "properties", new BsonDocument
{
{ "sourceArtifactSha", new BsonDocument("bsonType", "string") },
{ "fetchedAt", new BsonDocument("bsonType", "date") },
{ "ingestJobId", new BsonDocument("bsonType", new BsonArray { "string", "null" }) },
{ "signature", new BsonDocument("bsonType", new BsonArray { "object", "null" }) }
}
}
}
},
{ "ingestedAt", new BsonDocument("bsonType", "date") }
}
}
};
}
private static BsonDocument BuildLinksetSchema()
{
return new BsonDocument
{
{ "bsonType", "object" },
{ "required", new BsonArray { "_id", "tenantId", "source", "advisoryId", "observations", "createdAt" } },
{ "properties", new BsonDocument
{
{ "_id", new BsonDocument("bsonType", "objectId") },
{ "tenantId", new BsonDocument("bsonType", "string") },
{ "source", new BsonDocument("bsonType", "string") },
{ "advisoryId", new BsonDocument("bsonType", "string") },
{ "observations", new BsonDocument
{
{ "bsonType", "array" },
{ "items", new BsonDocument("bsonType", "string") }
}
},
{ "normalized", new BsonDocument
{
{ "bsonType", new BsonArray { "object", "null" } },
{ "properties", new BsonDocument
{
{ "purls", new BsonDocument { { "bsonType", new BsonArray { "array", "null" } }, { "items", new BsonDocument("bsonType", "string") } } },
{ "versions", new BsonDocument { { "bsonType", new BsonArray { "array", "null" } }, { "items", new BsonDocument("bsonType", "string") } } },
{ "ranges", new BsonDocument { { "bsonType", new BsonArray { "array", "null" } }, { "items", new BsonDocument("bsonType", "object") } } },
{ "severities", new BsonDocument { { "bsonType", new BsonArray { "array", "null" } }, { "items", new BsonDocument("bsonType", "object") } } }
}
}
}
},
{ "createdAt", new BsonDocument("bsonType", "date") },
{ "builtByJobId", new BsonDocument("bsonType", new BsonArray { "string", "null" }) },
{ "provenance", new BsonDocument
{
{ "bsonType", new BsonArray { "object", "null" } },
{ "properties", new BsonDocument
{
{ "observationHashes", new BsonDocument { { "bsonType", new BsonArray { "array", "null" } }, { "items", new BsonDocument("bsonType", "string") } } },
{ "toolVersion", new BsonDocument("bsonType", new BsonArray { "string", "null" }) },
{ "policyHash", new BsonDocument("bsonType", new BsonArray { "string", "null" }) }
}
}
}
}
}
}
};
}
}

22
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/Observations/AdvisoryObservationSink.cs Normal file
View File

@@ -0,0 +1,22 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Observations;
using StellaOps.Concelier.Models.Observations;
namespace StellaOps.Concelier.Storage.Mongo.Observations;
internal sealed class AdvisoryObservationSink : IAdvisoryObservationSink
{
private readonly IAdvisoryObservationStore _store;
public AdvisoryObservationSink(IAdvisoryObservationStore store)
{
_store = store ?? throw new ArgumentNullException(nameof(store));
}
public Task UpsertAsync(AdvisoryObservation observation, CancellationToken cancellationToken)
{
return _store.UpsertAsync(observation, cancellationToken);
}
}

7
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/ServiceCollectionExtensions.cs
View File

@@ -80,6 +80,13 @@ public static class ServiceCollectionExtensions
services.AddSingleton<IAdvisoryEventRepository, MongoAdvisoryEventRepository>();
services.AddSingleton<IAdvisoryEventLog, AdvisoryEventLog>();
services.AddSingleton<IAdvisoryRawRepository, MongoAdvisoryRawRepository>();
services.AddSingleton<StellaOps.Concelier.Storage.Mongo.Linksets.MongoAdvisoryLinksetStore>();
services.AddSingleton<StellaOps.Concelier.Core.Linksets.IAdvisoryLinksetStore>(sp =>
sp.GetRequiredService<StellaOps.Concelier.Storage.Mongo.Linksets.MongoAdvisoryLinksetStore>());
services.AddSingleton<StellaOps.Concelier.Core.Linksets.IAdvisoryLinksetLookup>(sp =>
sp.GetRequiredService<StellaOps.Concelier.Storage.Mongo.Linksets.MongoAdvisoryLinksetStore>());
services.AddSingleton<StellaOps.Concelier.Core.Linksets.IAdvisoryObservationSink, StellaOps.Concelier.Storage.Mongo.Linksets.AdvisoryObservationSink>();
services.AddSingleton<StellaOps.Concelier.Core.Linksets.IAdvisoryLinksetSink, StellaOps.Concelier.Storage.Mongo.Linksets.AdvisoryLinksetSink>();
services.AddSingleton<IExportStateStore, ExportStateStore>();
services.TryAddSingleton<ExportStateManager>();