Add unit tests and implementations for MongoDB index models and OpenAPI metadata

- Implemented `MongoIndexModelTests` to verify index models for various stores.
- Created `OpenApiMetadataFactory` with methods to generate OpenAPI metadata.
- Added tests for `OpenApiMetadataFactory` to ensure expected defaults and URL overrides.
- Introduced `ObserverSurfaceSecrets` and `WebhookSurfaceSecrets` for managing secrets.
- Developed `RuntimeSurfaceFsClient` and `WebhookSurfaceFsClient` for manifest retrieval.
- Added dependency injection tests for `SurfaceEnvironmentRegistration` in both Observer and Webhook contexts.
- Implemented tests for secret resolution in `ObserverSurfaceSecretsTests` and `WebhookSurfaceSecretsTests`.
- Created `EnsureLinkNotMergeCollectionsMigrationTests` to validate MongoDB migration logic.
- Added project files for MongoDB tests and NuGet package mirroring.
This commit is contained in:
master
2025-11-17 21:21:56 +02:00
parent d3128aec24
commit 9075bad2d9
146 changed files with 152183 additions and 82 deletions

View File

@@ -0,0 +1,87 @@
using System;
using System.Collections.Generic;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
namespace StellaOps.Concelier.Storage.Mongo.Linksets;
[BsonIgnoreExtraElements]
public sealed class AdvisoryLinksetDocument
{
[BsonId]
public ObjectId Id { get; set; }
= ObjectId.GenerateNewId();
[BsonElement("tenantId")]
public string TenantId { get; set; } = string.Empty;
[BsonElement("source")]
public string Source { get; set; } = string.Empty;
[BsonElement("advisoryId")]
public string AdvisoryId { get; set; } = string.Empty;
[BsonElement("observations")]
public List<string> Observations { get; set; } = new();
[BsonElement("normalized")]
[BsonIgnoreIfNull]
public AdvisoryLinksetNormalizedDocument? Normalized { get; set; }
= null;
[BsonElement("createdAt")]
public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
[BsonElement("builtByJobId")]
[BsonIgnoreIfNull]
public string? BuiltByJobId { get; set; }
= null;
[BsonElement("provenance")]
[BsonIgnoreIfNull]
public AdvisoryLinksetProvenanceDocument? Provenance { get; set; }
= null;
}
[BsonIgnoreExtraElements]
public sealed class AdvisoryLinksetNormalizedDocument
{
[BsonElement("purls")]
[BsonIgnoreIfNull]
public List<string>? Purls { get; set; }
= new();
[BsonElement("versions")]
[BsonIgnoreIfNull]
public List<string>? Versions { get; set; }
= new();
[BsonElement("ranges")]
[BsonIgnoreIfNull]
public List<BsonDocument>? Ranges { get; set; }
= new();
[BsonElement("severities")]
[BsonIgnoreIfNull]
public List<BsonDocument>? Severities { get; set; }
= new();
}
[BsonIgnoreExtraElements]
public sealed class AdvisoryLinksetProvenanceDocument
{
[BsonElement("observationHashes")]
[BsonIgnoreIfNull]
public List<string>? ObservationHashes { get; set; }
= new();
[BsonElement("toolVersion")]
[BsonIgnoreIfNull]
public string? ToolVersion { get; set; }
= null;
[BsonElement("policyHash")]
[BsonIgnoreIfNull]
public string? PolicyHash { get; set; }
= null;
}

View File

@@ -0,0 +1,22 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using CoreLinksets = StellaOps.Concelier.Core.Linksets;
namespace StellaOps.Concelier.Storage.Mongo.Linksets;
internal sealed class AdvisoryLinksetSink : CoreLinksets.IAdvisoryLinksetSink
{
private readonly IAdvisoryLinksetStore _store;
public AdvisoryLinksetSink(IAdvisoryLinksetStore store)
{
_store = store ?? throw new ArgumentNullException(nameof(store));
}
public Task UpsertAsync(CoreLinksets.AdvisoryLinkset linkset, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(linkset);
return _store.UpsertAsync(linkset, cancellationToken);
}
}

View File

@@ -0,0 +1,170 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Driver;
using CoreLinksets = StellaOps.Concelier.Core.Linksets;
namespace StellaOps.Concelier.Storage.Mongo.Linksets;
// Internal type kept in storage namespace to avoid name clash with core interface
internal sealed class MongoAdvisoryLinksetStore : CoreLinksets.IAdvisoryLinksetStore, CoreLinksets.IAdvisoryLinksetLookup
{
private readonly IMongoCollection<AdvisoryLinksetDocument> _collection;
public MongoAdvisoryLinksetStore(IMongoCollection<AdvisoryLinksetDocument> collection)
{
_collection = collection ?? throw new ArgumentNullException(nameof(collection));
}
public async Task UpsertAsync(CoreLinksets.AdvisoryLinkset linkset, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(linkset);
var document = MapToDocument(linkset);
var filter = Builders<AdvisoryLinksetDocument>.Filter.And(
Builders<AdvisoryLinksetDocument>.Filter.Eq(d => d.TenantId, linkset.TenantId),
Builders<AdvisoryLinksetDocument>.Filter.Eq(d => d.Source, linkset.Source),
Builders<AdvisoryLinksetDocument>.Filter.Eq(d => d.AdvisoryId, linkset.AdvisoryId));
var options = new ReplaceOptions { IsUpsert = true };
await _collection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false);
}
public async Task<IReadOnlyList<CoreLinksets.AdvisoryLinkset>> FindByTenantAsync(
string tenantId,
IEnumerable<string>? advisoryIds,
IEnumerable<string>? sources,
CoreLinksets.AdvisoryLinksetCursor? cursor,
int limit,
CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
if (limit <= 0)
{
throw new ArgumentOutOfRangeException(nameof(limit));
}
var builder = Builders<AdvisoryLinksetDocument>.Filter;
var filters = new List<FilterDefinition<AdvisoryLinksetDocument>>
{
builder.Eq(d => d.TenantId, tenantId.ToLowerInvariant())
};
if (advisoryIds is not null)
{
var ids = advisoryIds.Where(v => !string.IsNullOrWhiteSpace(v)).ToArray();
if (ids.Length > 0)
{
filters.Add(builder.In(d => d.AdvisoryId, ids));
}
}
if (sources is not null)
{
var srcs = sources.Where(v => !string.IsNullOrWhiteSpace(v)).ToArray();
if (srcs.Length > 0)
{
filters.Add(builder.In(d => d.Source, srcs));
}
}
var filter = builder.And(filters);
var sort = Builders<AdvisoryLinksetDocument>.Sort.Descending(d => d.CreatedAt).Ascending(d => d.AdvisoryId);
var findFilter = filter;
if (cursor is not null)
{
var cursorFilter = builder.Or(
builder.Lt(d => d.CreatedAt, cursor.CreatedAt.UtcDateTime),
builder.And(
builder.Eq(d => d.CreatedAt, cursor.CreatedAt.UtcDateTime),
builder.Gt(d => d.AdvisoryId, cursor.AdvisoryId)));
findFilter = builder.And(findFilter, cursorFilter);
}
var documents = await _collection.Find(findFilter)
.Sort(sort)
.Limit(limit)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return documents.Select(FromDocument).ToArray();
}
private static AdvisoryLinksetDocument MapToDocument(CoreLinksets.AdvisoryLinkset linkset)
{
var doc = new AdvisoryLinksetDocument
{
TenantId = linkset.TenantId,
Source = linkset.Source,
AdvisoryId = linkset.AdvisoryId,
Observations = new List<string>(linkset.ObservationIds),
CreatedAt = linkset.CreatedAt.UtcDateTime,
BuiltByJobId = linkset.BuiltByJobId,
Provenance = linkset.Provenance is null ? null : new AdvisoryLinksetProvenanceDocument
{
ObservationHashes = linkset.Provenance.ObservationHashes is null
? null
: new List<string>(linkset.Provenance.ObservationHashes),
ToolVersion = linkset.Provenance.ToolVersion,
PolicyHash = linkset.Provenance.PolicyHash,
},
Normalized = linkset.Normalized is null ? null : new AdvisoryLinksetNormalizedDocument
{
Purls = linkset.Normalized.Purls is null ? null : new List<string>(linkset.Normalized.Purls),
Versions = linkset.Normalized.Versions is null ? null : new List<string>(linkset.Normalized.Versions),
Ranges = linkset.Normalized.RangesToBson(),
Severities = linkset.Normalized.SeveritiesToBson(),
}
};
return doc;
}
private static CoreLinksets.AdvisoryLinkset FromDocument(AdvisoryLinksetDocument doc)
{
return new AdvisoryLinkset(
doc.TenantId,
doc.Source,
doc.AdvisoryId,
doc.Observations.ToImmutableArray(),
doc.Normalized is null ? null : new AdvisoryLinksetNormalized(
doc.Normalized.Purls,
doc.Normalized.Versions,
doc.Normalized.Ranges?.Select(ToDictionary).ToList(),
doc.Normalized.Severities?.Select(ToDictionary).ToList()),
doc.Provenance is null ? null : new AdvisoryLinksetProvenance(
doc.Provenance.ObservationHashes,
doc.Provenance.ToolVersion,
doc.Provenance.PolicyHash),
DateTime.SpecifyKind(doc.CreatedAt, DateTimeKind.Utc),
doc.BuiltByJobId);
}
private static Dictionary<string, object?> ToDictionary(MongoDB.Bson.BsonDocument bson)
{
var dict = new Dictionary<string, object?>(StringComparer.Ordinal);
foreach (var element in bson.Elements)
{
dict[element.Name] = element.Value switch
{
MongoDB.Bson.BsonString s => s.AsString,
MongoDB.Bson.BsonInt32 i => i.AsInt32,
MongoDB.Bson.BsonInt64 l => l.AsInt64,
MongoDB.Bson.BsonDouble d => d.AsDouble,
MongoDB.Bson.BsonDecimal128 dec => dec.ToDecimal(),
MongoDB.Bson.BsonBoolean b => b.AsBoolean,
MongoDB.Bson.BsonDateTime dt => dt.ToUniversalTime(),
MongoDB.Bson.BsonNull => (object?)null,
MongoDB.Bson.BsonArray arr => arr.Select(v => v.ToString()).ToArray(),
_ => element.Value.ToString()
};
}
return dict;
}
}

View File

@@ -0,0 +1,242 @@
using System.Collections.Generic;
using MongoDB.Bson;
using MongoDB.Driver;
namespace StellaOps.Concelier.Storage.Mongo.Migrations;
internal sealed class EnsureLinkNotMergeCollectionsMigration : IMongoMigration
{
public string Id => "20251116_link_not_merge_collections";
public string Description => "Ensure advisory_observations and advisory_linksets collections exist with validators and indexes for Link-Not-Merge";
public async Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(database);
await EnsureObservationsAsync(database, cancellationToken).ConfigureAwait(false);
await EnsureLinksetsAsync(database, cancellationToken).ConfigureAwait(false);
}
private static async Task EnsureObservationsAsync(IMongoDatabase database, CancellationToken ct)
{
var collectionName = MongoStorageDefaults.Collections.AdvisoryObservations;
var validator = new BsonDocument("$jsonSchema", BuildObservationSchema());
await EnsureCollectionWithValidatorAsync(database, collectionName, validator, ct).ConfigureAwait(false);
var collection = database.GetCollection<BsonDocument>(collectionName);
var indexes = new List<CreateIndexModel<BsonDocument>>
{
new(new BsonDocument
{
{"tenant", 1},
{"source", 1},
{"advisoryId", 1},
{"upstream.fetchedAt", -1},
},
new CreateIndexOptions { Name = "obs_tenant_source_adv_fetchedAt" }),
new(new BsonDocument
{
{"provenance.sourceArtifactSha", 1},
},
new CreateIndexOptions { Name = "obs_prov_sourceArtifactSha_unique", Unique = true }),
};
await collection.Indexes.CreateManyAsync(indexes, cancellationToken: ct).ConfigureAwait(false);
}
private static async Task EnsureLinksetsAsync(IMongoDatabase database, CancellationToken ct)
{
var collectionName = MongoStorageDefaults.Collections.AdvisoryLinksets;
var validator = new BsonDocument("$jsonSchema", BuildLinksetSchema());
await EnsureCollectionWithValidatorAsync(database, collectionName, validator, ct).ConfigureAwait(false);
var collection = database.GetCollection<BsonDocument>(collectionName);
var indexes = new List<CreateIndexModel<BsonDocument>>
{
new(new BsonDocument
{
{"tenantId", 1},
{"advisoryId", 1},
{"source", 1},
},
new CreateIndexOptions { Name = "linkset_tenant_advisory_source", Unique = true }),
new(new BsonDocument { { "observations", 1 } }, new CreateIndexOptions { Name = "linkset_observations" })
};
await collection.Indexes.CreateManyAsync(indexes, cancellationToken: ct).ConfigureAwait(false);
}
private static async Task EnsureCollectionWithValidatorAsync(
IMongoDatabase database,
string collectionName,
BsonDocument validator,
CancellationToken ct)
{
var filter = new BsonDocument("name", collectionName);
var existing = await database.ListCollectionsAsync(new ListCollectionsOptions { Filter = filter }, ct)
.ConfigureAwait(false);
var exists = await existing.AnyAsync(ct).ConfigureAwait(false);
if (!exists)
{
var options = new CreateCollectionOptions<BsonDocument>
{
Validator = validator,
ValidationLevel = DocumentValidationLevel.Moderate,
ValidationAction = DocumentValidationAction.Error,
};
await database.CreateCollectionAsync(collectionName, options, ct).ConfigureAwait(false);
}
else
{
var command = new BsonDocument
{
{ "collMod", collectionName },
{ "validator", validator },
{ "validationLevel", "moderate" },
{ "validationAction", "error" },
};
await database.RunCommandAsync<BsonDocument>(command, cancellationToken: ct).ConfigureAwait(false);
}
}
private static BsonDocument BuildObservationSchema()
{
return new BsonDocument
{
{ "bsonType", "object" },
{ "required", new BsonArray { "_id", "tenantId", "source", "advisoryId", "affected", "provenance", "ingestedAt" } },
{ "properties", new BsonDocument
{
{ "_id", new BsonDocument("bsonType", "string") },
{ "tenantId", new BsonDocument("bsonType", "string") },
{ "source", new BsonDocument("bsonType", "string") },
{ "advisoryId", new BsonDocument("bsonType", "string") },
{ "title", new BsonDocument("bsonType", new BsonArray { "string", "null" }) },
{ "summary", new BsonDocument("bsonType", new BsonArray { "string", "null" }) },
{ "severities", new BsonDocument
{
{ "bsonType", "array" },
{ "items", new BsonDocument
{
{ "bsonType", "object" },
{ "required", new BsonArray { "system", "score" } },
{ "properties", new BsonDocument
{
{ "system", new BsonDocument("bsonType", "string") },
{ "score", new BsonDocument("bsonType", new BsonArray { "double", "int", "long", "decimal" }) },
{ "vector", new BsonDocument("bsonType", new BsonArray { "string", "null" }) }
}
}
}
}
}
},
{ "affected", new BsonDocument
{
{ "bsonType", "array" },
{ "items", new BsonDocument
{
{ "bsonType", "object" },
{ "required", new BsonArray { "purl" } },
{ "properties", new BsonDocument
{
{ "purl", new BsonDocument("bsonType", "string") },
{ "package", new BsonDocument("bsonType", new BsonArray { "string", "null" }) },
{ "versions", new BsonDocument("bsonType", new BsonArray { "array", "null" }) },
{ "ranges", new BsonDocument("bsonType", new BsonArray { "array", "null" }) },
{ "ecosystem", new BsonDocument("bsonType", new BsonArray { "string", "null" }) },
{ "cpe", new BsonDocument("bsonType", new BsonArray { "array", "null" }) },
{ "cpes", new BsonDocument("bsonType", new BsonArray { "array", "null" }) }
}
}
}
}
}
},
{ "references", new BsonDocument
{
{ "bsonType", new BsonArray { "array", "null" } },
{ "items", new BsonDocument("bsonType", "string") }
}
},
{ "weaknesses", new BsonDocument
{
{ "bsonType", new BsonArray { "array", "null" } },
{ "items", new BsonDocument("bsonType", "string") }
}
},
{ "published", new BsonDocument("bsonType", new BsonArray { "date", "null" }) },
{ "modified", new BsonDocument("bsonType", new BsonArray { "date", "null" }) },
{ "provenance", new BsonDocument
{
{ "bsonType", "object" },
{ "required", new BsonArray { "sourceArtifactSha", "fetchedAt" } },
{ "properties", new BsonDocument
{
{ "sourceArtifactSha", new BsonDocument("bsonType", "string") },
{ "fetchedAt", new BsonDocument("bsonType", "date") },
{ "ingestJobId", new BsonDocument("bsonType", new BsonArray { "string", "null" }) },
{ "signature", new BsonDocument("bsonType", new BsonArray { "object", "null" }) }
}
}
}
},
{ "ingestedAt", new BsonDocument("bsonType", "date") }
}
}
};
}
private static BsonDocument BuildLinksetSchema()
{
return new BsonDocument
{
{ "bsonType", "object" },
{ "required", new BsonArray { "_id", "tenantId", "source", "advisoryId", "observations", "createdAt" } },
{ "properties", new BsonDocument
{
{ "_id", new BsonDocument("bsonType", "objectId") },
{ "tenantId", new BsonDocument("bsonType", "string") },
{ "source", new BsonDocument("bsonType", "string") },
{ "advisoryId", new BsonDocument("bsonType", "string") },
{ "observations", new BsonDocument
{
{ "bsonType", "array" },
{ "items", new BsonDocument("bsonType", "string") }
}
},
{ "normalized", new BsonDocument
{
{ "bsonType", new BsonArray { "object", "null" } },
{ "properties", new BsonDocument
{
{ "purls", new BsonDocument { { "bsonType", new BsonArray { "array", "null" } }, { "items", new BsonDocument("bsonType", "string") } } },
{ "versions", new BsonDocument { { "bsonType", new BsonArray { "array", "null" } }, { "items", new BsonDocument("bsonType", "string") } } },
{ "ranges", new BsonDocument { { "bsonType", new BsonArray { "array", "null" } }, { "items", new BsonDocument("bsonType", "object") } } },
{ "severities", new BsonDocument { { "bsonType", new BsonArray { "array", "null" } }, { "items", new BsonDocument("bsonType", "object") } } }
}
}
}
},
{ "createdAt", new BsonDocument("bsonType", "date") },
{ "builtByJobId", new BsonDocument("bsonType", new BsonArray { "string", "null" }) },
{ "provenance", new BsonDocument
{
{ "bsonType", new BsonArray { "object", "null" } },
{ "properties", new BsonDocument
{
{ "observationHashes", new BsonDocument { { "bsonType", new BsonArray { "array", "null" } }, { "items", new BsonDocument("bsonType", "string") } } },
{ "toolVersion", new BsonDocument("bsonType", new BsonArray { "string", "null" }) },
{ "policyHash", new BsonDocument("bsonType", new BsonArray { "string", "null" }) }
}
}
}
}
}
}
};
}
}

View File

@@ -0,0 +1,22 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Observations;
using StellaOps.Concelier.Models.Observations;
namespace StellaOps.Concelier.Storage.Mongo.Observations;
internal sealed class AdvisoryObservationSink : IAdvisoryObservationSink
{
private readonly IAdvisoryObservationStore _store;
public AdvisoryObservationSink(IAdvisoryObservationStore store)
{
_store = store ?? throw new ArgumentNullException(nameof(store));
}
public Task UpsertAsync(AdvisoryObservation observation, CancellationToken cancellationToken)
{
return _store.UpsertAsync(observation, cancellationToken);
}
}

View File

@@ -80,6 +80,13 @@ public static class ServiceCollectionExtensions
services.AddSingleton<IAdvisoryEventRepository, MongoAdvisoryEventRepository>();
services.AddSingleton<IAdvisoryEventLog, AdvisoryEventLog>();
services.AddSingleton<IAdvisoryRawRepository, MongoAdvisoryRawRepository>();
services.AddSingleton<StellaOps.Concelier.Storage.Mongo.Linksets.MongoAdvisoryLinksetStore>();
services.AddSingleton<StellaOps.Concelier.Core.Linksets.IAdvisoryLinksetStore>(sp =>
sp.GetRequiredService<StellaOps.Concelier.Storage.Mongo.Linksets.MongoAdvisoryLinksetStore>());
services.AddSingleton<StellaOps.Concelier.Core.Linksets.IAdvisoryLinksetLookup>(sp =>
sp.GetRequiredService<StellaOps.Concelier.Storage.Mongo.Linksets.MongoAdvisoryLinksetStore>());
services.AddSingleton<StellaOps.Concelier.Core.Linksets.IAdvisoryObservationSink, StellaOps.Concelier.Storage.Mongo.Linksets.AdvisoryObservationSink>();
services.AddSingleton<StellaOps.Concelier.Core.Linksets.IAdvisoryLinksetSink, StellaOps.Concelier.Storage.Mongo.Linksets.AdvisoryLinksetSink>();
services.AddSingleton<IExportStateStore, ExportStateStore>();
services.TryAddSingleton<ExportStateManager>();