feat: Add in-memory implementations for issuer audit, key, repository, and trust management
Some checks failed
devportal-offline / build-offline (push) Has been cancelled
Mirror Thin Bundle Sign & Verify / mirror-sign (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
api-governance / spectral-lint (push) Has been cancelled
oas-ci / oas-validate (push) Has been cancelled

- Introduced InMemoryIssuerAuditSink to retain audit entries for testing.
- Implemented InMemoryIssuerKeyRepository for deterministic key storage.
- Created InMemoryIssuerRepository to manage issuer records in memory.
- Added InMemoryIssuerTrustRepository for managing issuer trust overrides.
- Each repository utilizes concurrent collections for thread-safe operations.
- Enhanced deprecation tracking with a comprehensive YAML schema for API governance.
This commit is contained in:
master
2025-12-11 19:47:43 +02:00
parent ab22181e8b
commit ce5ec9c158
48 changed files with 1898 additions and 1580 deletions

View File

@@ -19,7 +19,6 @@
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="EphemeralMongo" Version="3.0.0" />
</ItemGroup>
<ItemGroup>

View File

@@ -9,8 +9,6 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="MongoDB.Driver" Version="3.5.0" />
<PackageReference Include="EphemeralMongo" Version="3.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0" />
</ItemGroup>
</Project>

View File

@@ -1,10 +1,8 @@
using MongoDB.Bson;
namespace StellaOps.Bench.LinkNotMerge.Vex;
internal sealed class VexLinksetAggregator
{
public VexAggregationResult Correlate(IEnumerable<BsonDocument> documents)
public VexAggregationResult Correlate(IEnumerable<VexObservationDocument> documents)
{
ArgumentNullException.ThrowIfNull(documents);
@@ -13,39 +11,21 @@ internal sealed class VexLinksetAggregator
foreach (var document in documents)
{
var tenant = document.GetValue("tenant", "unknown").AsString;
var linksetValue = document.GetValue("linkset", new BsonDocument());
var linkset = linksetValue.IsBsonDocument ? linksetValue.AsBsonDocument : new BsonDocument();
var aliases = linkset.GetValue("aliases", new BsonArray()).AsBsonArray;
var statementsValue = document.GetValue("statements", new BsonArray());
var statements = statementsValue.IsBsonArray ? statementsValue.AsBsonArray : new BsonArray();
var tenant = document.Tenant;
var aliases = document.Aliases;
var statements = document.Statements;
foreach (var statementValue in statements)
{
if (!statementValue.IsBsonDocument)
{
continue;
}
statementsSeen++;
var statement = statementValue.AsBsonDocument;
var status = statement.GetValue("status", "unknown").AsString;
var justification = statement.GetValue("justification", BsonNull.Value);
var lastUpdated = statement.GetValue("last_updated", BsonNull.Value);
var productValue = statement.GetValue("product", new BsonDocument());
var product = productValue.IsBsonDocument ? productValue.AsBsonDocument : new BsonDocument();
var productKey = product.GetValue("purl", "unknown").AsString;
var status = statementValue.Status;
var justification = statementValue.Justification;
var lastUpdated = statementValue.LastUpdated;
var productKey = statementValue.Product.Purl;
foreach (var aliasValue in aliases)
foreach (var alias in aliases)
{
if (!aliasValue.IsString)
{
continue;
}
var alias = aliasValue.AsString;
var key = string.Create(alias.Length + tenant.Length + productKey.Length + 2, (tenant, alias, productKey), static (span, data) =>
{
var (tenantValue, aliasValue, productValue) = data;
@@ -70,7 +50,7 @@ internal sealed class VexLinksetAggregator
}
}
var eventDocuments = new List<BsonDocument>(groups.Count);
var eventDocuments = new List<VexEvent>(groups.Count);
foreach (var accumulator in groups.Values)
{
if (accumulator.ShouldEmitEvent)
@@ -93,7 +73,7 @@ internal sealed class VexLinksetAggregator
private readonly string _tenant;
private readonly string _alias;
private readonly string _product;
private DateTime? _latest;
private DateTimeOffset? _latest;
public VexAccumulator(string tenant, string alias, string product)
{
@@ -102,22 +82,22 @@ internal sealed class VexLinksetAggregator
_product = product;
}
public void AddStatement(string status, BsonValue justification, BsonValue updatedAt)
public void AddStatement(string status, string justification, DateTimeOffset updatedAt)
{
if (!_statusCounts.TryAdd(status, 1))
{
_statusCounts[status]++;
}
if (justification.IsString)
if (!string.IsNullOrEmpty(justification))
{
_justifications.Add(justification.AsString);
_justifications.Add(justification);
}
if (updatedAt.IsValidDateTime)
if (updatedAt != default)
{
var value = updatedAt.ToUniversalTime();
if (!_latest.HasValue || value > _latest)
if (!_latest.HasValue || value > _latest.Value)
{
_latest = value;
}
@@ -142,19 +122,15 @@ internal sealed class VexLinksetAggregator
}
}
public BsonDocument ToEvent()
public VexEvent ToEvent()
{
var payload = new BsonDocument
{
["tenant"] = _tenant,
["alias"] = _alias,
["product"] = _product,
["statuses"] = new BsonDocument(_statusCounts.Select(kvp => new BsonElement(kvp.Key, kvp.Value))),
["justifications"] = new BsonArray(_justifications.Select(justification => justification)),
["last_updated"] = _latest.HasValue ? _latest.Value : (BsonValue)BsonNull.Value,
};
return payload;
return new VexEvent(
_tenant,
_alias,
_product,
new Dictionary<string, int>(_statusCounts, StringComparer.Ordinal),
_justifications.ToArray(),
_latest);
}
}
}
@@ -163,4 +139,12 @@ internal sealed record VexAggregationResult(
int LinksetCount,
int StatementCount,
int EventCount,
IReadOnlyList<BsonDocument> EventDocuments);
IReadOnlyList<VexEvent> EventDocuments);
internal sealed record VexEvent(
string Tenant,
string Alias,
string Product,
IReadOnlyDictionary<string, int> Statuses,
IReadOnlyCollection<string> Justifications,
DateTimeOffset? LastUpdated);

View File

@@ -1,252 +1,194 @@
using System.Collections.Immutable;
using System.Security.Cryptography;
using MongoDB.Bson;
namespace StellaOps.Bench.LinkNotMerge.Vex;
internal static class VexObservationGenerator
{
private static readonly ImmutableArray<string> StatusPool = ImmutableArray.Create(
"affected",
"not_affected",
"under_investigation");
private static readonly ImmutableArray<string> JustificationPool = ImmutableArray.Create(
"exploitation_mitigated",
"component_not_present",
"vulnerable_code_not_present",
"vulnerable_code_not_in_execute_path");
public static IReadOnlyList<VexObservationSeed> Generate(VexScenarioConfig config)
{
ArgumentNullException.ThrowIfNull(config);
var observationCount = config.ResolveObservationCount();
var aliasGroups = config.ResolveAliasGroups();
var statementsPerObservation = config.ResolveStatementsPerObservation();
var tenantCount = config.ResolveTenantCount();
var productsPerObservation = config.ResolveProductsPerObservation();
var seed = config.ResolveSeed();
var seeds = new VexObservationSeed[observationCount];
var random = new Random(seed);
var baseTime = new DateTimeOffset(2025, 10, 1, 0, 0, 0, TimeSpan.Zero);
for (var index = 0; index < observationCount; index++)
{
var tenantIndex = index % tenantCount;
var tenant = $"tenant-{tenantIndex:D2}";
var group = index % aliasGroups;
var revision = index / aliasGroups;
var vulnerabilityAlias = $"CVE-2025-{group:D4}";
var upstreamId = $"VEX-{group:D4}-{revision:D3}";
var observationId = $"{tenant}:vex:{group:D5}:{revision:D6}";
var fetchedAt = baseTime.AddMinutes(revision);
var receivedAt = fetchedAt.AddSeconds(2);
var documentVersion = fetchedAt.AddSeconds(15).ToString("O");
var products = CreateProducts(group, revision, productsPerObservation);
var statements = CreateStatements(vulnerabilityAlias, products, statementsPerObservation, random, fetchedAt);
var rawPayload = CreateRawPayload(upstreamId, vulnerabilityAlias, statements);
var contentHash = ComputeContentHash(rawPayload, tenant, group, revision);
var aliases = ImmutableArray.Create(vulnerabilityAlias, $"GHSA-{group:D4}-{revision % 26 + 'a'}{revision % 26 + 'a'}");
var references = ImmutableArray.Create(
new VexReference("advisory", $"https://vendor.example/advisories/{vulnerabilityAlias.ToLowerInvariant()}"),
new VexReference("fix", $"https://vendor.example/patch/{vulnerabilityAlias.ToLowerInvariant()}"));
seeds[index] = new VexObservationSeed(
ObservationId: observationId,
Tenant: tenant,
Vendor: "excititor-bench",
Stream: "simulated",
Api: $"https://bench.stella/vex/{group:D4}/{revision:D3}",
CollectorVersion: "1.0.0-bench",
UpstreamId: upstreamId,
DocumentVersion: documentVersion,
FetchedAt: fetchedAt,
ReceivedAt: receivedAt,
ContentHash: contentHash,
VulnerabilityAlias: vulnerabilityAlias,
Aliases: aliases,
Products: products,
Statements: statements,
References: references,
ContentFormat: "CycloneDX-VEX",
SpecVersion: "1.4",
RawPayload: rawPayload);
}
return seeds;
}
private static ImmutableArray<VexProduct> CreateProducts(int group, int revision, int count)
{
var builder = ImmutableArray.CreateBuilder<VexProduct>(count);
for (var index = 0; index < count; index++)
{
var purl = $"pkg:generic/stella/product-{group:D4}-{index}@{1 + revision % 5}.{index + 1}.{revision % 9}";
builder.Add(new VexProduct(purl, $"component-{group % 30:D2}", $"namespace-{group % 10:D2}"));
}
return builder.MoveToImmutable();
}
private static ImmutableArray<BsonDocument> CreateStatements(
string vulnerabilityAlias,
ImmutableArray<VexProduct> products,
int statementsPerObservation,
Random random,
DateTimeOffset baseTime)
{
var builder = ImmutableArray.CreateBuilder<BsonDocument>(statementsPerObservation);
for (var index = 0; index < statementsPerObservation; index++)
{
var statusIndex = random.Next(StatusPool.Length);
var status = StatusPool[statusIndex];
var justification = JustificationPool[random.Next(JustificationPool.Length)];
var product = products[index % products.Length];
var statementId = $"stmt-{vulnerabilityAlias}-{index:D2}";
var document = new BsonDocument
{
["statement_id"] = statementId,
["vulnerability_alias"] = vulnerabilityAlias,
["product"] = new BsonDocument
{
["purl"] = product.Purl,
["component"] = product.Component,
["namespace"] = product.Namespace,
},
["status"] = status,
["justification"] = justification,
["impact"] = status == "affected" ? "high" : "none",
["last_updated"] = baseTime.AddMinutes(index).UtcDateTime,
};
builder.Add(document);
}
return builder.MoveToImmutable();
}
private static BsonDocument CreateRawPayload(string upstreamId, string vulnerabilityAlias, ImmutableArray<BsonDocument> statements)
{
var doc = new BsonDocument
{
["documentId"] = upstreamId,
["title"] = $"Simulated VEX report {upstreamId}",
["summary"] = $"Synthetic VEX payload for {vulnerabilityAlias}.",
["statements"] = new BsonArray(statements),
};
return doc;
}
private static string ComputeContentHash(BsonDocument rawPayload, string tenant, int group, int revision)
{
using var sha256 = SHA256.Create();
var seed = $"{tenant}|{group}|{revision}";
var rawBytes = rawPayload.ToBson();
var seedBytes = System.Text.Encoding.UTF8.GetBytes(seed);
var combined = new byte[rawBytes.Length + seedBytes.Length];
Buffer.BlockCopy(rawBytes, 0, combined, 0, rawBytes.Length);
Buffer.BlockCopy(seedBytes, 0, combined, rawBytes.Length, seedBytes.Length);
var hash = sha256.ComputeHash(combined);
return $"sha256:{Convert.ToHexString(hash)}";
}
}
internal sealed record VexObservationSeed(
string ObservationId,
string Tenant,
string Vendor,
string Stream,
string Api,
string CollectorVersion,
string UpstreamId,
string DocumentVersion,
DateTimeOffset FetchedAt,
DateTimeOffset ReceivedAt,
string ContentHash,
string VulnerabilityAlias,
ImmutableArray<string> Aliases,
ImmutableArray<VexProduct> Products,
ImmutableArray<BsonDocument> Statements,
ImmutableArray<VexReference> References,
string ContentFormat,
string SpecVersion,
BsonDocument RawPayload)
{
public BsonDocument ToBsonDocument()
{
var aliases = new BsonArray(Aliases.Select(alias => alias));
var statements = new BsonArray(Statements);
var productsArray = new BsonArray(Products.Select(product => new BsonDocument
{
["purl"] = product.Purl,
["component"] = product.Component,
["namespace"] = product.Namespace,
}));
var references = new BsonArray(References.Select(reference => new BsonDocument
{
["type"] = reference.Type,
["url"] = reference.Url,
}));
var document = new BsonDocument
{
["_id"] = ObservationId,
["tenant"] = Tenant,
["source"] = new BsonDocument
{
["vendor"] = Vendor,
["stream"] = Stream,
["api"] = Api,
["collector_version"] = CollectorVersion,
},
["upstream"] = new BsonDocument
{
["upstream_id"] = UpstreamId,
["document_version"] = DocumentVersion,
["fetched_at"] = FetchedAt.UtcDateTime,
["received_at"] = ReceivedAt.UtcDateTime,
["content_hash"] = ContentHash,
["signature"] = new BsonDocument
{
["present"] = false,
["format"] = BsonNull.Value,
["key_id"] = BsonNull.Value,
["signature"] = BsonNull.Value,
},
},
["content"] = new BsonDocument
{
["format"] = ContentFormat,
["spec_version"] = SpecVersion,
["raw"] = RawPayload,
},
["identifiers"] = new BsonDocument
{
["aliases"] = aliases,
["primary"] = VulnerabilityAlias,
},
["statements"] = statements,
["linkset"] = new BsonDocument
{
["aliases"] = aliases,
["products"] = productsArray,
["references"] = references,
["reconciled_from"] = new BsonArray { "/statements" },
},
["supersedes"] = BsonNull.Value,
};
return document;
}
}
internal sealed record VexProduct(string Purl, string Component, string Namespace);
internal sealed record VexReference(string Type, string Url);
using System.Collections.Immutable;
using System.Security.Cryptography;
using System.Text;
namespace StellaOps.Bench.LinkNotMerge.Vex;
internal static class VexObservationGenerator
{
private static readonly ImmutableArray<string> StatusPool = ImmutableArray.Create(
"affected",
"not_affected",
"under_investigation");
private static readonly ImmutableArray<string> JustificationPool = ImmutableArray.Create(
"exploitation_mitigated",
"component_not_present",
"vulnerable_code_not_present",
"vulnerable_code_not_in_execute_path");
public static IReadOnlyList<VexObservationSeed> Generate(VexScenarioConfig config)
{
ArgumentNullException.ThrowIfNull(config);
var observationCount = config.ResolveObservationCount();
var aliasGroups = config.ResolveAliasGroups();
var statementsPerObservation = config.ResolveStatementsPerObservation();
var tenantCount = config.ResolveTenantCount();
var productsPerObservation = config.ResolveProductsPerObservation();
var seed = config.ResolveSeed();
var seeds = new VexObservationSeed[observationCount];
var random = new Random(seed);
var baseTime = new DateTimeOffset(2025, 10, 1, 0, 0, 0, TimeSpan.Zero);
for (var index = 0; index < observationCount; index++)
{
var tenantIndex = index % tenantCount;
var tenant = $"tenant-{tenantIndex:D2}";
var group = index % aliasGroups;
var revision = index / aliasGroups;
var vulnerabilityAlias = $"CVE-2025-{group:D4}";
var upstreamId = $"VEX-{group:D4}-{revision:D3}";
var observationId = $"{tenant}:vex:{group:D5}:{revision:D6}";
var fetchedAt = baseTime.AddMinutes(revision);
var receivedAt = fetchedAt.AddSeconds(2);
var documentVersion = fetchedAt.AddSeconds(15).ToString("O");
var products = CreateProducts(group, revision, productsPerObservation);
var statements = CreateStatements(vulnerabilityAlias, products, statementsPerObservation, random, fetchedAt);
var contentHash = ComputeContentHash(upstreamId, vulnerabilityAlias, statements, tenant, group, revision);
var aliases = ImmutableArray.Create(vulnerabilityAlias, $"GHSA-{group:D4}-{revision % 26 + 'a'}{revision % 26 + 'a'}");
var references = ImmutableArray.Create(
new VexReference("advisory", $"https://vendor.example/advisories/{vulnerabilityAlias.ToLowerInvariant()}"),
new VexReference("fix", $"https://vendor.example/patch/{vulnerabilityAlias.ToLowerInvariant()}"));
seeds[index] = new VexObservationSeed(
ObservationId: observationId,
Tenant: tenant,
Vendor: "excititor-bench",
Stream: "simulated",
Api: $"https://bench.stella/vex/{group:D4}/{revision:D3}",
CollectorVersion: "1.0.0-bench",
UpstreamId: upstreamId,
DocumentVersion: documentVersion,
FetchedAt: fetchedAt,
ReceivedAt: receivedAt,
ContentHash: contentHash,
VulnerabilityAlias: vulnerabilityAlias,
Aliases: aliases,
Products: products,
Statements: statements,
References: references,
ContentFormat: "CycloneDX-VEX",
SpecVersion: "1.4");
}
return seeds;
}
private static ImmutableArray<VexProduct> CreateProducts(int group, int revision, int count)
{
var builder = ImmutableArray.CreateBuilder<VexProduct>(count);
for (var index = 0; index < count; index++)
{
var purl = $"pkg:generic/stella/product-{group:D4}-{index}@{1 + revision % 5}.{index + 1}.{revision % 9}";
builder.Add(new VexProduct(purl, $"component-{group % 30:D2}", $"namespace-{group % 10:D2}"));
}
return builder.MoveToImmutable();
}
private static ImmutableArray<VexStatement> CreateStatements(
string vulnerabilityAlias,
ImmutableArray<VexProduct> products,
int statementsPerObservation,
Random random,
DateTimeOffset baseTime)
{
var builder = ImmutableArray.CreateBuilder<VexStatement>(statementsPerObservation);
for (var index = 0; index < statementsPerObservation; index++)
{
var statusIndex = random.Next(StatusPool.Length);
var status = StatusPool[statusIndex];
var justification = JustificationPool[random.Next(JustificationPool.Length)];
var product = products[index % products.Length];
var statementId = $"stmt-{vulnerabilityAlias}-{index:D2}";
var lastUpdated = baseTime.AddMinutes(index).ToUniversalTime();
builder.Add(new VexStatement(
StatementId: statementId,
VulnerabilityAlias: vulnerabilityAlias,
Product: product,
Status: status,
Justification: justification,
LastUpdated: lastUpdated));
}
return builder.MoveToImmutable();
}
private static string ComputeContentHash(
string upstreamId,
string vulnerabilityAlias,
ImmutableArray<VexStatement> statements,
string tenant,
int group,
int revision)
{
using var sha256 = SHA256.Create();
var builder = new StringBuilder();
builder.Append(tenant).Append('|').Append(group).Append('|').Append(revision).Append('|');
builder.Append(upstreamId).Append('|').Append(vulnerabilityAlias).Append('|');
foreach (var statement in statements)
{
builder.Append(statement.StatementId).Append('|')
.Append(statement.Status).Append('|')
.Append(statement.Product.Purl).Append('|')
.Append(statement.Justification).Append('|')
.Append(statement.LastUpdated.ToUniversalTime().ToString("O")).Append('|');
}
var data = Encoding.UTF8.GetBytes(builder.ToString());
var hash = sha256.ComputeHash(data);
return $"sha256:{Convert.ToHexString(hash)}";
}
}
internal sealed record VexObservationSeed(
string ObservationId,
string Tenant,
string Vendor,
string Stream,
string Api,
string CollectorVersion,
string UpstreamId,
string DocumentVersion,
DateTimeOffset FetchedAt,
DateTimeOffset ReceivedAt,
string ContentHash,
string VulnerabilityAlias,
ImmutableArray<string> Aliases,
ImmutableArray<VexProduct> Products,
ImmutableArray<VexStatement> Statements,
ImmutableArray<VexReference> References,
string ContentFormat,
string SpecVersion)
{
public VexObservationDocument ToDocument()
{
return new VexObservationDocument(
Tenant,
Aliases,
Statements);
}
}
internal sealed record VexObservationDocument(
string Tenant,
ImmutableArray<string> Aliases,
ImmutableArray<VexStatement> Statements);
internal sealed record VexStatement(
string StatementId,
string VulnerabilityAlias,
VexProduct Product,
string Status,
string Justification,
DateTimeOffset LastUpdated);
internal sealed record VexProduct(string Purl, string Component, string Namespace);
internal sealed record VexReference(string Type, string Url);

View File

@@ -1,7 +1,4 @@
using System.Diagnostics;
using EphemeralMongo;
using MongoDB.Bson;
using MongoDB.Driver;
namespace StellaOps.Bench.LinkNotMerge.Vex;
@@ -29,38 +26,19 @@ internal sealed class VexScenarioRunner
var allocated = new double[iterations];
var observationThroughputs = new double[iterations];
var eventThroughputs = new double[iterations];
VexAggregationResult lastAggregation = new(0, 0, 0, Array.Empty<BsonDocument>());
VexAggregationResult lastAggregation = new(0, 0, 0, Array.Empty<VexEvent>());
for (var iteration = 0; iteration < iterations; iteration++)
{
cancellationToken.ThrowIfCancellationRequested();
using var runner = MongoRunner.Run(new MongoRunnerOptions
{
UseSingleNodeReplicaSet = false,
});
var client = new MongoClient(runner.ConnectionString);
var database = client.GetDatabase("linknotmerge_vex_bench");
var collection = database.GetCollection<BsonDocument>("vex_observations");
CreateIndexes(collection, cancellationToken);
var beforeAllocated = GC.GetTotalAllocatedBytes();
var insertStopwatch = Stopwatch.StartNew();
InsertObservations(collection, _seeds, _config.ResolveBatchSize(), cancellationToken);
var documents = InsertObservations(_seeds, _config.ResolveBatchSize(), cancellationToken);
insertStopwatch.Stop();
var correlationStopwatch = Stopwatch.StartNew();
var documents = collection
.Find(FilterDefinition<BsonDocument>.Empty)
.Project(Builders<BsonDocument>.Projection
.Include("tenant")
.Include("statements")
.Include("linkset"))
.ToList(cancellationToken);
var aggregator = new VexLinksetAggregator();
lastAggregation = aggregator.Correlate(documents);
correlationStopwatch.Stop();
@@ -95,44 +73,26 @@ internal sealed class VexScenarioRunner
AggregationResult: lastAggregation);
}
private static void InsertObservations(
IMongoCollection<BsonDocument> collection,
private static IReadOnlyList<VexObservationDocument> InsertObservations(
IReadOnlyList<VexObservationSeed> seeds,
int batchSize,
CancellationToken cancellationToken)
{
var documents = new List<VexObservationDocument>(seeds.Count);
for (var offset = 0; offset < seeds.Count; offset += batchSize)
{
cancellationToken.ThrowIfCancellationRequested();
var remaining = Math.Min(batchSize, seeds.Count - offset);
var batch = new List<BsonDocument>(remaining);
var batch = new List<VexObservationDocument>(remaining);
for (var index = 0; index < remaining; index++)
{
batch.Add(seeds[offset + index].ToBsonDocument());
batch.Add(seeds[offset + index].ToDocument());
}
collection.InsertMany(batch, new InsertManyOptions
{
IsOrdered = false,
BypassDocumentValidation = true,
}, cancellationToken);
documents.AddRange(batch);
}
}
private static void CreateIndexes(IMongoCollection<BsonDocument> collection, CancellationToken cancellationToken)
{
var indexKeys = Builders<BsonDocument>.IndexKeys
.Ascending("tenant")
.Ascending("linkset.aliases");
try
{
collection.Indexes.CreateOne(new CreateIndexModel<BsonDocument>(indexKeys), cancellationToken: cancellationToken);
}
catch
{
// non-fatal
}
return documents;
}
}

View File

@@ -19,7 +19,6 @@
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="EphemeralMongo" Version="3.0.0" />
</ItemGroup>
<ItemGroup>

View File

@@ -1,7 +1,4 @@
using System.Diagnostics;
using EphemeralMongo;
using MongoDB.Bson;
using MongoDB.Driver;
namespace StellaOps.Bench.LinkNotMerge;
@@ -35,30 +32,12 @@ internal sealed class LinkNotMergeScenarioRunner
{
cancellationToken.ThrowIfCancellationRequested();
using var runner = MongoRunner.Run(new MongoRunnerOptions
{
UseSingleNodeReplicaSet = false,
});
var client = new MongoClient(runner.ConnectionString);
var database = client.GetDatabase("linknotmerge_bench");
var collection = database.GetCollection<BsonDocument>("advisory_observations");
CreateIndexes(collection, cancellationToken);
var beforeAllocated = GC.GetTotalAllocatedBytes();
var insertStopwatch = Stopwatch.StartNew();
InsertObservations(collection, _seeds, _config.ResolveBatchSize(), cancellationToken);
var documents = InsertObservations(_seeds, _config.ResolveBatchSize(), cancellationToken);
insertStopwatch.Stop();
var correlationStopwatch = Stopwatch.StartNew();
var documents = collection
.Find(FilterDefinition<BsonDocument>.Empty)
.Project(Builders<BsonDocument>.Projection
.Include("tenant")
.Include("linkset"))
.ToList(cancellationToken);
var correlator = new LinksetAggregator();
lastAggregation = correlator.Correlate(documents);
correlationStopwatch.Stop();
@@ -92,44 +71,26 @@ internal sealed class LinkNotMergeScenarioRunner
AggregationResult: lastAggregation);
}
private static void InsertObservations(
IMongoCollection<BsonDocument> collection,
private static IReadOnlyList<ObservationDocument> InsertObservations(
IReadOnlyList<ObservationSeed> seeds,
int batchSize,
CancellationToken cancellationToken)
{
var documents = new List<ObservationDocument>(seeds.Count);
for (var offset = 0; offset < seeds.Count; offset += batchSize)
{
cancellationToken.ThrowIfCancellationRequested();
var remaining = Math.Min(batchSize, seeds.Count - offset);
var batch = new List<BsonDocument>(remaining);
var batch = new List<ObservationDocument>(remaining);
for (var index = 0; index < remaining; index++)
{
batch.Add(seeds[offset + index].ToBsonDocument());
batch.Add(seeds[offset + index].ToDocument());
}
collection.InsertMany(batch, new InsertManyOptions
{
IsOrdered = false,
BypassDocumentValidation = true,
}, cancellationToken);
documents.AddRange(batch);
}
}
private static void CreateIndexes(IMongoCollection<BsonDocument> collection, CancellationToken cancellationToken)
{
var indexKeys = Builders<BsonDocument>.IndexKeys
.Ascending("tenant")
.Ascending("identifiers.aliases");
try
{
collection.Indexes.CreateOne(new CreateIndexModel<BsonDocument>(indexKeys), cancellationToken: cancellationToken);
}
catch
{
// Index creation failures should not abort the benchmark; they may occur when running multiple iterations concurrently.
}
return documents;
}
}

View File

@@ -1,10 +1,8 @@
using MongoDB.Bson;
namespace StellaOps.Bench.LinkNotMerge;
internal sealed class LinksetAggregator
{
public LinksetAggregationResult Correlate(IEnumerable<BsonDocument> documents)
public LinksetAggregationResult Correlate(IEnumerable<ObservationDocument> documents)
{
ArgumentNullException.ThrowIfNull(documents);
@@ -15,21 +13,16 @@ internal sealed class LinksetAggregator
{
totalObservations++;
var tenant = document.GetValue("tenant", "unknown").AsString;
var linkset = document.GetValue("linkset", new BsonDocument()).AsBsonDocument;
var aliases = linkset.GetValue("aliases", new BsonArray()).AsBsonArray;
var purls = linkset.GetValue("purls", new BsonArray()).AsBsonArray;
var cpes = linkset.GetValue("cpes", new BsonArray()).AsBsonArray;
var references = linkset.GetValue("references", new BsonArray()).AsBsonArray;
var tenant = document.Tenant;
var linkset = document.Linkset;
var aliases = linkset.Aliases;
var purls = linkset.Purls;
var cpes = linkset.Cpes;
var references = linkset.References;
foreach (var aliasValue in aliases)
{
if (!aliasValue.IsString)
{
continue;
}
var alias = aliasValue.AsString;
var alias = aliasValue;
var key = string.Create(alias.Length + tenant.Length + 1, (tenant, alias), static (span, data) =>
{
var (tenantValue, aliasValue) = data;
@@ -91,42 +84,30 @@ internal sealed class LinksetAggregator
public int ReferenceCount => _references.Count;
public void AddPurls(BsonArray array)
public void AddPurls(IEnumerable<string> array)
{
foreach (var item in array)
{
if (item.IsString)
{
_purls.Add(item.AsString);
}
if (!string.IsNullOrEmpty(item))
_purls.Add(item);
}
}
public void AddCpes(BsonArray array)
public void AddCpes(IEnumerable<string> array)
{
foreach (var item in array)
{
if (item.IsString)
{
_cpes.Add(item.AsString);
}
if (!string.IsNullOrEmpty(item))
_cpes.Add(item);
}
}
public void AddReferences(BsonArray array)
public void AddReferences(IEnumerable<ObservationReference> array)
{
foreach (var item in array)
{
if (!item.IsBsonDocument)
{
continue;
}
var document = item.AsBsonDocument;
if (document.TryGetValue("url", out var urlValue) && urlValue.IsString)
{
_references.Add(urlValue.AsString);
}
if (!string.IsNullOrEmpty(item.Url))
_references.Add(item.Url);
}
}
}

View File

@@ -1,270 +1,198 @@
using System.Collections.Immutable;
using System.Security.Cryptography;
using MongoDB.Bson;
namespace StellaOps.Bench.LinkNotMerge;
internal static class ObservationGenerator
{
public static IReadOnlyList<ObservationSeed> Generate(LinkNotMergeScenarioConfig config)
{
ArgumentNullException.ThrowIfNull(config);
var observationCount = config.ResolveObservationCount();
var aliasGroups = config.ResolveAliasGroups();
var purlsPerObservation = config.ResolvePurlsPerObservation();
var cpesPerObservation = config.ResolveCpesPerObservation();
var referencesPerObservation = config.ResolveReferencesPerObservation();
var tenantCount = config.ResolveTenantCount();
var seed = config.ResolveSeed();
var seeds = new ObservationSeed[observationCount];
var random = new Random(seed);
var baseTime = new DateTimeOffset(2025, 10, 1, 0, 0, 0, TimeSpan.Zero);
for (var index = 0; index < observationCount; index++)
{
var tenantIndex = index % tenantCount;
var tenant = $"tenant-{tenantIndex:D2}";
var group = index % aliasGroups;
var revision = index / aliasGroups;
var primaryAlias = $"CVE-2025-{group:D4}";
var vendorAlias = $"VENDOR-{group:D4}";
var thirdAlias = $"GHSA-{group:D4}-{(revision % 26 + 'a')}{(revision % 26 + 'a')}";
var aliases = ImmutableArray.Create(primaryAlias, vendorAlias, thirdAlias);
var observationId = $"{tenant}:advisory:{group:D5}:{revision:D6}";
var upstreamId = primaryAlias;
var documentVersion = baseTime.AddMinutes(revision).ToString("O");
var fetchedAt = baseTime.AddSeconds(index % 1_800);
var receivedAt = fetchedAt.AddSeconds(1);
var purls = CreatePurls(group, revision, purlsPerObservation);
var cpes = CreateCpes(group, revision, cpesPerObservation);
var references = CreateReferences(primaryAlias, referencesPerObservation);
var rawPayload = CreateRawPayload(primaryAlias, vendorAlias, purls, cpes, references);
var contentHash = ComputeContentHash(rawPayload, tenant, group, revision);
seeds[index] = new ObservationSeed(
ObservationId: observationId,
Tenant: tenant,
Vendor: "concelier-bench",
Stream: "simulated",
Api: $"https://bench.stella/{group:D4}/{revision:D2}",
CollectorVersion: "1.0.0-bench",
UpstreamId: upstreamId,
DocumentVersion: documentVersion,
FetchedAt: fetchedAt,
ReceivedAt: receivedAt,
ContentHash: contentHash,
Aliases: aliases,
Purls: purls,
Cpes: cpes,
References: references,
ContentFormat: "CSAF",
SpecVersion: "2.0",
RawPayload: rawPayload);
}
return seeds;
}
private static ImmutableArray<string> CreatePurls(int group, int revision, int count)
{
if (count <= 0)
{
return ImmutableArray<string>.Empty;
}
var builder = ImmutableArray.CreateBuilder<string>(count);
for (var index = 0; index < count; index++)
{
var version = $"{revision % 9 + 1}.{index + 1}.{group % 10}";
builder.Add($"pkg:generic/stella/sample-{group:D4}-{index}@{version}");
}
return builder.MoveToImmutable();
}
private static ImmutableArray<string> CreateCpes(int group, int revision, int count)
{
if (count <= 0)
{
return ImmutableArray<string>.Empty;
}
var builder = ImmutableArray.CreateBuilder<string>(count);
for (var index = 0; index < count; index++)
{
var component = $"benchtool{group % 50:D2}";
var version = $"{revision % 5}.{index}";
builder.Add($"cpe:2.3:a:stellaops:{component}:{version}:*:*:*:*:*:*:*");
}
return builder.MoveToImmutable();
}
private static ImmutableArray<ObservationReference> CreateReferences(string primaryAlias, int count)
{
if (count <= 0)
{
return ImmutableArray<ObservationReference>.Empty;
}
var builder = ImmutableArray.CreateBuilder<ObservationReference>(count);
for (var index = 0; index < count; index++)
{
builder.Add(new ObservationReference(
Type: index % 2 == 0 ? "advisory" : "patch",
Url: $"https://vendor.example/{primaryAlias.ToLowerInvariant()}/ref/{index:D2}"));
}
return builder.MoveToImmutable();
}
private static BsonDocument CreateRawPayload(
string primaryAlias,
string vendorAlias,
IReadOnlyCollection<string> purls,
IReadOnlyCollection<string> cpes,
IReadOnlyCollection<ObservationReference> references)
{
var document = new BsonDocument
{
["id"] = primaryAlias,
["vendorId"] = vendorAlias,
["title"] = $"Simulated advisory {primaryAlias}",
["summary"] = "Synthetic payload produced by Link-Not-Merge benchmark.",
["metrics"] = new BsonArray
{
new BsonDocument
{
["kind"] = "cvss:v3.1",
["score"] = 7.5,
["vector"] = "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N",
},
},
};
if (purls.Count > 0)
{
document["purls"] = new BsonArray(purls);
}
if (cpes.Count > 0)
{
document["cpes"] = new BsonArray(cpes);
}
if (references.Count > 0)
{
document["references"] = new BsonArray(references.Select(reference => new BsonDocument
{
["type"] = reference.Type,
["url"] = reference.Url,
}));
}
return document;
}
private static string ComputeContentHash(BsonDocument rawPayload, string tenant, int group, int revision)
{
using var sha256 = SHA256.Create();
var seed = $"{tenant}|{group}|{revision}";
var rawBytes = rawPayload.ToBson();
var seedBytes = System.Text.Encoding.UTF8.GetBytes(seed);
var combined = new byte[rawBytes.Length + seedBytes.Length];
Buffer.BlockCopy(rawBytes, 0, combined, 0, rawBytes.Length);
Buffer.BlockCopy(seedBytes, 0, combined, rawBytes.Length, seedBytes.Length);
var hash = sha256.ComputeHash(combined);
return $"sha256:{Convert.ToHexString(hash)}";
}
}
internal sealed record ObservationSeed(
string ObservationId,
string Tenant,
string Vendor,
string Stream,
string Api,
string CollectorVersion,
string UpstreamId,
string DocumentVersion,
DateTimeOffset FetchedAt,
DateTimeOffset ReceivedAt,
string ContentHash,
ImmutableArray<string> Aliases,
ImmutableArray<string> Purls,
ImmutableArray<string> Cpes,
ImmutableArray<ObservationReference> References,
string ContentFormat,
string SpecVersion,
BsonDocument RawPayload)
{
public BsonDocument ToBsonDocument()
{
var aliases = new BsonArray(Aliases.Select(alias => alias));
var purls = new BsonArray(Purls.Select(purl => purl));
var cpes = new BsonArray(Cpes.Select(cpe => cpe));
var references = new BsonArray(References.Select(reference => new BsonDocument
{
["type"] = reference.Type,
["url"] = reference.Url,
}));
var document = new BsonDocument
{
["_id"] = ObservationId,
["tenant"] = Tenant,
["source"] = new BsonDocument
{
["vendor"] = Vendor,
["stream"] = Stream,
["api"] = Api,
["collector_version"] = CollectorVersion,
},
["upstream"] = new BsonDocument
{
["upstream_id"] = UpstreamId,
["document_version"] = DocumentVersion,
["fetched_at"] = FetchedAt.UtcDateTime,
["received_at"] = ReceivedAt.UtcDateTime,
["content_hash"] = ContentHash,
["signature"] = new BsonDocument
{
["present"] = false,
["format"] = BsonNull.Value,
["key_id"] = BsonNull.Value,
["signature"] = BsonNull.Value,
},
},
["content"] = new BsonDocument
{
["format"] = ContentFormat,
["spec_version"] = SpecVersion,
["raw"] = RawPayload,
},
["identifiers"] = new BsonDocument
{
["aliases"] = aliases,
["primary"] = UpstreamId,
["cve"] = Aliases.FirstOrDefault(alias => alias.StartsWith("CVE-", StringComparison.Ordinal)) ?? UpstreamId,
},
["linkset"] = new BsonDocument
{
["aliases"] = aliases,
["purls"] = purls,
["cpes"] = cpes,
["references"] = references,
["reconciled_from"] = new BsonArray { "/content/product_tree" },
},
["supersedes"] = BsonNull.Value,
};
return document;
}
}
internal sealed record ObservationReference(string Type, string Url);
using System.Collections.Immutable;
using System.Security.Cryptography;
using System.Text;
namespace StellaOps.Bench.LinkNotMerge;
internal static class ObservationGenerator
{
public static IReadOnlyList<ObservationSeed> Generate(LinkNotMergeScenarioConfig config)
{
ArgumentNullException.ThrowIfNull(config);
var observationCount = config.ResolveObservationCount();
var aliasGroups = config.ResolveAliasGroups();
var purlsPerObservation = config.ResolvePurlsPerObservation();
var cpesPerObservation = config.ResolveCpesPerObservation();
var referencesPerObservation = config.ResolveReferencesPerObservation();
var tenantCount = config.ResolveTenantCount();
var seed = config.ResolveSeed();
var seeds = new ObservationSeed[observationCount];
var random = new Random(seed);
var baseTime = new DateTimeOffset(2025, 10, 1, 0, 0, 0, TimeSpan.Zero);
for (var index = 0; index < observationCount; index++)
{
var tenantIndex = index % tenantCount;
var tenant = $"tenant-{tenantIndex:D2}";
var group = index % aliasGroups;
var revision = index / aliasGroups;
var primaryAlias = $"CVE-2025-{group:D4}";
var vendorAlias = $"VENDOR-{group:D4}";
var thirdAlias = $"GHSA-{group:D4}-{(revision % 26 + 'a')}{(revision % 26 + 'a')}";
var aliases = ImmutableArray.Create(primaryAlias, vendorAlias, thirdAlias);
var observationId = $"{tenant}:advisory:{group:D5}:{revision:D6}";
var upstreamId = primaryAlias;
var documentVersion = baseTime.AddMinutes(revision).ToString("O");
var fetchedAt = baseTime.AddSeconds(index % 1_800);
var receivedAt = fetchedAt.AddSeconds(1);
var purls = CreatePurls(group, revision, purlsPerObservation);
var cpes = CreateCpes(group, revision, cpesPerObservation);
var references = CreateReferences(primaryAlias, referencesPerObservation);
var contentHash = ComputeContentHash(primaryAlias, vendorAlias, purls, cpes, references, tenant, group, revision);
seeds[index] = new ObservationSeed(
ObservationId: observationId,
Tenant: tenant,
Vendor: "concelier-bench",
Stream: "simulated",
Api: $"https://bench.stella/{group:D4}/{revision:D2}",
CollectorVersion: "1.0.0-bench",
UpstreamId: upstreamId,
DocumentVersion: documentVersion,
FetchedAt: fetchedAt,
ReceivedAt: receivedAt,
ContentHash: contentHash,
Aliases: aliases,
Purls: purls,
Cpes: cpes,
References: references,
ContentFormat: "CSAF",
SpecVersion: "2.0");
}
return seeds;
}
private static ImmutableArray<string> CreatePurls(int group, int revision, int count)
{
if (count <= 0)
{
return ImmutableArray<string>.Empty;
}
var builder = ImmutableArray.CreateBuilder<string>(count);
for (var index = 0; index < count; index++)
{
var version = $"{revision % 9 + 1}.{index + 1}.{group % 10}";
builder.Add($"pkg:generic/stella/sample-{group:D4}-{index}@{version}");
}
return builder.MoveToImmutable();
}
private static ImmutableArray<string> CreateCpes(int group, int revision, int count)
{
if (count <= 0)
{
return ImmutableArray<string>.Empty;
}
var builder = ImmutableArray.CreateBuilder<string>(count);
for (var index = 0; index < count; index++)
{
var component = $"benchtool{group % 50:D2}";
var version = $"{revision % 5}.{index}";
builder.Add($"cpe:2.3:a:stellaops:{component}:{version}:*:*:*:*:*:*:*");
}
return builder.MoveToImmutable();
}
private static ImmutableArray<ObservationReference> CreateReferences(string primaryAlias, int count)
{
if (count <= 0)
{
return ImmutableArray<ObservationReference>.Empty;
}
var builder = ImmutableArray.CreateBuilder<ObservationReference>(count);
for (var index = 0; index < count; index++)
{
builder.Add(new ObservationReference(
Type: index % 2 == 0 ? "advisory" : "patch",
Url: $"https://vendor.example/{primaryAlias.ToLowerInvariant()}/ref/{index:D2}"));
}
return builder.MoveToImmutable();
}
private static string ComputeContentHash(
string primaryAlias,
string vendorAlias,
IReadOnlyCollection<string> purls,
IReadOnlyCollection<string> cpes,
IReadOnlyCollection<ObservationReference> references,
string tenant,
int group,
int revision)
{
using var sha256 = SHA256.Create();
var builder = new StringBuilder();
builder.Append(tenant).Append('|').Append(group).Append('|').Append(revision).Append('|');
builder.Append(primaryAlias).Append('|').Append(vendorAlias).Append('|');
foreach (var purl in purls)
{
builder.Append(purl).Append('|');
}
foreach (var cpe in cpes)
{
builder.Append(cpe).Append('|');
}
foreach (var reference in references)
{
builder.Append(reference.Type).Append(':').Append(reference.Url).Append('|');
}
var data = Encoding.UTF8.GetBytes(builder.ToString());
var hash = sha256.ComputeHash(data);
return $"sha256:{Convert.ToHexString(hash)}";
}
}
internal sealed record ObservationSeed(
string ObservationId,
string Tenant,
string Vendor,
string Stream,
string Api,
string CollectorVersion,
string UpstreamId,
string DocumentVersion,
DateTimeOffset FetchedAt,
DateTimeOffset ReceivedAt,
string ContentHash,
ImmutableArray<string> Aliases,
ImmutableArray<string> Purls,
ImmutableArray<string> Cpes,
ImmutableArray<ObservationReference> References,
string ContentFormat,
string SpecVersion)
{
public ObservationDocument ToDocument()
{
return new ObservationDocument(
Tenant,
new LinksetDocument(
Aliases,
Purls,
Cpes,
References));
}
}
internal sealed record ObservationDocument(string Tenant, LinksetDocument Linkset);
internal sealed record LinksetDocument(
ImmutableArray<string> Aliases,
ImmutableArray<string> Purls,
ImmutableArray<string> Cpes,
ImmutableArray<ObservationReference> References);
internal sealed record ObservationReference(string Type, string Url);

View File

@@ -9,8 +9,6 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="MongoDB.Driver" Version="3.5.0" />
<PackageReference Include="EphemeralMongo" Version="3.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0" />
</ItemGroup>
</Project>