up
Some checks failed
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Manifest Integrity / Validate Schema Integrity (push) Has been cancelled
Manifest Integrity / Validate Contract Documents (push) Has been cancelled
Manifest Integrity / Validate Pack Fixtures (push) Has been cancelled
Manifest Integrity / Audit SHA256SUMS Files (push) Has been cancelled
Manifest Integrity / Verify Merkle Roots (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled

This commit is contained in:
StellaOps Bot
2025-12-12 09:35:37 +02:00
parent ce5ec9c158
commit efaf3cb789
238 changed files with 146274 additions and 5767 deletions

View File

@@ -27,8 +27,8 @@
- Offline posture: no external calls beyond allowlisted feeds; prefer cached schemas and local nugets in `local-nugets/`.
## Data & Environment
- Canonical store: MongoDB (>=3.0 driver). Tests use `STELLAOPS_TEST_MONGO_URI`; fallback `mongodb://127.0.0.1:27017`, then Mongo2Go.
- Collections: `graph_nodes`, `graph_edges`, `graph_overlays_cache`, `graph_snapshots`, `graph_saved_queries`.
- Storage is currently in-memory (MongoDB dependency removed); persistent backing store to be added in a follow-up sprint.
- Collections (historical naming) `graph_nodes`, `graph_edges`, `graph_overlays_cache`, `graph_snapshots`, `graph_saved_queries` map to in-memory structures for now.
- Tenant isolation mandatory on every query and export.
## Testing Expectations

View File

@@ -23,7 +23,7 @@ Provide tenant-scoped Graph Explorer APIs for search, query, paths, diffs, overl
## Tooling
- .NET 10 preview Minimal API with async streaming; pipeline pattern for parsing/planning/fetching.
- Mongo aggregation / adjacency store from Graph Indexer; optional caching layer.
- Graph Indexer currently exposes in-memory adjacency storage (Mongo removed); optional caching layer.
- SSE/WebSockets or chunked NDJSON responses for progressive loading.
## Definition of Done

View File

@@ -5,7 +5,7 @@ Project SBOM, advisory, VEX, and policy overlay data into a tenant-scoped proper
## Scope
- Service source under `src/Graph/StellaOps.Graph.Indexer` (workers, ingestion pipelines, schema builders).
- Mongo collections/object storage for `graph_nodes`, `graph_edges`, `graph_snapshots`, clustering metadata.
- In-memory graph storage for `graph_nodes`, `graph_edges`, `graph_snapshots`, clustering metadata (Mongo removed; durable store to follow).
- Event consumers: SBOM ingest, Conseiller advisories, Excitor VEX, Policy overlay materials.
- Incremental rebuild, diff, and cache warmers for graph overlays.
@@ -23,9 +23,9 @@ Project SBOM, advisory, VEX, and policy overlay data into a tenant-scoped proper
## Tooling
- .NET 10 preview workers (HostedService + channel pipelines).
- MongoDB for node/edge storage; S3-compatible buckets for layout tiles/snapshots if needed.
- In-memory node/edge storage (Mongo removed); S3-compatible buckets for layout tiles/snapshots if needed.
- Scheduler integration (jobs, change streams) to handle incremental updates.
- Analytics: clustering/centrality pipelines with Mongo-backed snapshot provider and overlays; change-stream/backfill worker with idempotency store (Mongo or in-memory) and retry/backoff.
- Analytics: clustering/centrality pipelines with in-memory snapshot provider and overlays; change-stream/backfill worker with in-memory idempotency store and retry/backoff.
## Definition of Done
- Pipelines deterministic and tested; fixtures validated.

View File

@@ -2,7 +2,6 @@ using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using MongoDB.Driver;
namespace StellaOps.Graph.Indexer.Analytics;
@@ -37,47 +36,4 @@ public static class GraphAnalyticsServiceCollectionExtensions
return services;
}
public static IServiceCollection AddGraphAnalyticsMongo(
this IServiceCollection services,
Action<GraphAnalyticsOptions>? configureOptions = null,
Action<MongoGraphSnapshotProviderOptions>? configureSnapshot = null,
Action<GraphAnalyticsWriterOptions>? configureWriter = null)
{
services.AddGraphAnalyticsPipeline(configureOptions);
if (configureSnapshot is not null)
{
services.Configure(configureSnapshot);
}
else
{
services.Configure<MongoGraphSnapshotProviderOptions>(_ => { });
}
if (configureWriter is not null)
{
services.Configure(configureWriter);
}
else
{
services.Configure<GraphAnalyticsWriterOptions>(_ => { });
}
services.Replace(ServiceDescriptor.Singleton<IGraphSnapshotProvider>(sp =>
{
var db = sp.GetRequiredService<IMongoDatabase>();
var options = sp.GetRequiredService<IOptions<MongoGraphSnapshotProviderOptions>>();
return new MongoGraphSnapshotProvider(db, options.Value);
}));
services.Replace(ServiceDescriptor.Singleton<IGraphAnalyticsWriter>(sp =>
{
var db = sp.GetRequiredService<IMongoDatabase>();
var options = sp.GetRequiredService<IOptions<GraphAnalyticsWriterOptions>>();
return new MongoGraphAnalyticsWriter(db, options.Value);
}));
return services;
}
}

View File

@@ -1,117 +0,0 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Globalization;
using System.Linq;
using System.Text.Json.Nodes;
using MongoDB.Bson;
using MongoDB.Driver;
namespace StellaOps.Graph.Indexer.Analytics;
public sealed class MongoGraphAnalyticsWriter : IGraphAnalyticsWriter
{
private readonly IMongoCollection<BsonDocument> _clusters;
private readonly IMongoCollection<BsonDocument> _centrality;
private readonly IMongoCollection<BsonDocument> _nodes;
private readonly GraphAnalyticsWriterOptions _options;
public MongoGraphAnalyticsWriter(IMongoDatabase database, GraphAnalyticsWriterOptions? options = null)
{
ArgumentNullException.ThrowIfNull(database);
_options = options ?? new GraphAnalyticsWriterOptions();
_clusters = database.GetCollection<BsonDocument>(_options.ClusterCollectionName);
_centrality = database.GetCollection<BsonDocument>(_options.CentralityCollectionName);
_nodes = database.GetCollection<BsonDocument>(_options.NodeCollectionName);
}
public async Task PersistClusterAssignmentsAsync(GraphAnalyticsSnapshot snapshot, ImmutableArray<ClusterAssignment> assignments, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
if (assignments.Length == 0)
{
return;
}
var models = new List<WriteModel<BsonDocument>>(assignments.Length);
foreach (var assignment in assignments)
{
var filter = Builders<BsonDocument>.Filter.And(
Builders<BsonDocument>.Filter.Eq("tenant", snapshot.Tenant),
Builders<BsonDocument>.Filter.Eq("snapshot_id", snapshot.SnapshotId),
Builders<BsonDocument>.Filter.Eq("node_id", assignment.NodeId));
var document = new BsonDocument
{
{ "tenant", snapshot.Tenant },
{ "snapshot_id", snapshot.SnapshotId },
{ "node_id", assignment.NodeId },
{ "cluster_id", assignment.ClusterId },
{ "kind", assignment.Kind },
{ "generated_at", snapshot.GeneratedAt.UtcDateTime }
};
models.Add(new ReplaceOneModel<BsonDocument>(filter, document) { IsUpsert = true });
}
await _clusters.BulkWriteAsync(models, new BulkWriteOptions { IsOrdered = false }, cancellationToken).ConfigureAwait(false);
if (_options.WriteClusterAssignmentsToNodes)
{
await WriteClustersToNodesAsync(assignments, cancellationToken).ConfigureAwait(false);
}
}
public async Task PersistCentralityAsync(GraphAnalyticsSnapshot snapshot, ImmutableArray<CentralityScore> scores, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
if (scores.Length == 0)
{
return;
}
var models = new List<WriteModel<BsonDocument>>(scores.Length);
foreach (var score in scores)
{
var filter = Builders<BsonDocument>.Filter.And(
Builders<BsonDocument>.Filter.Eq("tenant", snapshot.Tenant),
Builders<BsonDocument>.Filter.Eq("snapshot_id", snapshot.SnapshotId),
Builders<BsonDocument>.Filter.Eq("node_id", score.NodeId));
var document = new BsonDocument
{
{ "tenant", snapshot.Tenant },
{ "snapshot_id", snapshot.SnapshotId },
{ "node_id", score.NodeId },
{ "kind", score.Kind },
{ "degree", score.Degree },
{ "betweenness", score.Betweenness },
{ "generated_at", snapshot.GeneratedAt.UtcDateTime }
};
models.Add(new ReplaceOneModel<BsonDocument>(filter, document) { IsUpsert = true });
}
await _centrality.BulkWriteAsync(models, new BulkWriteOptions { IsOrdered = false }, cancellationToken).ConfigureAwait(false);
}
private async Task WriteClustersToNodesAsync(IEnumerable<ClusterAssignment> assignments, CancellationToken cancellationToken)
{
var models = new List<WriteModel<BsonDocument>>();
foreach (var assignment in assignments)
{
var filter = Builders<BsonDocument>.Filter.Eq("id", assignment.NodeId);
var update = Builders<BsonDocument>.Update.Set("attributes.cluster_id", assignment.ClusterId);
models.Add(new UpdateOneModel<BsonDocument>(filter, update) { IsUpsert = false });
}
if (models.Count == 0)
{
return;
}
await _nodes.BulkWriteAsync(models, new BulkWriteOptions { IsOrdered = false }, cancellationToken).ConfigureAwait(false);
}
}

View File

@@ -1,79 +0,0 @@
using System.Collections.Immutable;
using System.Text.Json.Nodes;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Graph.Indexer.Infrastructure;
namespace StellaOps.Graph.Indexer.Analytics;
public sealed class MongoGraphSnapshotProvider : IGraphSnapshotProvider
{
private readonly IMongoCollection<BsonDocument> _snapshots;
private readonly IMongoCollection<BsonDocument> _progress;
private readonly MongoGraphSnapshotProviderOptions _options;
public MongoGraphSnapshotProvider(IMongoDatabase database, MongoGraphSnapshotProviderOptions? options = null)
{
ArgumentNullException.ThrowIfNull(database);
_options = options ?? new MongoGraphSnapshotProviderOptions();
_snapshots = database.GetCollection<BsonDocument>(_options.SnapshotCollectionName);
_progress = database.GetCollection<BsonDocument>(_options.ProgressCollectionName);
}
public async Task<IReadOnlyList<GraphAnalyticsSnapshot>> GetPendingSnapshotsAsync(CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var processedIds = await _progress
.Find(FilterDefinition<BsonDocument>.Empty)
.Project(doc => doc["snapshot_id"].AsString)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
var filter = Builders<BsonDocument>.Filter.Nin("snapshot_id", processedIds);
var snapshots = await _snapshots
.Find(filter)
.Limit(_options.MaxBatch)
.Sort(Builders<BsonDocument>.Sort.Descending("generated_at"))
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
var result = new List<GraphAnalyticsSnapshot>(snapshots.Count);
foreach (var snapshot in snapshots)
{
var tenant = snapshot.GetValue("tenant", string.Empty).AsString;
var snapshotId = snapshot.GetValue("snapshot_id", string.Empty).AsString;
var generatedAt = snapshot.TryGetValue("generated_at", out var generated)
&& generated.BsonType == BsonType.DateTime
? DateTime.SpecifyKind(generated.ToUniversalTime(), DateTimeKind.Utc)
: DateTimeOffset.UtcNow;
var nodes = snapshot.TryGetValue("nodes", out var nodesValue) && nodesValue is BsonArray nodesArray
? BsonJsonConverter.ToJsonArray(nodesArray).Select(n => (JsonObject)n!).ToImmutableArray()
: ImmutableArray<JsonObject>.Empty;
var edges = snapshot.TryGetValue("edges", out var edgesValue) && edgesValue is BsonArray edgesArray
? BsonJsonConverter.ToJsonArray(edgesArray).Select(n => (JsonObject)n!).ToImmutableArray()
: ImmutableArray<JsonObject>.Empty;
result.Add(new GraphAnalyticsSnapshot(tenant, snapshotId, generatedAt, nodes, edges));
}
return result;
}
public async Task MarkProcessedAsync(string tenant, string snapshotId, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var filter = Builders<BsonDocument>.Filter.Eq("snapshot_id", snapshotId)
& Builders<BsonDocument>.Filter.Eq("tenant", tenant);
var update = Builders<BsonDocument>.Update.Set("snapshot_id", snapshotId)
.Set("tenant", tenant)
.SetOnInsert("processed_at", DateTimeOffset.UtcNow.UtcDateTime);
await _progress.UpdateOneAsync(filter, update, new UpdateOptions { IsUpsert = true }, cancellationToken)
.ConfigureAwait(false);
}
}

View File

@@ -1,8 +0,0 @@
namespace StellaOps.Graph.Indexer.Analytics;
public sealed class MongoGraphSnapshotProviderOptions
{
public string SnapshotCollectionName { get; set; } = "graph_snapshots";
public string ProgressCollectionName { get; set; } = "graph_analytics_progress";
public int MaxBatch { get; set; } = 5;
}

View File

@@ -1,8 +1,7 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using MongoDB.Driver;
using StellaOps.Graph.Indexer.Ingestion.Sbom;
namespace StellaOps.Graph.Indexer.Incremental;
@@ -23,58 +22,12 @@ public static class GraphChangeStreamServiceCollectionExtensions
services.Configure<GraphChangeStreamOptions>(_ => { });
}
services.TryAddSingleton<IGraphChangeEventSource, NoOpGraphChangeEventSource>();
services.TryAddSingleton<IGraphBackfillSource, NoOpGraphChangeEventSource>();
services.TryAddSingleton<IIdempotencyStore, InMemoryIdempotencyStore>();
services.TryAddSingleton<IGraphDocumentWriter, InMemoryGraphDocumentWriter>();
services.AddSingleton<GraphBackfillMetrics>();
services.AddHostedService<GraphChangeStreamProcessor>();
return services;
}
public static IServiceCollection AddGraphChangeStreamProcessorWithMongo(
this IServiceCollection services,
Action<GraphChangeStreamOptions>? configureOptions = null,
Action<MongoGraphChangeEventOptions>? configureChangeOptions = null,
Action<MongoIdempotencyStoreOptions>? configureIdempotency = null)
{
services.AddGraphChangeStreamProcessor(configureOptions);
if (configureChangeOptions is not null)
{
services.Configure(configureChangeOptions);
}
else
{
services.Configure<MongoGraphChangeEventOptions>(_ => { });
}
if (configureIdempotency is not null)
{
services.Configure(configureIdempotency);
}
else
{
services.Configure<MongoIdempotencyStoreOptions>(_ => { });
}
services.Replace(ServiceDescriptor.Singleton<IGraphChangeEventSource>(sp =>
{
var db = sp.GetRequiredService<IMongoDatabase>();
var opts = sp.GetRequiredService<IOptions<MongoGraphChangeEventOptions>>();
return new MongoGraphChangeEventSource(db, opts.Value);
}));
services.Replace(ServiceDescriptor.Singleton<IGraphBackfillSource>(sp =>
{
var db = sp.GetRequiredService<IMongoDatabase>();
var opts = sp.GetRequiredService<IOptions<MongoGraphChangeEventOptions>>();
return new MongoGraphChangeEventSource(db, opts.Value);
}));
services.Replace(ServiceDescriptor.Singleton<IIdempotencyStore>(sp =>
{
var db = sp.GetRequiredService<IMongoDatabase>();
var opts = sp.GetRequiredService<IOptions<MongoIdempotencyStoreOptions>>();
return new MongoIdempotencyStore(db, opts.Value);
}));
return services;
}
}

View File

@@ -1,10 +0,0 @@
namespace StellaOps.Graph.Indexer.Incremental;
public sealed class MongoGraphChangeEventOptions
{
public string CollectionName { get; set; } = "graph_change_events";
public string SequenceFieldName { get; set; } = "sequence_token";
public string NodeArrayFieldName { get; set; } = "nodes";
public string EdgeArrayFieldName { get; set; } = "edges";
public int MaxBatchSize { get; set; } = 256;
}

View File

@@ -1,72 +0,0 @@
using System.Collections.Immutable;
using System.Text.Json.Nodes;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Graph.Indexer.Infrastructure;
namespace StellaOps.Graph.Indexer.Incremental;
public sealed class MongoGraphChangeEventSource : IGraphChangeEventSource, IGraphBackfillSource
{
private readonly IMongoCollection<BsonDocument> _collection;
private readonly MongoGraphChangeEventOptions _options;
public MongoGraphChangeEventSource(IMongoDatabase database, MongoGraphChangeEventOptions? options = null)
{
ArgumentNullException.ThrowIfNull(database);
_options = options ?? new MongoGraphChangeEventOptions();
_collection = database.GetCollection<BsonDocument>(_options.CollectionName);
}
public async IAsyncEnumerable<GraphChangeEvent> ReadAsync([System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
var filter = Builders<BsonDocument>.Filter.Eq("is_backfill", false);
await foreach (var change in EnumerateAsync(filter, cancellationToken))
{
yield return change with { IsBackfill = false };
}
}
public async IAsyncEnumerable<GraphChangeEvent> ReadBackfillAsync([System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
var filter = Builders<BsonDocument>.Filter.Eq("is_backfill", true);
await foreach (var change in EnumerateAsync(filter, cancellationToken))
{
yield return change with { IsBackfill = true };
}
}
private async IAsyncEnumerable<GraphChangeEvent> EnumerateAsync(FilterDefinition<BsonDocument> filter, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
var sort = Builders<BsonDocument>.Sort.Ascending(_options.SequenceFieldName);
using var cursor = await _collection.FindAsync(filter, new FindOptions<BsonDocument> { Sort = sort, BatchSize = _options.MaxBatchSize }, cancellationToken).ConfigureAwait(false);
while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false))
{
foreach (var doc in cursor.Current)
{
cancellationToken.ThrowIfCancellationRequested();
var tenant = doc.GetValue("tenant", string.Empty).AsString;
var snapshotId = doc.GetValue("snapshot_id", string.Empty).AsString;
var sequence = doc.GetValue(_options.SequenceFieldName, string.Empty).AsString;
var nodes = doc.TryGetValue(_options.NodeArrayFieldName, out var nodesValue) && nodesValue is BsonArray nodeArray
? BsonJsonConverter.ToJsonArray(nodeArray).Select(n => (JsonObject)n!).ToImmutableArray()
: ImmutableArray<JsonObject>.Empty;
var edges = doc.TryGetValue(_options.EdgeArrayFieldName, out var edgesValue) && edgesValue is BsonArray edgeArray
? BsonJsonConverter.ToJsonArray(edgeArray).Select(n => (JsonObject)n!).ToImmutableArray()
: ImmutableArray<JsonObject>.Empty;
yield return new GraphChangeEvent(
tenant,
snapshotId,
sequence,
nodes,
edges,
doc.GetValue("is_backfill", false).ToBoolean());
}
}
}
}

View File

@@ -1,34 +0,0 @@
using MongoDB.Bson;
using MongoDB.Driver;
namespace StellaOps.Graph.Indexer.Incremental;
public sealed class MongoIdempotencyStore : IIdempotencyStore
{
private readonly IMongoCollection<BsonDocument> _collection;
public MongoIdempotencyStore(IMongoDatabase database, MongoIdempotencyStoreOptions? options = null)
{
ArgumentNullException.ThrowIfNull(database);
var resolved = options ?? new MongoIdempotencyStoreOptions();
_collection = database.GetCollection<BsonDocument>(resolved.CollectionName);
}
public async Task<bool> HasSeenAsync(string sequenceToken, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var filter = Builders<BsonDocument>.Filter.Eq("sequence_token", sequenceToken);
return await _collection.Find(filter).AnyAsync(cancellationToken).ConfigureAwait(false);
}
public async Task MarkSeenAsync(string sequenceToken, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var filter = Builders<BsonDocument>.Filter.Eq("sequence_token", sequenceToken);
var update = Builders<BsonDocument>.Update.Set("sequence_token", sequenceToken)
.SetOnInsert("recorded_at", DateTimeOffset.UtcNow.UtcDateTime);
await _collection.UpdateOneAsync(filter, update, new UpdateOptions { IsUpsert = true }, cancellationToken)
.ConfigureAwait(false);
}
}

View File

@@ -1,6 +0,0 @@
namespace StellaOps.Graph.Indexer.Incremental;
public sealed class MongoIdempotencyStoreOptions
{
public string CollectionName { get; set; } = "graph_change_idempotency";
}

View File

@@ -0,0 +1,23 @@
using System.Collections.Immutable;
using System.Runtime.CompilerServices;
using System.Text.Json.Nodes;
namespace StellaOps.Graph.Indexer.Incremental;
/// <summary>
/// No-op change/backfill source used when no external change feed is configured.
/// </summary>
public sealed class NoOpGraphChangeEventSource : IGraphChangeEventSource, IGraphBackfillSource
{
public IAsyncEnumerable<GraphChangeEvent> ReadAsync(CancellationToken cancellationToken) =>
ReadInternalAsync(cancellationToken);
public IAsyncEnumerable<GraphChangeEvent> ReadBackfillAsync(CancellationToken cancellationToken) =>
ReadInternalAsync(cancellationToken);
private static async IAsyncEnumerable<GraphChangeEvent> ReadInternalAsync([EnumeratorCancellation] CancellationToken cancellationToken)
{
await Task.CompletedTask.ConfigureAwait(false);
yield break;
}
}

View File

@@ -1,21 +0,0 @@
using System.Text.Json.Nodes;
using MongoDB.Bson;
namespace StellaOps.Graph.Indexer.Infrastructure;
internal static class BsonJsonConverter
{
public static JsonObject ToJsonObject(BsonDocument document)
{
ArgumentNullException.ThrowIfNull(document);
var parsed = JsonNode.Parse(document.ToJson());
return parsed as JsonObject ?? new JsonObject();
}
public static JsonArray ToJsonArray(BsonArray array)
{
ArgumentNullException.ThrowIfNull(array);
var parsed = JsonNode.Parse(array.ToJson());
return parsed as JsonArray ?? new JsonArray();
}
}

View File

@@ -1,7 +0,0 @@
namespace StellaOps.Graph.Indexer.Infrastructure;
public sealed class MongoDatabaseOptions
{
public string ConnectionString { get; set; } = string.Empty;
public string DatabaseName { get; set; } = "stellaops-graph";
}

View File

@@ -1,48 +0,0 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using MongoDB.Driver;
namespace StellaOps.Graph.Indexer.Infrastructure;
public static class MongoServiceCollectionExtensions
{
public static IServiceCollection AddGraphMongoDatabase(
this IServiceCollection services,
Action<MongoDatabaseOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.Configure(configure);
services.AddSingleton<IMongoClient>(sp =>
{
var opts = sp.GetRequiredService<IOptions<MongoDatabaseOptions>>().Value;
Validate(opts);
return new MongoClient(opts.ConnectionString);
});
services.AddSingleton<IMongoDatabase>(sp =>
{
var opts = sp.GetRequiredService<IOptions<MongoDatabaseOptions>>().Value;
Validate(opts);
return sp.GetRequiredService<IMongoClient>().GetDatabase(opts.DatabaseName);
});
return services;
}
private static void Validate(MongoDatabaseOptions options)
{
if (string.IsNullOrWhiteSpace(options.ConnectionString))
{
throw new InvalidOperationException("Mongo connection string must be provided.");
}
if (string.IsNullOrWhiteSpace(options.DatabaseName))
{
throw new InvalidOperationException("Mongo database name must be provided.");
}
}
}

View File

@@ -1,84 +0,0 @@
using System;
using System.Collections.Generic;
using System.Text.Encodings.Web;
using System.Text.Json;
using System.Text.Json.Nodes;
using StellaOps.Graph.Indexer.Ingestion.Sbom;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Bson;
using MongoDB.Driver;
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
public sealed class MongoGraphDocumentWriter : IGraphDocumentWriter
{
private static readonly JsonSerializerOptions SerializerOptions = new()
{
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
WriteIndented = false
};
private readonly IMongoCollection<BsonDocument> _nodes;
private readonly IMongoCollection<BsonDocument> _edges;
public MongoGraphDocumentWriter(IMongoDatabase database, MongoGraphDocumentWriterOptions? options = null)
{
ArgumentNullException.ThrowIfNull(database);
var resolved = options ?? new MongoGraphDocumentWriterOptions();
_nodes = database.GetCollection<BsonDocument>(resolved.NodeCollectionName);
_edges = database.GetCollection<BsonDocument>(resolved.EdgeCollectionName);
}
public async Task WriteAsync(GraphBuildBatch batch, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(batch);
cancellationToken.ThrowIfCancellationRequested();
if (batch.Nodes.Length > 0)
{
var nodeModels = CreateReplaceModels(_nodes, batch.Nodes);
if (nodeModels.Count > 0)
{
await _nodes.BulkWriteAsync(nodeModels, new BulkWriteOptions { IsOrdered = false }, cancellationToken)
.ConfigureAwait(false);
}
}
if (batch.Edges.Length > 0)
{
var edgeModels = CreateReplaceModels(_edges, batch.Edges);
if (edgeModels.Count > 0)
{
await _edges.BulkWriteAsync(edgeModels, new BulkWriteOptions { IsOrdered = false }, cancellationToken)
.ConfigureAwait(false);
}
}
}
private static List<WriteModel<BsonDocument>> CreateReplaceModels(IMongoCollection<BsonDocument> collection, IReadOnlyList<JsonObject> documents)
{
var models = new List<WriteModel<BsonDocument>>(documents.Count);
foreach (var document in documents)
{
if (!document.TryGetPropertyValue("id", out var idNode) || idNode is null)
{
continue;
}
var id = idNode.GetValue<string>();
var filter = Builders<BsonDocument>.Filter.Eq("id", id);
var bsonDocument = ToBsonDocument(document);
models.Add(new ReplaceOneModel<BsonDocument>(filter, bsonDocument) { IsUpsert = true });
}
return models;
}
private static BsonDocument ToBsonDocument(JsonObject json)
{
var jsonString = json.ToJsonString(SerializerOptions);
return BsonDocument.Parse(jsonString);
}
}

View File

@@ -1,7 +0,0 @@
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
public sealed class MongoGraphDocumentWriterOptions
{
public string NodeCollectionName { get; init; } = "graph_nodes";
public string EdgeCollectionName { get; init; } = "graph_edges";
}

View File

@@ -11,6 +11,7 @@ public static class InspectorIngestServiceCollectionExtensions
{
ArgumentNullException.ThrowIfNull(services);
services.TryAddSingleton<Ingestion.Sbom.IGraphDocumentWriter, Ingestion.Sbom.InMemoryGraphDocumentWriter>();
services.TryAddSingleton<GraphInspectorTransformer>();
services.TryAddSingleton<GraphInspectorProcessor>(provider =>
{

View File

@@ -0,0 +1,30 @@
using System.Collections.Concurrent;
using System.Collections.Immutable;
using System.Text.Json.Nodes;
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
/// <summary>
/// In-memory graph document writer used as a Mongo-free fallback.
/// </summary>
public sealed class InMemoryGraphDocumentWriter : IGraphDocumentWriter
{
private readonly ConcurrentBag<GraphBuildBatch> _batches = new();
public IReadOnlyCollection<GraphBuildBatch> Batches => _batches.ToArray();
public Task WriteAsync(GraphBuildBatch batch, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(batch);
_batches.Add(CloneBatch(batch));
return Task.CompletedTask;
}
private static GraphBuildBatch CloneBatch(GraphBuildBatch source)
{
static JsonObject CloneNode(JsonObject node) => (JsonObject)node.DeepClone();
return new GraphBuildBatch(
ImmutableArray.CreateRange(source.Nodes.Select(CloneNode)),
ImmutableArray.CreateRange(source.Edges.Select(CloneNode)));
}
}

View File

@@ -22,6 +22,7 @@ public static class SbomIngestServiceCollectionExtensions
services.Configure(configure);
}
services.TryAddSingleton<IGraphDocumentWriter, InMemoryGraphDocumentWriter>();
services.TryAddSingleton<SbomIngestTransformer>();
services.TryAddSingleton<ISbomIngestMetrics, SbomIngestMetrics>();

View File

@@ -13,8 +13,6 @@
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="10.0.0" />
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0" />
<PackageReference Include="MongoDB.Driver" Version="3.5.0" />
<PackageReference Include="MongoDB.Bson" Version="3.5.0" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="10.0.0" />
</ItemGroup>
</Project>

View File

@@ -1,125 +0,0 @@
using System.Collections.Immutable;
using System.Text.Json.Nodes;
using Mongo2Go;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Graph.Indexer.Analytics;
using StellaOps.Graph.Indexer.Incremental;
namespace StellaOps.Graph.Indexer.Tests;
public sealed class MongoProviderIntegrationTests : IAsyncLifetime
{
private readonly MongoDbRunner _runner;
private IMongoDatabase _database = default!;
public MongoProviderIntegrationTests()
{
_runner = MongoDbRunner.Start(singleNodeReplSet: true);
}
public Task InitializeAsync()
{
var client = new MongoClient(_runner.ConnectionString);
_database = client.GetDatabase("graph-indexer-tests");
return Task.CompletedTask;
}
public Task DisposeAsync()
{
_runner.Dispose();
return Task.CompletedTask;
}
[Fact]
public async Task SnapshotProvider_ReadsPendingSnapshots()
{
var snapshots = _database.GetCollection<BsonDocument>("graph_snapshots");
var nodes = new BsonArray
{
new BsonDocument
{
{ "id", "gn:tenant-a:component:1" },
{ "kind", "component" },
{ "attributes", new BsonDocument { { "purl", "pkg:npm/a@1.0.0" } } }
}
};
var edges = new BsonArray();
await snapshots.InsertOneAsync(new BsonDocument
{
{ "tenant", "tenant-a" },
{ "snapshot_id", "snap-1" },
{ "generated_at", DateTime.UtcNow },
{ "nodes", nodes },
{ "edges", edges }
});
var provider = new MongoGraphSnapshotProvider(_database);
var pending = await provider.GetPendingSnapshotsAsync(CancellationToken.None);
Assert.Single(pending);
Assert.Equal("snap-1", pending[0].SnapshotId);
Assert.Single(pending[0].Nodes);
await provider.MarkProcessedAsync("tenant-a", "snap-1", CancellationToken.None);
var none = await provider.GetPendingSnapshotsAsync(CancellationToken.None);
Assert.Empty(none);
}
[Fact]
public async Task ChangeEventSource_EnumeratesAndHonorsIdempotency()
{
var changes = _database.GetCollection<BsonDocument>("graph_change_events");
await changes.InsertManyAsync(new[]
{
new BsonDocument
{
{ "tenant", "tenant-a" },
{ "snapshot_id", "snap-1" },
{ "sequence_token", "seq-1" },
{ "is_backfill", false },
{ "nodes", new BsonArray { new BsonDocument { { "id", "gn:1" }, { "kind", "component" } } } },
{ "edges", new BsonArray() }
},
new BsonDocument
{
{ "tenant", "tenant-a" },
{ "snapshot_id", "snap-1" },
{ "sequence_token", "seq-2" },
{ "is_backfill", false },
{ "nodes", new BsonArray { new BsonDocument { { "id", "gn:2" }, { "kind", "component" } } } },
{ "edges", new BsonArray() }
}
});
var source = new MongoGraphChangeEventSource(_database);
var idempotency = new MongoIdempotencyStore(_database);
var events = new List<GraphChangeEvent>();
await foreach (var change in source.ReadAsync(CancellationToken.None))
{
if (await idempotency.HasSeenAsync(change.SequenceToken, CancellationToken.None))
{
continue;
}
events.Add(change);
await idempotency.MarkSeenAsync(change.SequenceToken, CancellationToken.None);
}
Assert.Equal(2, events.Count);
var secondPass = new List<GraphChangeEvent>();
await foreach (var change in source.ReadAsync(CancellationToken.None))
{
if (!await idempotency.HasSeenAsync(change.SequenceToken, CancellationToken.None))
{
secondPass.Add(change);
}
}
Assert.Empty(secondPass);
}
}

View File

@@ -1,44 +0,0 @@
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Graph.Indexer.Infrastructure;
using Mongo2Go;
using MongoDB.Driver;
namespace StellaOps.Graph.Indexer.Tests;
public sealed class MongoServiceCollectionExtensionsTests : IAsyncLifetime
{
private MongoDbRunner _runner = default!;
public Task InitializeAsync()
{
_runner = MongoDbRunner.Start(singleNodeReplSet: true);
return Task.CompletedTask;
}
public Task DisposeAsync()
{
_runner.Dispose();
return Task.CompletedTask;
}
[Fact]
public void AddGraphMongoDatabase_RegistersClientAndDatabase()
{
var services = new ServiceCollection();
services.AddGraphMongoDatabase(options =>
{
options.ConnectionString = _runner.ConnectionString;
options.DatabaseName = "graph-indexer-ext-tests";
});
var provider = services.BuildServiceProvider();
var client = provider.GetService<IMongoClient>();
var database = provider.GetService<IMongoDatabase>();
Assert.NotNull(client);
Assert.NotNull(database);
Assert.Equal("graph-indexer-ext-tests", database!.DatabaseNamespace.DatabaseName);
}
}

View File

@@ -12,6 +12,5 @@
<PackageReference Include="xunit" Version="2.9.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
<PackageReference Include="Mongo2Go" Version="3.1.3" />
</ItemGroup>
</Project>