up
Some checks failed
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Manifest Integrity / Validate Schema Integrity (push) Has been cancelled
Manifest Integrity / Validate Contract Documents (push) Has been cancelled
Manifest Integrity / Validate Pack Fixtures (push) Has been cancelled
Manifest Integrity / Audit SHA256SUMS Files (push) Has been cancelled
Manifest Integrity / Verify Merkle Roots (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Manifest Integrity / Validate Schema Integrity (push) Has been cancelled
Manifest Integrity / Validate Contract Documents (push) Has been cancelled
Manifest Integrity / Validate Pack Fixtures (push) Has been cancelled
Manifest Integrity / Audit SHA256SUMS Files (push) Has been cancelled
Manifest Integrity / Verify Merkle Roots (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
This commit is contained in:
@@ -27,8 +27,8 @@
|
||||
- Offline posture: no external calls beyond allowlisted feeds; prefer cached schemas and local nugets in `local-nugets/`.
|
||||
|
||||
## Data & Environment
|
||||
- Canonical store: MongoDB (>=3.0 driver). Tests use `STELLAOPS_TEST_MONGO_URI`; fallback `mongodb://127.0.0.1:27017`, then Mongo2Go.
|
||||
- Collections: `graph_nodes`, `graph_edges`, `graph_overlays_cache`, `graph_snapshots`, `graph_saved_queries`.
|
||||
- Storage is currently in-memory (MongoDB dependency removed); persistent backing store to be added in a follow-up sprint.
|
||||
- Collections (historical naming) `graph_nodes`, `graph_edges`, `graph_overlays_cache`, `graph_snapshots`, `graph_saved_queries` map to in-memory structures for now.
|
||||
- Tenant isolation mandatory on every query and export.
|
||||
|
||||
## Testing Expectations
|
||||
|
||||
@@ -23,7 +23,7 @@ Provide tenant-scoped Graph Explorer APIs for search, query, paths, diffs, overl
|
||||
|
||||
## Tooling
|
||||
- .NET 10 preview Minimal API with async streaming; pipeline pattern for parsing/planning/fetching.
|
||||
- Mongo aggregation / adjacency store from Graph Indexer; optional caching layer.
|
||||
- Graph Indexer currently exposes in-memory adjacency storage (Mongo removed); optional caching layer.
|
||||
- SSE/WebSockets or chunked NDJSON responses for progressive loading.
|
||||
|
||||
## Definition of Done
|
||||
|
||||
@@ -5,7 +5,7 @@ Project SBOM, advisory, VEX, and policy overlay data into a tenant-scoped proper
|
||||
|
||||
## Scope
|
||||
- Service source under `src/Graph/StellaOps.Graph.Indexer` (workers, ingestion pipelines, schema builders).
|
||||
- Mongo collections/object storage for `graph_nodes`, `graph_edges`, `graph_snapshots`, clustering metadata.
|
||||
- In-memory graph storage for `graph_nodes`, `graph_edges`, `graph_snapshots`, clustering metadata (Mongo removed; durable store to follow).
|
||||
- Event consumers: SBOM ingest, Conseiller advisories, Excitor VEX, Policy overlay materials.
|
||||
- Incremental rebuild, diff, and cache warmers for graph overlays.
|
||||
|
||||
@@ -23,9 +23,9 @@ Project SBOM, advisory, VEX, and policy overlay data into a tenant-scoped proper
|
||||
|
||||
## Tooling
|
||||
- .NET 10 preview workers (HostedService + channel pipelines).
|
||||
- MongoDB for node/edge storage; S3-compatible buckets for layout tiles/snapshots if needed.
|
||||
- In-memory node/edge storage (Mongo removed); S3-compatible buckets for layout tiles/snapshots if needed.
|
||||
- Scheduler integration (jobs, change streams) to handle incremental updates.
|
||||
- Analytics: clustering/centrality pipelines with Mongo-backed snapshot provider and overlays; change-stream/backfill worker with idempotency store (Mongo or in-memory) and retry/backoff.
|
||||
- Analytics: clustering/centrality pipelines with in-memory snapshot provider and overlays; change-stream/backfill worker with in-memory idempotency store and retry/backoff.
|
||||
|
||||
## Definition of Done
|
||||
- Pipelines deterministic and tested; fixtures validated.
|
||||
|
||||
@@ -2,7 +2,6 @@ using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Analytics;
|
||||
|
||||
@@ -37,47 +36,4 @@ public static class GraphAnalyticsServiceCollectionExtensions
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
public static IServiceCollection AddGraphAnalyticsMongo(
|
||||
this IServiceCollection services,
|
||||
Action<GraphAnalyticsOptions>? configureOptions = null,
|
||||
Action<MongoGraphSnapshotProviderOptions>? configureSnapshot = null,
|
||||
Action<GraphAnalyticsWriterOptions>? configureWriter = null)
|
||||
{
|
||||
services.AddGraphAnalyticsPipeline(configureOptions);
|
||||
|
||||
if (configureSnapshot is not null)
|
||||
{
|
||||
services.Configure(configureSnapshot);
|
||||
}
|
||||
else
|
||||
{
|
||||
services.Configure<MongoGraphSnapshotProviderOptions>(_ => { });
|
||||
}
|
||||
|
||||
if (configureWriter is not null)
|
||||
{
|
||||
services.Configure(configureWriter);
|
||||
}
|
||||
else
|
||||
{
|
||||
services.Configure<GraphAnalyticsWriterOptions>(_ => { });
|
||||
}
|
||||
|
||||
services.Replace(ServiceDescriptor.Singleton<IGraphSnapshotProvider>(sp =>
|
||||
{
|
||||
var db = sp.GetRequiredService<IMongoDatabase>();
|
||||
var options = sp.GetRequiredService<IOptions<MongoGraphSnapshotProviderOptions>>();
|
||||
return new MongoGraphSnapshotProvider(db, options.Value);
|
||||
}));
|
||||
|
||||
services.Replace(ServiceDescriptor.Singleton<IGraphAnalyticsWriter>(sp =>
|
||||
{
|
||||
var db = sp.GetRequiredService<IMongoDatabase>();
|
||||
var options = sp.GetRequiredService<IOptions<GraphAnalyticsWriterOptions>>();
|
||||
return new MongoGraphAnalyticsWriter(db, options.Value);
|
||||
}));
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Text.Json.Nodes;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Analytics;
|
||||
|
||||
public sealed class MongoGraphAnalyticsWriter : IGraphAnalyticsWriter
|
||||
{
|
||||
private readonly IMongoCollection<BsonDocument> _clusters;
|
||||
private readonly IMongoCollection<BsonDocument> _centrality;
|
||||
private readonly IMongoCollection<BsonDocument> _nodes;
|
||||
private readonly GraphAnalyticsWriterOptions _options;
|
||||
|
||||
public MongoGraphAnalyticsWriter(IMongoDatabase database, GraphAnalyticsWriterOptions? options = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
|
||||
_options = options ?? new GraphAnalyticsWriterOptions();
|
||||
_clusters = database.GetCollection<BsonDocument>(_options.ClusterCollectionName);
|
||||
_centrality = database.GetCollection<BsonDocument>(_options.CentralityCollectionName);
|
||||
_nodes = database.GetCollection<BsonDocument>(_options.NodeCollectionName);
|
||||
}
|
||||
|
||||
public async Task PersistClusterAssignmentsAsync(GraphAnalyticsSnapshot snapshot, ImmutableArray<ClusterAssignment> assignments, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (assignments.Length == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var models = new List<WriteModel<BsonDocument>>(assignments.Length);
|
||||
foreach (var assignment in assignments)
|
||||
{
|
||||
var filter = Builders<BsonDocument>.Filter.And(
|
||||
Builders<BsonDocument>.Filter.Eq("tenant", snapshot.Tenant),
|
||||
Builders<BsonDocument>.Filter.Eq("snapshot_id", snapshot.SnapshotId),
|
||||
Builders<BsonDocument>.Filter.Eq("node_id", assignment.NodeId));
|
||||
|
||||
var document = new BsonDocument
|
||||
{
|
||||
{ "tenant", snapshot.Tenant },
|
||||
{ "snapshot_id", snapshot.SnapshotId },
|
||||
{ "node_id", assignment.NodeId },
|
||||
{ "cluster_id", assignment.ClusterId },
|
||||
{ "kind", assignment.Kind },
|
||||
{ "generated_at", snapshot.GeneratedAt.UtcDateTime }
|
||||
};
|
||||
|
||||
models.Add(new ReplaceOneModel<BsonDocument>(filter, document) { IsUpsert = true });
|
||||
}
|
||||
|
||||
await _clusters.BulkWriteAsync(models, new BulkWriteOptions { IsOrdered = false }, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (_options.WriteClusterAssignmentsToNodes)
|
||||
{
|
||||
await WriteClustersToNodesAsync(assignments, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task PersistCentralityAsync(GraphAnalyticsSnapshot snapshot, ImmutableArray<CentralityScore> scores, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (scores.Length == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var models = new List<WriteModel<BsonDocument>>(scores.Length);
|
||||
foreach (var score in scores)
|
||||
{
|
||||
var filter = Builders<BsonDocument>.Filter.And(
|
||||
Builders<BsonDocument>.Filter.Eq("tenant", snapshot.Tenant),
|
||||
Builders<BsonDocument>.Filter.Eq("snapshot_id", snapshot.SnapshotId),
|
||||
Builders<BsonDocument>.Filter.Eq("node_id", score.NodeId));
|
||||
|
||||
var document = new BsonDocument
|
||||
{
|
||||
{ "tenant", snapshot.Tenant },
|
||||
{ "snapshot_id", snapshot.SnapshotId },
|
||||
{ "node_id", score.NodeId },
|
||||
{ "kind", score.Kind },
|
||||
{ "degree", score.Degree },
|
||||
{ "betweenness", score.Betweenness },
|
||||
{ "generated_at", snapshot.GeneratedAt.UtcDateTime }
|
||||
};
|
||||
|
||||
models.Add(new ReplaceOneModel<BsonDocument>(filter, document) { IsUpsert = true });
|
||||
}
|
||||
|
||||
await _centrality.BulkWriteAsync(models, new BulkWriteOptions { IsOrdered = false }, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task WriteClustersToNodesAsync(IEnumerable<ClusterAssignment> assignments, CancellationToken cancellationToken)
|
||||
{
|
||||
var models = new List<WriteModel<BsonDocument>>();
|
||||
foreach (var assignment in assignments)
|
||||
{
|
||||
var filter = Builders<BsonDocument>.Filter.Eq("id", assignment.NodeId);
|
||||
var update = Builders<BsonDocument>.Update.Set("attributes.cluster_id", assignment.ClusterId);
|
||||
models.Add(new UpdateOneModel<BsonDocument>(filter, update) { IsUpsert = false });
|
||||
}
|
||||
|
||||
if (models.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _nodes.BulkWriteAsync(models, new BulkWriteOptions { IsOrdered = false }, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Nodes;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Graph.Indexer.Infrastructure;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Analytics;
|
||||
|
||||
public sealed class MongoGraphSnapshotProvider : IGraphSnapshotProvider
|
||||
{
|
||||
private readonly IMongoCollection<BsonDocument> _snapshots;
|
||||
private readonly IMongoCollection<BsonDocument> _progress;
|
||||
private readonly MongoGraphSnapshotProviderOptions _options;
|
||||
|
||||
public MongoGraphSnapshotProvider(IMongoDatabase database, MongoGraphSnapshotProviderOptions? options = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
_options = options ?? new MongoGraphSnapshotProviderOptions();
|
||||
_snapshots = database.GetCollection<BsonDocument>(_options.SnapshotCollectionName);
|
||||
_progress = database.GetCollection<BsonDocument>(_options.ProgressCollectionName);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<GraphAnalyticsSnapshot>> GetPendingSnapshotsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var processedIds = await _progress
|
||||
.Find(FilterDefinition<BsonDocument>.Empty)
|
||||
.Project(doc => doc["snapshot_id"].AsString)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var filter = Builders<BsonDocument>.Filter.Nin("snapshot_id", processedIds);
|
||||
var snapshots = await _snapshots
|
||||
.Find(filter)
|
||||
.Limit(_options.MaxBatch)
|
||||
.Sort(Builders<BsonDocument>.Sort.Descending("generated_at"))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var result = new List<GraphAnalyticsSnapshot>(snapshots.Count);
|
||||
foreach (var snapshot in snapshots)
|
||||
{
|
||||
var tenant = snapshot.GetValue("tenant", string.Empty).AsString;
|
||||
var snapshotId = snapshot.GetValue("snapshot_id", string.Empty).AsString;
|
||||
var generatedAt = snapshot.TryGetValue("generated_at", out var generated)
|
||||
&& generated.BsonType == BsonType.DateTime
|
||||
? DateTime.SpecifyKind(generated.ToUniversalTime(), DateTimeKind.Utc)
|
||||
: DateTimeOffset.UtcNow;
|
||||
|
||||
var nodes = snapshot.TryGetValue("nodes", out var nodesValue) && nodesValue is BsonArray nodesArray
|
||||
? BsonJsonConverter.ToJsonArray(nodesArray).Select(n => (JsonObject)n!).ToImmutableArray()
|
||||
: ImmutableArray<JsonObject>.Empty;
|
||||
|
||||
var edges = snapshot.TryGetValue("edges", out var edgesValue) && edgesValue is BsonArray edgesArray
|
||||
? BsonJsonConverter.ToJsonArray(edgesArray).Select(n => (JsonObject)n!).ToImmutableArray()
|
||||
: ImmutableArray<JsonObject>.Empty;
|
||||
|
||||
result.Add(new GraphAnalyticsSnapshot(tenant, snapshotId, generatedAt, nodes, edges));
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public async Task MarkProcessedAsync(string tenant, string snapshotId, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var filter = Builders<BsonDocument>.Filter.Eq("snapshot_id", snapshotId)
|
||||
& Builders<BsonDocument>.Filter.Eq("tenant", tenant);
|
||||
|
||||
var update = Builders<BsonDocument>.Update.Set("snapshot_id", snapshotId)
|
||||
.Set("tenant", tenant)
|
||||
.SetOnInsert("processed_at", DateTimeOffset.UtcNow.UtcDateTime);
|
||||
|
||||
await _progress.UpdateOneAsync(filter, update, new UpdateOptions { IsUpsert = true }, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
namespace StellaOps.Graph.Indexer.Analytics;
|
||||
|
||||
public sealed class MongoGraphSnapshotProviderOptions
|
||||
{
|
||||
public string SnapshotCollectionName { get; set; } = "graph_snapshots";
|
||||
public string ProgressCollectionName { get; set; } = "graph_analytics_progress";
|
||||
public int MaxBatch { get; set; } = 5;
|
||||
}
|
||||
@@ -1,8 +1,7 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Incremental;
|
||||
|
||||
@@ -23,58 +22,12 @@ public static class GraphChangeStreamServiceCollectionExtensions
|
||||
services.Configure<GraphChangeStreamOptions>(_ => { });
|
||||
}
|
||||
|
||||
services.TryAddSingleton<IGraphChangeEventSource, NoOpGraphChangeEventSource>();
|
||||
services.TryAddSingleton<IGraphBackfillSource, NoOpGraphChangeEventSource>();
|
||||
services.TryAddSingleton<IIdempotencyStore, InMemoryIdempotencyStore>();
|
||||
services.TryAddSingleton<IGraphDocumentWriter, InMemoryGraphDocumentWriter>();
|
||||
services.AddSingleton<GraphBackfillMetrics>();
|
||||
services.AddHostedService<GraphChangeStreamProcessor>();
|
||||
return services;
|
||||
}
|
||||
|
||||
public static IServiceCollection AddGraphChangeStreamProcessorWithMongo(
|
||||
this IServiceCollection services,
|
||||
Action<GraphChangeStreamOptions>? configureOptions = null,
|
||||
Action<MongoGraphChangeEventOptions>? configureChangeOptions = null,
|
||||
Action<MongoIdempotencyStoreOptions>? configureIdempotency = null)
|
||||
{
|
||||
services.AddGraphChangeStreamProcessor(configureOptions);
|
||||
|
||||
if (configureChangeOptions is not null)
|
||||
{
|
||||
services.Configure(configureChangeOptions);
|
||||
}
|
||||
else
|
||||
{
|
||||
services.Configure<MongoGraphChangeEventOptions>(_ => { });
|
||||
}
|
||||
|
||||
if (configureIdempotency is not null)
|
||||
{
|
||||
services.Configure(configureIdempotency);
|
||||
}
|
||||
else
|
||||
{
|
||||
services.Configure<MongoIdempotencyStoreOptions>(_ => { });
|
||||
}
|
||||
|
||||
services.Replace(ServiceDescriptor.Singleton<IGraphChangeEventSource>(sp =>
|
||||
{
|
||||
var db = sp.GetRequiredService<IMongoDatabase>();
|
||||
var opts = sp.GetRequiredService<IOptions<MongoGraphChangeEventOptions>>();
|
||||
return new MongoGraphChangeEventSource(db, opts.Value);
|
||||
}));
|
||||
|
||||
services.Replace(ServiceDescriptor.Singleton<IGraphBackfillSource>(sp =>
|
||||
{
|
||||
var db = sp.GetRequiredService<IMongoDatabase>();
|
||||
var opts = sp.GetRequiredService<IOptions<MongoGraphChangeEventOptions>>();
|
||||
return new MongoGraphChangeEventSource(db, opts.Value);
|
||||
}));
|
||||
|
||||
services.Replace(ServiceDescriptor.Singleton<IIdempotencyStore>(sp =>
|
||||
{
|
||||
var db = sp.GetRequiredService<IMongoDatabase>();
|
||||
var opts = sp.GetRequiredService<IOptions<MongoIdempotencyStoreOptions>>();
|
||||
return new MongoIdempotencyStore(db, opts.Value);
|
||||
}));
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
namespace StellaOps.Graph.Indexer.Incremental;
|
||||
|
||||
public sealed class MongoGraphChangeEventOptions
|
||||
{
|
||||
public string CollectionName { get; set; } = "graph_change_events";
|
||||
public string SequenceFieldName { get; set; } = "sequence_token";
|
||||
public string NodeArrayFieldName { get; set; } = "nodes";
|
||||
public string EdgeArrayFieldName { get; set; } = "edges";
|
||||
public int MaxBatchSize { get; set; } = 256;
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Nodes;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Graph.Indexer.Infrastructure;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Incremental;
|
||||
|
||||
public sealed class MongoGraphChangeEventSource : IGraphChangeEventSource, IGraphBackfillSource
|
||||
{
|
||||
private readonly IMongoCollection<BsonDocument> _collection;
|
||||
private readonly MongoGraphChangeEventOptions _options;
|
||||
|
||||
public MongoGraphChangeEventSource(IMongoDatabase database, MongoGraphChangeEventOptions? options = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
_options = options ?? new MongoGraphChangeEventOptions();
|
||||
_collection = database.GetCollection<BsonDocument>(_options.CollectionName);
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<GraphChangeEvent> ReadAsync([System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
var filter = Builders<BsonDocument>.Filter.Eq("is_backfill", false);
|
||||
await foreach (var change in EnumerateAsync(filter, cancellationToken))
|
||||
{
|
||||
yield return change with { IsBackfill = false };
|
||||
}
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<GraphChangeEvent> ReadBackfillAsync([System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
var filter = Builders<BsonDocument>.Filter.Eq("is_backfill", true);
|
||||
await foreach (var change in EnumerateAsync(filter, cancellationToken))
|
||||
{
|
||||
yield return change with { IsBackfill = true };
|
||||
}
|
||||
}
|
||||
|
||||
private async IAsyncEnumerable<GraphChangeEvent> EnumerateAsync(FilterDefinition<BsonDocument> filter, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
var sort = Builders<BsonDocument>.Sort.Ascending(_options.SequenceFieldName);
|
||||
using var cursor = await _collection.FindAsync(filter, new FindOptions<BsonDocument> { Sort = sort, BatchSize = _options.MaxBatchSize }, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
foreach (var doc in cursor.Current)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var tenant = doc.GetValue("tenant", string.Empty).AsString;
|
||||
var snapshotId = doc.GetValue("snapshot_id", string.Empty).AsString;
|
||||
var sequence = doc.GetValue(_options.SequenceFieldName, string.Empty).AsString;
|
||||
|
||||
var nodes = doc.TryGetValue(_options.NodeArrayFieldName, out var nodesValue) && nodesValue is BsonArray nodeArray
|
||||
? BsonJsonConverter.ToJsonArray(nodeArray).Select(n => (JsonObject)n!).ToImmutableArray()
|
||||
: ImmutableArray<JsonObject>.Empty;
|
||||
|
||||
var edges = doc.TryGetValue(_options.EdgeArrayFieldName, out var edgesValue) && edgesValue is BsonArray edgeArray
|
||||
? BsonJsonConverter.ToJsonArray(edgeArray).Select(n => (JsonObject)n!).ToImmutableArray()
|
||||
: ImmutableArray<JsonObject>.Empty;
|
||||
|
||||
yield return new GraphChangeEvent(
|
||||
tenant,
|
||||
snapshotId,
|
||||
sequence,
|
||||
nodes,
|
||||
edges,
|
||||
doc.GetValue("is_backfill", false).ToBoolean());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Incremental;
|
||||
|
||||
public sealed class MongoIdempotencyStore : IIdempotencyStore
|
||||
{
|
||||
private readonly IMongoCollection<BsonDocument> _collection;
|
||||
|
||||
public MongoIdempotencyStore(IMongoDatabase database, MongoIdempotencyStoreOptions? options = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
var resolved = options ?? new MongoIdempotencyStoreOptions();
|
||||
_collection = database.GetCollection<BsonDocument>(resolved.CollectionName);
|
||||
}
|
||||
|
||||
public async Task<bool> HasSeenAsync(string sequenceToken, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var filter = Builders<BsonDocument>.Filter.Eq("sequence_token", sequenceToken);
|
||||
return await _collection.Find(filter).AnyAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task MarkSeenAsync(string sequenceToken, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var filter = Builders<BsonDocument>.Filter.Eq("sequence_token", sequenceToken);
|
||||
var update = Builders<BsonDocument>.Update.Set("sequence_token", sequenceToken)
|
||||
.SetOnInsert("recorded_at", DateTimeOffset.UtcNow.UtcDateTime);
|
||||
|
||||
await _collection.UpdateOneAsync(filter, update, new UpdateOptions { IsUpsert = true }, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
namespace StellaOps.Graph.Indexer.Incremental;
|
||||
|
||||
public sealed class MongoIdempotencyStoreOptions
|
||||
{
|
||||
public string CollectionName { get; set; } = "graph_change_idempotency";
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text.Json.Nodes;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Incremental;
|
||||
|
||||
/// <summary>
|
||||
/// No-op change/backfill source used when no external change feed is configured.
|
||||
/// </summary>
|
||||
public sealed class NoOpGraphChangeEventSource : IGraphChangeEventSource, IGraphBackfillSource
|
||||
{
|
||||
public IAsyncEnumerable<GraphChangeEvent> ReadAsync(CancellationToken cancellationToken) =>
|
||||
ReadInternalAsync(cancellationToken);
|
||||
|
||||
public IAsyncEnumerable<GraphChangeEvent> ReadBackfillAsync(CancellationToken cancellationToken) =>
|
||||
ReadInternalAsync(cancellationToken);
|
||||
|
||||
private static async IAsyncEnumerable<GraphChangeEvent> ReadInternalAsync([EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
await Task.CompletedTask.ConfigureAwait(false);
|
||||
yield break;
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
using System.Text.Json.Nodes;
|
||||
using MongoDB.Bson;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Infrastructure;
|
||||
|
||||
internal static class BsonJsonConverter
|
||||
{
|
||||
public static JsonObject ToJsonObject(BsonDocument document)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
var parsed = JsonNode.Parse(document.ToJson());
|
||||
return parsed as JsonObject ?? new JsonObject();
|
||||
}
|
||||
|
||||
public static JsonArray ToJsonArray(BsonArray array)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(array);
|
||||
var parsed = JsonNode.Parse(array.ToJson());
|
||||
return parsed as JsonArray ?? new JsonArray();
|
||||
}
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
namespace StellaOps.Graph.Indexer.Infrastructure;
|
||||
|
||||
public sealed class MongoDatabaseOptions
|
||||
{
|
||||
public string ConnectionString { get; set; } = string.Empty;
|
||||
public string DatabaseName { get; set; } = "stellaops-graph";
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Infrastructure;
|
||||
|
||||
public static class MongoServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddGraphMongoDatabase(
|
||||
this IServiceCollection services,
|
||||
Action<MongoDatabaseOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.Configure(configure);
|
||||
|
||||
services.AddSingleton<IMongoClient>(sp =>
|
||||
{
|
||||
var opts = sp.GetRequiredService<IOptions<MongoDatabaseOptions>>().Value;
|
||||
Validate(opts);
|
||||
return new MongoClient(opts.ConnectionString);
|
||||
});
|
||||
|
||||
services.AddSingleton<IMongoDatabase>(sp =>
|
||||
{
|
||||
var opts = sp.GetRequiredService<IOptions<MongoDatabaseOptions>>().Value;
|
||||
Validate(opts);
|
||||
return sp.GetRequiredService<IMongoClient>().GetDatabase(opts.DatabaseName);
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void Validate(MongoDatabaseOptions options)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(options.ConnectionString))
|
||||
{
|
||||
throw new InvalidOperationException("Mongo connection string must be provided.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(options.DatabaseName))
|
||||
{
|
||||
throw new InvalidOperationException("Mongo database name must be provided.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,84 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Encodings.Web;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
|
||||
|
||||
public sealed class MongoGraphDocumentWriter : IGraphDocumentWriter
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new()
|
||||
{
|
||||
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
|
||||
WriteIndented = false
|
||||
};
|
||||
|
||||
private readonly IMongoCollection<BsonDocument> _nodes;
|
||||
private readonly IMongoCollection<BsonDocument> _edges;
|
||||
|
||||
public MongoGraphDocumentWriter(IMongoDatabase database, MongoGraphDocumentWriterOptions? options = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
|
||||
var resolved = options ?? new MongoGraphDocumentWriterOptions();
|
||||
_nodes = database.GetCollection<BsonDocument>(resolved.NodeCollectionName);
|
||||
_edges = database.GetCollection<BsonDocument>(resolved.EdgeCollectionName);
|
||||
}
|
||||
|
||||
public async Task WriteAsync(GraphBuildBatch batch, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(batch);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (batch.Nodes.Length > 0)
|
||||
{
|
||||
var nodeModels = CreateReplaceModels(_nodes, batch.Nodes);
|
||||
if (nodeModels.Count > 0)
|
||||
{
|
||||
await _nodes.BulkWriteAsync(nodeModels, new BulkWriteOptions { IsOrdered = false }, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
if (batch.Edges.Length > 0)
|
||||
{
|
||||
var edgeModels = CreateReplaceModels(_edges, batch.Edges);
|
||||
if (edgeModels.Count > 0)
|
||||
{
|
||||
await _edges.BulkWriteAsync(edgeModels, new BulkWriteOptions { IsOrdered = false }, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static List<WriteModel<BsonDocument>> CreateReplaceModels(IMongoCollection<BsonDocument> collection, IReadOnlyList<JsonObject> documents)
|
||||
{
|
||||
var models = new List<WriteModel<BsonDocument>>(documents.Count);
|
||||
foreach (var document in documents)
|
||||
{
|
||||
if (!document.TryGetPropertyValue("id", out var idNode) || idNode is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var id = idNode.GetValue<string>();
|
||||
var filter = Builders<BsonDocument>.Filter.Eq("id", id);
|
||||
var bsonDocument = ToBsonDocument(document);
|
||||
models.Add(new ReplaceOneModel<BsonDocument>(filter, bsonDocument) { IsUpsert = true });
|
||||
}
|
||||
|
||||
return models;
|
||||
}
|
||||
|
||||
private static BsonDocument ToBsonDocument(JsonObject json)
|
||||
{
|
||||
var jsonString = json.ToJsonString(SerializerOptions);
|
||||
return BsonDocument.Parse(jsonString);
|
||||
}
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
|
||||
|
||||
public sealed class MongoGraphDocumentWriterOptions
|
||||
{
|
||||
public string NodeCollectionName { get; init; } = "graph_nodes";
|
||||
public string EdgeCollectionName { get; init; } = "graph_edges";
|
||||
}
|
||||
@@ -11,6 +11,7 @@ public static class InspectorIngestServiceCollectionExtensions
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
services.TryAddSingleton<Ingestion.Sbom.IGraphDocumentWriter, Ingestion.Sbom.InMemoryGraphDocumentWriter>();
|
||||
services.TryAddSingleton<GraphInspectorTransformer>();
|
||||
services.TryAddSingleton<GraphInspectorProcessor>(provider =>
|
||||
{
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Nodes;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
/// <summary>
|
||||
/// In-memory graph document writer used as a Mongo-free fallback.
|
||||
/// </summary>
|
||||
public sealed class InMemoryGraphDocumentWriter : IGraphDocumentWriter
|
||||
{
|
||||
private readonly ConcurrentBag<GraphBuildBatch> _batches = new();
|
||||
|
||||
public IReadOnlyCollection<GraphBuildBatch> Batches => _batches.ToArray();
|
||||
|
||||
public Task WriteAsync(GraphBuildBatch batch, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(batch);
|
||||
_batches.Add(CloneBatch(batch));
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private static GraphBuildBatch CloneBatch(GraphBuildBatch source)
|
||||
{
|
||||
static JsonObject CloneNode(JsonObject node) => (JsonObject)node.DeepClone();
|
||||
return new GraphBuildBatch(
|
||||
ImmutableArray.CreateRange(source.Nodes.Select(CloneNode)),
|
||||
ImmutableArray.CreateRange(source.Edges.Select(CloneNode)));
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,7 @@ public static class SbomIngestServiceCollectionExtensions
|
||||
services.Configure(configure);
|
||||
}
|
||||
|
||||
services.TryAddSingleton<IGraphDocumentWriter, InMemoryGraphDocumentWriter>();
|
||||
services.TryAddSingleton<SbomIngestTransformer>();
|
||||
services.TryAddSingleton<ISbomIngestMetrics, SbomIngestMetrics>();
|
||||
|
||||
|
||||
@@ -13,8 +13,6 @@
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="10.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0" />
|
||||
<PackageReference Include="MongoDB.Driver" Version="3.5.0" />
|
||||
<PackageReference Include="MongoDB.Bson" Version="3.5.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="10.0.0" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
@@ -1,125 +0,0 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Nodes;
|
||||
using Mongo2Go;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Graph.Indexer.Analytics;
|
||||
using StellaOps.Graph.Indexer.Incremental;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Tests;
|
||||
|
||||
public sealed class MongoProviderIntegrationTests : IAsyncLifetime
|
||||
{
|
||||
private readonly MongoDbRunner _runner;
|
||||
private IMongoDatabase _database = default!;
|
||||
|
||||
public MongoProviderIntegrationTests()
|
||||
{
|
||||
_runner = MongoDbRunner.Start(singleNodeReplSet: true);
|
||||
}
|
||||
|
||||
public Task InitializeAsync()
|
||||
{
|
||||
var client = new MongoClient(_runner.ConnectionString);
|
||||
_database = client.GetDatabase("graph-indexer-tests");
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task DisposeAsync()
|
||||
{
|
||||
_runner.Dispose();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SnapshotProvider_ReadsPendingSnapshots()
|
||||
{
|
||||
var snapshots = _database.GetCollection<BsonDocument>("graph_snapshots");
|
||||
var nodes = new BsonArray
|
||||
{
|
||||
new BsonDocument
|
||||
{
|
||||
{ "id", "gn:tenant-a:component:1" },
|
||||
{ "kind", "component" },
|
||||
{ "attributes", new BsonDocument { { "purl", "pkg:npm/a@1.0.0" } } }
|
||||
}
|
||||
};
|
||||
|
||||
var edges = new BsonArray();
|
||||
|
||||
await snapshots.InsertOneAsync(new BsonDocument
|
||||
{
|
||||
{ "tenant", "tenant-a" },
|
||||
{ "snapshot_id", "snap-1" },
|
||||
{ "generated_at", DateTime.UtcNow },
|
||||
{ "nodes", nodes },
|
||||
{ "edges", edges }
|
||||
});
|
||||
|
||||
var provider = new MongoGraphSnapshotProvider(_database);
|
||||
var pending = await provider.GetPendingSnapshotsAsync(CancellationToken.None);
|
||||
|
||||
Assert.Single(pending);
|
||||
Assert.Equal("snap-1", pending[0].SnapshotId);
|
||||
Assert.Single(pending[0].Nodes);
|
||||
|
||||
await provider.MarkProcessedAsync("tenant-a", "snap-1", CancellationToken.None);
|
||||
var none = await provider.GetPendingSnapshotsAsync(CancellationToken.None);
|
||||
Assert.Empty(none);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ChangeEventSource_EnumeratesAndHonorsIdempotency()
|
||||
{
|
||||
var changes = _database.GetCollection<BsonDocument>("graph_change_events");
|
||||
await changes.InsertManyAsync(new[]
|
||||
{
|
||||
new BsonDocument
|
||||
{
|
||||
{ "tenant", "tenant-a" },
|
||||
{ "snapshot_id", "snap-1" },
|
||||
{ "sequence_token", "seq-1" },
|
||||
{ "is_backfill", false },
|
||||
{ "nodes", new BsonArray { new BsonDocument { { "id", "gn:1" }, { "kind", "component" } } } },
|
||||
{ "edges", new BsonArray() }
|
||||
},
|
||||
new BsonDocument
|
||||
{
|
||||
{ "tenant", "tenant-a" },
|
||||
{ "snapshot_id", "snap-1" },
|
||||
{ "sequence_token", "seq-2" },
|
||||
{ "is_backfill", false },
|
||||
{ "nodes", new BsonArray { new BsonDocument { { "id", "gn:2" }, { "kind", "component" } } } },
|
||||
{ "edges", new BsonArray() }
|
||||
}
|
||||
});
|
||||
|
||||
var source = new MongoGraphChangeEventSource(_database);
|
||||
var idempotency = new MongoIdempotencyStore(_database);
|
||||
|
||||
var events = new List<GraphChangeEvent>();
|
||||
await foreach (var change in source.ReadAsync(CancellationToken.None))
|
||||
{
|
||||
if (await idempotency.HasSeenAsync(change.SequenceToken, CancellationToken.None))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
events.Add(change);
|
||||
await idempotency.MarkSeenAsync(change.SequenceToken, CancellationToken.None);
|
||||
}
|
||||
|
||||
Assert.Equal(2, events.Count);
|
||||
|
||||
var secondPass = new List<GraphChangeEvent>();
|
||||
await foreach (var change in source.ReadAsync(CancellationToken.None))
|
||||
{
|
||||
if (!await idempotency.HasSeenAsync(change.SequenceToken, CancellationToken.None))
|
||||
{
|
||||
secondPass.Add(change);
|
||||
}
|
||||
}
|
||||
|
||||
Assert.Empty(secondPass);
|
||||
}
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Graph.Indexer.Infrastructure;
|
||||
using Mongo2Go;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Tests;
|
||||
|
||||
public sealed class MongoServiceCollectionExtensionsTests : IAsyncLifetime
|
||||
{
|
||||
private MongoDbRunner _runner = default!;
|
||||
|
||||
public Task InitializeAsync()
|
||||
{
|
||||
_runner = MongoDbRunner.Start(singleNodeReplSet: true);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task DisposeAsync()
|
||||
{
|
||||
_runner.Dispose();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AddGraphMongoDatabase_RegistersClientAndDatabase()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
|
||||
services.AddGraphMongoDatabase(options =>
|
||||
{
|
||||
options.ConnectionString = _runner.ConnectionString;
|
||||
options.DatabaseName = "graph-indexer-ext-tests";
|
||||
});
|
||||
|
||||
var provider = services.BuildServiceProvider();
|
||||
|
||||
var client = provider.GetService<IMongoClient>();
|
||||
var database = provider.GetService<IMongoDatabase>();
|
||||
|
||||
Assert.NotNull(client);
|
||||
Assert.NotNull(database);
|
||||
Assert.Equal("graph-indexer-ext-tests", database!.DatabaseNamespace.DatabaseName);
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,5 @@
|
||||
<PackageReference Include="xunit" Version="2.9.2" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
|
||||
<PackageReference Include="Mongo2Go" Version="3.1.3" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
Reference in New Issue
Block a user