feat(graph-api): Add schema review notes for upcoming Graph API changes
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
feat(sbomservice): Add placeholder for SHA256SUMS in LNM v1 fixtures docs(devportal): Create README for SDK archives in public directory build(devportal): Implement offline bundle build script test(devportal): Add link checker script for validating links in documentation test(devportal): Create performance check script for dist folder size test(devportal): Implement accessibility check script using Playwright and Axe docs(devportal): Add SDK quickstart guide with examples for Node.js, Python, and cURL feat(excititor): Implement MongoDB storage for airgap import records test(findings): Add unit tests for export filters hash determinism feat(findings): Define attestation contracts for ledger web service feat(graph): Add MongoDB options and service collection extensions for graph indexing test(graph): Implement integration tests for MongoDB provider and service collection extensions feat(zastava): Define configuration options for Zastava surface secrets build(tests): Create script to run Concelier linkset tests with TRX output
This commit is contained in:
@@ -25,12 +25,13 @@ Project SBOM, advisory, VEX, and policy overlay data into a tenant-scoped proper
|
||||
- .NET 10 preview workers (HostedService + channel pipelines).
|
||||
- MongoDB for node/edge storage; S3-compatible buckets for layout tiles/snapshots if needed.
|
||||
- Scheduler integration (jobs, change streams) to handle incremental updates.
|
||||
- Analytics: clustering/centrality pipelines with Mongo-backed snapshot provider and overlays; change-stream/backfill worker with idempotency store (Mongo or in-memory) and retry/backoff.
|
||||
|
||||
## Definition of Done
|
||||
- Pipelines deterministic and tested; fixtures validated.
|
||||
- Metrics/logs/traces wired with tenant context.
|
||||
- Schema docs + OpenAPI (where applicable) updated; compliance checklist appended.
|
||||
- Offline kit includes seed data for air-gapped installs.
|
||||
- Offline kit includes seed data for air-gapped installs, including analytics overlays (`overlays/*.ndjson` with manifest) ordered deterministically.
|
||||
|
||||
## Required Reading
|
||||
- `docs/modules/graph/architecture.md`
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
namespace StellaOps.Graph.Indexer.Infrastructure;
|
||||
|
||||
public sealed class MongoDatabaseOptions
|
||||
{
|
||||
public string ConnectionString { get; set; } = string.Empty;
|
||||
public string DatabaseName { get; set; } = "stellaops-graph";
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Infrastructure;
|
||||
|
||||
public static class MongoServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddGraphMongoDatabase(
|
||||
this IServiceCollection services,
|
||||
Action<MongoDatabaseOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.Configure(configure);
|
||||
|
||||
services.AddSingleton<IMongoClient>(sp =>
|
||||
{
|
||||
var opts = sp.GetRequiredService<IOptions<MongoDatabaseOptions>>().Value;
|
||||
Validate(opts);
|
||||
return new MongoClient(opts.ConnectionString);
|
||||
});
|
||||
|
||||
services.AddSingleton<IMongoDatabase>(sp =>
|
||||
{
|
||||
var opts = sp.GetRequiredService<IOptions<MongoDatabaseOptions>>().Value;
|
||||
Validate(opts);
|
||||
return sp.GetRequiredService<IMongoClient>().GetDatabase(opts.DatabaseName);
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void Validate(MongoDatabaseOptions options)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(options.ConnectionString))
|
||||
{
|
||||
throw new InvalidOperationException("Mongo connection string must be provided.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(options.DatabaseName))
|
||||
{
|
||||
throw new InvalidOperationException("Mongo database name must be provided.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -15,5 +15,6 @@
|
||||
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0-rc.2.25502.107" />
|
||||
<PackageReference Include="MongoDB.Driver" Version="3.5.0" />
|
||||
<PackageReference Include="MongoDB.Bson" Version="3.5.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="10.0.0-rc.2.25502.107" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Nodes;
|
||||
using Mongo2Go;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Graph.Indexer.Analytics;
|
||||
using StellaOps.Graph.Indexer.Incremental;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Tests;
|
||||
|
||||
public sealed class MongoProviderIntegrationTests : IAsyncLifetime
|
||||
{
|
||||
private readonly MongoDbRunner _runner;
|
||||
private IMongoDatabase _database = default!;
|
||||
|
||||
public MongoProviderIntegrationTests()
|
||||
{
|
||||
_runner = MongoDbRunner.Start(singleNodeReplSet: true);
|
||||
}
|
||||
|
||||
public Task InitializeAsync()
|
||||
{
|
||||
var client = new MongoClient(_runner.ConnectionString);
|
||||
_database = client.GetDatabase("graph-indexer-tests");
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task DisposeAsync()
|
||||
{
|
||||
_runner.Dispose();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SnapshotProvider_ReadsPendingSnapshots()
|
||||
{
|
||||
var snapshots = _database.GetCollection<BsonDocument>("graph_snapshots");
|
||||
var nodes = new BsonArray
|
||||
{
|
||||
new BsonDocument
|
||||
{
|
||||
{ "id", "gn:tenant-a:component:1" },
|
||||
{ "kind", "component" },
|
||||
{ "attributes", new BsonDocument { { "purl", "pkg:npm/a@1.0.0" } } }
|
||||
}
|
||||
};
|
||||
|
||||
var edges = new BsonArray();
|
||||
|
||||
await snapshots.InsertOneAsync(new BsonDocument
|
||||
{
|
||||
{ "tenant", "tenant-a" },
|
||||
{ "snapshot_id", "snap-1" },
|
||||
{ "generated_at", DateTime.UtcNow },
|
||||
{ "nodes", nodes },
|
||||
{ "edges", edges }
|
||||
});
|
||||
|
||||
var provider = new MongoGraphSnapshotProvider(_database);
|
||||
var pending = await provider.GetPendingSnapshotsAsync(CancellationToken.None);
|
||||
|
||||
Assert.Single(pending);
|
||||
Assert.Equal("snap-1", pending[0].SnapshotId);
|
||||
Assert.Single(pending[0].Nodes);
|
||||
|
||||
await provider.MarkProcessedAsync("tenant-a", "snap-1", CancellationToken.None);
|
||||
var none = await provider.GetPendingSnapshotsAsync(CancellationToken.None);
|
||||
Assert.Empty(none);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ChangeEventSource_EnumeratesAndHonorsIdempotency()
|
||||
{
|
||||
var changes = _database.GetCollection<BsonDocument>("graph_change_events");
|
||||
await changes.InsertManyAsync(new[]
|
||||
{
|
||||
new BsonDocument
|
||||
{
|
||||
{ "tenant", "tenant-a" },
|
||||
{ "snapshot_id", "snap-1" },
|
||||
{ "sequence_token", "seq-1" },
|
||||
{ "is_backfill", false },
|
||||
{ "nodes", new BsonArray { new BsonDocument { { "id", "gn:1" }, { "kind", "component" } } } },
|
||||
{ "edges", new BsonArray() }
|
||||
},
|
||||
new BsonDocument
|
||||
{
|
||||
{ "tenant", "tenant-a" },
|
||||
{ "snapshot_id", "snap-1" },
|
||||
{ "sequence_token", "seq-2" },
|
||||
{ "is_backfill", false },
|
||||
{ "nodes", new BsonArray { new BsonDocument { { "id", "gn:2" }, { "kind", "component" } } } },
|
||||
{ "edges", new BsonArray() }
|
||||
}
|
||||
});
|
||||
|
||||
var source = new MongoGraphChangeEventSource(_database);
|
||||
var idempotency = new MongoIdempotencyStore(_database);
|
||||
|
||||
var events = new List<GraphChangeEvent>();
|
||||
await foreach (var change in source.ReadAsync(CancellationToken.None))
|
||||
{
|
||||
if (await idempotency.HasSeenAsync(change.SequenceToken, CancellationToken.None))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
events.Add(change);
|
||||
await idempotency.MarkSeenAsync(change.SequenceToken, CancellationToken.None);
|
||||
}
|
||||
|
||||
Assert.Equal(2, events.Count);
|
||||
|
||||
var secondPass = new List<GraphChangeEvent>();
|
||||
await foreach (var change in source.ReadAsync(CancellationToken.None))
|
||||
{
|
||||
if (!await idempotency.HasSeenAsync(change.SequenceToken, CancellationToken.None))
|
||||
{
|
||||
secondPass.Add(change);
|
||||
}
|
||||
}
|
||||
|
||||
Assert.Empty(secondPass);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Graph.Indexer.Infrastructure;
|
||||
using Mongo2Go;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Tests;
|
||||
|
||||
public sealed class MongoServiceCollectionExtensionsTests : IAsyncLifetime
|
||||
{
|
||||
private MongoDbRunner _runner = default!;
|
||||
|
||||
public Task InitializeAsync()
|
||||
{
|
||||
_runner = MongoDbRunner.Start(singleNodeReplSet: true);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task DisposeAsync()
|
||||
{
|
||||
_runner.Dispose();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AddGraphMongoDatabase_RegistersClientAndDatabase()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
|
||||
services.AddGraphMongoDatabase(options =>
|
||||
{
|
||||
options.ConnectionString = _runner.ConnectionString;
|
||||
options.DatabaseName = "graph-indexer-ext-tests";
|
||||
});
|
||||
|
||||
var provider = services.BuildServiceProvider();
|
||||
|
||||
var client = provider.GetService<IMongoClient>();
|
||||
var database = provider.GetService<IMongoDatabase>();
|
||||
|
||||
Assert.NotNull(client);
|
||||
Assert.NotNull(database);
|
||||
Assert.Equal("graph-indexer-ext-tests", database!.DatabaseNamespace.DatabaseName);
|
||||
}
|
||||
}
|
||||
@@ -12,5 +12,6 @@
|
||||
<PackageReference Include="xunit" Version="2.9.2" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
|
||||
<PackageReference Include="Mongo2Go" Version="3.1.3" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
Reference in New Issue
Block a user