feat(graph-api): Add schema review notes for upcoming Graph API changes
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

feat(sbomservice): Add placeholder for SHA256SUMS in LNM v1 fixtures

docs(devportal): Create README for SDK archives in public directory

build(devportal): Implement offline bundle build script

test(devportal): Add link checker script for validating links in documentation

test(devportal): Create performance check script for dist folder size

test(devportal): Implement accessibility check script using Playwright and Axe

docs(devportal): Add SDK quickstart guide with examples for Node.js, Python, and cURL

feat(excititor): Implement MongoDB storage for airgap import records

test(findings): Add unit tests for export filters hash determinism

feat(findings): Define attestation contracts for ledger web service

feat(graph): Add MongoDB options and service collection extensions for graph indexing

test(graph): Implement integration tests for MongoDB provider and service collection extensions

feat(zastava): Define configuration options for Zastava surface secrets

build(tests): Create script to run Concelier linkset tests with TRX output
This commit is contained in:
StellaOps Bot
2025-11-22 19:22:30 +02:00
parent ca09400069
commit 48702191be
76 changed files with 3878 additions and 1081 deletions

View File

@@ -25,12 +25,13 @@ Project SBOM, advisory, VEX, and policy overlay data into a tenant-scoped proper
- .NET 10 preview workers (HostedService + channel pipelines).
- MongoDB for node/edge storage; S3-compatible buckets for layout tiles/snapshots if needed.
- Scheduler integration (jobs, change streams) to handle incremental updates.
- Analytics: clustering/centrality pipelines with Mongo-backed snapshot provider and overlays; change-stream/backfill worker with idempotency store (Mongo or in-memory) and retry/backoff.
## Definition of Done
- Pipelines deterministic and tested; fixtures validated.
- Metrics/logs/traces wired with tenant context.
- Schema docs + OpenAPI (where applicable) updated; compliance checklist appended.
- Offline kit includes seed data for air-gapped installs.
- Offline kit includes seed data for air-gapped installs, including analytics overlays (`overlays/*.ndjson` with manifest) ordered deterministically.
## Required Reading
- `docs/modules/graph/architecture.md`

View File

@@ -0,0 +1,7 @@
namespace StellaOps.Graph.Indexer.Infrastructure;
public sealed class MongoDatabaseOptions
{
public string ConnectionString { get; set; } = string.Empty;
public string DatabaseName { get; set; } = "stellaops-graph";
}

View File

@@ -0,0 +1,48 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using MongoDB.Driver;
namespace StellaOps.Graph.Indexer.Infrastructure;
public static class MongoServiceCollectionExtensions
{
public static IServiceCollection AddGraphMongoDatabase(
this IServiceCollection services,
Action<MongoDatabaseOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.Configure(configure);
services.AddSingleton<IMongoClient>(sp =>
{
var opts = sp.GetRequiredService<IOptions<MongoDatabaseOptions>>().Value;
Validate(opts);
return new MongoClient(opts.ConnectionString);
});
services.AddSingleton<IMongoDatabase>(sp =>
{
var opts = sp.GetRequiredService<IOptions<MongoDatabaseOptions>>().Value;
Validate(opts);
return sp.GetRequiredService<IMongoClient>().GetDatabase(opts.DatabaseName);
});
return services;
}
private static void Validate(MongoDatabaseOptions options)
{
if (string.IsNullOrWhiteSpace(options.ConnectionString))
{
throw new InvalidOperationException("Mongo connection string must be provided.");
}
if (string.IsNullOrWhiteSpace(options.DatabaseName))
{
throw new InvalidOperationException("Mongo database name must be provided.");
}
}
}

View File

@@ -15,5 +15,6 @@
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="MongoDB.Driver" Version="3.5.0" />
<PackageReference Include="MongoDB.Bson" Version="3.5.0" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="10.0.0-rc.2.25502.107" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,125 @@
using System.Collections.Immutable;
using System.Text.Json.Nodes;
using Mongo2Go;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Graph.Indexer.Analytics;
using StellaOps.Graph.Indexer.Incremental;
namespace StellaOps.Graph.Indexer.Tests;
public sealed class MongoProviderIntegrationTests : IAsyncLifetime
{
private readonly MongoDbRunner _runner;
private IMongoDatabase _database = default!;
public MongoProviderIntegrationTests()
{
_runner = MongoDbRunner.Start(singleNodeReplSet: true);
}
public Task InitializeAsync()
{
var client = new MongoClient(_runner.ConnectionString);
_database = client.GetDatabase("graph-indexer-tests");
return Task.CompletedTask;
}
public Task DisposeAsync()
{
_runner.Dispose();
return Task.CompletedTask;
}
[Fact]
public async Task SnapshotProvider_ReadsPendingSnapshots()
{
var snapshots = _database.GetCollection<BsonDocument>("graph_snapshots");
var nodes = new BsonArray
{
new BsonDocument
{
{ "id", "gn:tenant-a:component:1" },
{ "kind", "component" },
{ "attributes", new BsonDocument { { "purl", "pkg:npm/a@1.0.0" } } }
}
};
var edges = new BsonArray();
await snapshots.InsertOneAsync(new BsonDocument
{
{ "tenant", "tenant-a" },
{ "snapshot_id", "snap-1" },
{ "generated_at", DateTime.UtcNow },
{ "nodes", nodes },
{ "edges", edges }
});
var provider = new MongoGraphSnapshotProvider(_database);
var pending = await provider.GetPendingSnapshotsAsync(CancellationToken.None);
Assert.Single(pending);
Assert.Equal("snap-1", pending[0].SnapshotId);
Assert.Single(pending[0].Nodes);
await provider.MarkProcessedAsync("tenant-a", "snap-1", CancellationToken.None);
var none = await provider.GetPendingSnapshotsAsync(CancellationToken.None);
Assert.Empty(none);
}
[Fact]
public async Task ChangeEventSource_EnumeratesAndHonorsIdempotency()
{
var changes = _database.GetCollection<BsonDocument>("graph_change_events");
await changes.InsertManyAsync(new[]
{
new BsonDocument
{
{ "tenant", "tenant-a" },
{ "snapshot_id", "snap-1" },
{ "sequence_token", "seq-1" },
{ "is_backfill", false },
{ "nodes", new BsonArray { new BsonDocument { { "id", "gn:1" }, { "kind", "component" } } } },
{ "edges", new BsonArray() }
},
new BsonDocument
{
{ "tenant", "tenant-a" },
{ "snapshot_id", "snap-1" },
{ "sequence_token", "seq-2" },
{ "is_backfill", false },
{ "nodes", new BsonArray { new BsonDocument { { "id", "gn:2" }, { "kind", "component" } } } },
{ "edges", new BsonArray() }
}
});
var source = new MongoGraphChangeEventSource(_database);
var idempotency = new MongoIdempotencyStore(_database);
var events = new List<GraphChangeEvent>();
await foreach (var change in source.ReadAsync(CancellationToken.None))
{
if (await idempotency.HasSeenAsync(change.SequenceToken, CancellationToken.None))
{
continue;
}
events.Add(change);
await idempotency.MarkSeenAsync(change.SequenceToken, CancellationToken.None);
}
Assert.Equal(2, events.Count);
var secondPass = new List<GraphChangeEvent>();
await foreach (var change in source.ReadAsync(CancellationToken.None))
{
if (!await idempotency.HasSeenAsync(change.SequenceToken, CancellationToken.None))
{
secondPass.Add(change);
}
}
Assert.Empty(secondPass);
}
}

View File

@@ -0,0 +1,44 @@
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Graph.Indexer.Infrastructure;
using Mongo2Go;
using MongoDB.Driver;
namespace StellaOps.Graph.Indexer.Tests;
public sealed class MongoServiceCollectionExtensionsTests : IAsyncLifetime
{
private MongoDbRunner _runner = default!;
public Task InitializeAsync()
{
_runner = MongoDbRunner.Start(singleNodeReplSet: true);
return Task.CompletedTask;
}
public Task DisposeAsync()
{
_runner.Dispose();
return Task.CompletedTask;
}
[Fact]
public void AddGraphMongoDatabase_RegistersClientAndDatabase()
{
var services = new ServiceCollection();
services.AddGraphMongoDatabase(options =>
{
options.ConnectionString = _runner.ConnectionString;
options.DatabaseName = "graph-indexer-ext-tests";
});
var provider = services.BuildServiceProvider();
var client = provider.GetService<IMongoClient>();
var database = provider.GetService<IMongoDatabase>();
Assert.NotNull(client);
Assert.NotNull(database);
Assert.Equal("graph-indexer-ext-tests", database!.DatabaseNamespace.DatabaseName);
}
}

View File

@@ -12,5 +12,6 @@
<PackageReference Include="xunit" Version="2.9.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
<PackageReference Include="Mongo2Go" Version="3.1.3" />
</ItemGroup>
</Project>