feat: Add MongoIdempotencyStoreOptions for MongoDB configuration
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

feat: Implement BsonJsonConverter for converting BsonDocument and BsonArray to JSON

fix: Update project file to include MongoDB.Bson package

test: Add GraphOverlayExporterTests to validate NDJSON export functionality

refactor: Refactor Program.cs in Attestation Tool for improved argument parsing and error handling

docs: Update README for stella-forensic-verify with usage instructions and exit codes

feat: Enhance HmacVerifier with clock skew and not-after checks

feat: Add MerkleRootVerifier and ChainOfCustodyVerifier for additional verification methods

fix: Update DenoRuntimeShim to correctly handle file paths

feat: Introduce ComposerAutoloadData and related parsing in ComposerLockReader

test: Add tests for Deno runtime execution and verification

test: Enhance PHP package tests to include autoload data verification

test: Add unit tests for HmacVerifier and verification logic
This commit is contained in:
StellaOps Bot
2025-11-22 16:42:56 +02:00
parent 967ae0ab16
commit dc7c75b496
85 changed files with 2272 additions and 917 deletions

View File

@@ -1,6 +1,8 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using MongoDB.Driver;
namespace StellaOps.Graph.Indexer.Analytics;
@@ -28,9 +30,54 @@ public static class GraphAnalyticsServiceCollectionExtensions
});
services.AddSingleton<GraphAnalyticsMetrics>();
services.TryAddSingleton<IGraphSnapshotProvider, InMemoryGraphSnapshotProvider>();
services.TryAddSingleton<IGraphAnalyticsWriter, InMemoryGraphAnalyticsWriter>();
services.AddSingleton<IGraphAnalyticsPipeline, GraphAnalyticsPipeline>();
services.AddHostedService<GraphAnalyticsHostedService>();
return services;
}
public static IServiceCollection AddGraphAnalyticsMongo(
this IServiceCollection services,
Action<GraphAnalyticsOptions>? configureOptions = null,
Action<MongoGraphSnapshotProviderOptions>? configureSnapshot = null,
Action<GraphAnalyticsWriterOptions>? configureWriter = null)
{
services.AddGraphAnalyticsPipeline(configureOptions);
if (configureSnapshot is not null)
{
services.Configure(configureSnapshot);
}
else
{
services.Configure<MongoGraphSnapshotProviderOptions>(_ => { });
}
if (configureWriter is not null)
{
services.Configure(configureWriter);
}
else
{
services.Configure<GraphAnalyticsWriterOptions>(_ => { });
}
services.Replace(ServiceDescriptor.Singleton<IGraphSnapshotProvider>(sp =>
{
var db = sp.GetRequiredService<IMongoDatabase>();
var options = sp.GetRequiredService<IOptions<MongoGraphSnapshotProviderOptions>>();
return new MongoGraphSnapshotProvider(db, options.Value);
}));
services.Replace(ServiceDescriptor.Singleton<IGraphAnalyticsWriter>(sp =>
{
var db = sp.GetRequiredService<IMongoDatabase>();
var options = sp.GetRequiredService<IOptions<GraphAnalyticsWriterOptions>>();
return new MongoGraphAnalyticsWriter(db, options.Value);
}));
return services;
}
}

View File

@@ -0,0 +1,76 @@
using System.Collections.Immutable;
using System.Text.Json.Nodes;
using StellaOps.Graph.Indexer.Ingestion.Sbom;
namespace StellaOps.Graph.Indexer.Analytics;
public sealed class GraphOverlayExporter
{
public async Task ExportAsync(
GraphAnalyticsSnapshot snapshot,
GraphAnalyticsResult result,
ISnapshotFileWriter fileWriter,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(snapshot);
ArgumentNullException.ThrowIfNull(fileWriter);
cancellationToken.ThrowIfCancellationRequested();
var clusters = result.Clusters
.OrderBy(c => c.NodeId, StringComparer.Ordinal)
.Select(c => CreateClusterOverlay(snapshot, c))
.ToImmutableArray();
var centrality = result.CentralityScores
.OrderBy(c => c.NodeId, StringComparer.Ordinal)
.Select(c => CreateCentralityOverlay(snapshot, c))
.ToImmutableArray();
await fileWriter.WriteJsonLinesAsync("overlays/clusters.ndjson", clusters, cancellationToken).ConfigureAwait(false);
await fileWriter.WriteJsonLinesAsync("overlays/centrality.ndjson", centrality, cancellationToken).ConfigureAwait(false);
var manifest = new JsonObject
{
["tenant"] = snapshot.Tenant,
["snapshot_id"] = snapshot.SnapshotId,
["generated_at"] = GraphTimestamp.Format(snapshot.GeneratedAt),
["cluster_count"] = clusters.Length,
["centrality_count"] = centrality.Length,
["files"] = new JsonObject
{
["clusters"] = "overlays/clusters.ndjson",
["centrality"] = "overlays/centrality.ndjson"
}
};
await fileWriter.WriteJsonAsync("overlays/manifest.json", manifest, cancellationToken).ConfigureAwait(false);
}
private static JsonObject CreateClusterOverlay(GraphAnalyticsSnapshot snapshot, ClusterAssignment assignment)
{
return new JsonObject
{
["tenant"] = snapshot.Tenant,
["snapshot_id"] = snapshot.SnapshotId,
["generated_at"] = GraphTimestamp.Format(snapshot.GeneratedAt),
["node_id"] = assignment.NodeId,
["cluster_id"] = assignment.ClusterId,
["kind"] = assignment.Kind
};
}
private static JsonObject CreateCentralityOverlay(GraphAnalyticsSnapshot snapshot, CentralityScore score)
{
return new JsonObject
{
["tenant"] = snapshot.Tenant,
["snapshot_id"] = snapshot.SnapshotId,
["generated_at"] = GraphTimestamp.Format(snapshot.GeneratedAt),
["node_id"] = score.NodeId,
["degree"] = score.Degree,
["betweenness"] = score.Betweenness,
["kind"] = score.Kind
};
}
}

View File

@@ -0,0 +1,79 @@
using System.Collections.Immutable;
using System.Text.Json.Nodes;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Graph.Indexer.Infrastructure;
namespace StellaOps.Graph.Indexer.Analytics;
public sealed class MongoGraphSnapshotProvider : IGraphSnapshotProvider
{
private readonly IMongoCollection<BsonDocument> _snapshots;
private readonly IMongoCollection<BsonDocument> _progress;
private readonly MongoGraphSnapshotProviderOptions _options;
public MongoGraphSnapshotProvider(IMongoDatabase database, MongoGraphSnapshotProviderOptions? options = null)
{
ArgumentNullException.ThrowIfNull(database);
_options = options ?? new MongoGraphSnapshotProviderOptions();
_snapshots = database.GetCollection<BsonDocument>(_options.SnapshotCollectionName);
_progress = database.GetCollection<BsonDocument>(_options.ProgressCollectionName);
}
public async Task<IReadOnlyList<GraphAnalyticsSnapshot>> GetPendingSnapshotsAsync(CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var processedIds = await _progress
.Find(FilterDefinition<BsonDocument>.Empty)
.Project(doc => doc["snapshot_id"].AsString)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
var filter = Builders<BsonDocument>.Filter.Nin("snapshot_id", processedIds);
var snapshots = await _snapshots
.Find(filter)
.Limit(_options.MaxBatch)
.Sort(Builders<BsonDocument>.Sort.Descending("generated_at"))
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
var result = new List<GraphAnalyticsSnapshot>(snapshots.Count);
foreach (var snapshot in snapshots)
{
var tenant = snapshot.GetValue("tenant", string.Empty).AsString;
var snapshotId = snapshot.GetValue("snapshot_id", string.Empty).AsString;
var generatedAt = snapshot.TryGetValue("generated_at", out var generated)
&& generated.TryToUniversalTime(out var dt)
? dt
: DateTimeOffset.UtcNow;
var nodes = snapshot.TryGetValue("nodes", out var nodesValue) && nodesValue is BsonArray nodesArray
? BsonJsonConverter.ToJsonArray(nodesArray).Select(n => (JsonObject)n!).ToImmutableArray()
: ImmutableArray<JsonObject>.Empty;
var edges = snapshot.TryGetValue("edges", out var edgesValue) && edgesValue is BsonArray edgesArray
? BsonJsonConverter.ToJsonArray(edgesArray).Select(n => (JsonObject)n!).ToImmutableArray()
: ImmutableArray<JsonObject>.Empty;
result.Add(new GraphAnalyticsSnapshot(tenant, snapshotId, generatedAt, nodes, edges));
}
return result;
}
public async Task MarkProcessedAsync(string tenant, string snapshotId, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var filter = Builders<BsonDocument>.Filter.Eq("snapshot_id", snapshotId)
& Builders<BsonDocument>.Filter.Eq("tenant", tenant);
var update = Builders<BsonDocument>.Update.Set("snapshot_id", snapshotId)
.Set("tenant", tenant)
.SetOnInsert("processed_at", DateTimeOffset.UtcNow.UtcDateTime);
await _progress.UpdateOneAsync(filter, update, new UpdateOptions { IsUpsert = true }, cancellationToken)
.ConfigureAwait(false);
}
}

View File

@@ -0,0 +1,8 @@
namespace StellaOps.Graph.Indexer.Analytics;
public sealed class MongoGraphSnapshotProviderOptions
{
public string SnapshotCollectionName { get; set; } = "graph_snapshots";
public string ProgressCollectionName { get; set; } = "graph_analytics_progress";
public int MaxBatch { get; set; } = 5;
}

View File

@@ -1,6 +1,8 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using MongoDB.Driver;
namespace StellaOps.Graph.Indexer.Incremental;
@@ -25,4 +27,54 @@ public static class GraphChangeStreamServiceCollectionExtensions
services.AddHostedService<GraphChangeStreamProcessor>();
return services;
}
public static IServiceCollection AddGraphChangeStreamProcessorWithMongo(
this IServiceCollection services,
Action<GraphChangeStreamOptions>? configureOptions = null,
Action<MongoGraphChangeEventOptions>? configureChangeOptions = null,
Action<MongoIdempotencyStoreOptions>? configureIdempotency = null)
{
services.AddGraphChangeStreamProcessor(configureOptions);
if (configureChangeOptions is not null)
{
services.Configure(configureChangeOptions);
}
else
{
services.Configure<MongoGraphChangeEventOptions>(_ => { });
}
if (configureIdempotency is not null)
{
services.Configure(configureIdempotency);
}
else
{
services.Configure<MongoIdempotencyStoreOptions>(_ => { });
}
services.Replace(ServiceDescriptor.Singleton<IGraphChangeEventSource>(sp =>
{
var db = sp.GetRequiredService<IMongoDatabase>();
var opts = sp.GetRequiredService<IOptions<MongoGraphChangeEventOptions>>();
return new MongoGraphChangeEventSource(db, opts.Value);
}));
services.Replace(ServiceDescriptor.Singleton<IGraphBackfillSource>(sp =>
{
var db = sp.GetRequiredService<IMongoDatabase>();
var opts = sp.GetRequiredService<IOptions<MongoGraphChangeEventOptions>>();
return new MongoGraphChangeEventSource(db, opts.Value);
}));
services.Replace(ServiceDescriptor.Singleton<IIdempotencyStore>(sp =>
{
var db = sp.GetRequiredService<IMongoDatabase>();
var opts = sp.GetRequiredService<IOptions<MongoIdempotencyStoreOptions>>();
return new MongoIdempotencyStore(db, opts.Value);
}));
return services;
}
}

View File

@@ -0,0 +1,10 @@
namespace StellaOps.Graph.Indexer.Incremental;
public sealed class MongoGraphChangeEventOptions
{
public string CollectionName { get; set; } = "graph_change_events";
public string SequenceFieldName { get; set; } = "sequence_token";
public string NodeArrayFieldName { get; set; } = "nodes";
public string EdgeArrayFieldName { get; set; } = "edges";
public int MaxBatchSize { get; set; } = 256;
}

View File

@@ -0,0 +1,72 @@
using System.Collections.Immutable;
using System.Text.Json.Nodes;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Graph.Indexer.Infrastructure;
namespace StellaOps.Graph.Indexer.Incremental;
public sealed class MongoGraphChangeEventSource : IGraphChangeEventSource, IGraphBackfillSource
{
private readonly IMongoCollection<BsonDocument> _collection;
private readonly MongoGraphChangeEventOptions _options;
public MongoGraphChangeEventSource(IMongoDatabase database, MongoGraphChangeEventOptions? options = null)
{
ArgumentNullException.ThrowIfNull(database);
_options = options ?? new MongoGraphChangeEventOptions();
_collection = database.GetCollection<BsonDocument>(_options.CollectionName);
}
public async IAsyncEnumerable<GraphChangeEvent> ReadAsync([System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
var filter = Builders<BsonDocument>.Filter.Eq("is_backfill", false);
await foreach (var change in EnumerateAsync(filter, cancellationToken))
{
yield return change with { IsBackfill = false };
}
}
public async IAsyncEnumerable<GraphChangeEvent> ReadBackfillAsync([System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
var filter = Builders<BsonDocument>.Filter.Eq("is_backfill", true);
await foreach (var change in EnumerateAsync(filter, cancellationToken))
{
yield return change with { IsBackfill = true };
}
}
private async IAsyncEnumerable<GraphChangeEvent> EnumerateAsync(FilterDefinition<BsonDocument> filter, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
var sort = Builders<BsonDocument>.Sort.Ascending(_options.SequenceFieldName);
using var cursor = await _collection.FindAsync(filter, new FindOptions<BsonDocument> { Sort = sort, BatchSize = _options.MaxBatchSize }, cancellationToken).ConfigureAwait(false);
while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false))
{
foreach (var doc in cursor.Current)
{
cancellationToken.ThrowIfCancellationRequested();
var tenant = doc.GetValue("tenant", string.Empty).AsString;
var snapshotId = doc.GetValue("snapshot_id", string.Empty).AsString;
var sequence = doc.GetValue(_options.SequenceFieldName, string.Empty).AsString;
var nodes = doc.TryGetValue(_options.NodeArrayFieldName, out var nodesValue) && nodesValue is BsonArray nodeArray
? BsonJsonConverter.ToJsonArray(nodeArray).Select(n => (JsonObject)n!).ToImmutableArray()
: ImmutableArray<JsonObject>.Empty;
var edges = doc.TryGetValue(_options.EdgeArrayFieldName, out var edgesValue) && edgesValue is BsonArray edgeArray
? BsonJsonConverter.ToJsonArray(edgeArray).Select(n => (JsonObject)n!).ToImmutableArray()
: ImmutableArray<JsonObject>.Empty;
yield return new GraphChangeEvent(
tenant,
snapshotId,
sequence,
nodes,
edges,
doc.GetValue("is_backfill", false).ToBoolean());
}
}
}
}

View File

@@ -0,0 +1,34 @@
using MongoDB.Bson;
using MongoDB.Driver;
namespace StellaOps.Graph.Indexer.Incremental;
public sealed class MongoIdempotencyStore : IIdempotencyStore
{
private readonly IMongoCollection<BsonDocument> _collection;
public MongoIdempotencyStore(IMongoDatabase database, MongoIdempotencyStoreOptions? options = null)
{
ArgumentNullException.ThrowIfNull(database);
var resolved = options ?? new MongoIdempotencyStoreOptions();
_collection = database.GetCollection<BsonDocument>(resolved.CollectionName);
}
public async Task<bool> HasSeenAsync(string sequenceToken, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var filter = Builders<BsonDocument>.Filter.Eq("sequence_token", sequenceToken);
return await _collection.Find(filter).AnyAsync(cancellationToken).ConfigureAwait(false);
}
public async Task MarkSeenAsync(string sequenceToken, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var filter = Builders<BsonDocument>.Filter.Eq("sequence_token", sequenceToken);
var update = Builders<BsonDocument>.Update.Set("sequence_token", sequenceToken)
.SetOnInsert("recorded_at", DateTimeOffset.UtcNow.UtcDateTime);
await _collection.UpdateOneAsync(filter, update, new UpdateOptions { IsUpsert = true }, cancellationToken)
.ConfigureAwait(false);
}
}

View File

@@ -0,0 +1,6 @@
namespace StellaOps.Graph.Indexer.Incremental;
public sealed class MongoIdempotencyStoreOptions
{
public string CollectionName { get; set; } = "graph_change_idempotency";
}

View File

@@ -0,0 +1,21 @@
using System.Text.Json.Nodes;
using MongoDB.Bson;
namespace StellaOps.Graph.Indexer.Infrastructure;
internal static class BsonJsonConverter
{
public static JsonObject ToJsonObject(BsonDocument document)
{
ArgumentNullException.ThrowIfNull(document);
var parsed = JsonNode.Parse(document.ToJson());
return parsed as JsonObject ?? new JsonObject();
}
public static JsonArray ToJsonArray(BsonArray array)
{
ArgumentNullException.ThrowIfNull(array);
var parsed = JsonNode.Parse(array.ToJson());
return parsed as JsonArray ?? new JsonArray();
}
}

View File

@@ -14,5 +14,6 @@
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="MongoDB.Driver" Version="3.5.0" />
<PackageReference Include="MongoDB.Bson" Version="3.5.0" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,49 @@
using System.IO;
using System.Linq;
using StellaOps.Graph.Indexer.Analytics;
using StellaOps.Graph.Indexer.Ingestion.Sbom;
namespace StellaOps.Graph.Indexer.Tests;
public sealed class GraphOverlayExporterTests
{
[Fact]
public async Task ExportAsync_WritesDeterministicNdjson()
{
var snapshot = GraphAnalyticsTestData.CreateLinearSnapshot();
var engine = new GraphAnalyticsEngine(new GraphAnalyticsOptions { MaxPropagationIterations = 3 });
var result = engine.Compute(snapshot);
var tempDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(tempDir);
try
{
var writer = new FileSystemSnapshotFileWriter(tempDir);
var exporter = new GraphOverlayExporter();
await exporter.ExportAsync(snapshot, result, writer, CancellationToken.None);
var clustersPath = Path.Combine(tempDir, "overlays", "clusters.ndjson");
var centralityPath = Path.Combine(tempDir, "overlays", "centrality.ndjson");
var clusterLines = await File.ReadAllLinesAsync(clustersPath);
var centralityLines = await File.ReadAllLinesAsync(centralityPath);
Assert.Equal(result.Clusters.Length, clusterLines.Length);
Assert.Equal(result.CentralityScores.Length, centralityLines.Length);
// Ensure deterministic ordering by node id
var clusterNodeIds = clusterLines.Select(line => System.Text.Json.JsonDocument.Parse(line).RootElement.GetProperty("node_id").GetString()).ToArray();
var sorted = clusterNodeIds.OrderBy(id => id, StringComparer.Ordinal).ToArray();
Assert.Equal(sorted, clusterNodeIds);
}
finally
{
if (Directory.Exists(tempDir))
{
Directory.Delete(tempDir, recursive: true);
}
}
}
}