up
Some checks failed
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Manifest Integrity / Validate Schema Integrity (push) Has been cancelled
Manifest Integrity / Validate Contract Documents (push) Has been cancelled
Manifest Integrity / Validate Pack Fixtures (push) Has been cancelled
Manifest Integrity / Audit SHA256SUMS Files (push) Has been cancelled
Manifest Integrity / Verify Merkle Roots (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Manifest Integrity / Validate Schema Integrity (push) Has been cancelled
Manifest Integrity / Validate Contract Documents (push) Has been cancelled
Manifest Integrity / Validate Pack Fixtures (push) Has been cancelled
Manifest Integrity / Audit SHA256SUMS Files (push) Has been cancelled
Manifest Integrity / Verify Merkle Roots (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
This commit is contained in:
@@ -1,84 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Encodings.Web;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
|
||||
|
||||
public sealed class MongoGraphDocumentWriter : IGraphDocumentWriter
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new()
|
||||
{
|
||||
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
|
||||
WriteIndented = false
|
||||
};
|
||||
|
||||
private readonly IMongoCollection<BsonDocument> _nodes;
|
||||
private readonly IMongoCollection<BsonDocument> _edges;
|
||||
|
||||
public MongoGraphDocumentWriter(IMongoDatabase database, MongoGraphDocumentWriterOptions? options = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
|
||||
var resolved = options ?? new MongoGraphDocumentWriterOptions();
|
||||
_nodes = database.GetCollection<BsonDocument>(resolved.NodeCollectionName);
|
||||
_edges = database.GetCollection<BsonDocument>(resolved.EdgeCollectionName);
|
||||
}
|
||||
|
||||
public async Task WriteAsync(GraphBuildBatch batch, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(batch);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (batch.Nodes.Length > 0)
|
||||
{
|
||||
var nodeModels = CreateReplaceModels(_nodes, batch.Nodes);
|
||||
if (nodeModels.Count > 0)
|
||||
{
|
||||
await _nodes.BulkWriteAsync(nodeModels, new BulkWriteOptions { IsOrdered = false }, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
if (batch.Edges.Length > 0)
|
||||
{
|
||||
var edgeModels = CreateReplaceModels(_edges, batch.Edges);
|
||||
if (edgeModels.Count > 0)
|
||||
{
|
||||
await _edges.BulkWriteAsync(edgeModels, new BulkWriteOptions { IsOrdered = false }, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static List<WriteModel<BsonDocument>> CreateReplaceModels(IMongoCollection<BsonDocument> collection, IReadOnlyList<JsonObject> documents)
|
||||
{
|
||||
var models = new List<WriteModel<BsonDocument>>(documents.Count);
|
||||
foreach (var document in documents)
|
||||
{
|
||||
if (!document.TryGetPropertyValue("id", out var idNode) || idNode is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var id = idNode.GetValue<string>();
|
||||
var filter = Builders<BsonDocument>.Filter.Eq("id", id);
|
||||
var bsonDocument = ToBsonDocument(document);
|
||||
models.Add(new ReplaceOneModel<BsonDocument>(filter, bsonDocument) { IsUpsert = true });
|
||||
}
|
||||
|
||||
return models;
|
||||
}
|
||||
|
||||
private static BsonDocument ToBsonDocument(JsonObject json)
|
||||
{
|
||||
var jsonString = json.ToJsonString(SerializerOptions);
|
||||
return BsonDocument.Parse(jsonString);
|
||||
}
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
|
||||
|
||||
public sealed class MongoGraphDocumentWriterOptions
|
||||
{
|
||||
public string NodeCollectionName { get; init; } = "graph_nodes";
|
||||
public string EdgeCollectionName { get; init; } = "graph_edges";
|
||||
}
|
||||
@@ -11,6 +11,7 @@ public static class InspectorIngestServiceCollectionExtensions
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
services.TryAddSingleton<Ingestion.Sbom.IGraphDocumentWriter, Ingestion.Sbom.InMemoryGraphDocumentWriter>();
|
||||
services.TryAddSingleton<GraphInspectorTransformer>();
|
||||
services.TryAddSingleton<GraphInspectorProcessor>(provider =>
|
||||
{
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Nodes;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
/// <summary>
|
||||
/// In-memory graph document writer used as a Mongo-free fallback.
|
||||
/// </summary>
|
||||
public sealed class InMemoryGraphDocumentWriter : IGraphDocumentWriter
|
||||
{
|
||||
private readonly ConcurrentBag<GraphBuildBatch> _batches = new();
|
||||
|
||||
public IReadOnlyCollection<GraphBuildBatch> Batches => _batches.ToArray();
|
||||
|
||||
public Task WriteAsync(GraphBuildBatch batch, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(batch);
|
||||
_batches.Add(CloneBatch(batch));
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private static GraphBuildBatch CloneBatch(GraphBuildBatch source)
|
||||
{
|
||||
static JsonObject CloneNode(JsonObject node) => (JsonObject)node.DeepClone();
|
||||
return new GraphBuildBatch(
|
||||
ImmutableArray.CreateRange(source.Nodes.Select(CloneNode)),
|
||||
ImmutableArray.CreateRange(source.Edges.Select(CloneNode)));
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,7 @@ public static class SbomIngestServiceCollectionExtensions
|
||||
services.Configure(configure);
|
||||
}
|
||||
|
||||
services.TryAddSingleton<IGraphDocumentWriter, InMemoryGraphDocumentWriter>();
|
||||
services.TryAddSingleton<SbomIngestTransformer>();
|
||||
services.TryAddSingleton<ISbomIngestMetrics, SbomIngestMetrics>();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user