Add unit tests for SBOM ingestion and transformation
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
- Implement `SbomIngestServiceCollectionExtensionsTests` to verify the SBOM ingestion pipeline exports snapshots correctly. - Create `SbomIngestTransformerTests` to ensure the transformation produces expected nodes and edges, including deduplication of license nodes and normalization of timestamps. - Add `SbomSnapshotExporterTests` to test the export functionality for manifest, adjacency, nodes, and edges. - Introduce `VexOverlayTransformerTests` to validate the transformation of VEX nodes and edges. - Set up project file for the test project with necessary dependencies and configurations. - Include JSON fixture files for testing purposes.
This commit is contained in:
master
@@ -0,0 +1,106 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
|
||||
|
||||
public sealed class AdvisoryLinksetMetrics : IAdvisoryLinksetMetrics, IDisposable
|
||||
{
|
||||
public const string MeterName = "StellaOps.Graph.Indexer";
|
||||
public const string MeterVersion = "1.0.0";
|
||||
|
||||
private readonly Meter _meter;
|
||||
private readonly bool _ownsMeter;
|
||||
private readonly Counter<long> _batchesTotal;
|
||||
private readonly Histogram<double> _batchDurationSeconds;
|
||||
private readonly Counter<long> _nodesTotal;
|
||||
private readonly Counter<long> _edgesTotal;
|
||||
private bool _disposed;
|
||||
|
||||
public AdvisoryLinksetMetrics()
|
||||
: this(null)
|
||||
{
|
||||
}
|
||||
|
||||
public AdvisoryLinksetMetrics(Meter? meter)
|
||||
{
|
||||
_meter = meter ?? new Meter(MeterName, MeterVersion);
|
||||
_ownsMeter = meter is null;
|
||||
|
||||
_batchesTotal = _meter.CreateCounter<long>(
|
||||
name: "graph_advisory_ingest_batches_total",
|
||||
unit: "count",
|
||||
description: "Advisory linkset ingest batches processed grouped by source, tenant, and result.");
|
||||
|
||||
_batchDurationSeconds = _meter.CreateHistogram<double>(
|
||||
name: "graph_advisory_ingest_duration_seconds",
|
||||
unit: "s",
|
||||
description: "Latency to transform and persist advisory linkset batches grouped by source, tenant, and result.");
|
||||
|
||||
_nodesTotal = _meter.CreateCounter<long>(
|
||||
name: "graph_advisory_ingest_nodes_total",
|
||||
unit: "count",
|
||||
description: "Advisory nodes produced by linkset ingest grouped by source and tenant.");
|
||||
|
||||
_edgesTotal = _meter.CreateCounter<long>(
|
||||
name: "graph_advisory_ingest_edges_total",
|
||||
unit: "count",
|
||||
description: "Affected_by edges produced by linkset ingest grouped by source and tenant.");
|
||||
}
|
||||
|
||||
public void RecordBatch(string source, string tenant, int nodeCount, int edgeCount, TimeSpan duration, bool success)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(source);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
|
||||
|
||||
var normalizedDurationSeconds = Math.Max(duration.TotalSeconds, 0d);
|
||||
var resultTag = success ? "success" : "failure";
|
||||
|
||||
var tags = new[]
|
||||
{
|
||||
new KeyValuePair<string, object?>("source", source),
|
||||
new KeyValuePair<string, object?>("tenant", tenant),
|
||||
new KeyValuePair<string, object?>("result", resultTag)
|
||||
};
|
||||
|
||||
_batchesTotal.Add(1, tags);
|
||||
_batchDurationSeconds.Record(normalizedDurationSeconds, tags);
|
||||
|
||||
if (!success)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var volumeTags = new[]
|
||||
{
|
||||
new KeyValuePair<string, object?>("source", source),
|
||||
new KeyValuePair<string, object?>("tenant", tenant)
|
||||
};
|
||||
|
||||
if (nodeCount > 0)
|
||||
{
|
||||
_nodesTotal.Add(nodeCount, volumeTags);
|
||||
}
|
||||
|
||||
if (edgeCount > 0)
|
||||
{
|
||||
_edgesTotal.Add(edgeCount, volumeTags);
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (_ownsMeter)
|
||||
{
|
||||
_meter.Dispose();
|
||||
}
|
||||
|
||||
_disposed = true;
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
using System.Diagnostics;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
|
||||
|
||||
public sealed class AdvisoryLinksetProcessor
|
||||
{
|
||||
private readonly AdvisoryLinksetTransformer _transformer;
|
||||
private readonly IGraphDocumentWriter _writer;
|
||||
private readonly IAdvisoryLinksetMetrics _metrics;
|
||||
private readonly ILogger<AdvisoryLinksetProcessor> _logger;
|
||||
|
||||
public AdvisoryLinksetProcessor(
|
||||
AdvisoryLinksetTransformer transformer,
|
||||
IGraphDocumentWriter writer,
|
||||
IAdvisoryLinksetMetrics metrics,
|
||||
ILogger<AdvisoryLinksetProcessor> logger)
|
||||
{
|
||||
_transformer = transformer ?? throw new ArgumentNullException(nameof(transformer));
|
||||
_writer = writer ?? throw new ArgumentNullException(nameof(writer));
|
||||
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task ProcessAsync(AdvisoryLinksetSnapshot snapshot, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(snapshot);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
GraphBuildBatch batch;
|
||||
|
||||
try
|
||||
{
|
||||
batch = _transformer.Transform(snapshot);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, 0, 0, stopwatch.Elapsed, success: false);
|
||||
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"graph-indexer: failed to transform advisory linkset {LinksetDigest} for tenant {Tenant}",
|
||||
snapshot.LinksetDigest,
|
||||
snapshot.Tenant);
|
||||
|
||||
throw;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
await _writer.WriteAsync(batch, cancellationToken).ConfigureAwait(false);
|
||||
stopwatch.Stop();
|
||||
|
||||
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, batch.Nodes.Length, batch.Edges.Length, stopwatch.Elapsed, success: true);
|
||||
|
||||
_logger.LogInformation(
|
||||
"graph-indexer: indexed advisory linkset {LinksetDigest} for tenant {Tenant} with {NodeCount} nodes and {EdgeCount} edges in {DurationMs:F2} ms",
|
||||
snapshot.LinksetDigest,
|
||||
snapshot.Tenant,
|
||||
batch.Nodes.Length,
|
||||
batch.Edges.Length,
|
||||
stopwatch.Elapsed.TotalMilliseconds);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, batch.Nodes.Length, batch.Edges.Length, stopwatch.Elapsed, success: false);
|
||||
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"graph-indexer: failed to persist advisory linkset {LinksetDigest} for tenant {Tenant}",
|
||||
snapshot.LinksetDigest,
|
||||
snapshot.Tenant);
|
||||
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
|
||||
|
||||
public sealed class AdvisoryLinksetSnapshot
|
||||
{
|
||||
[JsonPropertyName("tenant")]
|
||||
public string Tenant { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("linksetDigest")]
|
||||
public string LinksetDigest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long EventOffset { get; init; } = 0;
|
||||
|
||||
[JsonPropertyName("advisory")]
|
||||
public AdvisoryDetails Advisory { get; init; } = new();
|
||||
|
||||
[JsonPropertyName("components")]
|
||||
public IReadOnlyList<AdvisoryComponentImpact> Components { get; init; }
|
||||
= Array.Empty<AdvisoryComponentImpact>();
|
||||
}
|
||||
|
||||
public sealed class AdvisoryDetails
|
||||
{
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("advisorySource")]
|
||||
public string AdvisorySource { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("advisoryId")]
|
||||
public string AdvisoryId { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("severity")]
|
||||
public string Severity { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("publishedAt")]
|
||||
public DateTimeOffset PublishedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("contentHash")]
|
||||
public string ContentHash { get; init; } = string.Empty;
|
||||
}
|
||||
|
||||
public sealed class AdvisoryComponentImpact
|
||||
{
|
||||
[JsonPropertyName("purl")]
|
||||
public string ComponentPurl { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("sourceType")]
|
||||
public string ComponentSourceType { get; init; } = "inventory";
|
||||
|
||||
[JsonPropertyName("evidenceDigest")]
|
||||
public string EvidenceDigest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("matchedVersions")]
|
||||
public IReadOnlyList<string> MatchedVersions { get; init; }
|
||||
= Array.Empty<string>();
|
||||
|
||||
[JsonPropertyName("cvss")]
|
||||
public double? Cvss { get; init; }
|
||||
= null;
|
||||
|
||||
[JsonPropertyName("confidence")]
|
||||
public double? Confidence { get; init; }
|
||||
= null;
|
||||
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long EventOffset { get; init; } = 0;
|
||||
|
||||
[JsonPropertyName("sbomDigest")]
|
||||
public string? SbomDigest { get; init; }
|
||||
= null;
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Text.Json.Nodes;
|
||||
using StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
using StellaOps.Graph.Indexer.Schema;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
|
||||
|
||||
public sealed class AdvisoryLinksetTransformer
|
||||
{
|
||||
private const string AdvisoryNodeKind = "advisory";
|
||||
private const string ComponentNodeKind = "component";
|
||||
private const string AffectedByEdgeKind = "AFFECTED_BY";
|
||||
|
||||
public GraphBuildBatch Transform(AdvisoryLinksetSnapshot snapshot)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(snapshot);
|
||||
|
||||
var nodes = new List<JsonObject>();
|
||||
var edgesById = new Dictionary<string, JsonObject>(StringComparer.Ordinal);
|
||||
|
||||
var advisoryNode = CreateAdvisoryNode(snapshot);
|
||||
nodes.Add(advisoryNode);
|
||||
|
||||
foreach (var component in snapshot.Components ?? Array.Empty<AdvisoryComponentImpact>())
|
||||
{
|
||||
if (component is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var edge = CreateAffectedByEdge(snapshot, advisoryNode, component);
|
||||
edgesById[edge["id"]!.GetValue<string>()] = edge;
|
||||
}
|
||||
|
||||
var orderedNodes = nodes
|
||||
.OrderBy(node => node["kind"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ThenBy(node => node["id"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ToImmutableArray();
|
||||
|
||||
var orderedEdges = edgesById.Values
|
||||
.OrderBy(edge => edge["kind"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ThenBy(edge => edge["id"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ToImmutableArray();
|
||||
|
||||
return new GraphBuildBatch(orderedNodes, orderedEdges);
|
||||
}
|
||||
|
||||
private static JsonObject CreateAdvisoryNode(AdvisoryLinksetSnapshot snapshot)
|
||||
{
|
||||
var details = snapshot.Advisory ?? new AdvisoryDetails();
|
||||
var advisorySource = (details.AdvisorySource ?? string.Empty).Trim();
|
||||
var advisoryId = (details.AdvisoryId ?? string.Empty).Trim();
|
||||
var severity = (details.Severity ?? string.Empty).Trim();
|
||||
var contentHash = (details.ContentHash ?? string.Empty).Trim();
|
||||
var linksetDigest = (snapshot.LinksetDigest ?? string.Empty).Trim();
|
||||
var provenanceSource = ResolveSource(details.Source, snapshot.Source);
|
||||
|
||||
var validFrom = details.PublishedAt == DateTimeOffset.UnixEpoch
|
||||
? snapshot.CollectedAt
|
||||
: details.PublishedAt;
|
||||
|
||||
var attributes = new JsonObject
|
||||
{
|
||||
["advisory_source"] = advisorySource,
|
||||
["advisory_id"] = advisoryId,
|
||||
["severity"] = severity,
|
||||
["published_at"] = GraphTimestamp.Format(validFrom),
|
||||
["content_hash"] = contentHash,
|
||||
["linkset_digest"] = linksetDigest
|
||||
};
|
||||
|
||||
var canonicalKey = new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["advisory_source"] = advisorySource,
|
||||
["advisory_id"] = advisoryId,
|
||||
["content_hash"] = contentHash
|
||||
};
|
||||
|
||||
var node = GraphDocumentFactory.CreateNode(new GraphNodeSpec(
|
||||
Tenant: snapshot.Tenant,
|
||||
Kind: AdvisoryNodeKind,
|
||||
CanonicalKey: canonicalKey,
|
||||
Attributes: attributes,
|
||||
Provenance: new GraphProvenanceSpec(provenanceSource, snapshot.CollectedAt, null, snapshot.EventOffset),
|
||||
ValidFrom: validFrom,
|
||||
ValidTo: null));
|
||||
|
||||
var provenance = node["provenance"]!.AsObject();
|
||||
var sourceNode = provenance["source"]!.DeepClone();
|
||||
var collectedAtNode = provenance["collected_at"]!.DeepClone();
|
||||
var eventOffsetNode = provenance.ContainsKey("event_offset")
|
||||
? provenance["event_offset"]!.DeepClone()
|
||||
: null;
|
||||
|
||||
var reorderedProvenance = new JsonObject
|
||||
{
|
||||
["source"] = sourceNode,
|
||||
["collected_at"] = collectedAtNode,
|
||||
["sbom_digest"] = null
|
||||
};
|
||||
|
||||
if (eventOffsetNode is not null)
|
||||
{
|
||||
reorderedProvenance["event_offset"] = eventOffsetNode;
|
||||
}
|
||||
|
||||
node["provenance"] = reorderedProvenance;
|
||||
|
||||
node.Remove("hash");
|
||||
node["hash"] = GraphIdentity.ComputeDocumentHash(node);
|
||||
|
||||
return node;
|
||||
}
|
||||
|
||||
private static JsonObject CreateAffectedByEdge(
|
||||
AdvisoryLinksetSnapshot snapshot,
|
||||
JsonObject advisoryNode,
|
||||
AdvisoryComponentImpact component)
|
||||
{
|
||||
var advisoryNodeId = advisoryNode["id"]!.GetValue<string>();
|
||||
var componentSourceType = string.IsNullOrWhiteSpace(component.ComponentSourceType)
|
||||
? "inventory"
|
||||
: component.ComponentSourceType.Trim();
|
||||
var componentPurl = (component.ComponentPurl ?? string.Empty).Trim();
|
||||
var linksetDigest = (snapshot.LinksetDigest ?? string.Empty).Trim();
|
||||
|
||||
var componentIdentity = new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["purl"] = componentPurl,
|
||||
["source_type"] = componentSourceType
|
||||
};
|
||||
|
||||
var componentNodeId = GraphIdentity.ComputeNodeId(snapshot.Tenant, ComponentNodeKind, componentIdentity);
|
||||
|
||||
var canonicalKey = new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["component_node_id"] = componentNodeId,
|
||||
["advisory_node_id"] = advisoryNodeId,
|
||||
["linkset_digest"] = linksetDigest
|
||||
};
|
||||
|
||||
var attributes = new JsonObject
|
||||
{
|
||||
["evidence_digest"] = component.EvidenceDigest?.Trim() ?? string.Empty,
|
||||
["matched_versions"] = CreateJsonArray(component.MatchedVersions ?? Array.Empty<string>())
|
||||
};
|
||||
|
||||
if (component.Cvss is { } cvss)
|
||||
{
|
||||
attributes["cvss"] = cvss;
|
||||
}
|
||||
|
||||
if (component.Confidence is { } confidence)
|
||||
{
|
||||
attributes["confidence"] = confidence;
|
||||
}
|
||||
|
||||
var collectedAt = component.CollectedAt == DateTimeOffset.UnixEpoch
|
||||
? snapshot.CollectedAt
|
||||
: component.CollectedAt;
|
||||
|
||||
var eventOffset = component.EventOffset != 0 ? component.EventOffset : snapshot.EventOffset;
|
||||
var source = ResolveSource(component.Source, snapshot.Source);
|
||||
|
||||
return GraphDocumentFactory.CreateEdge(new GraphEdgeSpec(
|
||||
Tenant: snapshot.Tenant,
|
||||
Kind: AffectedByEdgeKind,
|
||||
CanonicalKey: canonicalKey,
|
||||
Attributes: attributes,
|
||||
Provenance: new GraphProvenanceSpec(source, collectedAt, component.SbomDigest, eventOffset),
|
||||
ValidFrom: collectedAt,
|
||||
ValidTo: null));
|
||||
}
|
||||
|
||||
private static JsonArray CreateJsonArray(IEnumerable<string> values)
|
||||
{
|
||||
var array = new JsonArray();
|
||||
foreach (var value in values
|
||||
.Where(v => !string.IsNullOrWhiteSpace(v))
|
||||
.Select(v => v.Trim())
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.OrderBy(v => v, StringComparer.Ordinal))
|
||||
{
|
||||
array.Add(value);
|
||||
}
|
||||
|
||||
return array;
|
||||
}
|
||||
|
||||
private static string ResolveSource(params string?[] candidates)
|
||||
{
|
||||
foreach (var candidate in candidates)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(candidate))
|
||||
{
|
||||
return candidate.Trim();
|
||||
}
|
||||
}
|
||||
|
||||
return string.Empty;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
|
||||
|
||||
public interface IAdvisoryLinksetMetrics
|
||||
{
|
||||
void RecordBatch(string source, string tenant, int nodeCount, int edgeCount, TimeSpan duration, bool success);
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Encodings.Web;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
|
||||
|
||||
public sealed class MongoGraphDocumentWriter : IGraphDocumentWriter
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new()
|
||||
{
|
||||
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
|
||||
WriteIndented = false
|
||||
};
|
||||
|
||||
private readonly IMongoCollection<BsonDocument> _nodes;
|
||||
private readonly IMongoCollection<BsonDocument> _edges;
|
||||
|
||||
public MongoGraphDocumentWriter(IMongoDatabase database, MongoGraphDocumentWriterOptions? options = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
|
||||
var resolved = options ?? new MongoGraphDocumentWriterOptions();
|
||||
_nodes = database.GetCollection<BsonDocument>(resolved.NodeCollectionName);
|
||||
_edges = database.GetCollection<BsonDocument>(resolved.EdgeCollectionName);
|
||||
}
|
||||
|
||||
public async Task WriteAsync(GraphBuildBatch batch, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(batch);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (batch.Nodes.Length > 0)
|
||||
{
|
||||
var nodeModels = CreateReplaceModels(_nodes, batch.Nodes);
|
||||
if (nodeModels.Count > 0)
|
||||
{
|
||||
await _nodes.BulkWriteAsync(nodeModels, new BulkWriteOptions { IsOrdered = false }, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
if (batch.Edges.Length > 0)
|
||||
{
|
||||
var edgeModels = CreateReplaceModels(_edges, batch.Edges);
|
||||
if (edgeModels.Count > 0)
|
||||
{
|
||||
await _edges.BulkWriteAsync(edgeModels, new BulkWriteOptions { IsOrdered = false }, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static List<WriteModel<BsonDocument>> CreateReplaceModels(IMongoCollection<BsonDocument> collection, IReadOnlyList<JsonObject> documents)
|
||||
{
|
||||
var models = new List<WriteModel<BsonDocument>>(documents.Count);
|
||||
foreach (var document in documents)
|
||||
{
|
||||
if (!document.TryGetPropertyValue("id", out var idNode) || idNode is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var id = idNode.GetValue<string>();
|
||||
var filter = Builders<BsonDocument>.Filter.Eq("id", id);
|
||||
var bsonDocument = ToBsonDocument(document);
|
||||
models.Add(new ReplaceOneModel<BsonDocument>(filter, bsonDocument) { IsUpsert = true });
|
||||
}
|
||||
|
||||
return models;
|
||||
}
|
||||
|
||||
private static BsonDocument ToBsonDocument(JsonObject json)
|
||||
{
|
||||
var jsonString = json.ToJsonString(SerializerOptions);
|
||||
return BsonDocument.Parse(jsonString);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
|
||||
|
||||
public sealed class MongoGraphDocumentWriterOptions
|
||||
{
|
||||
public string NodeCollectionName { get; init; } = "graph_nodes";
|
||||
public string EdgeCollectionName { get; init; } = "graph_edges";
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Policy;
|
||||
|
||||
public interface IPolicyOverlayMetrics
|
||||
{
|
||||
void RecordBatch(string source, string tenant, int nodeCount, int edgeCount, TimeSpan duration, bool success);
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Policy;
|
||||
|
||||
public sealed class PolicyOverlayMetrics : IPolicyOverlayMetrics, IDisposable
|
||||
{
|
||||
public const string MeterName = "StellaOps.Graph.Indexer";
|
||||
public const string MeterVersion = "1.0.0";
|
||||
|
||||
private const string BatchesTotalName = "graph_policy_overlay_batches_total";
|
||||
private const string BatchDurationSecondsName = "graph_policy_overlay_duration_seconds";
|
||||
private const string NodesTotalName = "graph_policy_overlay_nodes_total";
|
||||
private const string EdgesTotalName = "graph_policy_overlay_edges_total";
|
||||
|
||||
private const string UnitCount = "count";
|
||||
private const string UnitSeconds = "s";
|
||||
|
||||
private readonly Meter _meter;
|
||||
private readonly bool _ownsMeter;
|
||||
private readonly Counter<long> _batchesTotal;
|
||||
private readonly Histogram<double> _batchDurationSeconds;
|
||||
private readonly Counter<long> _nodesTotal;
|
||||
private readonly Counter<long> _edgesTotal;
|
||||
private bool _disposed;
|
||||
|
||||
public PolicyOverlayMetrics()
|
||||
: this(null)
|
||||
{
|
||||
}
|
||||
|
||||
public PolicyOverlayMetrics(Meter? meter)
|
||||
{
|
||||
_meter = meter ?? new Meter(MeterName, MeterVersion);
|
||||
_ownsMeter = meter is null;
|
||||
|
||||
_batchesTotal = _meter.CreateCounter<long>(
|
||||
name: BatchesTotalName,
|
||||
unit: UnitCount,
|
||||
description: "Policy overlay batches processed grouped by source, tenant, and result.");
|
||||
|
||||
_batchDurationSeconds = _meter.CreateHistogram<double>(
|
||||
name: BatchDurationSecondsName,
|
||||
unit: UnitSeconds,
|
||||
description: "Latency to transform and persist policy overlay batches grouped by source, tenant, and result.");
|
||||
|
||||
_nodesTotal = _meter.CreateCounter<long>(
|
||||
name: NodesTotalName,
|
||||
unit: UnitCount,
|
||||
description: "Policy overlay nodes produced grouped by source and tenant.");
|
||||
|
||||
_edgesTotal = _meter.CreateCounter<long>(
|
||||
name: EdgesTotalName,
|
||||
unit: UnitCount,
|
||||
description: "GOVERNS_WITH edges produced grouped by source and tenant.");
|
||||
}
|
||||
|
||||
public void RecordBatch(string source, string tenant, int nodeCount, int edgeCount, TimeSpan duration, bool success)
|
||||
{
|
||||
ThrowIfDisposed();
|
||||
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(source);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
|
||||
|
||||
var normalizedDuration = Math.Max(duration.TotalSeconds, 0d);
|
||||
var resultTag = success ? "success" : "failure";
|
||||
|
||||
var tags = new[]
|
||||
{
|
||||
new KeyValuePair<string, object?>("source", source),
|
||||
new KeyValuePair<string, object?>("tenant", tenant),
|
||||
new KeyValuePair<string, object?>("result", resultTag)
|
||||
};
|
||||
|
||||
_batchesTotal.Add(1, tags);
|
||||
_batchDurationSeconds.Record(normalizedDuration, tags);
|
||||
|
||||
if (!success)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var volumeTags = new[]
|
||||
{
|
||||
new KeyValuePair<string, object?>("source", source),
|
||||
new KeyValuePair<string, object?>("tenant", tenant)
|
||||
};
|
||||
|
||||
if (nodeCount > 0)
|
||||
{
|
||||
_nodesTotal.Add(nodeCount, volumeTags);
|
||||
}
|
||||
|
||||
if (edgeCount > 0)
|
||||
{
|
||||
_edgesTotal.Add(edgeCount, volumeTags);
|
||||
}
|
||||
}
|
||||
|
||||
private void ThrowIfDisposed()
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
throw new ObjectDisposedException(nameof(PolicyOverlayMetrics));
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (_ownsMeter)
|
||||
{
|
||||
_meter.Dispose();
|
||||
}
|
||||
|
||||
_disposed = true;
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
using System.Diagnostics;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Policy;
|
||||
|
||||
public sealed class PolicyOverlayProcessor
|
||||
{
|
||||
private readonly PolicyOverlayTransformer _transformer;
|
||||
private readonly IGraphDocumentWriter _writer;
|
||||
private readonly IPolicyOverlayMetrics _metrics;
|
||||
private readonly ILogger<PolicyOverlayProcessor> _logger;
|
||||
|
||||
public PolicyOverlayProcessor(
|
||||
PolicyOverlayTransformer transformer,
|
||||
IGraphDocumentWriter writer,
|
||||
IPolicyOverlayMetrics metrics,
|
||||
ILogger<PolicyOverlayProcessor> logger)
|
||||
{
|
||||
_transformer = transformer ?? throw new ArgumentNullException(nameof(transformer));
|
||||
_writer = writer ?? throw new ArgumentNullException(nameof(writer));
|
||||
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task ProcessAsync(PolicyOverlaySnapshot snapshot, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(snapshot);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
GraphBuildBatch batch;
|
||||
|
||||
try
|
||||
{
|
||||
batch = _transformer.Transform(snapshot);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, 0, 0, stopwatch.Elapsed, success: false);
|
||||
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"graph-indexer: failed to transform policy overlay {PolicyPackDigest} for tenant {Tenant}",
|
||||
snapshot.Policy?.PolicyPackDigest ?? string.Empty,
|
||||
snapshot.Tenant);
|
||||
|
||||
throw;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
await _writer.WriteAsync(batch, cancellationToken).ConfigureAwait(false);
|
||||
stopwatch.Stop();
|
||||
|
||||
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, batch.Nodes.Length, batch.Edges.Length, stopwatch.Elapsed, success: true);
|
||||
|
||||
_logger.LogInformation(
|
||||
"graph-indexer: indexed policy overlay {PolicyPackDigest} (effective {EffectiveFrom}) for tenant {Tenant} with {NodeCount} nodes and {EdgeCount} edges in {DurationMs:F2} ms",
|
||||
snapshot.Policy?.PolicyPackDigest ?? string.Empty,
|
||||
(snapshot.Policy?.EffectiveFrom ?? snapshot.CollectedAt).ToUniversalTime(),
|
||||
snapshot.Tenant,
|
||||
batch.Nodes.Length,
|
||||
batch.Edges.Length,
|
||||
stopwatch.Elapsed.TotalMilliseconds);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, batch.Nodes.Length, batch.Edges.Length, stopwatch.Elapsed, success: false);
|
||||
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"graph-indexer: failed to persist policy overlay {PolicyPackDigest} for tenant {Tenant}",
|
||||
snapshot.Policy?.PolicyPackDigest ?? string.Empty,
|
||||
snapshot.Tenant);
|
||||
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Policy;
|
||||
|
||||
public sealed class PolicyOverlaySnapshot
|
||||
{
|
||||
[JsonPropertyName("tenant")]
|
||||
public string Tenant { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long EventOffset { get; init; }
|
||||
|
||||
[JsonPropertyName("policy")]
|
||||
public PolicyVersionDetails Policy { get; init; } = new();
|
||||
|
||||
[JsonPropertyName("evaluations")]
|
||||
public IReadOnlyList<PolicyEvaluation> Evaluations { get; init; }
|
||||
= Array.Empty<PolicyEvaluation>();
|
||||
}
|
||||
|
||||
public sealed class PolicyVersionDetails
|
||||
{
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("policyPackDigest")]
|
||||
public string PolicyPackDigest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("policyName")]
|
||||
public string PolicyName { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("effectiveFrom")]
|
||||
public DateTimeOffset EffectiveFrom { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("expiresAt")]
|
||||
public DateTimeOffset ExpiresAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("explainHash")]
|
||||
public string ExplainHash { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long EventOffset { get; init; }
|
||||
}
|
||||
|
||||
public sealed class PolicyEvaluation
|
||||
{
|
||||
[JsonPropertyName("componentPurl")]
|
||||
public string ComponentPurl { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("componentSourceType")]
|
||||
public string ComponentSourceType { get; init; } = "inventory";
|
||||
|
||||
[JsonPropertyName("findingExplainHash")]
|
||||
public string FindingExplainHash { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("explainHash")]
|
||||
public string? ExplainHash { get; init; }
|
||||
|
||||
[JsonPropertyName("policyRuleId")]
|
||||
public string PolicyRuleId { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("verdict")]
|
||||
public string Verdict { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("evaluationTimestamp")]
|
||||
public DateTimeOffset EvaluationTimestamp { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("sbomDigest")]
|
||||
public string? SbomDigest { get; init; }
|
||||
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long EventOffset { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Text.Json.Nodes;
|
||||
using StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
using StellaOps.Graph.Indexer.Schema;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Policy;
|
||||
|
||||
public sealed class PolicyOverlayTransformer
|
||||
{
|
||||
private const string PolicyNodeKind = "policy_version";
|
||||
private const string ComponentNodeKind = "component";
|
||||
private const string GovernsWithEdgeKind = "GOVERNS_WITH";
|
||||
|
||||
public GraphBuildBatch Transform(PolicyOverlaySnapshot snapshot)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(snapshot);
|
||||
|
||||
var tenant = snapshot.Tenant?.Trim() ?? string.Empty;
|
||||
var nodes = new List<JsonObject>();
|
||||
var edges = new List<JsonObject>();
|
||||
var seenEdgeIds = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
var policyDetails = snapshot.Policy ?? new PolicyVersionDetails();
|
||||
|
||||
var policyNode = CreatePolicyNode(tenant, snapshot, policyDetails);
|
||||
nodes.Add(policyNode);
|
||||
|
||||
var policyNodeId = policyNode["id"]!.GetValue<string>();
|
||||
|
||||
foreach (var evaluation in snapshot.Evaluations ?? Array.Empty<PolicyEvaluation>())
|
||||
{
|
||||
if (!IsEvaluationCandidate(evaluation))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var edge = CreateGovernsWithEdge(
|
||||
tenant,
|
||||
snapshot,
|
||||
policyDetails,
|
||||
policyNodeId,
|
||||
evaluation!);
|
||||
|
||||
var edgeId = edge["id"]!.GetValue<string>();
|
||||
if (seenEdgeIds.Add(edgeId))
|
||||
{
|
||||
edges.Add(edge);
|
||||
}
|
||||
}
|
||||
|
||||
return new GraphBuildBatch(
|
||||
nodes
|
||||
.OrderBy(node => node["kind"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ThenBy(node => node["id"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ToImmutableArray(),
|
||||
edges
|
||||
.OrderBy(edge => edge["kind"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ThenBy(edge => edge["id"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ToImmutableArray());
|
||||
}
|
||||
|
||||
private static bool IsEvaluationCandidate(PolicyEvaluation? evaluation)
|
||||
{
|
||||
return evaluation is not null
|
||||
&& !string.IsNullOrWhiteSpace(evaluation.ComponentPurl)
|
||||
&& !string.IsNullOrWhiteSpace(evaluation.FindingExplainHash);
|
||||
}
|
||||
|
||||
private static JsonObject CreatePolicyNode(
|
||||
string tenant,
|
||||
PolicyOverlaySnapshot snapshot,
|
||||
PolicyVersionDetails policy)
|
||||
{
|
||||
var policyPackDigest = policy.PolicyPackDigest?.Trim() ?? string.Empty;
|
||||
var policyName = policy.PolicyName?.Trim() ?? string.Empty;
|
||||
var explainHash = policy.ExplainHash?.Trim() ?? string.Empty;
|
||||
|
||||
var effectiveFrom = policy.EffectiveFrom == DateTimeOffset.UnixEpoch
|
||||
? snapshot.CollectedAt
|
||||
: policy.EffectiveFrom;
|
||||
|
||||
var expiresAt = policy.ExpiresAt == DateTimeOffset.UnixEpoch
|
||||
? (DateTimeOffset?)null
|
||||
: policy.ExpiresAt;
|
||||
|
||||
var policyCollectedAt = policy.CollectedAt == DateTimeOffset.UnixEpoch
|
||||
? snapshot.CollectedAt
|
||||
: policy.CollectedAt;
|
||||
|
||||
var eventOffset = policy.EventOffset != 0 ? policy.EventOffset : snapshot.EventOffset;
|
||||
var source = ResolveSource(policy.Source, snapshot.Source);
|
||||
|
||||
var canonicalKey = new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = tenant,
|
||||
["policy_pack_digest"] = policyPackDigest,
|
||||
["effective_from"] = GraphTimestamp.Format(effectiveFrom)
|
||||
};
|
||||
|
||||
var attributes = new JsonObject
|
||||
{
|
||||
["policy_pack_digest"] = policyPackDigest,
|
||||
["policy_name"] = policyName,
|
||||
["effective_from"] = GraphTimestamp.Format(effectiveFrom),
|
||||
["expires_at"] = expiresAt is null ? null : GraphTimestamp.Format(expiresAt.Value),
|
||||
["explain_hash"] = explainHash
|
||||
};
|
||||
|
||||
var node = GraphDocumentFactory.CreateNode(new GraphNodeSpec(
|
||||
Tenant: tenant,
|
||||
Kind: PolicyNodeKind,
|
||||
CanonicalKey: canonicalKey,
|
||||
Attributes: attributes,
|
||||
Provenance: new GraphProvenanceSpec(source, policyCollectedAt, SbomDigest: null, EventOffset: eventOffset),
|
||||
ValidFrom: effectiveFrom,
|
||||
ValidTo: expiresAt));
|
||||
|
||||
NormalizeOverlayProvenance(node);
|
||||
|
||||
return node;
|
||||
}
|
||||
|
||||
private static JsonObject CreateGovernsWithEdge(
|
||||
string tenant,
|
||||
PolicyOverlaySnapshot snapshot,
|
||||
PolicyVersionDetails policy,
|
||||
string policyNodeId,
|
||||
PolicyEvaluation evaluation)
|
||||
{
|
||||
var componentSourceType = string.IsNullOrWhiteSpace(evaluation.ComponentSourceType)
|
||||
? "inventory"
|
||||
: evaluation.ComponentSourceType.Trim();
|
||||
|
||||
var componentIdentity = new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = tenant,
|
||||
["purl"] = evaluation.ComponentPurl.Trim(),
|
||||
["source_type"] = componentSourceType
|
||||
};
|
||||
|
||||
var componentNodeId = GraphIdentity.ComputeNodeId(tenant, ComponentNodeKind, componentIdentity);
|
||||
|
||||
var findingExplainHash = evaluation.FindingExplainHash.Trim();
|
||||
var edgeCanonicalKey = new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = tenant,
|
||||
["policy_node_id"] = policyNodeId,
|
||||
["component_node_id"] = componentNodeId,
|
||||
["finding_explain_hash"] = findingExplainHash
|
||||
};
|
||||
|
||||
var evaluationTimestamp = evaluation.EvaluationTimestamp == DateTimeOffset.UnixEpoch
|
||||
? snapshot.CollectedAt
|
||||
: evaluation.EvaluationTimestamp;
|
||||
|
||||
var explainHash = !string.IsNullOrWhiteSpace(evaluation.ExplainHash)
|
||||
? evaluation.ExplainHash.Trim()
|
||||
: !string.IsNullOrWhiteSpace(policy.ExplainHash)
|
||||
? policy.ExplainHash.Trim()
|
||||
: findingExplainHash;
|
||||
|
||||
var attributes = new JsonObject
|
||||
{
|
||||
["verdict"] = evaluation.Verdict?.Trim() ?? string.Empty,
|
||||
["explain_hash"] = explainHash,
|
||||
["policy_rule_id"] = evaluation.PolicyRuleId?.Trim() ?? string.Empty,
|
||||
["evaluation_timestamp"] = GraphTimestamp.Format(evaluationTimestamp)
|
||||
};
|
||||
|
||||
var collectedAt = evaluation.CollectedAt == DateTimeOffset.UnixEpoch
|
||||
? snapshot.CollectedAt
|
||||
: evaluation.CollectedAt;
|
||||
|
||||
var eventOffset = evaluation.EventOffset != 0 ? evaluation.EventOffset : snapshot.EventOffset;
|
||||
var source = ResolveSource(evaluation.Source, policy.Source, snapshot.Source);
|
||||
|
||||
return GraphDocumentFactory.CreateEdge(new GraphEdgeSpec(
|
||||
Tenant: tenant,
|
||||
Kind: GovernsWithEdgeKind,
|
||||
CanonicalKey: edgeCanonicalKey,
|
||||
Attributes: attributes,
|
||||
Provenance: new GraphProvenanceSpec(
|
||||
source,
|
||||
collectedAt,
|
||||
NormalizeOptional(evaluation.SbomDigest),
|
||||
eventOffset),
|
||||
ValidFrom: evaluationTimestamp,
|
||||
ValidTo: null));
|
||||
}
|
||||
|
||||
private static void NormalizeOverlayProvenance(JsonObject node)
|
||||
{
|
||||
var provenance = node["provenance"]!.AsObject();
|
||||
var sourceNode = provenance["source"]!.DeepClone();
|
||||
var collectedAtNode = provenance["collected_at"]!.DeepClone();
|
||||
var eventOffsetNode = provenance.ContainsKey("event_offset")
|
||||
? provenance["event_offset"]!.DeepClone()
|
||||
: null;
|
||||
|
||||
var normalized = new JsonObject
|
||||
{
|
||||
["source"] = sourceNode,
|
||||
["collected_at"] = collectedAtNode,
|
||||
["sbom_digest"] = null
|
||||
};
|
||||
|
||||
if (eventOffsetNode is not null)
|
||||
{
|
||||
normalized["event_offset"] = eventOffsetNode;
|
||||
}
|
||||
|
||||
node["provenance"] = normalized;
|
||||
node.Remove("hash");
|
||||
node["hash"] = GraphIdentity.ComputeDocumentHash(node);
|
||||
}
|
||||
|
||||
private static string ResolveSource(params string?[] candidates)
|
||||
{
|
||||
foreach (var candidate in candidates)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(candidate))
|
||||
{
|
||||
return candidate.Trim();
|
||||
}
|
||||
}
|
||||
|
||||
return "policy.engine.v1";
|
||||
}
|
||||
|
||||
private static string? NormalizeOptional(string? value)
|
||||
{
|
||||
return string.IsNullOrWhiteSpace(value) ? null : value.Trim();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
using System.Text;
|
||||
using System.Text.Json.Nodes;
|
||||
using StellaOps.Graph.Indexer.Documents;
|
||||
using StellaOps.Graph.Indexer.Schema;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
public sealed class FileSystemSnapshotFileWriter : ISnapshotFileWriter
|
||||
{
|
||||
private readonly string _root;
|
||||
|
||||
public FileSystemSnapshotFileWriter(string rootDirectory)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(rootDirectory))
|
||||
{
|
||||
throw new ArgumentException("Snapshot root directory must be provided.", nameof(rootDirectory));
|
||||
}
|
||||
|
||||
_root = Path.GetFullPath(rootDirectory);
|
||||
}
|
||||
|
||||
public async Task WriteJsonAsync(string relativePath, JsonObject content, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(content);
|
||||
|
||||
var fullPath = ResolvePath(relativePath);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!);
|
||||
|
||||
var canonicalBytes = CanonicalJson.ToCanonicalUtf8Bytes(content);
|
||||
await File.WriteAllBytesAsync(fullPath, canonicalBytes, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task WriteJsonLinesAsync(string relativePath, IEnumerable<JsonObject> items, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(items);
|
||||
|
||||
var fullPath = ResolvePath(relativePath);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!);
|
||||
|
||||
await using var stream = new FileStream(fullPath, FileMode.Create, FileAccess.Write, FileShare.None, 4096, useAsync: true);
|
||||
await using var writer = new StreamWriter(stream, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false));
|
||||
|
||||
foreach (var item in items)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var canonicalBytes = CanonicalJson.ToCanonicalUtf8Bytes(item);
|
||||
await writer.WriteAsync(Encoding.UTF8.GetString(canonicalBytes)).ConfigureAwait(false);
|
||||
await writer.WriteLineAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private string ResolvePath(string relativePath)
|
||||
{
|
||||
var sanitized = relativePath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar);
|
||||
return Path.Combine(_root, sanitized);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Nodes;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
public sealed record GraphBuildBatch(
|
||||
ImmutableArray<JsonObject> Nodes,
|
||||
ImmutableArray<JsonObject> Edges);
|
||||
@@ -0,0 +1,9 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
public interface IGraphDocumentWriter
|
||||
{
|
||||
Task WriteAsync(GraphBuildBatch batch, CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
public interface ISbomIngestMetrics
|
||||
{
|
||||
void RecordBatch(string source, string tenant, int nodeCount, int edgeCount, TimeSpan duration, bool success);
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
public sealed class SbomIngestMetrics : ISbomIngestMetrics, IDisposable
|
||||
{
|
||||
public const string MeterName = "StellaOps.Graph.Indexer";
|
||||
public const string MeterVersion = "1.0.0";
|
||||
|
||||
private const string BatchesTotalName = "graph_sbom_ingest_batches_total";
|
||||
private const string BatchDurationSecondsName = "graph_sbom_ingest_duration_seconds";
|
||||
private const string NodesTotalName = "graph_sbom_ingest_nodes_total";
|
||||
private const string EdgesTotalName = "graph_sbom_ingest_edges_total";
|
||||
|
||||
private const string UnitCount = "count";
|
||||
private const string UnitSeconds = "s";
|
||||
|
||||
private readonly Meter _meter;
|
||||
private readonly bool _ownsMeter;
|
||||
private readonly Counter<long> _batchesTotal;
|
||||
private readonly Histogram<double> _batchDurationSeconds;
|
||||
private readonly Counter<long> _nodesTotal;
|
||||
private readonly Counter<long> _edgesTotal;
|
||||
private bool _disposed;
|
||||
|
||||
public SbomIngestMetrics()
|
||||
: this(null)
|
||||
{
|
||||
}
|
||||
|
||||
public SbomIngestMetrics(Meter? meter)
|
||||
{
|
||||
_meter = meter ?? new Meter(MeterName, MeterVersion);
|
||||
_ownsMeter = meter is null;
|
||||
|
||||
_batchesTotal = _meter.CreateCounter<long>(
|
||||
name: BatchesTotalName,
|
||||
unit: UnitCount,
|
||||
description: "Total SBOM ingest batches processed.");
|
||||
|
||||
_batchDurationSeconds = _meter.CreateHistogram<double>(
|
||||
name: BatchDurationSecondsName,
|
||||
unit: UnitSeconds,
|
||||
description: "Duration, in seconds, for SBOM ingest batches.");
|
||||
|
||||
_nodesTotal = _meter.CreateCounter<long>(
|
||||
name: NodesTotalName,
|
||||
unit: UnitCount,
|
||||
description: "Total graph nodes emitted from SBOM ingest.");
|
||||
|
||||
_edgesTotal = _meter.CreateCounter<long>(
|
||||
name: EdgesTotalName,
|
||||
unit: UnitCount,
|
||||
description: "Total graph edges emitted from SBOM ingest.");
|
||||
}
|
||||
|
||||
public void RecordBatch(string source, string tenant, int nodeCount, int edgeCount, TimeSpan duration, bool success)
|
||||
{
|
||||
ThrowIfDisposed();
|
||||
|
||||
var tags = new KeyValuePair<string, object?>[]
|
||||
{
|
||||
new("source", source ?? string.Empty),
|
||||
new("tenant", tenant ?? string.Empty),
|
||||
new("success", success)
|
||||
};
|
||||
|
||||
var tagSpan = tags.AsSpan();
|
||||
_batchesTotal.Add(1, tagSpan);
|
||||
_nodesTotal.Add(nodeCount, tagSpan);
|
||||
_edgesTotal.Add(edgeCount, tagSpan);
|
||||
_batchDurationSeconds.Record(duration.TotalSeconds, tagSpan);
|
||||
}
|
||||
|
||||
private void ThrowIfDisposed()
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
throw new ObjectDisposedException(nameof(SbomIngestMetrics));
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (_ownsMeter)
|
||||
{
|
||||
_meter.Dispose();
|
||||
}
|
||||
|
||||
_disposed = true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
public sealed class SbomIngestOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Optional override for the snapshot export root directory. When null or whitespace,
|
||||
/// <c>STELLAOPS_GRAPH_SNAPSHOT_DIR</c> or the default <c>artifacts/graph-snapshots</c>
|
||||
/// location will be used.
|
||||
/// </summary>
|
||||
public string? SnapshotRootDirectory { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
using System.Diagnostics;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
public sealed class SbomIngestProcessor
|
||||
{
|
||||
private readonly SbomIngestTransformer _transformer;
|
||||
private readonly IGraphDocumentWriter _writer;
|
||||
private readonly ISbomIngestMetrics _metrics;
|
||||
private readonly ISbomSnapshotExporter _snapshotExporter;
|
||||
private readonly ILogger<SbomIngestProcessor> _logger;
|
||||
|
||||
public SbomIngestProcessor(
|
||||
SbomIngestTransformer transformer,
|
||||
IGraphDocumentWriter writer,
|
||||
ISbomIngestMetrics metrics,
|
||||
ISbomSnapshotExporter snapshotExporter,
|
||||
ILogger<SbomIngestProcessor> logger)
|
||||
{
|
||||
_transformer = transformer ?? throw new ArgumentNullException(nameof(transformer));
|
||||
_writer = writer ?? throw new ArgumentNullException(nameof(writer));
|
||||
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
|
||||
_snapshotExporter = snapshotExporter ?? throw new ArgumentNullException(nameof(snapshotExporter));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task ProcessAsync(SbomSnapshot snapshot, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(snapshot);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
GraphBuildBatch batch;
|
||||
|
||||
try
|
||||
{
|
||||
batch = _transformer.Transform(snapshot);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, 0, 0, stopwatch.Elapsed, success: false);
|
||||
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"graph-indexer: failed to transform SBOM {SbomDigest} for tenant {Tenant}",
|
||||
snapshot.SbomDigest,
|
||||
snapshot.Tenant);
|
||||
|
||||
throw;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
await _writer.WriteAsync(batch, cancellationToken).ConfigureAwait(false);
|
||||
await _snapshotExporter.ExportAsync(snapshot, batch, cancellationToken).ConfigureAwait(false);
|
||||
stopwatch.Stop();
|
||||
|
||||
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, batch.Nodes.Length, batch.Edges.Length, stopwatch.Elapsed, success: true);
|
||||
|
||||
_logger.LogInformation(
|
||||
"graph-indexer: indexed SBOM {SbomDigest} for tenant {Tenant} with {NodeCount} nodes and {EdgeCount} edges in {DurationMs:F2} ms",
|
||||
snapshot.SbomDigest,
|
||||
snapshot.Tenant,
|
||||
batch.Nodes.Length,
|
||||
batch.Edges.Length,
|
||||
stopwatch.Elapsed.TotalMilliseconds);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, batch.Nodes.Length, batch.Edges.Length, stopwatch.Elapsed, success: false);
|
||||
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"graph-indexer: failed to persist SBOM {SbomDigest} for tenant {Tenant}",
|
||||
snapshot.SbomDigest,
|
||||
snapshot.Tenant);
|
||||
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Graph.Indexer.Documents;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
public static class SbomIngestProcessorFactory
|
||||
{
|
||||
private const string SnapshotDirEnv = "STELLAOPS_GRAPH_SNAPSHOT_DIR";
|
||||
|
||||
public static SbomIngestProcessor CreateDefault(
|
||||
SbomIngestTransformer transformer,
|
||||
IGraphDocumentWriter writer,
|
||||
ISbomIngestMetrics metrics,
|
||||
ILogger<SbomIngestProcessor> logger,
|
||||
string? snapshotRoot = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(transformer);
|
||||
ArgumentNullException.ThrowIfNull(writer);
|
||||
ArgumentNullException.ThrowIfNull(metrics);
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
|
||||
var root = ResolveSnapshotRoot(snapshotRoot);
|
||||
var exporter = new SbomSnapshotExporter(new GraphSnapshotBuilder(), new FileSystemSnapshotFileWriter(root));
|
||||
return new SbomIngestProcessor(transformer, writer, metrics, exporter, logger);
|
||||
}
|
||||
|
||||
private static string ResolveSnapshotRoot(string? snapshotRoot)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(snapshotRoot))
|
||||
{
|
||||
return snapshotRoot!;
|
||||
}
|
||||
|
||||
var envRoot = Environment.GetEnvironmentVariable(SnapshotDirEnv);
|
||||
if (!string.IsNullOrWhiteSpace(envRoot))
|
||||
{
|
||||
return envRoot!;
|
||||
}
|
||||
|
||||
return Path.Combine(Environment.CurrentDirectory, "artifacts", "graph-snapshots");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
public static class SbomIngestServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddSbomIngestPipeline(
|
||||
this IServiceCollection services,
|
||||
Action<SbomIngestOptions>? configure = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
services.AddOptions<SbomIngestOptions>();
|
||||
|
||||
if (configure is not null)
|
||||
{
|
||||
services.Configure(configure);
|
||||
}
|
||||
|
||||
services.TryAddSingleton<SbomIngestTransformer>();
|
||||
services.TryAddSingleton<ISbomIngestMetrics, SbomIngestMetrics>();
|
||||
|
||||
services.TryAddSingleton<SbomIngestProcessor>(provider =>
|
||||
{
|
||||
var transformer = provider.GetRequiredService<SbomIngestTransformer>();
|
||||
var writer = provider.GetRequiredService<IGraphDocumentWriter>();
|
||||
var metrics = provider.GetRequiredService<ISbomIngestMetrics>();
|
||||
var logger = provider.GetService<ILogger<SbomIngestProcessor>>() ?? NullLogger<SbomIngestProcessor>.Instance;
|
||||
var options = provider.GetService<IOptions<SbomIngestOptions>>();
|
||||
var snapshotRoot = options?.Value.SnapshotRootDirectory;
|
||||
|
||||
return SbomIngestProcessorFactory.CreateDefault(
|
||||
transformer,
|
||||
writer,
|
||||
metrics,
|
||||
logger,
|
||||
snapshotRoot);
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,453 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Nodes;
|
||||
using StellaOps.Graph.Indexer.Schema;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
public sealed class SbomIngestTransformer
|
||||
{
|
||||
private const string ContainsEdgeKind = "CONTAINS";
|
||||
private const string DependsOnEdgeKind = "DEPENDS_ON";
|
||||
private const string DeclaredInEdgeKind = "DECLARED_IN";
|
||||
private const string BuiltFromEdgeKind = "BUILT_FROM";
|
||||
|
||||
public GraphBuildBatch Transform(SbomSnapshot snapshot)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(snapshot);
|
||||
|
||||
var nodes = new List<JsonObject>();
|
||||
var edges = new List<JsonObject>();
|
||||
|
||||
var artifactNodes = new Dictionary<string, JsonObject>(StringComparer.OrdinalIgnoreCase);
|
||||
var componentNodes = new Dictionary<string, JsonObject>(StringComparer.OrdinalIgnoreCase);
|
||||
var fileNodes = new Dictionary<string, JsonObject>(StringComparer.OrdinalIgnoreCase);
|
||||
var licenseCandidates = new Dictionary<(string License, string SourceDigest), LicenseCandidate>(LicenseKeyComparer.Instance);
|
||||
|
||||
long nextEdgeOffset = snapshot.EventOffset + 918;
|
||||
long NextEdgeOffset() => nextEdgeOffset++;
|
||||
|
||||
var artifactNode = CreateArtifactNode(snapshot);
|
||||
nodes.Add(artifactNode);
|
||||
artifactNodes[GetArtifactKey(snapshot.ArtifactDigest, snapshot.SbomDigest)] = artifactNode;
|
||||
|
||||
foreach (var component in snapshot.Components)
|
||||
{
|
||||
var componentNode = CreateComponentNode(snapshot, component);
|
||||
nodes.Add(componentNode);
|
||||
componentNodes[GetComponentKey(component.Purl, component.SourceType)] = componentNode;
|
||||
|
||||
if (string.Equals(component.Usage, "direct", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
var containsEdge = CreateContainsEdge(snapshot, artifactNode, componentNode, component, NextEdgeOffset());
|
||||
edges.Add(containsEdge);
|
||||
}
|
||||
|
||||
foreach (var dependency in component.Dependencies)
|
||||
{
|
||||
var dependsOnEdge = CreateDependsOnEdge(snapshot, componentNode, dependency, NextEdgeOffset());
|
||||
edges.Add(dependsOnEdge);
|
||||
}
|
||||
|
||||
foreach (var file in component.Files)
|
||||
{
|
||||
var fileNodeKey = GetFileKey(snapshot.ArtifactDigest, file.Path, file.ContentSha256);
|
||||
if (!fileNodes.TryGetValue(fileNodeKey, out var fileNode))
|
||||
{
|
||||
fileNode = CreateFileNode(snapshot, file);
|
||||
nodes.Add(fileNode);
|
||||
fileNodes[fileNodeKey] = fileNode;
|
||||
}
|
||||
|
||||
var declaredInEdge = CreateDeclaredInEdge(snapshot, componentNode, component, fileNode, file, NextEdgeOffset());
|
||||
edges.Add(declaredInEdge);
|
||||
}
|
||||
|
||||
if (HasLicenseMetadata(component.License))
|
||||
{
|
||||
var licenseKey = (component.License.Spdx, component.License.SourceDigest);
|
||||
var candidate = CreateLicenseCandidate(snapshot, component);
|
||||
if (!licenseCandidates.TryGetValue(licenseKey, out var existing) || existing.EventOffset > candidate.EventOffset)
|
||||
{
|
||||
licenseCandidates[licenseKey] = candidate;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var candidate in licenseCandidates.Values)
|
||||
{
|
||||
nodes.Add(CreateLicenseNode(snapshot, candidate));
|
||||
}
|
||||
|
||||
foreach (var baseArtifact in snapshot.BaseArtifacts)
|
||||
{
|
||||
var node = CreateBaseArtifactNode(snapshot, baseArtifact);
|
||||
if (!artifactNodes.ContainsKey(GetArtifactKey(baseArtifact.ArtifactDigest, baseArtifact.SbomDigest)))
|
||||
{
|
||||
nodes.Add(node);
|
||||
artifactNodes[GetArtifactKey(baseArtifact.ArtifactDigest, baseArtifact.SbomDigest)] = node;
|
||||
}
|
||||
|
||||
var edge = CreateBuiltFromEdge(snapshot, artifactNode, baseArtifact, NextEdgeOffset());
|
||||
edges.Add(edge);
|
||||
}
|
||||
|
||||
var orderedNodes = nodes
|
||||
.OrderBy(node => node["kind"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ThenBy(node => node["id"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ToImmutableArray();
|
||||
|
||||
var orderedEdges = edges
|
||||
.OrderBy(edge => edge["kind"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ThenBy(edge => edge["id"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ToImmutableArray();
|
||||
|
||||
return new GraphBuildBatch(orderedNodes, orderedEdges);
|
||||
}
|
||||
|
||||
private static JsonObject CreateArtifactNode(SbomSnapshot snapshot)
|
||||
{
|
||||
var labels = snapshot.Artifact.Labels?.OrderBy(x => x, StringComparer.Ordinal).ToArray() ?? Array.Empty<string>();
|
||||
|
||||
var attributes = new JsonObject
|
||||
{
|
||||
["display_name"] = snapshot.Artifact.DisplayName,
|
||||
["artifact_digest"] = snapshot.ArtifactDigest,
|
||||
["sbom_digest"] = snapshot.SbomDigest,
|
||||
["environment"] = snapshot.Artifact.Environment,
|
||||
["labels"] = CreateJsonArray(labels),
|
||||
["origin_registry"] = snapshot.Artifact.OriginRegistry,
|
||||
["supply_chain_stage"] = snapshot.Artifact.SupplyChainStage
|
||||
};
|
||||
|
||||
return GraphDocumentFactory.CreateNode(new GraphNodeSpec(
|
||||
Tenant: snapshot.Tenant,
|
||||
Kind: "artifact",
|
||||
CanonicalKey: new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["artifact_digest"] = snapshot.ArtifactDigest,
|
||||
["sbom_digest"] = snapshot.SbomDigest
|
||||
},
|
||||
Attributes: attributes,
|
||||
Provenance: new GraphProvenanceSpec(snapshot.Source, snapshot.CollectedAt, snapshot.SbomDigest, snapshot.EventOffset),
|
||||
ValidFrom: snapshot.CollectedAt,
|
||||
ValidTo: null));
|
||||
}
|
||||
|
||||
private static JsonObject CreateBaseArtifactNode(SbomSnapshot snapshot, SbomBaseArtifact baseArtifact)
|
||||
{
|
||||
var labels = baseArtifact.Labels?.OrderBy(x => x, StringComparer.Ordinal).ToArray() ?? Array.Empty<string>();
|
||||
|
||||
var attributes = new JsonObject
|
||||
{
|
||||
["display_name"] = baseArtifact.DisplayName,
|
||||
["artifact_digest"] = baseArtifact.ArtifactDigest,
|
||||
["sbom_digest"] = baseArtifact.SbomDigest,
|
||||
["environment"] = baseArtifact.Environment,
|
||||
["labels"] = CreateJsonArray(labels),
|
||||
["origin_registry"] = baseArtifact.OriginRegistry,
|
||||
["supply_chain_stage"] = baseArtifact.SupplyChainStage
|
||||
};
|
||||
|
||||
return GraphDocumentFactory.CreateNode(new GraphNodeSpec(
|
||||
Tenant: snapshot.Tenant,
|
||||
Kind: "artifact",
|
||||
CanonicalKey: new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["artifact_digest"] = baseArtifact.ArtifactDigest,
|
||||
["sbom_digest"] = baseArtifact.SbomDigest
|
||||
},
|
||||
Attributes: attributes,
|
||||
Provenance: new GraphProvenanceSpec(
|
||||
ResolveSource(baseArtifact.Source, snapshot.Source),
|
||||
baseArtifact.CollectedAt,
|
||||
baseArtifact.SbomDigest,
|
||||
baseArtifact.EventOffset),
|
||||
ValidFrom: baseArtifact.CollectedAt,
|
||||
ValidTo: null));
|
||||
}
|
||||
|
||||
private static JsonObject CreateComponentNode(SbomSnapshot snapshot, SbomComponent component)
|
||||
{
|
||||
var attributes = new JsonObject
|
||||
{
|
||||
["purl"] = component.Purl,
|
||||
["version"] = component.Version,
|
||||
["ecosystem"] = component.Ecosystem,
|
||||
["scope"] = component.Scope,
|
||||
["license_spdx"] = component.License.Spdx,
|
||||
["usage"] = component.Usage
|
||||
};
|
||||
|
||||
return GraphDocumentFactory.CreateNode(new GraphNodeSpec(
|
||||
Tenant: snapshot.Tenant,
|
||||
Kind: "component",
|
||||
CanonicalKey: new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["purl"] = component.Purl,
|
||||
["source_type"] = component.SourceType
|
||||
},
|
||||
Attributes: attributes,
|
||||
Provenance: new GraphProvenanceSpec(
|
||||
ResolveSource(component.Source, snapshot.Source),
|
||||
component.CollectedAt,
|
||||
snapshot.SbomDigest,
|
||||
component.EventOffset),
|
||||
ValidFrom: component.CollectedAt,
|
||||
ValidTo: null));
|
||||
}
|
||||
|
||||
private static JsonObject CreateFileNode(SbomSnapshot snapshot, SbomComponentFile file)
|
||||
{
|
||||
var attributes = new JsonObject
|
||||
{
|
||||
["normalized_path"] = file.Path,
|
||||
["content_sha256"] = file.ContentSha256,
|
||||
["language_hint"] = file.LanguageHint,
|
||||
["size_bytes"] = file.SizeBytes,
|
||||
["scope"] = file.Scope
|
||||
};
|
||||
|
||||
return GraphDocumentFactory.CreateNode(new GraphNodeSpec(
|
||||
Tenant: snapshot.Tenant,
|
||||
Kind: "file",
|
||||
CanonicalKey: new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["artifact_digest"] = snapshot.ArtifactDigest,
|
||||
["normalized_path"] = file.Path,
|
||||
["content_sha256"] = file.ContentSha256
|
||||
},
|
||||
Attributes: attributes,
|
||||
Provenance: new GraphProvenanceSpec(
|
||||
ResolveSource(file.Source, file.DetectedBy, snapshot.Source),
|
||||
file.CollectedAt,
|
||||
snapshot.SbomDigest,
|
||||
file.EventOffset),
|
||||
ValidFrom: file.CollectedAt,
|
||||
ValidTo: null));
|
||||
}
|
||||
|
||||
private static JsonObject CreateContainsEdge(SbomSnapshot snapshot, JsonObject artifactNode, JsonObject componentNode, SbomComponent component, long eventOffset)
|
||||
{
|
||||
return GraphDocumentFactory.CreateEdge(new GraphEdgeSpec(
|
||||
Tenant: snapshot.Tenant,
|
||||
Kind: ContainsEdgeKind,
|
||||
CanonicalKey: new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["artifact_node_id"] = artifactNode["id"]!.GetValue<string>(),
|
||||
["component_node_id"] = componentNode["id"]!.GetValue<string>(),
|
||||
["sbom_digest"] = snapshot.SbomDigest
|
||||
},
|
||||
Attributes: new JsonObject
|
||||
{
|
||||
["detected_by"] = component.DetectedBy,
|
||||
["layer_digest"] = component.LayerDigest,
|
||||
["scope"] = component.Scope,
|
||||
["evidence_digest"] = component.EvidenceDigest
|
||||
},
|
||||
Provenance: new GraphProvenanceSpec(
|
||||
ResolveSource(component.Source, component.DetectedBy, snapshot.Source),
|
||||
component.CollectedAt.AddSeconds(1),
|
||||
snapshot.SbomDigest,
|
||||
eventOffset),
|
||||
ValidFrom: component.CollectedAt.AddSeconds(1),
|
||||
ValidTo: null));
|
||||
}
|
||||
|
||||
private static JsonObject CreateDependsOnEdge(SbomSnapshot snapshot, JsonObject componentNode, SbomDependency dependency, long eventOffset)
|
||||
{
|
||||
return GraphDocumentFactory.CreateEdge(new GraphEdgeSpec(
|
||||
Tenant: snapshot.Tenant,
|
||||
Kind: DependsOnEdgeKind,
|
||||
CanonicalKey: new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["component_node_id"] = componentNode["id"]!.GetValue<string>(),
|
||||
["dependency_purl"] = dependency.Purl,
|
||||
["sbom_digest"] = snapshot.SbomDigest
|
||||
},
|
||||
Attributes: new JsonObject
|
||||
{
|
||||
["dependency_purl"] = dependency.Purl,
|
||||
["dependency_version"] = dependency.Version,
|
||||
["relationship"] = dependency.Relationship,
|
||||
["evidence_digest"] = dependency.EvidenceDigest
|
||||
},
|
||||
Provenance: new GraphProvenanceSpec(
|
||||
ResolveSource(dependency.Source, snapshot.Source),
|
||||
dependency.CollectedAt.AddSeconds(1),
|
||||
snapshot.SbomDigest,
|
||||
eventOffset),
|
||||
ValidFrom: dependency.CollectedAt.AddSeconds(1),
|
||||
ValidTo: null));
|
||||
}
|
||||
|
||||
private static JsonObject CreateDeclaredInEdge(SbomSnapshot snapshot, JsonObject componentNode, SbomComponent component, JsonObject fileNode, SbomComponentFile file, long eventOffset)
|
||||
{
|
||||
return GraphDocumentFactory.CreateEdge(new GraphEdgeSpec(
|
||||
Tenant: snapshot.Tenant,
|
||||
Kind: DeclaredInEdgeKind,
|
||||
CanonicalKey: new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["component_node_id"] = componentNode["id"]!.GetValue<string>(),
|
||||
["file_node_id"] = fileNode["id"]!.GetValue<string>(),
|
||||
["sbom_digest"] = snapshot.SbomDigest
|
||||
},
|
||||
Attributes: new JsonObject
|
||||
{
|
||||
["detected_by"] = file.DetectedBy,
|
||||
["scope"] = ResolveScope(component.Scope, file.Scope),
|
||||
["evidence_digest"] = file.EvidenceDigest
|
||||
},
|
||||
Provenance: new GraphProvenanceSpec(
|
||||
ResolveSource(file.Source, file.DetectedBy, snapshot.Source),
|
||||
file.CollectedAt.AddSeconds(1),
|
||||
snapshot.SbomDigest,
|
||||
eventOffset),
|
||||
ValidFrom: file.CollectedAt.AddSeconds(1),
|
||||
ValidTo: null));
|
||||
}
|
||||
|
||||
private static JsonObject CreateBuiltFromEdge(SbomSnapshot snapshot, JsonObject parentNode, SbomBaseArtifact baseArtifact, long eventOffset)
|
||||
{
|
||||
return GraphDocumentFactory.CreateEdge(new GraphEdgeSpec(
|
||||
Tenant: snapshot.Tenant,
|
||||
Kind: BuiltFromEdgeKind,
|
||||
CanonicalKey: new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["parent_artifact_node_id"] = parentNode["id"]!.GetValue<string>(),
|
||||
["child_artifact_digest"] = baseArtifact.ArtifactDigest
|
||||
},
|
||||
Attributes: new JsonObject
|
||||
{
|
||||
["build_type"] = snapshot.Build.BuildType,
|
||||
["builder_id"] = snapshot.Build.BuilderId,
|
||||
["attestation_digest"] = snapshot.Build.AttestationDigest
|
||||
},
|
||||
Provenance: new GraphProvenanceSpec(
|
||||
ResolveSource(snapshot.Build.Source, snapshot.Source),
|
||||
snapshot.Build.CollectedAt,
|
||||
snapshot.SbomDigest,
|
||||
eventOffset),
|
||||
ValidFrom: snapshot.Build.CollectedAt,
|
||||
ValidTo: null));
|
||||
}
|
||||
|
||||
private static string GetComponentKey(string purl, string sourceType)
|
||||
{
|
||||
var normalizedPurl = (purl ?? string.Empty).Trim();
|
||||
var normalizedSourceType = (sourceType ?? string.Empty).Trim();
|
||||
return $"{normalizedPurl}|{normalizedSourceType}";
|
||||
}
|
||||
|
||||
private static bool HasLicenseMetadata(SbomLicense license)
|
||||
=> !string.IsNullOrWhiteSpace(license.Spdx)
|
||||
&& !string.IsNullOrWhiteSpace(license.SourceDigest);
|
||||
|
||||
private static string ResolveScope(string componentScope, string fileScope)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(componentScope))
|
||||
{
|
||||
return componentScope.Trim();
|
||||
}
|
||||
|
||||
return string.IsNullOrWhiteSpace(fileScope) ? string.Empty : fileScope.Trim();
|
||||
}
|
||||
|
||||
private static string ResolveSource(params string?[] sources)
|
||||
{
|
||||
foreach (var source in sources)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(source))
|
||||
{
|
||||
return source!.Trim();
|
||||
}
|
||||
}
|
||||
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
private static string GetArtifactKey(string artifactDigest, string sbomDigest)
|
||||
=> $"{(artifactDigest ?? string.Empty).Trim()}|{(sbomDigest ?? string.Empty).Trim()}";
|
||||
|
||||
private static string GetFileKey(string artifactDigest, string path, string contentSha)
|
||||
=> $"{(artifactDigest ?? string.Empty).Trim()}|{(path ?? string.Empty).Trim()}|{(contentSha ?? string.Empty).Trim()}";
|
||||
|
||||
private static JsonArray CreateJsonArray(IEnumerable<string> values)
|
||||
{
|
||||
var array = new JsonArray();
|
||||
foreach (var value in values)
|
||||
{
|
||||
array.Add(value);
|
||||
}
|
||||
|
||||
return array;
|
||||
}
|
||||
|
||||
private static LicenseCandidate CreateLicenseCandidate(SbomSnapshot snapshot, SbomComponent component)
|
||||
{
|
||||
var collectedAt = component.CollectedAt.AddSeconds(2);
|
||||
var eventOffset = component.EventOffset + 3;
|
||||
return new LicenseCandidate(
|
||||
License: component.License,
|
||||
CollectedAt: collectedAt,
|
||||
EventOffset: eventOffset,
|
||||
SbomDigest: snapshot.SbomDigest,
|
||||
Source: ResolveSource(component.Source, snapshot.Source));
|
||||
}
|
||||
|
||||
private static JsonObject CreateLicenseNode(SbomSnapshot snapshot, LicenseCandidate candidate)
|
||||
{
|
||||
var attributes = new JsonObject
|
||||
{
|
||||
["license_spdx"] = candidate.License.Spdx,
|
||||
["name"] = candidate.License.Name,
|
||||
["classification"] = candidate.License.Classification,
|
||||
["notice_uri"] = candidate.License.NoticeUri is null ? null : candidate.License.NoticeUri
|
||||
};
|
||||
|
||||
return GraphDocumentFactory.CreateNode(new GraphNodeSpec(
|
||||
Tenant: snapshot.Tenant,
|
||||
Kind: "license",
|
||||
CanonicalKey: new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["license_spdx"] = candidate.License.Spdx,
|
||||
["source_digest"] = candidate.License.SourceDigest
|
||||
},
|
||||
Attributes: attributes,
|
||||
Provenance: new GraphProvenanceSpec(candidate.Source, candidate.CollectedAt, candidate.SbomDigest, candidate.EventOffset),
|
||||
ValidFrom: candidate.CollectedAt,
|
||||
ValidTo: null));
|
||||
}
|
||||
|
||||
private sealed record LicenseCandidate(
|
||||
SbomLicense License,
|
||||
DateTimeOffset CollectedAt,
|
||||
long EventOffset,
|
||||
string SbomDigest,
|
||||
string Source);
|
||||
|
||||
private sealed class LicenseKeyComparer : IEqualityComparer<(string License, string SourceDigest)>
|
||||
{
|
||||
public static readonly LicenseKeyComparer Instance = new();
|
||||
|
||||
public bool Equals((string License, string SourceDigest) x, (string License, string SourceDigest) y)
|
||||
=> string.Equals(x.License, y.License, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(x.SourceDigest, y.SourceDigest, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
public int GetHashCode((string License, string SourceDigest) obj)
|
||||
{
|
||||
var hash = new HashCode();
|
||||
hash.Add(obj.License, StringComparer.OrdinalIgnoreCase);
|
||||
hash.Add(obj.SourceDigest, StringComparer.OrdinalIgnoreCase);
|
||||
return hash.ToHashCode();
|
||||
}
|
||||
}
|
||||
}
|
||||
231
src/Graph/StellaOps.Graph.Indexer/Ingestion/Sbom/SbomSnapshot.cs
Normal file
231
src/Graph/StellaOps.Graph.Indexer/Ingestion/Sbom/SbomSnapshot.cs
Normal file
@@ -0,0 +1,231 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
public sealed class SbomSnapshot
|
||||
{
|
||||
[JsonPropertyName("tenant")]
|
||||
public string Tenant { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("artifactDigest")]
|
||||
public string ArtifactDigest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("sbomDigest")]
|
||||
public string SbomDigest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long EventOffset { get; init; }
|
||||
|
||||
[JsonPropertyName("artifact")]
|
||||
public SbomArtifactMetadata Artifact { get; init; } = new();
|
||||
|
||||
[JsonPropertyName("build")]
|
||||
public SbomBuildMetadata Build { get; init; } = new();
|
||||
|
||||
[JsonPropertyName("components")]
|
||||
public IReadOnlyList<SbomComponent> Components { get; init; } = Array.Empty<SbomComponent>();
|
||||
|
||||
[JsonPropertyName("baseArtifacts")]
|
||||
public IReadOnlyList<SbomBaseArtifact> BaseArtifacts { get; init; } = Array.Empty<SbomBaseArtifact>();
|
||||
}
|
||||
|
||||
public sealed class SbomArtifactMetadata
|
||||
{
|
||||
[JsonPropertyName("displayName")]
|
||||
public string DisplayName { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("environment")]
|
||||
public string Environment { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("labels")]
|
||||
public IReadOnlyList<string> Labels { get; init; } = Array.Empty<string>();
|
||||
|
||||
[JsonPropertyName("originRegistry")]
|
||||
public string OriginRegistry { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("supplyChainStage")]
|
||||
public string SupplyChainStage { get; init; } = string.Empty;
|
||||
}
|
||||
|
||||
public sealed class SbomBuildMetadata
|
||||
{
|
||||
[JsonPropertyName("builderId")]
|
||||
public string BuilderId { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("buildType")]
|
||||
public string BuildType { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("attestationDigest")]
|
||||
public string AttestationDigest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long EventOffset { get; init; }
|
||||
}
|
||||
|
||||
public sealed class SbomComponent
|
||||
{
|
||||
[JsonPropertyName("purl")]
|
||||
public string Purl { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("version")]
|
||||
public string Version { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("ecosystem")]
|
||||
public string Ecosystem { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("scope")]
|
||||
public string Scope { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("license")]
|
||||
public SbomLicense License { get; init; } = new();
|
||||
|
||||
[JsonPropertyName("usage")]
|
||||
public string Usage { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("detectedBy")]
|
||||
public string DetectedBy { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("layerDigest")]
|
||||
public string LayerDigest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("evidenceDigest")]
|
||||
public string EvidenceDigest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long EventOffset { get; init; }
|
||||
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("files")]
|
||||
public IReadOnlyList<SbomComponentFile> Files { get; init; } = Array.Empty<SbomComponentFile>();
|
||||
|
||||
[JsonPropertyName("dependencies")]
|
||||
public IReadOnlyList<SbomDependency> Dependencies { get; init; } = Array.Empty<SbomDependency>();
|
||||
|
||||
[JsonPropertyName("sourceType")]
|
||||
public string SourceType { get; init; } = "inventory";
|
||||
}
|
||||
|
||||
public sealed class SbomLicense
|
||||
{
|
||||
[JsonPropertyName("spdx")]
|
||||
public string Spdx { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("name")]
|
||||
public string Name { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("classification")]
|
||||
public string Classification { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("noticeUri")]
|
||||
public string? NoticeUri { get; init; }
|
||||
|
||||
[JsonPropertyName("sourceDigest")]
|
||||
public string SourceDigest { get; init; } = string.Empty;
|
||||
}
|
||||
|
||||
public sealed class SbomComponentFile
|
||||
{
|
||||
[JsonPropertyName("path")]
|
||||
public string Path { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("contentSha256")]
|
||||
public string ContentSha256 { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("languageHint")]
|
||||
public string LanguageHint { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("sizeBytes")]
|
||||
public long SizeBytes { get; init; }
|
||||
|
||||
[JsonPropertyName("scope")]
|
||||
public string Scope { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("detectedBy")]
|
||||
public string DetectedBy { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("evidenceDigest")]
|
||||
public string EvidenceDigest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long EventOffset { get; init; }
|
||||
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; init; } = string.Empty;
|
||||
}
|
||||
|
||||
public sealed class SbomDependency
|
||||
{
|
||||
[JsonPropertyName("purl")]
|
||||
public string Purl { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("version")]
|
||||
public string Version { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("relationship")]
|
||||
public string Relationship { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("evidenceDigest")]
|
||||
public string EvidenceDigest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long EventOffset { get; init; }
|
||||
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; init; } = string.Empty;
|
||||
}
|
||||
|
||||
public sealed class SbomBaseArtifact
|
||||
{
|
||||
[JsonPropertyName("artifactDigest")]
|
||||
public string ArtifactDigest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("sbomDigest")]
|
||||
public string SbomDigest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("displayName")]
|
||||
public string DisplayName { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("environment")]
|
||||
public string Environment { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("labels")]
|
||||
public IReadOnlyList<string> Labels { get; init; } = Array.Empty<string>();
|
||||
|
||||
[JsonPropertyName("originRegistry")]
|
||||
public string OriginRegistry { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("supplyChainStage")]
|
||||
public string SupplyChainStage { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long EventOffset { get; init; }
|
||||
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; init; } = string.Empty;
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using StellaOps.Graph.Indexer.Documents;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
public interface ISnapshotFileWriter
|
||||
{
|
||||
Task WriteJsonAsync(string relativePath, JsonObject content, CancellationToken cancellationToken);
|
||||
Task WriteJsonLinesAsync(string relativePath, IEnumerable<JsonObject> items, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public interface ISbomSnapshotExporter
|
||||
{
|
||||
Task ExportAsync(SbomSnapshot snapshot, GraphBuildBatch batch, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed class SbomSnapshotExporter : ISbomSnapshotExporter
|
||||
{
|
||||
private readonly GraphSnapshotBuilder _snapshotBuilder;
|
||||
private readonly ISnapshotFileWriter _fileWriter;
|
||||
|
||||
public SbomSnapshotExporter(GraphSnapshotBuilder snapshotBuilder, ISnapshotFileWriter fileWriter)
|
||||
{
|
||||
_snapshotBuilder = snapshotBuilder ?? throw new ArgumentNullException(nameof(snapshotBuilder));
|
||||
_fileWriter = fileWriter ?? throw new ArgumentNullException(nameof(fileWriter));
|
||||
}
|
||||
|
||||
public async Task ExportAsync(SbomSnapshot snapshot, GraphBuildBatch batch, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(snapshot);
|
||||
ArgumentNullException.ThrowIfNull(batch);
|
||||
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var graphSnapshot = _snapshotBuilder.Build(snapshot, batch, DateTimeOffset.UtcNow);
|
||||
|
||||
await _fileWriter.WriteJsonAsync("manifest.json", graphSnapshot.Manifest.ToJson(), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await _fileWriter.WriteJsonAsync("adjacency.json", graphSnapshot.Adjacency.ToJson(), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await _fileWriter.WriteJsonLinesAsync("nodes.jsonl", batch.Nodes, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await _fileWriter.WriteJsonLinesAsync("edges.jsonl", batch.Edges, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Vex;
|
||||
|
||||
public sealed class VexOverlaySnapshot
|
||||
{
|
||||
[JsonPropertyName("tenant")]
|
||||
public string Tenant { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long EventOffset { get; init; }
|
||||
|
||||
[JsonPropertyName("statement")]
|
||||
public VexStatementDetails Statement { get; init; } = new();
|
||||
|
||||
[JsonPropertyName("exemptions")]
|
||||
public IReadOnlyList<VexComponentExemption> Exemptions { get; init; } = Array.Empty<VexComponentExemption>();
|
||||
}
|
||||
|
||||
public sealed class VexStatementDetails
|
||||
{
|
||||
[JsonPropertyName("vexSource")]
|
||||
public string VexSource { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("statementId")]
|
||||
public string StatementId { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("status")]
|
||||
public string Status { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("justification")]
|
||||
public string Justification { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("impactStatement")]
|
||||
public string ImpactStatement { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("issuedAt")]
|
||||
public DateTimeOffset IssuedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("expiresAt")]
|
||||
public DateTimeOffset ExpiresAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("contentHash")]
|
||||
public string ContentHash { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("provenanceSource")]
|
||||
public string? ProvenanceSource { get; init; }
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset StatementCollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long StatementEventOffset { get; init; }
|
||||
}
|
||||
|
||||
public sealed class VexComponentExemption
|
||||
{
|
||||
[JsonPropertyName("componentPurl")]
|
||||
public string ComponentPurl { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("componentSourceType")]
|
||||
public string ComponentSourceType { get; init; } = "inventory";
|
||||
|
||||
[JsonPropertyName("sbomDigest")]
|
||||
public string? SbomDigest { get; init; }
|
||||
|
||||
[JsonPropertyName("statementHash")]
|
||||
public string? StatementHash { get; init; }
|
||||
|
||||
[JsonPropertyName("status")]
|
||||
public string Status { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("justification")]
|
||||
public string Justification { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("impactStatement")]
|
||||
public string ImpactStatement { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("evidenceDigest")]
|
||||
public string EvidenceDigest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("provenanceSource")]
|
||||
public string? ProvenanceSource { get; init; }
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long EventOffset { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Text.Json.Nodes;
|
||||
using StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
using StellaOps.Graph.Indexer.Schema;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Vex;
|
||||
|
||||
public sealed class VexOverlayTransformer
|
||||
{
|
||||
private const string VexNodeKind = "vex_statement";
|
||||
private const string ComponentNodeKind = "component";
|
||||
private const string VexExemptsEdgeKind = "VEX_EXEMPTS";
|
||||
|
||||
public GraphBuildBatch Transform(VexOverlaySnapshot snapshot)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(snapshot);
|
||||
|
||||
var nodes = new List<JsonObject>();
|
||||
var edges = new List<JsonObject>();
|
||||
var seenEdgeIds = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
var statement = snapshot.Statement ?? new VexStatementDetails();
|
||||
var canonicalStatementHash = statement.ContentHash?.Trim() ?? string.Empty;
|
||||
|
||||
var vexNode = CreateVexStatementNode(snapshot, statement, canonicalStatementHash);
|
||||
nodes.Add(vexNode);
|
||||
|
||||
var vexNodeId = vexNode["id"]!.GetValue<string>();
|
||||
foreach (var exemption in snapshot.Exemptions ?? Array.Empty<VexComponentExemption>())
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(exemption.ComponentPurl))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var edge = CreateVexExemptsEdge(snapshot, statement, canonicalStatementHash, vexNodeId, exemption);
|
||||
var edgeId = edge["id"]!.GetValue<string>();
|
||||
|
||||
if (seenEdgeIds.Add(edgeId))
|
||||
{
|
||||
edges.Add(edge);
|
||||
}
|
||||
}
|
||||
|
||||
return new GraphBuildBatch(
|
||||
nodes
|
||||
.OrderBy(node => node["kind"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ThenBy(node => node["id"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ToImmutableArray(),
|
||||
edges
|
||||
.OrderBy(edge => edge["kind"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ThenBy(edge => edge["id"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ToImmutableArray());
|
||||
}
|
||||
|
||||
private static JsonObject CreateVexStatementNode(
|
||||
VexOverlaySnapshot snapshot,
|
||||
VexStatementDetails statement,
|
||||
string canonicalStatementHash)
|
||||
{
|
||||
var vexSource = statement.VexSource?.Trim() ?? string.Empty;
|
||||
var statementId = statement.StatementId?.Trim() ?? string.Empty;
|
||||
var status = statement.Status?.Trim() ?? string.Empty;
|
||||
var justification = statement.Justification?.Trim() ?? string.Empty;
|
||||
|
||||
var issuedAt = statement.IssuedAt == DateTimeOffset.UnixEpoch
|
||||
? snapshot.CollectedAt
|
||||
: statement.IssuedAt;
|
||||
|
||||
var expiresAt = statement.ExpiresAt == DateTimeOffset.UnixEpoch
|
||||
? (DateTimeOffset?)null
|
||||
: statement.ExpiresAt;
|
||||
|
||||
var attributes = new JsonObject
|
||||
{
|
||||
["status"] = status,
|
||||
["statement_id"] = statementId,
|
||||
["justification"] = justification,
|
||||
["issued_at"] = GraphTimestamp.Format(issuedAt)
|
||||
};
|
||||
|
||||
if (expiresAt.HasValue)
|
||||
{
|
||||
attributes["expires_at"] = GraphTimestamp.Format(expiresAt.Value);
|
||||
}
|
||||
|
||||
attributes["content_hash"] = canonicalStatementHash;
|
||||
|
||||
var canonicalKey = new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["vex_source"] = vexSource,
|
||||
["statement_id"] = statementId,
|
||||
["content_hash"] = canonicalStatementHash
|
||||
};
|
||||
|
||||
var provenanceSource = string.IsNullOrWhiteSpace(statement.ProvenanceSource)
|
||||
? NormalizeVexSource(snapshot.Source)
|
||||
: statement.ProvenanceSource.Trim();
|
||||
|
||||
var collectedAt = statement.StatementCollectedAt == DateTimeOffset.UnixEpoch
|
||||
? snapshot.CollectedAt
|
||||
: statement.StatementCollectedAt;
|
||||
|
||||
var eventOffset = statement.StatementEventOffset != 0
|
||||
? statement.StatementEventOffset
|
||||
: snapshot.EventOffset;
|
||||
|
||||
var node = GraphDocumentFactory.CreateNode(new GraphNodeSpec(
|
||||
Tenant: snapshot.Tenant,
|
||||
Kind: VexNodeKind,
|
||||
CanonicalKey: canonicalKey,
|
||||
Attributes: attributes,
|
||||
Provenance: new GraphProvenanceSpec(provenanceSource, collectedAt, null, eventOffset),
|
||||
ValidFrom: issuedAt,
|
||||
ValidTo: null));
|
||||
|
||||
var provenance = node["provenance"]!.AsObject();
|
||||
var sourceNode = provenance["source"]!.DeepClone();
|
||||
var collectedAtNode = provenance["collected_at"]!.DeepClone();
|
||||
var eventOffsetNode = provenance.ContainsKey("event_offset")
|
||||
? provenance["event_offset"]!.DeepClone()
|
||||
: null;
|
||||
|
||||
var reorderedProvenance = new JsonObject
|
||||
{
|
||||
["source"] = sourceNode,
|
||||
["collected_at"] = collectedAtNode,
|
||||
["sbom_digest"] = null
|
||||
};
|
||||
|
||||
if (eventOffsetNode is not null)
|
||||
{
|
||||
reorderedProvenance["event_offset"] = eventOffsetNode;
|
||||
}
|
||||
|
||||
node["provenance"] = reorderedProvenance;
|
||||
|
||||
node.Remove("hash");
|
||||
node["hash"] = GraphIdentity.ComputeDocumentHash(node);
|
||||
|
||||
return node;
|
||||
}
|
||||
|
||||
private static JsonObject CreateVexExemptsEdge(
|
||||
VexOverlaySnapshot snapshot,
|
||||
VexStatementDetails statement,
|
||||
string canonicalStatementHash,
|
||||
string vexNodeId,
|
||||
VexComponentExemption exemption)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(exemption);
|
||||
|
||||
var normalizedSourceType = string.IsNullOrWhiteSpace(exemption.ComponentSourceType)
|
||||
? "inventory"
|
||||
: exemption.ComponentSourceType.Trim();
|
||||
var componentPurl = exemption.ComponentPurl.Trim();
|
||||
var statementHash = !string.IsNullOrWhiteSpace(exemption.StatementHash)
|
||||
? exemption.StatementHash.Trim()
|
||||
: canonicalStatementHash;
|
||||
|
||||
var componentIdentity = new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["purl"] = componentPurl,
|
||||
["source_type"] = normalizedSourceType
|
||||
};
|
||||
|
||||
var componentNodeId = GraphIdentity.ComputeNodeId(snapshot.Tenant, ComponentNodeKind, componentIdentity);
|
||||
|
||||
var canonicalKey = new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["component_node_id"] = componentNodeId,
|
||||
["vex_node_id"] = vexNodeId,
|
||||
["statement_hash"] = statementHash
|
||||
};
|
||||
|
||||
var impactStatement = !string.IsNullOrWhiteSpace(exemption.ImpactStatement)
|
||||
? exemption.ImpactStatement.Trim()
|
||||
: statement.ImpactStatement?.Trim() ?? string.Empty;
|
||||
|
||||
var status = !string.IsNullOrWhiteSpace(exemption.Status)
|
||||
? exemption.Status.Trim()
|
||||
: statement.Status?.Trim() ?? string.Empty;
|
||||
|
||||
var justification = !string.IsNullOrWhiteSpace(exemption.Justification)
|
||||
? exemption.Justification.Trim()
|
||||
: statement.Justification?.Trim() ?? string.Empty;
|
||||
|
||||
var attributes = new JsonObject
|
||||
{
|
||||
["status"] = status,
|
||||
["justification"] = justification,
|
||||
["impact_statement"] = impactStatement,
|
||||
["evidence_digest"] = exemption.EvidenceDigest?.Trim() ?? string.Empty
|
||||
};
|
||||
|
||||
var collectedAt = exemption.CollectedAt == DateTimeOffset.UnixEpoch
|
||||
? snapshot.CollectedAt
|
||||
: exemption.CollectedAt;
|
||||
|
||||
var eventOffset = exemption.EventOffset != 0 ? exemption.EventOffset : snapshot.EventOffset;
|
||||
var source = ResolveSource(exemption.ProvenanceSource, snapshot.Source);
|
||||
|
||||
return GraphDocumentFactory.CreateEdge(new GraphEdgeSpec(
|
||||
Tenant: snapshot.Tenant,
|
||||
Kind: VexExemptsEdgeKind,
|
||||
CanonicalKey: canonicalKey,
|
||||
Attributes: attributes,
|
||||
Provenance: new GraphProvenanceSpec(source, collectedAt, exemption.SbomDigest?.Trim(), eventOffset),
|
||||
ValidFrom: collectedAt,
|
||||
ValidTo: null));
|
||||
}
|
||||
|
||||
private static string ResolveSource(string? candidate, string? fallback)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(candidate))
|
||||
{
|
||||
return candidate.Trim();
|
||||
}
|
||||
|
||||
return string.IsNullOrWhiteSpace(fallback)
|
||||
? "excititor.overlay.v1"
|
||||
: fallback.Trim();
|
||||
}
|
||||
|
||||
private static string NormalizeVexSource(string? source)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(source))
|
||||
{
|
||||
return "excititor.vex.v1";
|
||||
}
|
||||
|
||||
var trimmed = source.Trim();
|
||||
return trimmed.Contains(".overlay.", StringComparison.Ordinal)
|
||||
? trimmed.Replace(".overlay.", ".vex.", StringComparison.Ordinal)
|
||||
: trimmed;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user