Add unit tests for SBOM ingestion and transformation
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Implement `SbomIngestServiceCollectionExtensionsTests` to verify the SBOM ingestion pipeline exports snapshots correctly.
- Create `SbomIngestTransformerTests` to ensure the transformation produces expected nodes and edges, including deduplication of license nodes and normalization of timestamps.
- Add `SbomSnapshotExporterTests` to test the export functionality for manifest, adjacency, nodes, and edges.
- Introduce `VexOverlayTransformerTests` to validate the transformation of VEX nodes and edges.
- Set up project file for the test project with necessary dependencies and configurations.
- Include JSON fixture files for testing purposes.
This commit is contained in:
master
2025-11-04 07:49:39 +02:00
parent f72c5c513a
commit 2eb6852d34
491 changed files with 39445 additions and 3917 deletions

View File

@@ -0,0 +1,261 @@
using System.Collections.Immutable;
using System.Text.Json.Nodes;
using System.Text.Json.Serialization;
using StellaOps.Graph.Indexer.Schema;
namespace StellaOps.Graph.Indexer.Documents;
public sealed record GraphSnapshot(
GraphSnapshotManifest Manifest,
GraphAdjacencyManifest Adjacency);
public sealed class GraphSnapshotManifest
{
[JsonPropertyName("tenant")]
public string Tenant { get; }
[JsonPropertyName("artifact_digest")]
public string ArtifactDigest { get; }
[JsonPropertyName("sbom_digest")]
public string SbomDigest { get; }
[JsonPropertyName("snapshot_id")]
public string SnapshotId { get; }
[JsonPropertyName("generated_at")]
public DateTimeOffset GeneratedAt { get; }
[JsonPropertyName("node_count")]
public int NodeCount { get; }
[JsonPropertyName("edge_count")]
public int EdgeCount { get; }
[JsonPropertyName("hash")]
public string Hash { get; }
[JsonPropertyName("lineage")]
public GraphSnapshotLineage Lineage { get; }
[JsonPropertyName("files")]
public GraphSnapshotFiles Files { get; }
public GraphSnapshotManifest(
string tenant,
string artifactDigest,
string sbomDigest,
string snapshotId,
DateTimeOffset generatedAt,
int nodeCount,
int edgeCount,
GraphSnapshotLineage lineage,
GraphSnapshotFiles files,
string hash)
{
Tenant = tenant;
ArtifactDigest = artifactDigest;
SbomDigest = sbomDigest;
SnapshotId = snapshotId;
GeneratedAt = generatedAt;
NodeCount = nodeCount;
EdgeCount = edgeCount;
Lineage = lineage;
Files = files;
Hash = hash;
}
public JsonObject ToJson()
{
var obj = new JsonObject
{
["tenant"] = Tenant,
["artifact_digest"] = ArtifactDigest,
["sbom_digest"] = SbomDigest,
["snapshot_id"] = SnapshotId,
["generated_at"] = GraphTimestamp.Format(GeneratedAt),
["node_count"] = NodeCount,
["edge_count"] = EdgeCount,
["lineage"] = Lineage.ToJson(),
["files"] = Files.ToJson(),
["hash"] = Hash
};
return obj;
}
}
public sealed class GraphSnapshotLineage
{
[JsonPropertyName("derived_from_sbom_digests")]
public ImmutableArray<string> DerivedFromSbomDigests { get; }
[JsonPropertyName("base_artifact_digests")]
public ImmutableArray<string> BaseArtifactDigests { get; }
[JsonPropertyName("source_snapshot_id")]
public string? SourceSnapshotId { get; }
public GraphSnapshotLineage(
ImmutableArray<string> derivedFromSbomDigests,
ImmutableArray<string> baseArtifactDigests,
string? sourceSnapshotId)
{
DerivedFromSbomDigests = derivedFromSbomDigests;
BaseArtifactDigests = baseArtifactDigests;
SourceSnapshotId = sourceSnapshotId;
}
public JsonObject ToJson()
{
var obj = new JsonObject
{
["derived_from_sbom_digests"] = CreateArray(DerivedFromSbomDigests),
["base_artifact_digests"] = CreateArray(BaseArtifactDigests),
["source_snapshot_id"] = SourceSnapshotId
};
return obj;
}
private static JsonArray CreateArray(ImmutableArray<string> values)
{
var array = new JsonArray();
foreach (var value in values)
{
array.Add(value);
}
return array;
}
}
public sealed class GraphSnapshotFiles
{
[JsonPropertyName("nodes")]
public string Nodes { get; }
[JsonPropertyName("edges")]
public string Edges { get; }
[JsonPropertyName("adjacency")]
public string Adjacency { get; }
public GraphSnapshotFiles(string nodes, string edges, string adjacency)
{
Nodes = nodes;
Edges = edges;
Adjacency = adjacency;
}
public JsonObject ToJson()
{
return new JsonObject
{
["nodes"] = Nodes,
["edges"] = Edges,
["adjacency"] = Adjacency
};
}
}
public sealed class GraphAdjacencyManifest
{
[JsonPropertyName("tenant")]
public string Tenant { get; }
[JsonPropertyName("artifact_node_id")]
public string ArtifactNodeId { get; }
[JsonPropertyName("generated_at")]
public DateTimeOffset GeneratedAt { get; }
[JsonPropertyName("snapshot_id")]
public string SnapshotId { get; }
[JsonPropertyName("nodes")]
public ImmutableArray<GraphAdjacencyNode> Nodes { get; }
public GraphAdjacencyManifest(
string tenant,
string artifactNodeId,
DateTimeOffset generatedAt,
string snapshotId,
ImmutableArray<GraphAdjacencyNode> nodes)
{
Tenant = tenant;
ArtifactNodeId = artifactNodeId;
GeneratedAt = generatedAt;
SnapshotId = snapshotId;
Nodes = nodes;
}
public JsonObject ToJson()
{
var obj = new JsonObject
{
["tenant"] = Tenant,
["artifact_node_id"] = ArtifactNodeId,
["generated_at"] = GraphTimestamp.Format(GeneratedAt),
["snapshot_id"] = SnapshotId
};
var nodesArray = new JsonArray();
foreach (var node in Nodes)
{
nodesArray.Add(node.ToJson());
}
obj["nodes"] = nodesArray;
return obj;
}
}
public sealed class GraphAdjacencyNode
{
[JsonPropertyName("node_id")]
public string NodeId { get; }
[JsonPropertyName("kind")]
public string Kind { get; }
[JsonPropertyName("outgoing_edges")]
public ImmutableArray<string> OutgoingEdges { get; }
[JsonPropertyName("incoming_edges")]
public ImmutableArray<string> IncomingEdges { get; }
public GraphAdjacencyNode(
string nodeId,
string kind,
ImmutableArray<string> outgoingEdges,
ImmutableArray<string> incomingEdges)
{
NodeId = nodeId;
Kind = kind;
OutgoingEdges = outgoingEdges;
IncomingEdges = incomingEdges;
}
public JsonObject ToJson()
{
return new JsonObject
{
["node_id"] = NodeId,
["kind"] = Kind,
["outgoing_edges"] = CreateArray(OutgoingEdges),
["incoming_edges"] = CreateArray(IncomingEdges)
};
}
private static JsonArray CreateArray(ImmutableArray<string> values)
{
var array = new JsonArray();
foreach (var value in values)
{
array.Add(value);
}
return array;
}
}

View File

@@ -0,0 +1,449 @@
using System.Collections.Immutable;
using System.Text.Json.Nodes;
using StellaOps.Graph.Indexer.Ingestion.Sbom;
using StellaOps.Graph.Indexer.Schema;
namespace StellaOps.Graph.Indexer.Documents;
public sealed class GraphSnapshotBuilder
{
private const string NodesFileName = "nodes.jsonl";
private const string EdgesFileName = "edges.jsonl";
private const string AdjacencyFileName = "adjacency.json";
public GraphSnapshot Build(SbomSnapshot sbomSnapshot, GraphBuildBatch batch, DateTimeOffset generatedAt)
{
ArgumentNullException.ThrowIfNull(sbomSnapshot);
ArgumentNullException.ThrowIfNull(batch);
var tenant = sbomSnapshot.Tenant ?? string.Empty;
var nodes = batch.Nodes;
var edges = batch.Edges;
var nodesById = nodes.ToImmutableDictionary(
node => node["id"]!.GetValue<string>(),
node => node,
StringComparer.Ordinal);
var artifactNodeId = ResolveArtifactNodeId(sbomSnapshot, nodes);
var snapshotId = ComputeSnapshotId(sbomSnapshot.Tenant, sbomSnapshot.ArtifactDigest, sbomSnapshot.SbomDigest);
var derivedSbomDigests = sbomSnapshot.BaseArtifacts
.Select(baseArtifact => baseArtifact.SbomDigest)
.Where(static digest => !string.IsNullOrWhiteSpace(digest))
.Select(static digest => digest.Trim())
.Distinct(StringComparer.Ordinal)
.OrderBy(static digest => digest, StringComparer.Ordinal)
.ToImmutableArray();
var baseArtifactDigests = sbomSnapshot.BaseArtifacts
.Select(baseArtifact => baseArtifact.ArtifactDigest)
.Where(static digest => !string.IsNullOrWhiteSpace(digest))
.Select(static digest => digest.Trim())
.Distinct(StringComparer.Ordinal)
.OrderBy(static digest => digest, StringComparer.Ordinal)
.ToImmutableArray();
var lineage = new GraphSnapshotLineage(derivedSbomDigests, baseArtifactDigests, null);
var files = new GraphSnapshotFiles(NodesFileName, EdgesFileName, AdjacencyFileName);
var manifest = CreateManifest(
tenant,
sbomSnapshot.ArtifactDigest,
sbomSnapshot.SbomDigest,
snapshotId,
generatedAt,
nodes.Length,
edges.Length,
lineage,
files);
var adjacency = BuildAdjacencyManifest(
sbomSnapshot,
snapshotId,
generatedAt,
artifactNodeId,
nodes,
edges,
nodesById);
return new GraphSnapshot(manifest, adjacency);
}
private static GraphSnapshotManifest CreateManifest(
string tenant,
string artifactDigest,
string sbomDigest,
string snapshotId,
DateTimeOffset generatedAt,
int nodeCount,
int edgeCount,
GraphSnapshotLineage lineage,
GraphSnapshotFiles files)
{
var json = new JsonObject
{
["tenant"] = tenant,
["artifact_digest"] = artifactDigest,
["sbom_digest"] = sbomDigest,
["snapshot_id"] = snapshotId,
["generated_at"] = GraphTimestamp.Format(generatedAt),
["node_count"] = nodeCount,
["edge_count"] = edgeCount,
["lineage"] = lineage.ToJson(),
["files"] = files.ToJson()
};
var hash = GraphIdentity.ComputeDocumentHash(json);
json["hash"] = hash;
return new GraphSnapshotManifest(
tenant,
artifactDigest,
sbomDigest,
snapshotId,
generatedAt,
nodeCount,
edgeCount,
lineage,
files,
hash);
}
private static GraphAdjacencyManifest BuildAdjacencyManifest(
SbomSnapshot snapshot,
string snapshotId,
DateTimeOffset generatedAt,
string artifactNodeId,
ImmutableArray<JsonObject> nodes,
ImmutableArray<JsonObject> edges,
IReadOnlyDictionary<string, JsonObject> nodesById)
{
var nodeEntries = new Dictionary<string, AdjacencyNodeBuilder>(StringComparer.Ordinal);
foreach (var node in nodes)
{
var nodeId = node["id"]!.GetValue<string>();
var kind = node["kind"]!.GetValue<string>();
nodeEntries[nodeId] = new AdjacencyNodeBuilder(nodeId, kind);
}
var componentNodeByPurl = BuildComponentIndex(nodes);
var artifactNodeByDigest = BuildArtifactIndex(nodes);
foreach (var edge in edges)
{
if (!TryResolveEdgeEndpoints(
edge,
nodesById,
componentNodeByPurl,
artifactNodeByDigest,
out var sourceNodeId,
out var targetNodeId))
{
continue;
}
if (!nodeEntries.TryGetValue(sourceNodeId, out var source))
{
continue;
}
if (!nodeEntries.TryGetValue(targetNodeId, out var target))
{
continue;
}
var edgeId = edge["id"]!.GetValue<string>();
source.AddOutgoing(edgeId);
target.AddIncoming(edgeId);
}
var nodesArray = nodeEntries.Values
.Select(builder => builder.ToNode())
.OrderBy(node => node.NodeId, StringComparer.Ordinal)
.ToImmutableArray();
return new GraphAdjacencyManifest(
snapshot.Tenant,
artifactNodeId,
generatedAt,
snapshotId,
nodesArray);
}
private static string ResolveArtifactNodeId(SbomSnapshot snapshot, ImmutableArray<JsonObject> nodes)
{
foreach (var node in nodes)
{
if (!string.Equals(node["kind"]!.GetValue<string>(), "artifact", StringComparison.Ordinal))
{
continue;
}
if (TryMatchArtifact(node, snapshot.ArtifactDigest, snapshot.SbomDigest))
{
return node["id"]!.GetValue<string>();
}
}
throw new InvalidOperationException($"Unable to locate artifact node for digest '{snapshot.ArtifactDigest}' and SBOM '{snapshot.SbomDigest}'.");
}
private static bool TryMatchArtifact(JsonObject node, string artifactDigest, string sbomDigest)
{
if (!node.TryGetPropertyValue("attributes", out var attributesNode) || attributesNode is not JsonObject attributes)
{
return false;
}
var nodeArtifactDigest = attributes.TryGetPropertyValue("artifact_digest", out var digestValue)
? digestValue?.GetValue<string>() ?? string.Empty
: string.Empty;
var nodeSbomDigest = attributes.TryGetPropertyValue("sbom_digest", out var sbomValue)
? sbomValue?.GetValue<string>() ?? string.Empty
: string.Empty;
return string.Equals(nodeArtifactDigest, artifactDigest, StringComparison.Ordinal)
&& string.Equals(nodeSbomDigest, sbomDigest, StringComparison.Ordinal);
}
private static string ComputeSnapshotId(string tenant, string artifactDigest, string sbomDigest)
{
var identity = new JsonObject
{
["tenant"] = tenant.Trim().ToLowerInvariant(),
["artifact_digest"] = artifactDigest.Trim().ToLowerInvariant(),
["sbom_digest"] = sbomDigest.Trim().ToLowerInvariant()
};
var hash = GraphIdentity.ComputeDocumentHash(identity);
return $"gs:{tenant.Trim().ToLowerInvariant()}:{hash}";
}
private static Dictionary<string, string> BuildComponentIndex(ImmutableArray<JsonObject> nodes)
{
var components = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var node in nodes)
{
if (!string.Equals(node["kind"]!.GetValue<string>(), "component", StringComparison.Ordinal))
{
continue;
}
if (!node.TryGetPropertyValue("attributes", out var attributesNode) || attributesNode is not JsonObject attributes)
{
continue;
}
if (!attributes.TryGetPropertyValue("purl", out var purlNode) || purlNode is null)
{
continue;
}
var purl = purlNode.GetValue<string>();
if (!string.IsNullOrWhiteSpace(purl))
{
components.TryAdd(purl.Trim(), node["id"]!.GetValue<string>());
}
}
return components;
}
private static Dictionary<string, string> BuildArtifactIndex(ImmutableArray<JsonObject> nodes)
{
var artifacts = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var node in nodes)
{
if (!string.Equals(node["kind"]!.GetValue<string>(), "artifact", StringComparison.Ordinal))
{
continue;
}
if (!node.TryGetPropertyValue("attributes", out var attributesNode) || attributesNode is not JsonObject attributes)
{
continue;
}
if (!attributes.TryGetPropertyValue("artifact_digest", out var digestNode) || digestNode is null)
{
continue;
}
var digest = digestNode.GetValue<string>();
if (!string.IsNullOrWhiteSpace(digest))
{
artifacts.TryAdd(digest.Trim(), node["id"]!.GetValue<string>());
}
}
return artifacts;
}
private static bool TryResolveEdgeEndpoints(
JsonObject edge,
IReadOnlyDictionary<string, JsonObject> nodesById,
IReadOnlyDictionary<string, string> componentNodeByPurl,
IReadOnlyDictionary<string, string> artifactNodeByDigest,
out string sourceNodeId,
out string targetNodeId)
{
var kind = edge["kind"]!.GetValue<string>();
var canonicalKey = edge["canonical_key"]!.AsObject();
string? source = null;
string? target = null;
switch (kind)
{
case "CONTAINS":
source = canonicalKey.TryGetPropertyValue("artifact_node_id", out var containsSource) ? containsSource?.GetValue<string>() : null;
target = canonicalKey.TryGetPropertyValue("component_node_id", out var containsTarget) ? containsTarget?.GetValue<string>() : null;
break;
case "DECLARED_IN":
source = canonicalKey.TryGetPropertyValue("component_node_id", out var declaredSource) ? declaredSource?.GetValue<string>() : null;
target = canonicalKey.TryGetPropertyValue("file_node_id", out var declaredTarget) ? declaredTarget?.GetValue<string>() : null;
break;
case "AFFECTED_BY":
source = canonicalKey.TryGetPropertyValue("component_node_id", out var affectedSource) ? affectedSource?.GetValue<string>() : null;
target = canonicalKey.TryGetPropertyValue("advisory_node_id", out var affectedTarget) ? affectedTarget?.GetValue<string>() : null;
break;
case "VEX_EXEMPTS":
source = canonicalKey.TryGetPropertyValue("component_node_id", out var vexSource) ? vexSource?.GetValue<string>() : null;
target = canonicalKey.TryGetPropertyValue("vex_node_id", out var vexTarget) ? vexTarget?.GetValue<string>() : null;
break;
case "GOVERNS_WITH":
source = canonicalKey.TryGetPropertyValue("policy_node_id", out var policySource) ? policySource?.GetValue<string>() : null;
target = canonicalKey.TryGetPropertyValue("component_node_id", out var policyTarget) ? policyTarget?.GetValue<string>() : null;
break;
case "OBSERVED_RUNTIME":
source = canonicalKey.TryGetPropertyValue("runtime_node_id", out var runtimeSource) ? runtimeSource?.GetValue<string>() : null;
target = canonicalKey.TryGetPropertyValue("component_node_id", out var runtimeTarget) ? runtimeTarget?.GetValue<string>() : null;
break;
case "BUILT_FROM":
source = canonicalKey.TryGetPropertyValue("parent_artifact_node_id", out var builtSource) ? builtSource?.GetValue<string>() : null;
if (canonicalKey.TryGetPropertyValue("child_artifact_node_id", out var builtTargetNode) && builtTargetNode is not null)
{
target = builtTargetNode.GetValue<string>();
}
else if (canonicalKey.TryGetPropertyValue("child_artifact_digest", out var builtTargetDigest) && builtTargetDigest is not null)
{
artifactNodeByDigest.TryGetValue(builtTargetDigest.GetValue<string>(), out target);
}
break;
case "DEPENDS_ON":
source = canonicalKey.TryGetPropertyValue("component_node_id", out var dependsSource) ? dependsSource?.GetValue<string>() : null;
if (canonicalKey.TryGetPropertyValue("dependency_node_id", out var dependsTargetNode) && dependsTargetNode is not null)
{
target = dependsTargetNode.GetValue<string>();
}
else if (canonicalKey.TryGetPropertyValue("dependency_purl", out var dependencyPurl) && dependencyPurl is not null)
{
componentNodeByPurl.TryGetValue(dependencyPurl.GetValue<string>(), out target);
}
break;
default:
source = ExtractFirstNodeId(canonicalKey);
target = ExtractSecondNodeId(canonicalKey);
break;
}
if (source is null || target is null)
{
sourceNodeId = string.Empty;
targetNodeId = string.Empty;
return false;
}
if (!nodesById.ContainsKey(source) || !nodesById.ContainsKey(target))
{
sourceNodeId = string.Empty;
targetNodeId = string.Empty;
return false;
}
sourceNodeId = source;
targetNodeId = target;
return true;
}
private static string? ExtractFirstNodeId(JsonObject canonicalKey)
{
foreach (var property in canonicalKey)
{
if (property.Value is JsonValue value
&& value.TryGetValue(out string? candidate)
&& candidate is not null
&& candidate.StartsWith("gn:", StringComparison.Ordinal))
{
return candidate;
}
}
return null;
}
private static string? ExtractSecondNodeId(JsonObject canonicalKey)
{
var encountered = false;
foreach (var property in canonicalKey)
{
if (property.Value is JsonValue value
&& value.TryGetValue(out string? candidate)
&& candidate is not null
&& candidate.StartsWith("gn:", StringComparison.Ordinal))
{
if (!encountered)
{
encountered = true;
continue;
}
return candidate;
}
}
return null;
}
private sealed class AdjacencyNodeBuilder
{
private readonly SortedSet<string> _outgoing = new(StringComparer.Ordinal);
private readonly SortedSet<string> _incoming = new(StringComparer.Ordinal);
public AdjacencyNodeBuilder(string nodeId, string kind)
{
NodeId = nodeId;
Kind = kind;
}
public string NodeId { get; }
public string Kind { get; }
public void AddOutgoing(string edgeId)
{
if (!string.IsNullOrWhiteSpace(edgeId))
{
_outgoing.Add(edgeId);
}
}
public void AddIncoming(string edgeId)
{
if (!string.IsNullOrWhiteSpace(edgeId))
{
_incoming.Add(edgeId);
}
}
public GraphAdjacencyNode ToNode()
{
return new GraphAdjacencyNode(
NodeId,
Kind,
_outgoing.ToImmutableArray(),
_incoming.ToImmutableArray());
}
}
}

View File

@@ -0,0 +1,106 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
public sealed class AdvisoryLinksetMetrics : IAdvisoryLinksetMetrics, IDisposable
{
public const string MeterName = "StellaOps.Graph.Indexer";
public const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly bool _ownsMeter;
private readonly Counter<long> _batchesTotal;
private readonly Histogram<double> _batchDurationSeconds;
private readonly Counter<long> _nodesTotal;
private readonly Counter<long> _edgesTotal;
private bool _disposed;
public AdvisoryLinksetMetrics()
: this(null)
{
}
public AdvisoryLinksetMetrics(Meter? meter)
{
_meter = meter ?? new Meter(MeterName, MeterVersion);
_ownsMeter = meter is null;
_batchesTotal = _meter.CreateCounter<long>(
name: "graph_advisory_ingest_batches_total",
unit: "count",
description: "Advisory linkset ingest batches processed grouped by source, tenant, and result.");
_batchDurationSeconds = _meter.CreateHistogram<double>(
name: "graph_advisory_ingest_duration_seconds",
unit: "s",
description: "Latency to transform and persist advisory linkset batches grouped by source, tenant, and result.");
_nodesTotal = _meter.CreateCounter<long>(
name: "graph_advisory_ingest_nodes_total",
unit: "count",
description: "Advisory nodes produced by linkset ingest grouped by source and tenant.");
_edgesTotal = _meter.CreateCounter<long>(
name: "graph_advisory_ingest_edges_total",
unit: "count",
description: "Affected_by edges produced by linkset ingest grouped by source and tenant.");
}
public void RecordBatch(string source, string tenant, int nodeCount, int edgeCount, TimeSpan duration, bool success)
{
ArgumentException.ThrowIfNullOrWhiteSpace(source);
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
var normalizedDurationSeconds = Math.Max(duration.TotalSeconds, 0d);
var resultTag = success ? "success" : "failure";
var tags = new[]
{
new KeyValuePair<string, object?>("source", source),
new KeyValuePair<string, object?>("tenant", tenant),
new KeyValuePair<string, object?>("result", resultTag)
};
_batchesTotal.Add(1, tags);
_batchDurationSeconds.Record(normalizedDurationSeconds, tags);
if (!success)
{
return;
}
var volumeTags = new[]
{
new KeyValuePair<string, object?>("source", source),
new KeyValuePair<string, object?>("tenant", tenant)
};
if (nodeCount > 0)
{
_nodesTotal.Add(nodeCount, volumeTags);
}
if (edgeCount > 0)
{
_edgesTotal.Add(edgeCount, volumeTags);
}
}
public void Dispose()
{
if (_disposed)
{
return;
}
if (_ownsMeter)
{
_meter.Dispose();
}
_disposed = true;
GC.SuppressFinalize(this);
}
}

View File

@@ -0,0 +1,84 @@
using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Graph.Indexer.Ingestion.Sbom;
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
public sealed class AdvisoryLinksetProcessor
{
private readonly AdvisoryLinksetTransformer _transformer;
private readonly IGraphDocumentWriter _writer;
private readonly IAdvisoryLinksetMetrics _metrics;
private readonly ILogger<AdvisoryLinksetProcessor> _logger;
public AdvisoryLinksetProcessor(
AdvisoryLinksetTransformer transformer,
IGraphDocumentWriter writer,
IAdvisoryLinksetMetrics metrics,
ILogger<AdvisoryLinksetProcessor> logger)
{
_transformer = transformer ?? throw new ArgumentNullException(nameof(transformer));
_writer = writer ?? throw new ArgumentNullException(nameof(writer));
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task ProcessAsync(AdvisoryLinksetSnapshot snapshot, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(snapshot);
cancellationToken.ThrowIfCancellationRequested();
var stopwatch = Stopwatch.StartNew();
GraphBuildBatch batch;
try
{
batch = _transformer.Transform(snapshot);
}
catch (Exception ex)
{
stopwatch.Stop();
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, 0, 0, stopwatch.Elapsed, success: false);
_logger.LogError(
ex,
"graph-indexer: failed to transform advisory linkset {LinksetDigest} for tenant {Tenant}",
snapshot.LinksetDigest,
snapshot.Tenant);
throw;
}
try
{
cancellationToken.ThrowIfCancellationRequested();
await _writer.WriteAsync(batch, cancellationToken).ConfigureAwait(false);
stopwatch.Stop();
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, batch.Nodes.Length, batch.Edges.Length, stopwatch.Elapsed, success: true);
_logger.LogInformation(
"graph-indexer: indexed advisory linkset {LinksetDigest} for tenant {Tenant} with {NodeCount} nodes and {EdgeCount} edges in {DurationMs:F2} ms",
snapshot.LinksetDigest,
snapshot.Tenant,
batch.Nodes.Length,
batch.Edges.Length,
stopwatch.Elapsed.TotalMilliseconds);
}
catch (Exception ex)
{
stopwatch.Stop();
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, batch.Nodes.Length, batch.Edges.Length, stopwatch.Elapsed, success: false);
_logger.LogError(
ex,
"graph-indexer: failed to persist advisory linkset {LinksetDigest} for tenant {Tenant}",
snapshot.LinksetDigest,
snapshot.Tenant);
throw;
}
}
}

View File

@@ -0,0 +1,88 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
public sealed class AdvisoryLinksetSnapshot
{
[JsonPropertyName("tenant")]
public string Tenant { get; init; } = string.Empty;
[JsonPropertyName("source")]
public string Source { get; init; } = string.Empty;
[JsonPropertyName("linksetDigest")]
public string LinksetDigest { get; init; } = string.Empty;
[JsonPropertyName("collectedAt")]
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
[JsonPropertyName("eventOffset")]
public long EventOffset { get; init; } = 0;
[JsonPropertyName("advisory")]
public AdvisoryDetails Advisory { get; init; } = new();
[JsonPropertyName("components")]
public IReadOnlyList<AdvisoryComponentImpact> Components { get; init; }
= Array.Empty<AdvisoryComponentImpact>();
}
public sealed class AdvisoryDetails
{
[JsonPropertyName("source")]
public string Source { get; init; } = string.Empty;
[JsonPropertyName("advisorySource")]
public string AdvisorySource { get; init; } = string.Empty;
[JsonPropertyName("advisoryId")]
public string AdvisoryId { get; init; } = string.Empty;
[JsonPropertyName("severity")]
public string Severity { get; init; } = string.Empty;
[JsonPropertyName("publishedAt")]
public DateTimeOffset PublishedAt { get; init; } = DateTimeOffset.UnixEpoch;
[JsonPropertyName("contentHash")]
public string ContentHash { get; init; } = string.Empty;
}
public sealed class AdvisoryComponentImpact
{
[JsonPropertyName("purl")]
public string ComponentPurl { get; init; } = string.Empty;
[JsonPropertyName("sourceType")]
public string ComponentSourceType { get; init; } = "inventory";
[JsonPropertyName("evidenceDigest")]
public string EvidenceDigest { get; init; } = string.Empty;
[JsonPropertyName("matchedVersions")]
public IReadOnlyList<string> MatchedVersions { get; init; }
= Array.Empty<string>();
[JsonPropertyName("cvss")]
public double? Cvss { get; init; }
= null;
[JsonPropertyName("confidence")]
public double? Confidence { get; init; }
= null;
[JsonPropertyName("source")]
public string Source { get; init; } = string.Empty;
[JsonPropertyName("collectedAt")]
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
[JsonPropertyName("eventOffset")]
public long EventOffset { get; init; } = 0;
[JsonPropertyName("sbomDigest")]
public string? SbomDigest { get; init; }
= null;
}

View File

@@ -0,0 +1,208 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text.Json.Nodes;
using StellaOps.Graph.Indexer.Ingestion.Sbom;
using StellaOps.Graph.Indexer.Schema;
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
public sealed class AdvisoryLinksetTransformer
{
private const string AdvisoryNodeKind = "advisory";
private const string ComponentNodeKind = "component";
private const string AffectedByEdgeKind = "AFFECTED_BY";
public GraphBuildBatch Transform(AdvisoryLinksetSnapshot snapshot)
{
ArgumentNullException.ThrowIfNull(snapshot);
var nodes = new List<JsonObject>();
var edgesById = new Dictionary<string, JsonObject>(StringComparer.Ordinal);
var advisoryNode = CreateAdvisoryNode(snapshot);
nodes.Add(advisoryNode);
foreach (var component in snapshot.Components ?? Array.Empty<AdvisoryComponentImpact>())
{
if (component is null)
{
continue;
}
var edge = CreateAffectedByEdge(snapshot, advisoryNode, component);
edgesById[edge["id"]!.GetValue<string>()] = edge;
}
var orderedNodes = nodes
.OrderBy(node => node["kind"]!.GetValue<string>(), StringComparer.Ordinal)
.ThenBy(node => node["id"]!.GetValue<string>(), StringComparer.Ordinal)
.ToImmutableArray();
var orderedEdges = edgesById.Values
.OrderBy(edge => edge["kind"]!.GetValue<string>(), StringComparer.Ordinal)
.ThenBy(edge => edge["id"]!.GetValue<string>(), StringComparer.Ordinal)
.ToImmutableArray();
return new GraphBuildBatch(orderedNodes, orderedEdges);
}
private static JsonObject CreateAdvisoryNode(AdvisoryLinksetSnapshot snapshot)
{
var details = snapshot.Advisory ?? new AdvisoryDetails();
var advisorySource = (details.AdvisorySource ?? string.Empty).Trim();
var advisoryId = (details.AdvisoryId ?? string.Empty).Trim();
var severity = (details.Severity ?? string.Empty).Trim();
var contentHash = (details.ContentHash ?? string.Empty).Trim();
var linksetDigest = (snapshot.LinksetDigest ?? string.Empty).Trim();
var provenanceSource = ResolveSource(details.Source, snapshot.Source);
var validFrom = details.PublishedAt == DateTimeOffset.UnixEpoch
? snapshot.CollectedAt
: details.PublishedAt;
var attributes = new JsonObject
{
["advisory_source"] = advisorySource,
["advisory_id"] = advisoryId,
["severity"] = severity,
["published_at"] = GraphTimestamp.Format(validFrom),
["content_hash"] = contentHash,
["linkset_digest"] = linksetDigest
};
var canonicalKey = new Dictionary<string, string>
{
["tenant"] = snapshot.Tenant,
["advisory_source"] = advisorySource,
["advisory_id"] = advisoryId,
["content_hash"] = contentHash
};
var node = GraphDocumentFactory.CreateNode(new GraphNodeSpec(
Tenant: snapshot.Tenant,
Kind: AdvisoryNodeKind,
CanonicalKey: canonicalKey,
Attributes: attributes,
Provenance: new GraphProvenanceSpec(provenanceSource, snapshot.CollectedAt, null, snapshot.EventOffset),
ValidFrom: validFrom,
ValidTo: null));
var provenance = node["provenance"]!.AsObject();
var sourceNode = provenance["source"]!.DeepClone();
var collectedAtNode = provenance["collected_at"]!.DeepClone();
var eventOffsetNode = provenance.ContainsKey("event_offset")
? provenance["event_offset"]!.DeepClone()
: null;
var reorderedProvenance = new JsonObject
{
["source"] = sourceNode,
["collected_at"] = collectedAtNode,
["sbom_digest"] = null
};
if (eventOffsetNode is not null)
{
reorderedProvenance["event_offset"] = eventOffsetNode;
}
node["provenance"] = reorderedProvenance;
node.Remove("hash");
node["hash"] = GraphIdentity.ComputeDocumentHash(node);
return node;
}
private static JsonObject CreateAffectedByEdge(
AdvisoryLinksetSnapshot snapshot,
JsonObject advisoryNode,
AdvisoryComponentImpact component)
{
var advisoryNodeId = advisoryNode["id"]!.GetValue<string>();
var componentSourceType = string.IsNullOrWhiteSpace(component.ComponentSourceType)
? "inventory"
: component.ComponentSourceType.Trim();
var componentPurl = (component.ComponentPurl ?? string.Empty).Trim();
var linksetDigest = (snapshot.LinksetDigest ?? string.Empty).Trim();
var componentIdentity = new Dictionary<string, string>
{
["tenant"] = snapshot.Tenant,
["purl"] = componentPurl,
["source_type"] = componentSourceType
};
var componentNodeId = GraphIdentity.ComputeNodeId(snapshot.Tenant, ComponentNodeKind, componentIdentity);
var canonicalKey = new Dictionary<string, string>
{
["tenant"] = snapshot.Tenant,
["component_node_id"] = componentNodeId,
["advisory_node_id"] = advisoryNodeId,
["linkset_digest"] = linksetDigest
};
var attributes = new JsonObject
{
["evidence_digest"] = component.EvidenceDigest?.Trim() ?? string.Empty,
["matched_versions"] = CreateJsonArray(component.MatchedVersions ?? Array.Empty<string>())
};
if (component.Cvss is { } cvss)
{
attributes["cvss"] = cvss;
}
if (component.Confidence is { } confidence)
{
attributes["confidence"] = confidence;
}
var collectedAt = component.CollectedAt == DateTimeOffset.UnixEpoch
? snapshot.CollectedAt
: component.CollectedAt;
var eventOffset = component.EventOffset != 0 ? component.EventOffset : snapshot.EventOffset;
var source = ResolveSource(component.Source, snapshot.Source);
return GraphDocumentFactory.CreateEdge(new GraphEdgeSpec(
Tenant: snapshot.Tenant,
Kind: AffectedByEdgeKind,
CanonicalKey: canonicalKey,
Attributes: attributes,
Provenance: new GraphProvenanceSpec(source, collectedAt, component.SbomDigest, eventOffset),
ValidFrom: collectedAt,
ValidTo: null));
}
private static JsonArray CreateJsonArray(IEnumerable<string> values)
{
var array = new JsonArray();
foreach (var value in values
.Where(v => !string.IsNullOrWhiteSpace(v))
.Select(v => v.Trim())
.Distinct(StringComparer.Ordinal)
.OrderBy(v => v, StringComparer.Ordinal))
{
array.Add(value);
}
return array;
}
private static string ResolveSource(params string?[] candidates)
{
foreach (var candidate in candidates)
{
if (!string.IsNullOrWhiteSpace(candidate))
{
return candidate.Trim();
}
}
return string.Empty;
}
}

View File

@@ -0,0 +1,8 @@
using System;
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
public interface IAdvisoryLinksetMetrics
{
void RecordBatch(string source, string tenant, int nodeCount, int edgeCount, TimeSpan duration, bool success);
}

View File

@@ -0,0 +1,84 @@
using System;
using System.Collections.Generic;
using System.Text.Encodings.Web;
using System.Text.Json;
using System.Text.Json.Nodes;
using StellaOps.Graph.Indexer.Ingestion.Sbom;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Bson;
using MongoDB.Driver;
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
public sealed class MongoGraphDocumentWriter : IGraphDocumentWriter
{
private static readonly JsonSerializerOptions SerializerOptions = new()
{
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
WriteIndented = false
};
private readonly IMongoCollection<BsonDocument> _nodes;
private readonly IMongoCollection<BsonDocument> _edges;
public MongoGraphDocumentWriter(IMongoDatabase database, MongoGraphDocumentWriterOptions? options = null)
{
ArgumentNullException.ThrowIfNull(database);
var resolved = options ?? new MongoGraphDocumentWriterOptions();
_nodes = database.GetCollection<BsonDocument>(resolved.NodeCollectionName);
_edges = database.GetCollection<BsonDocument>(resolved.EdgeCollectionName);
}
public async Task WriteAsync(GraphBuildBatch batch, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(batch);
cancellationToken.ThrowIfCancellationRequested();
if (batch.Nodes.Length > 0)
{
var nodeModels = CreateReplaceModels(_nodes, batch.Nodes);
if (nodeModels.Count > 0)
{
await _nodes.BulkWriteAsync(nodeModels, new BulkWriteOptions { IsOrdered = false }, cancellationToken)
.ConfigureAwait(false);
}
}
if (batch.Edges.Length > 0)
{
var edgeModels = CreateReplaceModels(_edges, batch.Edges);
if (edgeModels.Count > 0)
{
await _edges.BulkWriteAsync(edgeModels, new BulkWriteOptions { IsOrdered = false }, cancellationToken)
.ConfigureAwait(false);
}
}
}
private static List<WriteModel<BsonDocument>> CreateReplaceModels(IMongoCollection<BsonDocument> collection, IReadOnlyList<JsonObject> documents)
{
var models = new List<WriteModel<BsonDocument>>(documents.Count);
foreach (var document in documents)
{
if (!document.TryGetPropertyValue("id", out var idNode) || idNode is null)
{
continue;
}
var id = idNode.GetValue<string>();
var filter = Builders<BsonDocument>.Filter.Eq("id", id);
var bsonDocument = ToBsonDocument(document);
models.Add(new ReplaceOneModel<BsonDocument>(filter, bsonDocument) { IsUpsert = true });
}
return models;
}
private static BsonDocument ToBsonDocument(JsonObject json)
{
var jsonString = json.ToJsonString(SerializerOptions);
return BsonDocument.Parse(jsonString);
}
}

View File

@@ -0,0 +1,7 @@
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
public sealed class MongoGraphDocumentWriterOptions
{
public string NodeCollectionName { get; init; } = "graph_nodes";
public string EdgeCollectionName { get; init; } = "graph_edges";
}

View File

@@ -0,0 +1,8 @@
using System;
namespace StellaOps.Graph.Indexer.Ingestion.Policy;
public interface IPolicyOverlayMetrics
{
void RecordBatch(string source, string tenant, int nodeCount, int edgeCount, TimeSpan duration, bool success);
}

View File

@@ -0,0 +1,124 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
namespace StellaOps.Graph.Indexer.Ingestion.Policy;
public sealed class PolicyOverlayMetrics : IPolicyOverlayMetrics, IDisposable
{
public const string MeterName = "StellaOps.Graph.Indexer";
public const string MeterVersion = "1.0.0";
private const string BatchesTotalName = "graph_policy_overlay_batches_total";
private const string BatchDurationSecondsName = "graph_policy_overlay_duration_seconds";
private const string NodesTotalName = "graph_policy_overlay_nodes_total";
private const string EdgesTotalName = "graph_policy_overlay_edges_total";
private const string UnitCount = "count";
private const string UnitSeconds = "s";
private readonly Meter _meter;
private readonly bool _ownsMeter;
private readonly Counter<long> _batchesTotal;
private readonly Histogram<double> _batchDurationSeconds;
private readonly Counter<long> _nodesTotal;
private readonly Counter<long> _edgesTotal;
private bool _disposed;
public PolicyOverlayMetrics()
: this(null)
{
}
public PolicyOverlayMetrics(Meter? meter)
{
_meter = meter ?? new Meter(MeterName, MeterVersion);
_ownsMeter = meter is null;
_batchesTotal = _meter.CreateCounter<long>(
name: BatchesTotalName,
unit: UnitCount,
description: "Policy overlay batches processed grouped by source, tenant, and result.");
_batchDurationSeconds = _meter.CreateHistogram<double>(
name: BatchDurationSecondsName,
unit: UnitSeconds,
description: "Latency to transform and persist policy overlay batches grouped by source, tenant, and result.");
_nodesTotal = _meter.CreateCounter<long>(
name: NodesTotalName,
unit: UnitCount,
description: "Policy overlay nodes produced grouped by source and tenant.");
_edgesTotal = _meter.CreateCounter<long>(
name: EdgesTotalName,
unit: UnitCount,
description: "GOVERNS_WITH edges produced grouped by source and tenant.");
}
public void RecordBatch(string source, string tenant, int nodeCount, int edgeCount, TimeSpan duration, bool success)
{
ThrowIfDisposed();
ArgumentException.ThrowIfNullOrWhiteSpace(source);
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
var normalizedDuration = Math.Max(duration.TotalSeconds, 0d);
var resultTag = success ? "success" : "failure";
var tags = new[]
{
new KeyValuePair<string, object?>("source", source),
new KeyValuePair<string, object?>("tenant", tenant),
new KeyValuePair<string, object?>("result", resultTag)
};
_batchesTotal.Add(1, tags);
_batchDurationSeconds.Record(normalizedDuration, tags);
if (!success)
{
return;
}
var volumeTags = new[]
{
new KeyValuePair<string, object?>("source", source),
new KeyValuePair<string, object?>("tenant", tenant)
};
if (nodeCount > 0)
{
_nodesTotal.Add(nodeCount, volumeTags);
}
if (edgeCount > 0)
{
_edgesTotal.Add(edgeCount, volumeTags);
}
}
private void ThrowIfDisposed()
{
if (_disposed)
{
throw new ObjectDisposedException(nameof(PolicyOverlayMetrics));
}
}
public void Dispose()
{
if (_disposed)
{
return;
}
if (_ownsMeter)
{
_meter.Dispose();
}
_disposed = true;
GC.SuppressFinalize(this);
}
}

View File

@@ -0,0 +1,85 @@
using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Graph.Indexer.Ingestion.Sbom;
namespace StellaOps.Graph.Indexer.Ingestion.Policy;
public sealed class PolicyOverlayProcessor
{
private readonly PolicyOverlayTransformer _transformer;
private readonly IGraphDocumentWriter _writer;
private readonly IPolicyOverlayMetrics _metrics;
private readonly ILogger<PolicyOverlayProcessor> _logger;
public PolicyOverlayProcessor(
PolicyOverlayTransformer transformer,
IGraphDocumentWriter writer,
IPolicyOverlayMetrics metrics,
ILogger<PolicyOverlayProcessor> logger)
{
_transformer = transformer ?? throw new ArgumentNullException(nameof(transformer));
_writer = writer ?? throw new ArgumentNullException(nameof(writer));
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task ProcessAsync(PolicyOverlaySnapshot snapshot, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(snapshot);
cancellationToken.ThrowIfCancellationRequested();
var stopwatch = Stopwatch.StartNew();
GraphBuildBatch batch;
try
{
batch = _transformer.Transform(snapshot);
}
catch (Exception ex)
{
stopwatch.Stop();
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, 0, 0, stopwatch.Elapsed, success: false);
_logger.LogError(
ex,
"graph-indexer: failed to transform policy overlay {PolicyPackDigest} for tenant {Tenant}",
snapshot.Policy?.PolicyPackDigest ?? string.Empty,
snapshot.Tenant);
throw;
}
try
{
cancellationToken.ThrowIfCancellationRequested();
await _writer.WriteAsync(batch, cancellationToken).ConfigureAwait(false);
stopwatch.Stop();
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, batch.Nodes.Length, batch.Edges.Length, stopwatch.Elapsed, success: true);
_logger.LogInformation(
"graph-indexer: indexed policy overlay {PolicyPackDigest} (effective {EffectiveFrom}) for tenant {Tenant} with {NodeCount} nodes and {EdgeCount} edges in {DurationMs:F2} ms",
snapshot.Policy?.PolicyPackDigest ?? string.Empty,
(snapshot.Policy?.EffectiveFrom ?? snapshot.CollectedAt).ToUniversalTime(),
snapshot.Tenant,
batch.Nodes.Length,
batch.Edges.Length,
stopwatch.Elapsed.TotalMilliseconds);
}
catch (Exception ex)
{
stopwatch.Stop();
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, batch.Nodes.Length, batch.Edges.Length, stopwatch.Elapsed, success: false);
_logger.LogError(
ex,
"graph-indexer: failed to persist policy overlay {PolicyPackDigest} for tenant {Tenant}",
snapshot.Policy?.PolicyPackDigest ?? string.Empty,
snapshot.Tenant);
throw;
}
}
}

View File

@@ -0,0 +1,90 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Graph.Indexer.Ingestion.Policy;
public sealed class PolicyOverlaySnapshot
{
[JsonPropertyName("tenant")]
public string Tenant { get; init; } = string.Empty;
[JsonPropertyName("source")]
public string Source { get; init; } = string.Empty;
[JsonPropertyName("collectedAt")]
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
[JsonPropertyName("eventOffset")]
public long EventOffset { get; init; }
[JsonPropertyName("policy")]
public PolicyVersionDetails Policy { get; init; } = new();
[JsonPropertyName("evaluations")]
public IReadOnlyList<PolicyEvaluation> Evaluations { get; init; }
= Array.Empty<PolicyEvaluation>();
}
public sealed class PolicyVersionDetails
{
[JsonPropertyName("source")]
public string Source { get; init; } = string.Empty;
[JsonPropertyName("policyPackDigest")]
public string PolicyPackDigest { get; init; } = string.Empty;
[JsonPropertyName("policyName")]
public string PolicyName { get; init; } = string.Empty;
[JsonPropertyName("effectiveFrom")]
public DateTimeOffset EffectiveFrom { get; init; } = DateTimeOffset.UnixEpoch;
[JsonPropertyName("expiresAt")]
public DateTimeOffset ExpiresAt { get; init; } = DateTimeOffset.UnixEpoch;
[JsonPropertyName("explainHash")]
public string ExplainHash { get; init; } = string.Empty;
[JsonPropertyName("collectedAt")]
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
[JsonPropertyName("eventOffset")]
public long EventOffset { get; init; }
}
public sealed class PolicyEvaluation
{
[JsonPropertyName("componentPurl")]
public string ComponentPurl { get; init; } = string.Empty;
[JsonPropertyName("componentSourceType")]
public string ComponentSourceType { get; init; } = "inventory";
[JsonPropertyName("findingExplainHash")]
public string FindingExplainHash { get; init; } = string.Empty;
[JsonPropertyName("explainHash")]
public string? ExplainHash { get; init; }
[JsonPropertyName("policyRuleId")]
public string PolicyRuleId { get; init; } = string.Empty;
[JsonPropertyName("verdict")]
public string Verdict { get; init; } = string.Empty;
[JsonPropertyName("evaluationTimestamp")]
public DateTimeOffset EvaluationTimestamp { get; init; } = DateTimeOffset.UnixEpoch;
[JsonPropertyName("sbomDigest")]
public string? SbomDigest { get; init; }
[JsonPropertyName("source")]
public string Source { get; init; } = string.Empty;
[JsonPropertyName("collectedAt")]
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
[JsonPropertyName("eventOffset")]
public long EventOffset { get; init; }
}

View File

@@ -0,0 +1,237 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text.Json.Nodes;
using StellaOps.Graph.Indexer.Ingestion.Sbom;
using StellaOps.Graph.Indexer.Schema;
namespace StellaOps.Graph.Indexer.Ingestion.Policy;
public sealed class PolicyOverlayTransformer
{
private const string PolicyNodeKind = "policy_version";
private const string ComponentNodeKind = "component";
private const string GovernsWithEdgeKind = "GOVERNS_WITH";
public GraphBuildBatch Transform(PolicyOverlaySnapshot snapshot)
{
ArgumentNullException.ThrowIfNull(snapshot);
var tenant = snapshot.Tenant?.Trim() ?? string.Empty;
var nodes = new List<JsonObject>();
var edges = new List<JsonObject>();
var seenEdgeIds = new HashSet<string>(StringComparer.Ordinal);
var policyDetails = snapshot.Policy ?? new PolicyVersionDetails();
var policyNode = CreatePolicyNode(tenant, snapshot, policyDetails);
nodes.Add(policyNode);
var policyNodeId = policyNode["id"]!.GetValue<string>();
foreach (var evaluation in snapshot.Evaluations ?? Array.Empty<PolicyEvaluation>())
{
if (!IsEvaluationCandidate(evaluation))
{
continue;
}
var edge = CreateGovernsWithEdge(
tenant,
snapshot,
policyDetails,
policyNodeId,
evaluation!);
var edgeId = edge["id"]!.GetValue<string>();
if (seenEdgeIds.Add(edgeId))
{
edges.Add(edge);
}
}
return new GraphBuildBatch(
nodes
.OrderBy(node => node["kind"]!.GetValue<string>(), StringComparer.Ordinal)
.ThenBy(node => node["id"]!.GetValue<string>(), StringComparer.Ordinal)
.ToImmutableArray(),
edges
.OrderBy(edge => edge["kind"]!.GetValue<string>(), StringComparer.Ordinal)
.ThenBy(edge => edge["id"]!.GetValue<string>(), StringComparer.Ordinal)
.ToImmutableArray());
}
private static bool IsEvaluationCandidate(PolicyEvaluation? evaluation)
{
return evaluation is not null
&& !string.IsNullOrWhiteSpace(evaluation.ComponentPurl)
&& !string.IsNullOrWhiteSpace(evaluation.FindingExplainHash);
}
private static JsonObject CreatePolicyNode(
string tenant,
PolicyOverlaySnapshot snapshot,
PolicyVersionDetails policy)
{
var policyPackDigest = policy.PolicyPackDigest?.Trim() ?? string.Empty;
var policyName = policy.PolicyName?.Trim() ?? string.Empty;
var explainHash = policy.ExplainHash?.Trim() ?? string.Empty;
var effectiveFrom = policy.EffectiveFrom == DateTimeOffset.UnixEpoch
? snapshot.CollectedAt
: policy.EffectiveFrom;
var expiresAt = policy.ExpiresAt == DateTimeOffset.UnixEpoch
? (DateTimeOffset?)null
: policy.ExpiresAt;
var policyCollectedAt = policy.CollectedAt == DateTimeOffset.UnixEpoch
? snapshot.CollectedAt
: policy.CollectedAt;
var eventOffset = policy.EventOffset != 0 ? policy.EventOffset : snapshot.EventOffset;
var source = ResolveSource(policy.Source, snapshot.Source);
var canonicalKey = new Dictionary<string, string>
{
["tenant"] = tenant,
["policy_pack_digest"] = policyPackDigest,
["effective_from"] = GraphTimestamp.Format(effectiveFrom)
};
var attributes = new JsonObject
{
["policy_pack_digest"] = policyPackDigest,
["policy_name"] = policyName,
["effective_from"] = GraphTimestamp.Format(effectiveFrom),
["expires_at"] = expiresAt is null ? null : GraphTimestamp.Format(expiresAt.Value),
["explain_hash"] = explainHash
};
var node = GraphDocumentFactory.CreateNode(new GraphNodeSpec(
Tenant: tenant,
Kind: PolicyNodeKind,
CanonicalKey: canonicalKey,
Attributes: attributes,
Provenance: new GraphProvenanceSpec(source, policyCollectedAt, SbomDigest: null, EventOffset: eventOffset),
ValidFrom: effectiveFrom,
ValidTo: expiresAt));
NormalizeOverlayProvenance(node);
return node;
}
private static JsonObject CreateGovernsWithEdge(
string tenant,
PolicyOverlaySnapshot snapshot,
PolicyVersionDetails policy,
string policyNodeId,
PolicyEvaluation evaluation)
{
var componentSourceType = string.IsNullOrWhiteSpace(evaluation.ComponentSourceType)
? "inventory"
: evaluation.ComponentSourceType.Trim();
var componentIdentity = new Dictionary<string, string>
{
["tenant"] = tenant,
["purl"] = evaluation.ComponentPurl.Trim(),
["source_type"] = componentSourceType
};
var componentNodeId = GraphIdentity.ComputeNodeId(tenant, ComponentNodeKind, componentIdentity);
var findingExplainHash = evaluation.FindingExplainHash.Trim();
var edgeCanonicalKey = new Dictionary<string, string>
{
["tenant"] = tenant,
["policy_node_id"] = policyNodeId,
["component_node_id"] = componentNodeId,
["finding_explain_hash"] = findingExplainHash
};
var evaluationTimestamp = evaluation.EvaluationTimestamp == DateTimeOffset.UnixEpoch
? snapshot.CollectedAt
: evaluation.EvaluationTimestamp;
var explainHash = !string.IsNullOrWhiteSpace(evaluation.ExplainHash)
? evaluation.ExplainHash.Trim()
: !string.IsNullOrWhiteSpace(policy.ExplainHash)
? policy.ExplainHash.Trim()
: findingExplainHash;
var attributes = new JsonObject
{
["verdict"] = evaluation.Verdict?.Trim() ?? string.Empty,
["explain_hash"] = explainHash,
["policy_rule_id"] = evaluation.PolicyRuleId?.Trim() ?? string.Empty,
["evaluation_timestamp"] = GraphTimestamp.Format(evaluationTimestamp)
};
var collectedAt = evaluation.CollectedAt == DateTimeOffset.UnixEpoch
? snapshot.CollectedAt
: evaluation.CollectedAt;
var eventOffset = evaluation.EventOffset != 0 ? evaluation.EventOffset : snapshot.EventOffset;
var source = ResolveSource(evaluation.Source, policy.Source, snapshot.Source);
return GraphDocumentFactory.CreateEdge(new GraphEdgeSpec(
Tenant: tenant,
Kind: GovernsWithEdgeKind,
CanonicalKey: edgeCanonicalKey,
Attributes: attributes,
Provenance: new GraphProvenanceSpec(
source,
collectedAt,
NormalizeOptional(evaluation.SbomDigest),
eventOffset),
ValidFrom: evaluationTimestamp,
ValidTo: null));
}
private static void NormalizeOverlayProvenance(JsonObject node)
{
var provenance = node["provenance"]!.AsObject();
var sourceNode = provenance["source"]!.DeepClone();
var collectedAtNode = provenance["collected_at"]!.DeepClone();
var eventOffsetNode = provenance.ContainsKey("event_offset")
? provenance["event_offset"]!.DeepClone()
: null;
var normalized = new JsonObject
{
["source"] = sourceNode,
["collected_at"] = collectedAtNode,
["sbom_digest"] = null
};
if (eventOffsetNode is not null)
{
normalized["event_offset"] = eventOffsetNode;
}
node["provenance"] = normalized;
node.Remove("hash");
node["hash"] = GraphIdentity.ComputeDocumentHash(node);
}
private static string ResolveSource(params string?[] candidates)
{
foreach (var candidate in candidates)
{
if (!string.IsNullOrWhiteSpace(candidate))
{
return candidate.Trim();
}
}
return "policy.engine.v1";
}
private static string? NormalizeOptional(string? value)
{
return string.IsNullOrWhiteSpace(value) ? null : value.Trim();
}
}

View File

@@ -0,0 +1,58 @@
using System.Text;
using System.Text.Json.Nodes;
using StellaOps.Graph.Indexer.Documents;
using StellaOps.Graph.Indexer.Schema;
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
public sealed class FileSystemSnapshotFileWriter : ISnapshotFileWriter
{
private readonly string _root;
public FileSystemSnapshotFileWriter(string rootDirectory)
{
if (string.IsNullOrWhiteSpace(rootDirectory))
{
throw new ArgumentException("Snapshot root directory must be provided.", nameof(rootDirectory));
}
_root = Path.GetFullPath(rootDirectory);
}
public async Task WriteJsonAsync(string relativePath, JsonObject content, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(content);
var fullPath = ResolvePath(relativePath);
Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!);
var canonicalBytes = CanonicalJson.ToCanonicalUtf8Bytes(content);
await File.WriteAllBytesAsync(fullPath, canonicalBytes, cancellationToken).ConfigureAwait(false);
}
public async Task WriteJsonLinesAsync(string relativePath, IEnumerable<JsonObject> items, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(items);
var fullPath = ResolvePath(relativePath);
Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!);
await using var stream = new FileStream(fullPath, FileMode.Create, FileAccess.Write, FileShare.None, 4096, useAsync: true);
await using var writer = new StreamWriter(stream, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false));
foreach (var item in items)
{
cancellationToken.ThrowIfCancellationRequested();
var canonicalBytes = CanonicalJson.ToCanonicalUtf8Bytes(item);
await writer.WriteAsync(Encoding.UTF8.GetString(canonicalBytes)).ConfigureAwait(false);
await writer.WriteLineAsync().ConfigureAwait(false);
}
}
private string ResolvePath(string relativePath)
{
var sanitized = relativePath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar);
return Path.Combine(_root, sanitized);
}
}

View File

@@ -0,0 +1,8 @@
using System.Collections.Immutable;
using System.Text.Json.Nodes;
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
public sealed record GraphBuildBatch(
ImmutableArray<JsonObject> Nodes,
ImmutableArray<JsonObject> Edges);

View File

@@ -0,0 +1,9 @@
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
public interface IGraphDocumentWriter
{
Task WriteAsync(GraphBuildBatch batch, CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,8 @@
using System;
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
public interface ISbomIngestMetrics
{
void RecordBatch(string source, string tenant, int nodeCount, int edgeCount, TimeSpan duration, bool success);
}

View File

@@ -0,0 +1,99 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
public sealed class SbomIngestMetrics : ISbomIngestMetrics, IDisposable
{
public const string MeterName = "StellaOps.Graph.Indexer";
public const string MeterVersion = "1.0.0";
private const string BatchesTotalName = "graph_sbom_ingest_batches_total";
private const string BatchDurationSecondsName = "graph_sbom_ingest_duration_seconds";
private const string NodesTotalName = "graph_sbom_ingest_nodes_total";
private const string EdgesTotalName = "graph_sbom_ingest_edges_total";
private const string UnitCount = "count";
private const string UnitSeconds = "s";
private readonly Meter _meter;
private readonly bool _ownsMeter;
private readonly Counter<long> _batchesTotal;
private readonly Histogram<double> _batchDurationSeconds;
private readonly Counter<long> _nodesTotal;
private readonly Counter<long> _edgesTotal;
private bool _disposed;
public SbomIngestMetrics()
: this(null)
{
}
public SbomIngestMetrics(Meter? meter)
{
_meter = meter ?? new Meter(MeterName, MeterVersion);
_ownsMeter = meter is null;
_batchesTotal = _meter.CreateCounter<long>(
name: BatchesTotalName,
unit: UnitCount,
description: "Total SBOM ingest batches processed.");
_batchDurationSeconds = _meter.CreateHistogram<double>(
name: BatchDurationSecondsName,
unit: UnitSeconds,
description: "Duration, in seconds, for SBOM ingest batches.");
_nodesTotal = _meter.CreateCounter<long>(
name: NodesTotalName,
unit: UnitCount,
description: "Total graph nodes emitted from SBOM ingest.");
_edgesTotal = _meter.CreateCounter<long>(
name: EdgesTotalName,
unit: UnitCount,
description: "Total graph edges emitted from SBOM ingest.");
}
public void RecordBatch(string source, string tenant, int nodeCount, int edgeCount, TimeSpan duration, bool success)
{
ThrowIfDisposed();
var tags = new KeyValuePair<string, object?>[]
{
new("source", source ?? string.Empty),
new("tenant", tenant ?? string.Empty),
new("success", success)
};
var tagSpan = tags.AsSpan();
_batchesTotal.Add(1, tagSpan);
_nodesTotal.Add(nodeCount, tagSpan);
_edgesTotal.Add(edgeCount, tagSpan);
_batchDurationSeconds.Record(duration.TotalSeconds, tagSpan);
}
private void ThrowIfDisposed()
{
if (_disposed)
{
throw new ObjectDisposedException(nameof(SbomIngestMetrics));
}
}
public void Dispose()
{
if (_disposed)
{
return;
}
if (_ownsMeter)
{
_meter.Dispose();
}
_disposed = true;
}
}

View File

@@ -0,0 +1,11 @@
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
public sealed class SbomIngestOptions
{
/// <summary>
/// Optional override for the snapshot export root directory. When null or whitespace,
/// <c>STELLAOPS_GRAPH_SNAPSHOT_DIR</c> or the default <c>artifacts/graph-snapshots</c>
/// location will be used.
/// </summary>
public string? SnapshotRootDirectory { get; set; }
}

View File

@@ -0,0 +1,87 @@
using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
public sealed class SbomIngestProcessor
{
private readonly SbomIngestTransformer _transformer;
private readonly IGraphDocumentWriter _writer;
private readonly ISbomIngestMetrics _metrics;
private readonly ISbomSnapshotExporter _snapshotExporter;
private readonly ILogger<SbomIngestProcessor> _logger;
public SbomIngestProcessor(
SbomIngestTransformer transformer,
IGraphDocumentWriter writer,
ISbomIngestMetrics metrics,
ISbomSnapshotExporter snapshotExporter,
ILogger<SbomIngestProcessor> logger)
{
_transformer = transformer ?? throw new ArgumentNullException(nameof(transformer));
_writer = writer ?? throw new ArgumentNullException(nameof(writer));
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
_snapshotExporter = snapshotExporter ?? throw new ArgumentNullException(nameof(snapshotExporter));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task ProcessAsync(SbomSnapshot snapshot, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(snapshot);
cancellationToken.ThrowIfCancellationRequested();
var stopwatch = Stopwatch.StartNew();
GraphBuildBatch batch;
try
{
batch = _transformer.Transform(snapshot);
}
catch (Exception ex)
{
stopwatch.Stop();
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, 0, 0, stopwatch.Elapsed, success: false);
_logger.LogError(
ex,
"graph-indexer: failed to transform SBOM {SbomDigest} for tenant {Tenant}",
snapshot.SbomDigest,
snapshot.Tenant);
throw;
}
try
{
cancellationToken.ThrowIfCancellationRequested();
await _writer.WriteAsync(batch, cancellationToken).ConfigureAwait(false);
await _snapshotExporter.ExportAsync(snapshot, batch, cancellationToken).ConfigureAwait(false);
stopwatch.Stop();
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, batch.Nodes.Length, batch.Edges.Length, stopwatch.Elapsed, success: true);
_logger.LogInformation(
"graph-indexer: indexed SBOM {SbomDigest} for tenant {Tenant} with {NodeCount} nodes and {EdgeCount} edges in {DurationMs:F2} ms",
snapshot.SbomDigest,
snapshot.Tenant,
batch.Nodes.Length,
batch.Edges.Length,
stopwatch.Elapsed.TotalMilliseconds);
}
catch (Exception ex)
{
stopwatch.Stop();
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, batch.Nodes.Length, batch.Edges.Length, stopwatch.Elapsed, success: false);
_logger.LogError(
ex,
"graph-indexer: failed to persist SBOM {SbomDigest} for tenant {Tenant}",
snapshot.SbomDigest,
snapshot.Tenant);
throw;
}
}
}

View File

@@ -0,0 +1,42 @@
using Microsoft.Extensions.Logging;
using StellaOps.Graph.Indexer.Documents;
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
public static class SbomIngestProcessorFactory
{
private const string SnapshotDirEnv = "STELLAOPS_GRAPH_SNAPSHOT_DIR";
public static SbomIngestProcessor CreateDefault(
SbomIngestTransformer transformer,
IGraphDocumentWriter writer,
ISbomIngestMetrics metrics,
ILogger<SbomIngestProcessor> logger,
string? snapshotRoot = null)
{
ArgumentNullException.ThrowIfNull(transformer);
ArgumentNullException.ThrowIfNull(writer);
ArgumentNullException.ThrowIfNull(metrics);
ArgumentNullException.ThrowIfNull(logger);
var root = ResolveSnapshotRoot(snapshotRoot);
var exporter = new SbomSnapshotExporter(new GraphSnapshotBuilder(), new FileSystemSnapshotFileWriter(root));
return new SbomIngestProcessor(transformer, writer, metrics, exporter, logger);
}
private static string ResolveSnapshotRoot(string? snapshotRoot)
{
if (!string.IsNullOrWhiteSpace(snapshotRoot))
{
return snapshotRoot!;
}
var envRoot = Environment.GetEnvironmentVariable(SnapshotDirEnv);
if (!string.IsNullOrWhiteSpace(envRoot))
{
return envRoot!;
}
return Path.Combine(Environment.CurrentDirectory, "artifacts", "graph-snapshots");
}
}

View File

@@ -0,0 +1,47 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
public static class SbomIngestServiceCollectionExtensions
{
public static IServiceCollection AddSbomIngestPipeline(
this IServiceCollection services,
Action<SbomIngestOptions>? configure = null)
{
ArgumentNullException.ThrowIfNull(services);
services.AddOptions<SbomIngestOptions>();
if (configure is not null)
{
services.Configure(configure);
}
services.TryAddSingleton<SbomIngestTransformer>();
services.TryAddSingleton<ISbomIngestMetrics, SbomIngestMetrics>();
services.TryAddSingleton<SbomIngestProcessor>(provider =>
{
var transformer = provider.GetRequiredService<SbomIngestTransformer>();
var writer = provider.GetRequiredService<IGraphDocumentWriter>();
var metrics = provider.GetRequiredService<ISbomIngestMetrics>();
var logger = provider.GetService<ILogger<SbomIngestProcessor>>() ?? NullLogger<SbomIngestProcessor>.Instance;
var options = provider.GetService<IOptions<SbomIngestOptions>>();
var snapshotRoot = options?.Value.SnapshotRootDirectory;
return SbomIngestProcessorFactory.CreateDefault(
transformer,
writer,
metrics,
logger,
snapshotRoot);
});
return services;
}
}

View File

@@ -0,0 +1,453 @@
using System.Collections.Immutable;
using System.Text.Json.Nodes;
using StellaOps.Graph.Indexer.Schema;
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
public sealed class SbomIngestTransformer
{
private const string ContainsEdgeKind = "CONTAINS";
private const string DependsOnEdgeKind = "DEPENDS_ON";
private const string DeclaredInEdgeKind = "DECLARED_IN";
private const string BuiltFromEdgeKind = "BUILT_FROM";
public GraphBuildBatch Transform(SbomSnapshot snapshot)
{
ArgumentNullException.ThrowIfNull(snapshot);
var nodes = new List<JsonObject>();
var edges = new List<JsonObject>();
var artifactNodes = new Dictionary<string, JsonObject>(StringComparer.OrdinalIgnoreCase);
var componentNodes = new Dictionary<string, JsonObject>(StringComparer.OrdinalIgnoreCase);
var fileNodes = new Dictionary<string, JsonObject>(StringComparer.OrdinalIgnoreCase);
var licenseCandidates = new Dictionary<(string License, string SourceDigest), LicenseCandidate>(LicenseKeyComparer.Instance);
long nextEdgeOffset = snapshot.EventOffset + 918;
long NextEdgeOffset() => nextEdgeOffset++;
var artifactNode = CreateArtifactNode(snapshot);
nodes.Add(artifactNode);
artifactNodes[GetArtifactKey(snapshot.ArtifactDigest, snapshot.SbomDigest)] = artifactNode;
foreach (var component in snapshot.Components)
{
var componentNode = CreateComponentNode(snapshot, component);
nodes.Add(componentNode);
componentNodes[GetComponentKey(component.Purl, component.SourceType)] = componentNode;
if (string.Equals(component.Usage, "direct", StringComparison.OrdinalIgnoreCase))
{
var containsEdge = CreateContainsEdge(snapshot, artifactNode, componentNode, component, NextEdgeOffset());
edges.Add(containsEdge);
}
foreach (var dependency in component.Dependencies)
{
var dependsOnEdge = CreateDependsOnEdge(snapshot, componentNode, dependency, NextEdgeOffset());
edges.Add(dependsOnEdge);
}
foreach (var file in component.Files)
{
var fileNodeKey = GetFileKey(snapshot.ArtifactDigest, file.Path, file.ContentSha256);
if (!fileNodes.TryGetValue(fileNodeKey, out var fileNode))
{
fileNode = CreateFileNode(snapshot, file);
nodes.Add(fileNode);
fileNodes[fileNodeKey] = fileNode;
}
var declaredInEdge = CreateDeclaredInEdge(snapshot, componentNode, component, fileNode, file, NextEdgeOffset());
edges.Add(declaredInEdge);
}
if (HasLicenseMetadata(component.License))
{
var licenseKey = (component.License.Spdx, component.License.SourceDigest);
var candidate = CreateLicenseCandidate(snapshot, component);
if (!licenseCandidates.TryGetValue(licenseKey, out var existing) || existing.EventOffset > candidate.EventOffset)
{
licenseCandidates[licenseKey] = candidate;
}
}
}
foreach (var candidate in licenseCandidates.Values)
{
nodes.Add(CreateLicenseNode(snapshot, candidate));
}
foreach (var baseArtifact in snapshot.BaseArtifacts)
{
var node = CreateBaseArtifactNode(snapshot, baseArtifact);
if (!artifactNodes.ContainsKey(GetArtifactKey(baseArtifact.ArtifactDigest, baseArtifact.SbomDigest)))
{
nodes.Add(node);
artifactNodes[GetArtifactKey(baseArtifact.ArtifactDigest, baseArtifact.SbomDigest)] = node;
}
var edge = CreateBuiltFromEdge(snapshot, artifactNode, baseArtifact, NextEdgeOffset());
edges.Add(edge);
}
var orderedNodes = nodes
.OrderBy(node => node["kind"]!.GetValue<string>(), StringComparer.Ordinal)
.ThenBy(node => node["id"]!.GetValue<string>(), StringComparer.Ordinal)
.ToImmutableArray();
var orderedEdges = edges
.OrderBy(edge => edge["kind"]!.GetValue<string>(), StringComparer.Ordinal)
.ThenBy(edge => edge["id"]!.GetValue<string>(), StringComparer.Ordinal)
.ToImmutableArray();
return new GraphBuildBatch(orderedNodes, orderedEdges);
}
private static JsonObject CreateArtifactNode(SbomSnapshot snapshot)
{
var labels = snapshot.Artifact.Labels?.OrderBy(x => x, StringComparer.Ordinal).ToArray() ?? Array.Empty<string>();
var attributes = new JsonObject
{
["display_name"] = snapshot.Artifact.DisplayName,
["artifact_digest"] = snapshot.ArtifactDigest,
["sbom_digest"] = snapshot.SbomDigest,
["environment"] = snapshot.Artifact.Environment,
["labels"] = CreateJsonArray(labels),
["origin_registry"] = snapshot.Artifact.OriginRegistry,
["supply_chain_stage"] = snapshot.Artifact.SupplyChainStage
};
return GraphDocumentFactory.CreateNode(new GraphNodeSpec(
Tenant: snapshot.Tenant,
Kind: "artifact",
CanonicalKey: new Dictionary<string, string>
{
["tenant"] = snapshot.Tenant,
["artifact_digest"] = snapshot.ArtifactDigest,
["sbom_digest"] = snapshot.SbomDigest
},
Attributes: attributes,
Provenance: new GraphProvenanceSpec(snapshot.Source, snapshot.CollectedAt, snapshot.SbomDigest, snapshot.EventOffset),
ValidFrom: snapshot.CollectedAt,
ValidTo: null));
}
private static JsonObject CreateBaseArtifactNode(SbomSnapshot snapshot, SbomBaseArtifact baseArtifact)
{
var labels = baseArtifact.Labels?.OrderBy(x => x, StringComparer.Ordinal).ToArray() ?? Array.Empty<string>();
var attributes = new JsonObject
{
["display_name"] = baseArtifact.DisplayName,
["artifact_digest"] = baseArtifact.ArtifactDigest,
["sbom_digest"] = baseArtifact.SbomDigest,
["environment"] = baseArtifact.Environment,
["labels"] = CreateJsonArray(labels),
["origin_registry"] = baseArtifact.OriginRegistry,
["supply_chain_stage"] = baseArtifact.SupplyChainStage
};
return GraphDocumentFactory.CreateNode(new GraphNodeSpec(
Tenant: snapshot.Tenant,
Kind: "artifact",
CanonicalKey: new Dictionary<string, string>
{
["tenant"] = snapshot.Tenant,
["artifact_digest"] = baseArtifact.ArtifactDigest,
["sbom_digest"] = baseArtifact.SbomDigest
},
Attributes: attributes,
Provenance: new GraphProvenanceSpec(
ResolveSource(baseArtifact.Source, snapshot.Source),
baseArtifact.CollectedAt,
baseArtifact.SbomDigest,
baseArtifact.EventOffset),
ValidFrom: baseArtifact.CollectedAt,
ValidTo: null));
}
private static JsonObject CreateComponentNode(SbomSnapshot snapshot, SbomComponent component)
{
var attributes = new JsonObject
{
["purl"] = component.Purl,
["version"] = component.Version,
["ecosystem"] = component.Ecosystem,
["scope"] = component.Scope,
["license_spdx"] = component.License.Spdx,
["usage"] = component.Usage
};
return GraphDocumentFactory.CreateNode(new GraphNodeSpec(
Tenant: snapshot.Tenant,
Kind: "component",
CanonicalKey: new Dictionary<string, string>
{
["tenant"] = snapshot.Tenant,
["purl"] = component.Purl,
["source_type"] = component.SourceType
},
Attributes: attributes,
Provenance: new GraphProvenanceSpec(
ResolveSource(component.Source, snapshot.Source),
component.CollectedAt,
snapshot.SbomDigest,
component.EventOffset),
ValidFrom: component.CollectedAt,
ValidTo: null));
}
private static JsonObject CreateFileNode(SbomSnapshot snapshot, SbomComponentFile file)
{
var attributes = new JsonObject
{
["normalized_path"] = file.Path,
["content_sha256"] = file.ContentSha256,
["language_hint"] = file.LanguageHint,
["size_bytes"] = file.SizeBytes,
["scope"] = file.Scope
};
return GraphDocumentFactory.CreateNode(new GraphNodeSpec(
Tenant: snapshot.Tenant,
Kind: "file",
CanonicalKey: new Dictionary<string, string>
{
["tenant"] = snapshot.Tenant,
["artifact_digest"] = snapshot.ArtifactDigest,
["normalized_path"] = file.Path,
["content_sha256"] = file.ContentSha256
},
Attributes: attributes,
Provenance: new GraphProvenanceSpec(
ResolveSource(file.Source, file.DetectedBy, snapshot.Source),
file.CollectedAt,
snapshot.SbomDigest,
file.EventOffset),
ValidFrom: file.CollectedAt,
ValidTo: null));
}
private static JsonObject CreateContainsEdge(SbomSnapshot snapshot, JsonObject artifactNode, JsonObject componentNode, SbomComponent component, long eventOffset)
{
return GraphDocumentFactory.CreateEdge(new GraphEdgeSpec(
Tenant: snapshot.Tenant,
Kind: ContainsEdgeKind,
CanonicalKey: new Dictionary<string, string>
{
["tenant"] = snapshot.Tenant,
["artifact_node_id"] = artifactNode["id"]!.GetValue<string>(),
["component_node_id"] = componentNode["id"]!.GetValue<string>(),
["sbom_digest"] = snapshot.SbomDigest
},
Attributes: new JsonObject
{
["detected_by"] = component.DetectedBy,
["layer_digest"] = component.LayerDigest,
["scope"] = component.Scope,
["evidence_digest"] = component.EvidenceDigest
},
Provenance: new GraphProvenanceSpec(
ResolveSource(component.Source, component.DetectedBy, snapshot.Source),
component.CollectedAt.AddSeconds(1),
snapshot.SbomDigest,
eventOffset),
ValidFrom: component.CollectedAt.AddSeconds(1),
ValidTo: null));
}
private static JsonObject CreateDependsOnEdge(SbomSnapshot snapshot, JsonObject componentNode, SbomDependency dependency, long eventOffset)
{
return GraphDocumentFactory.CreateEdge(new GraphEdgeSpec(
Tenant: snapshot.Tenant,
Kind: DependsOnEdgeKind,
CanonicalKey: new Dictionary<string, string>
{
["tenant"] = snapshot.Tenant,
["component_node_id"] = componentNode["id"]!.GetValue<string>(),
["dependency_purl"] = dependency.Purl,
["sbom_digest"] = snapshot.SbomDigest
},
Attributes: new JsonObject
{
["dependency_purl"] = dependency.Purl,
["dependency_version"] = dependency.Version,
["relationship"] = dependency.Relationship,
["evidence_digest"] = dependency.EvidenceDigest
},
Provenance: new GraphProvenanceSpec(
ResolveSource(dependency.Source, snapshot.Source),
dependency.CollectedAt.AddSeconds(1),
snapshot.SbomDigest,
eventOffset),
ValidFrom: dependency.CollectedAt.AddSeconds(1),
ValidTo: null));
}
private static JsonObject CreateDeclaredInEdge(SbomSnapshot snapshot, JsonObject componentNode, SbomComponent component, JsonObject fileNode, SbomComponentFile file, long eventOffset)
{
return GraphDocumentFactory.CreateEdge(new GraphEdgeSpec(
Tenant: snapshot.Tenant,
Kind: DeclaredInEdgeKind,
CanonicalKey: new Dictionary<string, string>
{
["tenant"] = snapshot.Tenant,
["component_node_id"] = componentNode["id"]!.GetValue<string>(),
["file_node_id"] = fileNode["id"]!.GetValue<string>(),
["sbom_digest"] = snapshot.SbomDigest
},
Attributes: new JsonObject
{
["detected_by"] = file.DetectedBy,
["scope"] = ResolveScope(component.Scope, file.Scope),
["evidence_digest"] = file.EvidenceDigest
},
Provenance: new GraphProvenanceSpec(
ResolveSource(file.Source, file.DetectedBy, snapshot.Source),
file.CollectedAt.AddSeconds(1),
snapshot.SbomDigest,
eventOffset),
ValidFrom: file.CollectedAt.AddSeconds(1),
ValidTo: null));
}
private static JsonObject CreateBuiltFromEdge(SbomSnapshot snapshot, JsonObject parentNode, SbomBaseArtifact baseArtifact, long eventOffset)
{
return GraphDocumentFactory.CreateEdge(new GraphEdgeSpec(
Tenant: snapshot.Tenant,
Kind: BuiltFromEdgeKind,
CanonicalKey: new Dictionary<string, string>
{
["tenant"] = snapshot.Tenant,
["parent_artifact_node_id"] = parentNode["id"]!.GetValue<string>(),
["child_artifact_digest"] = baseArtifact.ArtifactDigest
},
Attributes: new JsonObject
{
["build_type"] = snapshot.Build.BuildType,
["builder_id"] = snapshot.Build.BuilderId,
["attestation_digest"] = snapshot.Build.AttestationDigest
},
Provenance: new GraphProvenanceSpec(
ResolveSource(snapshot.Build.Source, snapshot.Source),
snapshot.Build.CollectedAt,
snapshot.SbomDigest,
eventOffset),
ValidFrom: snapshot.Build.CollectedAt,
ValidTo: null));
}
private static string GetComponentKey(string purl, string sourceType)
{
var normalizedPurl = (purl ?? string.Empty).Trim();
var normalizedSourceType = (sourceType ?? string.Empty).Trim();
return $"{normalizedPurl}|{normalizedSourceType}";
}
private static bool HasLicenseMetadata(SbomLicense license)
=> !string.IsNullOrWhiteSpace(license.Spdx)
&& !string.IsNullOrWhiteSpace(license.SourceDigest);
private static string ResolveScope(string componentScope, string fileScope)
{
if (!string.IsNullOrWhiteSpace(componentScope))
{
return componentScope.Trim();
}
return string.IsNullOrWhiteSpace(fileScope) ? string.Empty : fileScope.Trim();
}
private static string ResolveSource(params string?[] sources)
{
foreach (var source in sources)
{
if (!string.IsNullOrWhiteSpace(source))
{
return source!.Trim();
}
}
return string.Empty;
}
private static string GetArtifactKey(string artifactDigest, string sbomDigest)
=> $"{(artifactDigest ?? string.Empty).Trim()}|{(sbomDigest ?? string.Empty).Trim()}";
private static string GetFileKey(string artifactDigest, string path, string contentSha)
=> $"{(artifactDigest ?? string.Empty).Trim()}|{(path ?? string.Empty).Trim()}|{(contentSha ?? string.Empty).Trim()}";
private static JsonArray CreateJsonArray(IEnumerable<string> values)
{
var array = new JsonArray();
foreach (var value in values)
{
array.Add(value);
}
return array;
}
private static LicenseCandidate CreateLicenseCandidate(SbomSnapshot snapshot, SbomComponent component)
{
var collectedAt = component.CollectedAt.AddSeconds(2);
var eventOffset = component.EventOffset + 3;
return new LicenseCandidate(
License: component.License,
CollectedAt: collectedAt,
EventOffset: eventOffset,
SbomDigest: snapshot.SbomDigest,
Source: ResolveSource(component.Source, snapshot.Source));
}
private static JsonObject CreateLicenseNode(SbomSnapshot snapshot, LicenseCandidate candidate)
{
var attributes = new JsonObject
{
["license_spdx"] = candidate.License.Spdx,
["name"] = candidate.License.Name,
["classification"] = candidate.License.Classification,
["notice_uri"] = candidate.License.NoticeUri is null ? null : candidate.License.NoticeUri
};
return GraphDocumentFactory.CreateNode(new GraphNodeSpec(
Tenant: snapshot.Tenant,
Kind: "license",
CanonicalKey: new Dictionary<string, string>
{
["tenant"] = snapshot.Tenant,
["license_spdx"] = candidate.License.Spdx,
["source_digest"] = candidate.License.SourceDigest
},
Attributes: attributes,
Provenance: new GraphProvenanceSpec(candidate.Source, candidate.CollectedAt, candidate.SbomDigest, candidate.EventOffset),
ValidFrom: candidate.CollectedAt,
ValidTo: null));
}
private sealed record LicenseCandidate(
SbomLicense License,
DateTimeOffset CollectedAt,
long EventOffset,
string SbomDigest,
string Source);
private sealed class LicenseKeyComparer : IEqualityComparer<(string License, string SourceDigest)>
{
public static readonly LicenseKeyComparer Instance = new();
public bool Equals((string License, string SourceDigest) x, (string License, string SourceDigest) y)
=> string.Equals(x.License, y.License, StringComparison.OrdinalIgnoreCase)
&& string.Equals(x.SourceDigest, y.SourceDigest, StringComparison.OrdinalIgnoreCase);
public int GetHashCode((string License, string SourceDigest) obj)
{
var hash = new HashCode();
hash.Add(obj.License, StringComparer.OrdinalIgnoreCase);
hash.Add(obj.SourceDigest, StringComparer.OrdinalIgnoreCase);
return hash.ToHashCode();
}
}
}

View File

@@ -0,0 +1,231 @@
using System.Text.Json.Serialization;
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
public sealed class SbomSnapshot
{
[JsonPropertyName("tenant")]
public string Tenant { get; init; } = string.Empty;
[JsonPropertyName("source")]
public string Source { get; init; } = string.Empty;
[JsonPropertyName("artifactDigest")]
public string ArtifactDigest { get; init; } = string.Empty;
[JsonPropertyName("sbomDigest")]
public string SbomDigest { get; init; } = string.Empty;
[JsonPropertyName("collectedAt")]
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
[JsonPropertyName("eventOffset")]
public long EventOffset { get; init; }
[JsonPropertyName("artifact")]
public SbomArtifactMetadata Artifact { get; init; } = new();
[JsonPropertyName("build")]
public SbomBuildMetadata Build { get; init; } = new();
[JsonPropertyName("components")]
public IReadOnlyList<SbomComponent> Components { get; init; } = Array.Empty<SbomComponent>();
[JsonPropertyName("baseArtifacts")]
public IReadOnlyList<SbomBaseArtifact> BaseArtifacts { get; init; } = Array.Empty<SbomBaseArtifact>();
}
public sealed class SbomArtifactMetadata
{
[JsonPropertyName("displayName")]
public string DisplayName { get; init; } = string.Empty;
[JsonPropertyName("environment")]
public string Environment { get; init; } = string.Empty;
[JsonPropertyName("labels")]
public IReadOnlyList<string> Labels { get; init; } = Array.Empty<string>();
[JsonPropertyName("originRegistry")]
public string OriginRegistry { get; init; } = string.Empty;
[JsonPropertyName("supplyChainStage")]
public string SupplyChainStage { get; init; } = string.Empty;
}
public sealed class SbomBuildMetadata
{
[JsonPropertyName("builderId")]
public string BuilderId { get; init; } = string.Empty;
[JsonPropertyName("buildType")]
public string BuildType { get; init; } = string.Empty;
[JsonPropertyName("attestationDigest")]
public string AttestationDigest { get; init; } = string.Empty;
[JsonPropertyName("source")]
public string Source { get; init; } = string.Empty;
[JsonPropertyName("collectedAt")]
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
[JsonPropertyName("eventOffset")]
public long EventOffset { get; init; }
}
public sealed class SbomComponent
{
[JsonPropertyName("purl")]
public string Purl { get; init; } = string.Empty;
[JsonPropertyName("version")]
public string Version { get; init; } = string.Empty;
[JsonPropertyName("ecosystem")]
public string Ecosystem { get; init; } = string.Empty;
[JsonPropertyName("scope")]
public string Scope { get; init; } = string.Empty;
[JsonPropertyName("license")]
public SbomLicense License { get; init; } = new();
[JsonPropertyName("usage")]
public string Usage { get; init; } = string.Empty;
[JsonPropertyName("detectedBy")]
public string DetectedBy { get; init; } = string.Empty;
[JsonPropertyName("layerDigest")]
public string LayerDigest { get; init; } = string.Empty;
[JsonPropertyName("evidenceDigest")]
public string EvidenceDigest { get; init; } = string.Empty;
[JsonPropertyName("collectedAt")]
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
[JsonPropertyName("eventOffset")]
public long EventOffset { get; init; }
[JsonPropertyName("source")]
public string Source { get; init; } = string.Empty;
[JsonPropertyName("files")]
public IReadOnlyList<SbomComponentFile> Files { get; init; } = Array.Empty<SbomComponentFile>();
[JsonPropertyName("dependencies")]
public IReadOnlyList<SbomDependency> Dependencies { get; init; } = Array.Empty<SbomDependency>();
[JsonPropertyName("sourceType")]
public string SourceType { get; init; } = "inventory";
}
public sealed class SbomLicense
{
[JsonPropertyName("spdx")]
public string Spdx { get; init; } = string.Empty;
[JsonPropertyName("name")]
public string Name { get; init; } = string.Empty;
[JsonPropertyName("classification")]
public string Classification { get; init; } = string.Empty;
[JsonPropertyName("noticeUri")]
public string? NoticeUri { get; init; }
[JsonPropertyName("sourceDigest")]
public string SourceDigest { get; init; } = string.Empty;
}
public sealed class SbomComponentFile
{
[JsonPropertyName("path")]
public string Path { get; init; } = string.Empty;
[JsonPropertyName("contentSha256")]
public string ContentSha256 { get; init; } = string.Empty;
[JsonPropertyName("languageHint")]
public string LanguageHint { get; init; } = string.Empty;
[JsonPropertyName("sizeBytes")]
public long SizeBytes { get; init; }
[JsonPropertyName("scope")]
public string Scope { get; init; } = string.Empty;
[JsonPropertyName("detectedBy")]
public string DetectedBy { get; init; } = string.Empty;
[JsonPropertyName("evidenceDigest")]
public string EvidenceDigest { get; init; } = string.Empty;
[JsonPropertyName("collectedAt")]
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
[JsonPropertyName("eventOffset")]
public long EventOffset { get; init; }
[JsonPropertyName("source")]
public string Source { get; init; } = string.Empty;
}
public sealed class SbomDependency
{
[JsonPropertyName("purl")]
public string Purl { get; init; } = string.Empty;
[JsonPropertyName("version")]
public string Version { get; init; } = string.Empty;
[JsonPropertyName("relationship")]
public string Relationship { get; init; } = string.Empty;
[JsonPropertyName("evidenceDigest")]
public string EvidenceDigest { get; init; } = string.Empty;
[JsonPropertyName("collectedAt")]
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
[JsonPropertyName("eventOffset")]
public long EventOffset { get; init; }
[JsonPropertyName("source")]
public string Source { get; init; } = string.Empty;
}
public sealed class SbomBaseArtifact
{
[JsonPropertyName("artifactDigest")]
public string ArtifactDigest { get; init; } = string.Empty;
[JsonPropertyName("sbomDigest")]
public string SbomDigest { get; init; } = string.Empty;
[JsonPropertyName("displayName")]
public string DisplayName { get; init; } = string.Empty;
[JsonPropertyName("environment")]
public string Environment { get; init; } = string.Empty;
[JsonPropertyName("labels")]
public IReadOnlyList<string> Labels { get; init; } = Array.Empty<string>();
[JsonPropertyName("originRegistry")]
public string OriginRegistry { get; init; } = string.Empty;
[JsonPropertyName("supplyChainStage")]
public string SupplyChainStage { get; init; } = string.Empty;
[JsonPropertyName("collectedAt")]
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
[JsonPropertyName("eventOffset")]
public long EventOffset { get; init; }
[JsonPropertyName("source")]
public string Source { get; init; } = string.Empty;
}

View File

@@ -0,0 +1,50 @@
using System.Text.Json;
using System.Text.Json.Nodes;
using StellaOps.Graph.Indexer.Documents;
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
public interface ISnapshotFileWriter
{
Task WriteJsonAsync(string relativePath, JsonObject content, CancellationToken cancellationToken);
Task WriteJsonLinesAsync(string relativePath, IEnumerable<JsonObject> items, CancellationToken cancellationToken);
}
public interface ISbomSnapshotExporter
{
Task ExportAsync(SbomSnapshot snapshot, GraphBuildBatch batch, CancellationToken cancellationToken);
}
public sealed class SbomSnapshotExporter : ISbomSnapshotExporter
{
private readonly GraphSnapshotBuilder _snapshotBuilder;
private readonly ISnapshotFileWriter _fileWriter;
public SbomSnapshotExporter(GraphSnapshotBuilder snapshotBuilder, ISnapshotFileWriter fileWriter)
{
_snapshotBuilder = snapshotBuilder ?? throw new ArgumentNullException(nameof(snapshotBuilder));
_fileWriter = fileWriter ?? throw new ArgumentNullException(nameof(fileWriter));
}
public async Task ExportAsync(SbomSnapshot snapshot, GraphBuildBatch batch, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(snapshot);
ArgumentNullException.ThrowIfNull(batch);
cancellationToken.ThrowIfCancellationRequested();
var graphSnapshot = _snapshotBuilder.Build(snapshot, batch, DateTimeOffset.UtcNow);
await _fileWriter.WriteJsonAsync("manifest.json", graphSnapshot.Manifest.ToJson(), cancellationToken)
.ConfigureAwait(false);
await _fileWriter.WriteJsonAsync("adjacency.json", graphSnapshot.Adjacency.ToJson(), cancellationToken)
.ConfigureAwait(false);
await _fileWriter.WriteJsonLinesAsync("nodes.jsonl", batch.Nodes, cancellationToken)
.ConfigureAwait(false);
await _fileWriter.WriteJsonLinesAsync("edges.jsonl", batch.Edges, cancellationToken)
.ConfigureAwait(false);
}
}

View File

@@ -0,0 +1,96 @@
using System.Text.Json.Serialization;
namespace StellaOps.Graph.Indexer.Ingestion.Vex;
public sealed class VexOverlaySnapshot
{
[JsonPropertyName("tenant")]
public string Tenant { get; init; } = string.Empty;
[JsonPropertyName("source")]
public string Source { get; init; } = string.Empty;
[JsonPropertyName("collectedAt")]
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
[JsonPropertyName("eventOffset")]
public long EventOffset { get; init; }
[JsonPropertyName("statement")]
public VexStatementDetails Statement { get; init; } = new();
[JsonPropertyName("exemptions")]
public IReadOnlyList<VexComponentExemption> Exemptions { get; init; } = Array.Empty<VexComponentExemption>();
}
public sealed class VexStatementDetails
{
[JsonPropertyName("vexSource")]
public string VexSource { get; init; } = string.Empty;
[JsonPropertyName("statementId")]
public string StatementId { get; init; } = string.Empty;
[JsonPropertyName("status")]
public string Status { get; init; } = string.Empty;
[JsonPropertyName("justification")]
public string Justification { get; init; } = string.Empty;
[JsonPropertyName("impactStatement")]
public string ImpactStatement { get; init; } = string.Empty;
[JsonPropertyName("issuedAt")]
public DateTimeOffset IssuedAt { get; init; } = DateTimeOffset.UnixEpoch;
[JsonPropertyName("expiresAt")]
public DateTimeOffset ExpiresAt { get; init; } = DateTimeOffset.UnixEpoch;
[JsonPropertyName("contentHash")]
public string ContentHash { get; init; } = string.Empty;
[JsonPropertyName("provenanceSource")]
public string? ProvenanceSource { get; init; }
[JsonPropertyName("collectedAt")]
public DateTimeOffset StatementCollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
[JsonPropertyName("eventOffset")]
public long StatementEventOffset { get; init; }
}
public sealed class VexComponentExemption
{
[JsonPropertyName("componentPurl")]
public string ComponentPurl { get; init; } = string.Empty;
[JsonPropertyName("componentSourceType")]
public string ComponentSourceType { get; init; } = "inventory";
[JsonPropertyName("sbomDigest")]
public string? SbomDigest { get; init; }
[JsonPropertyName("statementHash")]
public string? StatementHash { get; init; }
[JsonPropertyName("status")]
public string Status { get; init; } = string.Empty;
[JsonPropertyName("justification")]
public string Justification { get; init; } = string.Empty;
[JsonPropertyName("impactStatement")]
public string ImpactStatement { get; init; } = string.Empty;
[JsonPropertyName("evidenceDigest")]
public string EvidenceDigest { get; init; } = string.Empty;
[JsonPropertyName("provenanceSource")]
public string? ProvenanceSource { get; init; }
[JsonPropertyName("collectedAt")]
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
[JsonPropertyName("eventOffset")]
public long EventOffset { get; init; }
}

View File

@@ -0,0 +1,243 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text.Json.Nodes;
using StellaOps.Graph.Indexer.Ingestion.Sbom;
using StellaOps.Graph.Indexer.Schema;
namespace StellaOps.Graph.Indexer.Ingestion.Vex;
public sealed class VexOverlayTransformer
{
private const string VexNodeKind = "vex_statement";
private const string ComponentNodeKind = "component";
private const string VexExemptsEdgeKind = "VEX_EXEMPTS";
public GraphBuildBatch Transform(VexOverlaySnapshot snapshot)
{
ArgumentNullException.ThrowIfNull(snapshot);
var nodes = new List<JsonObject>();
var edges = new List<JsonObject>();
var seenEdgeIds = new HashSet<string>(StringComparer.Ordinal);
var statement = snapshot.Statement ?? new VexStatementDetails();
var canonicalStatementHash = statement.ContentHash?.Trim() ?? string.Empty;
var vexNode = CreateVexStatementNode(snapshot, statement, canonicalStatementHash);
nodes.Add(vexNode);
var vexNodeId = vexNode["id"]!.GetValue<string>();
foreach (var exemption in snapshot.Exemptions ?? Array.Empty<VexComponentExemption>())
{
if (string.IsNullOrWhiteSpace(exemption.ComponentPurl))
{
continue;
}
var edge = CreateVexExemptsEdge(snapshot, statement, canonicalStatementHash, vexNodeId, exemption);
var edgeId = edge["id"]!.GetValue<string>();
if (seenEdgeIds.Add(edgeId))
{
edges.Add(edge);
}
}
return new GraphBuildBatch(
nodes
.OrderBy(node => node["kind"]!.GetValue<string>(), StringComparer.Ordinal)
.ThenBy(node => node["id"]!.GetValue<string>(), StringComparer.Ordinal)
.ToImmutableArray(),
edges
.OrderBy(edge => edge["kind"]!.GetValue<string>(), StringComparer.Ordinal)
.ThenBy(edge => edge["id"]!.GetValue<string>(), StringComparer.Ordinal)
.ToImmutableArray());
}
private static JsonObject CreateVexStatementNode(
VexOverlaySnapshot snapshot,
VexStatementDetails statement,
string canonicalStatementHash)
{
var vexSource = statement.VexSource?.Trim() ?? string.Empty;
var statementId = statement.StatementId?.Trim() ?? string.Empty;
var status = statement.Status?.Trim() ?? string.Empty;
var justification = statement.Justification?.Trim() ?? string.Empty;
var issuedAt = statement.IssuedAt == DateTimeOffset.UnixEpoch
? snapshot.CollectedAt
: statement.IssuedAt;
var expiresAt = statement.ExpiresAt == DateTimeOffset.UnixEpoch
? (DateTimeOffset?)null
: statement.ExpiresAt;
var attributes = new JsonObject
{
["status"] = status,
["statement_id"] = statementId,
["justification"] = justification,
["issued_at"] = GraphTimestamp.Format(issuedAt)
};
if (expiresAt.HasValue)
{
attributes["expires_at"] = GraphTimestamp.Format(expiresAt.Value);
}
attributes["content_hash"] = canonicalStatementHash;
var canonicalKey = new Dictionary<string, string>
{
["tenant"] = snapshot.Tenant,
["vex_source"] = vexSource,
["statement_id"] = statementId,
["content_hash"] = canonicalStatementHash
};
var provenanceSource = string.IsNullOrWhiteSpace(statement.ProvenanceSource)
? NormalizeVexSource(snapshot.Source)
: statement.ProvenanceSource.Trim();
var collectedAt = statement.StatementCollectedAt == DateTimeOffset.UnixEpoch
? snapshot.CollectedAt
: statement.StatementCollectedAt;
var eventOffset = statement.StatementEventOffset != 0
? statement.StatementEventOffset
: snapshot.EventOffset;
var node = GraphDocumentFactory.CreateNode(new GraphNodeSpec(
Tenant: snapshot.Tenant,
Kind: VexNodeKind,
CanonicalKey: canonicalKey,
Attributes: attributes,
Provenance: new GraphProvenanceSpec(provenanceSource, collectedAt, null, eventOffset),
ValidFrom: issuedAt,
ValidTo: null));
var provenance = node["provenance"]!.AsObject();
var sourceNode = provenance["source"]!.DeepClone();
var collectedAtNode = provenance["collected_at"]!.DeepClone();
var eventOffsetNode = provenance.ContainsKey("event_offset")
? provenance["event_offset"]!.DeepClone()
: null;
var reorderedProvenance = new JsonObject
{
["source"] = sourceNode,
["collected_at"] = collectedAtNode,
["sbom_digest"] = null
};
if (eventOffsetNode is not null)
{
reorderedProvenance["event_offset"] = eventOffsetNode;
}
node["provenance"] = reorderedProvenance;
node.Remove("hash");
node["hash"] = GraphIdentity.ComputeDocumentHash(node);
return node;
}
private static JsonObject CreateVexExemptsEdge(
VexOverlaySnapshot snapshot,
VexStatementDetails statement,
string canonicalStatementHash,
string vexNodeId,
VexComponentExemption exemption)
{
ArgumentNullException.ThrowIfNull(exemption);
var normalizedSourceType = string.IsNullOrWhiteSpace(exemption.ComponentSourceType)
? "inventory"
: exemption.ComponentSourceType.Trim();
var componentPurl = exemption.ComponentPurl.Trim();
var statementHash = !string.IsNullOrWhiteSpace(exemption.StatementHash)
? exemption.StatementHash.Trim()
: canonicalStatementHash;
var componentIdentity = new Dictionary<string, string>
{
["tenant"] = snapshot.Tenant,
["purl"] = componentPurl,
["source_type"] = normalizedSourceType
};
var componentNodeId = GraphIdentity.ComputeNodeId(snapshot.Tenant, ComponentNodeKind, componentIdentity);
var canonicalKey = new Dictionary<string, string>
{
["tenant"] = snapshot.Tenant,
["component_node_id"] = componentNodeId,
["vex_node_id"] = vexNodeId,
["statement_hash"] = statementHash
};
var impactStatement = !string.IsNullOrWhiteSpace(exemption.ImpactStatement)
? exemption.ImpactStatement.Trim()
: statement.ImpactStatement?.Trim() ?? string.Empty;
var status = !string.IsNullOrWhiteSpace(exemption.Status)
? exemption.Status.Trim()
: statement.Status?.Trim() ?? string.Empty;
var justification = !string.IsNullOrWhiteSpace(exemption.Justification)
? exemption.Justification.Trim()
: statement.Justification?.Trim() ?? string.Empty;
var attributes = new JsonObject
{
["status"] = status,
["justification"] = justification,
["impact_statement"] = impactStatement,
["evidence_digest"] = exemption.EvidenceDigest?.Trim() ?? string.Empty
};
var collectedAt = exemption.CollectedAt == DateTimeOffset.UnixEpoch
? snapshot.CollectedAt
: exemption.CollectedAt;
var eventOffset = exemption.EventOffset != 0 ? exemption.EventOffset : snapshot.EventOffset;
var source = ResolveSource(exemption.ProvenanceSource, snapshot.Source);
return GraphDocumentFactory.CreateEdge(new GraphEdgeSpec(
Tenant: snapshot.Tenant,
Kind: VexExemptsEdgeKind,
CanonicalKey: canonicalKey,
Attributes: attributes,
Provenance: new GraphProvenanceSpec(source, collectedAt, exemption.SbomDigest?.Trim(), eventOffset),
ValidFrom: collectedAt,
ValidTo: null));
}
private static string ResolveSource(string? candidate, string? fallback)
{
if (!string.IsNullOrWhiteSpace(candidate))
{
return candidate.Trim();
}
return string.IsNullOrWhiteSpace(fallback)
? "excititor.overlay.v1"
: fallback.Trim();
}
private static string NormalizeVexSource(string? source)
{
if (string.IsNullOrWhiteSpace(source))
{
return "excititor.vex.v1";
}
var trimmed = source.Trim();
return trimmed.Contains(".overlay.", StringComparison.Ordinal)
? trimmed.Replace(".overlay.", ".vex.", StringComparison.Ordinal)
: trimmed;
}
}

View File

@@ -0,0 +1,44 @@
using System.Text;
namespace StellaOps.Graph.Indexer.Schema;
/// <summary>
/// Base32 encoder using the Crockford alphabet (0-9A-HJKMNPQRSTVWXYZ).
/// </summary>
internal static class Base32Crockford
{
private const string Alphabet = "0123456789ABCDEFGHJKMNPQRSTVWXYZ";
public static string Encode(ReadOnlySpan<byte> data)
{
if (data.IsEmpty)
{
return string.Empty;
}
var output = new StringBuilder((data.Length * 8 + 4) / 5);
var buffer = 0;
var bitsLeft = 0;
foreach (var b in data)
{
buffer = (buffer << 8) | b;
bitsLeft += 8;
while (bitsLeft >= 5)
{
bitsLeft -= 5;
var index = (buffer >> bitsLeft) & 0x1F;
output.Append(Alphabet[index]);
}
}
if (bitsLeft > 0)
{
var index = (buffer << (5 - bitsLeft)) & 0x1F;
output.Append(Alphabet[index]);
}
return output.ToString();
}
}

View File

@@ -0,0 +1,134 @@
using System.Text.Json;
using System.Text.Json.Nodes;
namespace StellaOps.Graph.Indexer.Schema;
/// <summary>
/// Canonical JSON serialiser used for deterministic hashing.
/// </summary>
public static class CanonicalJson
{
public static byte[] ToCanonicalUtf8Bytes(JsonNode node)
{
ArgumentNullException.ThrowIfNull(node);
using var stream = new MemoryStream();
using var writer = new Utf8JsonWriter(stream, new JsonWriterOptions
{
Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
Indented = false
});
WriteNode(node, writer);
writer.Flush();
return stream.ToArray();
}
private static void WriteNode(JsonNode node, Utf8JsonWriter writer)
{
switch (node)
{
case JsonObject obj:
writer.WriteStartObject();
foreach (var property in obj.OrderBy(static p => p.Key, StringComparer.Ordinal))
{
writer.WritePropertyName(property.Key);
WriteNode(property.Value!, writer);
}
writer.WriteEndObject();
break;
case JsonArray array:
writer.WriteStartArray();
foreach (var item in array)
{
if (item is null)
{
writer.WriteNullValue();
}
else
{
WriteNode(item, writer);
}
}
writer.WriteEndArray();
break;
case JsonValue value:
WriteValue(value, writer);
break;
default:
writer.WriteNullValue();
break;
}
}
private static void WriteValue(JsonValue value, Utf8JsonWriter writer)
{
if (value.TryGetValue(out string? stringValue))
{
writer.WriteStringValue(stringValue);
return;
}
if (value.TryGetValue(out bool boolValue))
{
writer.WriteBooleanValue(boolValue);
return;
}
if (value.TryGetValue(out long longValue))
{
writer.WriteNumberValue(longValue);
return;
}
if (value.TryGetValue(out int intValue))
{
writer.WriteNumberValue(intValue);
return;
}
if (value.TryGetValue(out double doubleValue))
{
writer.WriteNumberValue(doubleValue);
return;
}
if (value.TryGetValue(out decimal decimalValue))
{
writer.WriteNumberValue(decimalValue);
return;
}
if (value.TryGetValue(out float floatValue))
{
writer.WriteNumberValue(floatValue);
return;
}
if (value.TryGetValue(out Guid guidValue))
{
writer.WriteStringValue(guidValue);
return;
}
if (value.TryGetValue(out DateTime dateTimeValue))
{
writer.WriteStringValue(dateTimeValue.ToUniversalTime());
return;
}
if (value.TryGetValue(out DateTimeOffset dateTimeOffsetValue))
{
writer.WriteStringValue(dateTimeOffsetValue.ToUniversalTime());
return;
}
// Fallback to raw text.
writer.WriteStringValue(value.ToJsonString());
}
}

View File

@@ -0,0 +1,132 @@
using System.Collections.Immutable;
using System.Globalization;
using System.Text.Json;
using System.Text.Json.Nodes;
namespace StellaOps.Graph.Indexer.Schema;
public readonly record struct GraphProvenanceSpec(
string Source,
DateTimeOffset CollectedAt,
string? SbomDigest,
long? EventOffset)
{
public JsonObject ToJson()
{
var obj = new JsonObject
{
["source"] = Source,
["collected_at"] = GraphTimestamp.Format(CollectedAt)
};
if (!string.IsNullOrWhiteSpace(SbomDigest))
{
obj["sbom_digest"] = SbomDigest;
}
if (EventOffset.HasValue)
{
obj["event_offset"] = EventOffset.Value;
}
return obj;
}
}
public readonly record struct GraphNodeSpec(
string Tenant,
string Kind,
IReadOnlyDictionary<string, string> CanonicalKey,
JsonObject Attributes,
GraphProvenanceSpec Provenance,
DateTimeOffset ValidFrom,
DateTimeOffset? ValidTo);
public readonly record struct GraphEdgeSpec(
string Tenant,
string Kind,
IReadOnlyDictionary<string, string> CanonicalKey,
JsonObject Attributes,
GraphProvenanceSpec Provenance,
DateTimeOffset ValidFrom,
DateTimeOffset? ValidTo);
public static class GraphDocumentFactory
{
public static JsonObject CreateNode(GraphNodeSpec spec)
{
var canonicalKey = CreateCanonicalKey(spec.CanonicalKey);
var node = new JsonObject
{
["tenant"] = spec.Tenant,
["kind"] = spec.Kind,
["canonical_key"] = canonicalKey,
["attributes"] = (JsonObject)spec.Attributes.DeepClone(),
["provenance"] = spec.Provenance.ToJson(),
["valid_from"] = GraphTimestamp.Format(spec.ValidFrom),
["valid_to"] = spec.ValidTo.HasValue ? GraphTimestamp.Format(spec.ValidTo.Value) : null
};
var identityTuple = GraphIdentity.ExtractIdentityTuple(canonicalKey);
node["id"] = GraphIdentity.ComputeNodeId(spec.Tenant, spec.Kind, identityTuple);
var hash = GraphIdentity.ComputeDocumentHash(node);
node["hash"] = hash;
return node;
}
public static JsonObject CreateEdge(GraphEdgeSpec spec)
{
var canonicalKey = CreateCanonicalKey(spec.CanonicalKey);
var edge = new JsonObject
{
["tenant"] = spec.Tenant,
["kind"] = spec.Kind,
["canonical_key"] = canonicalKey,
["attributes"] = (JsonObject)spec.Attributes.DeepClone(),
["provenance"] = spec.Provenance.ToJson(),
["valid_from"] = GraphTimestamp.Format(spec.ValidFrom),
["valid_to"] = spec.ValidTo.HasValue ? GraphTimestamp.Format(spec.ValidTo.Value) : null
};
var identityTuple = GraphIdentity.ExtractIdentityTuple(canonicalKey);
edge["id"] = GraphIdentity.ComputeEdgeId(spec.Tenant, spec.Kind, identityTuple);
var hash = GraphIdentity.ComputeDocumentHash(edge);
edge["hash"] = hash;
return edge;
}
private static JsonObject CreateCanonicalKey(IReadOnlyDictionary<string, string> entries)
{
var builder = ImmutableSortedDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
foreach (var (key, value) in entries)
{
builder[key.Trim()] = value.Trim();
}
var obj = new JsonObject();
foreach (var (key, value) in builder)
{
obj[key] = value;
}
return obj;
}
}
internal static class GraphTimestamp
{
public static string Format(DateTimeOffset value)
{
var utc = value.UtcDateTime;
return utc.Ticks % TimeSpan.TicksPerSecond == 0
? utc.ToString("yyyy-MM-dd'T'HH:mm:ss'Z'", CultureInfo.InvariantCulture)
: utc.ToString("yyyy-MM-dd'T'HH:mm:ss.fffffff'Z'", CultureInfo.InvariantCulture);
}
}

View File

@@ -0,0 +1,152 @@
using System.Collections.Immutable;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
namespace StellaOps.Graph.Indexer.Schema;
/// <summary>
/// Helpers for computing deterministic identifiers and hashes for graph documents.
/// </summary>
public static class GraphIdentity
{
private const string NodePrefix = "gn:";
private const StringComparison OrdinalIgnoreCase = StringComparison.OrdinalIgnoreCase;
private static readonly string[] CaseSensitiveHints = { "digest", "hash", "fingerprint", "id" };
/// <summary>
/// Computes a deterministic node identifier using the canonical identity tuple.
/// </summary>
public static string ComputeNodeId(string tenant, string kind, IReadOnlyDictionary<string, string> identityTuple)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
ArgumentException.ThrowIfNullOrWhiteSpace(kind);
ArgumentNullException.ThrowIfNull(identityTuple);
var normalizedKind = kind.Trim().ToLowerInvariant();
var normalizedTenant = tenant.Trim().ToLowerInvariant();
var tuplePayload = JoinTuple(identityTuple);
var hash = HashBase32(tuplePayload);
return $"{NodePrefix}{normalizedTenant}:{normalizedKind}:{hash}";
}
/// <summary>
/// Computes a deterministic edge identifier using the canonical identity tuple.
/// </summary>
public static string ComputeEdgeId(string tenant, string kind, IReadOnlyDictionary<string, string> identityTuple)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
ArgumentException.ThrowIfNullOrWhiteSpace(kind);
ArgumentNullException.ThrowIfNull(identityTuple);
var normalizedKind = kind.Trim().ToUpperInvariant();
var normalizedTenant = tenant.Trim().ToLowerInvariant();
var tuplePayload = JoinTuple(identityTuple);
var hash = HashBase32(tuplePayload);
return $"ge:{normalizedTenant}:{normalizedKind}:{hash}";
}
/// <summary>
/// Computes the canonical SHA-256 hash for a JSON document. Canonicalisation sorts object keys
/// and serialises using UTF-8 JSON without extra whitespace.
/// </summary>
public static string ComputeDocumentHash(JsonNode document)
{
ArgumentNullException.ThrowIfNull(document);
var canonicalBytes = CanonicalJson.ToCanonicalUtf8Bytes(document);
var hash = SHA256.HashData(canonicalBytes);
return Convert.ToHexString(hash).ToLowerInvariant();
}
/// <summary>
/// Converts a JSON object into an ordered dictionary of string components used for identifiers.
/// </summary>
public static IReadOnlyDictionary<string, string> ExtractIdentityTuple(JsonObject source)
{
ArgumentNullException.ThrowIfNull(source);
var builder = ImmutableSortedDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
foreach (var kvp in source)
{
if (kvp.Value is null)
{
continue;
}
if (kvp.Value is JsonValue value)
{
builder[kvp.Key.Trim()] = value.ToJsonString().Trim('"');
}
else
{
builder[kvp.Key.Trim()] = kvp.Value.ToJsonString();
}
}
return builder.ToImmutable();
}
private static string JoinTuple(IReadOnlyDictionary<string, string> tuple)
{
var builder = new StringBuilder();
var sorted = tuple.OrderBy(static pair => pair.Key, StringComparer.Ordinal);
var first = true;
foreach (var (key, value) in sorted)
{
if (!first)
{
builder.Append('|');
}
first = false;
var normalizedKey = key.Trim().ToLowerInvariant();
var normalizedValue = NormalizeValue(normalizedKey, value);
builder.Append(normalizedKey);
builder.Append('=');
builder.Append(normalizedValue);
}
return builder.ToString();
}
private static string NormalizeValue(string key, string value)
{
if (string.IsNullOrWhiteSpace(value))
{
return string.Empty;
}
if (IsCaseSensitiveKey(key))
{
return value.Trim();
}
return value.Trim().ToLowerInvariant();
}
private static bool IsCaseSensitiveKey(string key)
{
foreach (var hint in CaseSensitiveHints)
{
if (key.Contains(hint, OrdinalIgnoreCase))
{
return true;
}
}
return false;
}
private static string HashBase32(string payload)
{
var data = SHA256.HashData(Encoding.UTF8.GetBytes(payload));
return Base32Crockford.Encode(data);
}
}

View File

@@ -0,0 +1,17 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<LangVersion>preview</LangVersion>
<AssemblyName>StellaOps.Graph.Indexer</AssemblyName>
<RootNamespace>StellaOps.Graph.Indexer</RootNamespace>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="MongoDB.Driver" Version="3.5.0" />
</ItemGroup>
</Project>

View File

@@ -1,13 +1,14 @@
# Graph Indexer Task Board — Epic 5: SBOM Graph Explorer
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|----|--------|----------|------------|-------------|---------------|
| GRAPH-INDEX-28-001 | TODO | Graph Indexer Guild | SBOM-SERVICE-21-001, CARTO-GRAPH-21-001 | Define canonical node/edge schemas, attribute dictionaries, identity rules, and seed fixtures; publish schema doc. | Schema doc merged; identity property tests pass; fixtures committed for CI usage. |
| GRAPH-INDEX-28-002 | TODO | Graph Indexer Guild | GRAPH-INDEX-28-001, SBOM-SERVICE-21-002 | Implement SBOM ingest consumer producing artifact/package/file nodes and edges with `valid_from/valid_to`, scope metadata, and provenance links. | Ingest pipeline processes sample SBOMs deterministically; metrics recorded; unit tests cover identity stability. |
| GRAPH-INDEX-28-003 | TODO | Graph Indexer Guild | GRAPH-INDEX-28-001, CONCELIER-CONSOLE-23-001 | Project Concelier linksets into overlay tiles (`affected_by` edges, evidence refs) without mutating source observations; keep advisory aggregates in overlay store only. | Overlay documents generated deterministically; raw node/edge collections remain immutable; tests cover overlay refresh and eviction. |
| GRAPH-INDEX-28-004 | TODO | Graph Indexer Guild | GRAPH-INDEX-28-001, EXCITITOR-CONSOLE-23-001 | Integrate VEX statements (`vex_exempts` edges) with justification metadata and precedence markers for overlays. | VEX edges generated; conflicts resolved deterministically; tests cover status transitions. |
| GRAPH-INDEX-28-005 | TODO | Graph Indexer Guild, Policy Guild | POLICY-ENGINE-27-001, POLICY-ENGINE-27-002 | Hydrate policy overlays into graph (`governs_with` nodes/edges) referencing effective findings and explain hashes for sampled nodes. | Overlay nodes stored with policy version id, severity, status; explain references captured; validation tests pass. |
| GRAPH-INDEX-28-006 | TODO | Graph Indexer Guild | GRAPH-INDEX-28-002..005 | Generate graph snapshots per SBOM with lineage (`derived_from`), adjacency manifests, and metadata for diff jobs. | Snapshot documents produced; lineage recorded; tests assert diff readiness; metrics emitted. |
| GRAPH-INDEX-28-007 | TODO | Graph Indexer Guild, Observability Guild | GRAPH-INDEX-28-002..006 | Implement clustering/centrality background jobs (Louvain/degree/betweenness approximations) with configurable schedules and store cluster ids on nodes. | Clustering jobs run on fixtures; metrics logged; cluster ids accessible via API; SLA documented. |
| GRAPH-INDEX-28-008 | TODO | Graph Indexer Guild | GRAPH-INDEX-28-002..007 | Provide incremental update + backfill pipeline with change streams, retry/backoff, idempotent operations, and backlog metrics. | Incremental updates replay sample change logs; retries/backoff validated; backlog metrics exported. |
| GRAPH-INDEX-28-009 | TODO | Graph Indexer Guild, QA Guild | GRAPH-INDEX-28-002..008 | Add unit/property/integration tests, synthetic large graph fixtures, chaos testing (missing overlays, cycles), and determinism checks across runs. | Test suite green; determinism harness passes across two runs; perf metrics recorded. |
| GRAPH-INDEX-28-010 | TODO | Graph Indexer Guild, DevOps Guild | GRAPH-INDEX-28-008 | Package deployment artifacts (Helm/Compose), offline seed bundles, and configuration docs; integrate Offline Kit. | Deployment descriptors merged; offline seed bundle documented; smoke deploy tested. |
# Graph Indexer Task Board — Epic 5: SBOM Graph Explorer
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|----|--------|----------|------------|-------------|---------------|
| GRAPH-INDEX-28-001 | DONE (2025-11-03) | Graph Indexer Guild | SBOM-SERVICE-21-001, CARTO-GRAPH-21-001 | Define canonical node/edge schemas, attribute dictionaries, identity rules, and seed fixtures; publish schema doc.<br>2025-11-03: Schema doc v1 published, fixtures added (`nodes.json`, `edges.json`, `schema-matrix.json`), GraphIdentity determinism tests green. | Schema doc merged; identity property tests pass; fixtures committed for CI usage. |
| GRAPH-INDEX-28-002 | DONE (2025-11-03) | Graph Indexer Guild | GRAPH-INDEX-28-001, SBOM-SERVICE-21-002 | Implement SBOM ingest consumer producing artifact/package/file nodes and edges with `valid_from/valid_to`, scope metadata, and provenance links.<br>2025-11-03: Snapshot models repaired, provenance resolution tightened, ingest processor/metrics surfaces added, and transformer/fixtures/tests expanded for license/base artifact determinism. | Ingest pipeline processes sample SBOMs deterministically; metrics recorded; unit tests cover identity stability. |
| GRAPH-INDEX-28-003 | DONE (2025-11-03) | Graph Indexer Guild | GRAPH-INDEX-28-001, CONCELIER-CONSOLE-23-001 | Project Concelier linksets into overlay tiles (`affected_by` edges, evidence refs) without mutating source observations; keep advisory aggregates in overlay store only.<br>2025-11-03: Snapshot model repaired, transformer finalized with dedupe + provenance normalization, fixtures/tests refreshed, full graph suite green. | Overlay documents generated deterministically; raw node/edge collections remain immutable; tests cover overlay refresh and eviction. |
| GRAPH-INDEX-28-004 | DONE (2025-11-03) | Graph Indexer Guild | GRAPH-INDEX-28-001, EXCITITOR-CONSOLE-23-001 | Integrate VEX statements (`vex_exempts` edges) with justification metadata and precedence markers for overlays.<br>2025-11-03: VEX snapshot/transformer emit deterministic VEX_EXEMPTS overlays with provenance hashes; fixtures and tests updated; full graph indexer suite green. | VEX edges generated; conflicts resolved deterministically; tests cover status transitions. |
| GRAPH-INDEX-28-005 | DONE (2025-11-03) | Graph Indexer Guild, Policy Guild | POLICY-ENGINE-27-001, POLICY-ENGINE-27-002 | Hydrate policy overlays into graph (`governs_with` nodes/edges) referencing effective findings and explain hashes for sampled nodes.<br>2025-11-03: Policy overlay snapshot/transformer added with deterministic nodes/edges, fixtures + tests updated, targeted graph tests pass; Mongo writer tests now probe `STELLAOPS_TEST_MONGO_URI` or localhost before falling back to Mongo2Go and skip with guidance when neither path is available.<br>2025-11-03: Processor + metrics wired atop Mongo writer; unit tests cover success/failure paths. | Overlay nodes stored with policy version id, severity, status; explain references captured; validation tests pass. |
| GRAPH-INDEX-28-006 | DONE (2025-11-03) | Graph Indexer Guild | GRAPH-INDEX-28-002..005 | Generate graph snapshots per SBOM with lineage (`derived_from`), adjacency manifests, and metadata for diff jobs.<br>2025-11-03: Snapshot builder emits hashed manifest + adjacency (incoming/outgoing edges), integration tests cover lineage/diff readiness, docs updated with required Mongo env.<br>2025-11-03: Snapshot exporter writes manifest/adjacency/nodes/edges to snapshot directory with deterministic ordering. | Snapshot documents produced; lineage recorded; tests assert diff readiness; metrics emitted. |
| GRAPH-INDEX-28-007 | TODO | Graph Indexer Guild, Observability Guild | GRAPH-INDEX-28-002..006 | Implement clustering/centrality background jobs (Louvain/degree/betweenness approximations) with configurable schedules and store cluster ids on nodes. | Clustering jobs run on fixtures; metrics logged; cluster ids accessible via API; SLA documented. |
| GRAPH-INDEX-28-008 | TODO | Graph Indexer Guild | GRAPH-INDEX-28-002..007 | Provide incremental update + backfill pipeline with change streams, retry/backoff, idempotent operations, and backlog metrics. | Incremental updates replay sample change logs; retries/backoff validated; backlog metrics exported. |
| GRAPH-INDEX-28-009 | TODO | Graph Indexer Guild, QA Guild | GRAPH-INDEX-28-002..008 | Add unit/property/integration tests, synthetic large graph fixtures, chaos testing (missing overlays, cycles), and determinism checks across runs. | Test suite green; determinism harness passes across two runs; perf metrics recorded. |
| GRAPH-INDEX-28-010 | TODO | Graph Indexer Guild, DevOps Guild | GRAPH-INDEX-28-008 | Package deployment artifacts (Helm/Compose), offline seed bundles, and configuration docs; integrate Offline Kit. | Deployment descriptors merged; offline seed bundle documented; smoke deploy tested. |
| GRAPH-INDEX-28-011 | DONE (2025-11-04) | Graph Indexer Guild | GRAPH-INDEX-28-002..006 | Wire SBOM ingest runtime to emit graph snapshot artifacts and harden Mongo test configuration.<br>2025-11-04: Adopted `SbomIngestProcessorFactory.CreateDefault` inside a DI extension, added configurable snapshot root (`STELLAOPS_GRAPH_SNAPSHOT_DIR` or options), documented Mongo/snapshot env guidance, and verified Graph Indexer tests (Mongo writer skipped when no URI). | Composition root uses factory/exporter, snapshot files land in configured artifacts directory, and dev/CI guidance ensures Mongo availability without manual edits. |