Add unit tests for SBOM ingestion and transformation
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
- Implement `SbomIngestServiceCollectionExtensionsTests` to verify the SBOM ingestion pipeline exports snapshots correctly. - Create `SbomIngestTransformerTests` to ensure the transformation produces expected nodes and edges, including deduplication of license nodes and normalization of timestamps. - Add `SbomSnapshotExporterTests` to test the export functionality for manifest, adjacency, nodes, and edges. - Introduce `VexOverlayTransformerTests` to validate the transformation of VEX nodes and edges. - Set up project file for the test project with necessary dependencies and configurations. - Include JSON fixture files for testing purposes.
This commit is contained in:
261
src/Graph/StellaOps.Graph.Indexer/Documents/GraphSnapshot.cs
Normal file
261
src/Graph/StellaOps.Graph.Indexer/Documents/GraphSnapshot.cs
Normal file
@@ -0,0 +1,261 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Nodes;
|
||||
using System.Text.Json.Serialization;
|
||||
using StellaOps.Graph.Indexer.Schema;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Documents;
|
||||
|
||||
public sealed record GraphSnapshot(
|
||||
GraphSnapshotManifest Manifest,
|
||||
GraphAdjacencyManifest Adjacency);
|
||||
|
||||
public sealed class GraphSnapshotManifest
|
||||
{
|
||||
[JsonPropertyName("tenant")]
|
||||
public string Tenant { get; }
|
||||
|
||||
[JsonPropertyName("artifact_digest")]
|
||||
public string ArtifactDigest { get; }
|
||||
|
||||
[JsonPropertyName("sbom_digest")]
|
||||
public string SbomDigest { get; }
|
||||
|
||||
[JsonPropertyName("snapshot_id")]
|
||||
public string SnapshotId { get; }
|
||||
|
||||
[JsonPropertyName("generated_at")]
|
||||
public DateTimeOffset GeneratedAt { get; }
|
||||
|
||||
[JsonPropertyName("node_count")]
|
||||
public int NodeCount { get; }
|
||||
|
||||
[JsonPropertyName("edge_count")]
|
||||
public int EdgeCount { get; }
|
||||
|
||||
[JsonPropertyName("hash")]
|
||||
public string Hash { get; }
|
||||
|
||||
[JsonPropertyName("lineage")]
|
||||
public GraphSnapshotLineage Lineage { get; }
|
||||
|
||||
[JsonPropertyName("files")]
|
||||
public GraphSnapshotFiles Files { get; }
|
||||
|
||||
public GraphSnapshotManifest(
|
||||
string tenant,
|
||||
string artifactDigest,
|
||||
string sbomDigest,
|
||||
string snapshotId,
|
||||
DateTimeOffset generatedAt,
|
||||
int nodeCount,
|
||||
int edgeCount,
|
||||
GraphSnapshotLineage lineage,
|
||||
GraphSnapshotFiles files,
|
||||
string hash)
|
||||
{
|
||||
Tenant = tenant;
|
||||
ArtifactDigest = artifactDigest;
|
||||
SbomDigest = sbomDigest;
|
||||
SnapshotId = snapshotId;
|
||||
GeneratedAt = generatedAt;
|
||||
NodeCount = nodeCount;
|
||||
EdgeCount = edgeCount;
|
||||
Lineage = lineage;
|
||||
Files = files;
|
||||
Hash = hash;
|
||||
}
|
||||
|
||||
public JsonObject ToJson()
|
||||
{
|
||||
var obj = new JsonObject
|
||||
{
|
||||
["tenant"] = Tenant,
|
||||
["artifact_digest"] = ArtifactDigest,
|
||||
["sbom_digest"] = SbomDigest,
|
||||
["snapshot_id"] = SnapshotId,
|
||||
["generated_at"] = GraphTimestamp.Format(GeneratedAt),
|
||||
["node_count"] = NodeCount,
|
||||
["edge_count"] = EdgeCount,
|
||||
["lineage"] = Lineage.ToJson(),
|
||||
["files"] = Files.ToJson(),
|
||||
["hash"] = Hash
|
||||
};
|
||||
|
||||
return obj;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class GraphSnapshotLineage
|
||||
{
|
||||
[JsonPropertyName("derived_from_sbom_digests")]
|
||||
public ImmutableArray<string> DerivedFromSbomDigests { get; }
|
||||
|
||||
[JsonPropertyName("base_artifact_digests")]
|
||||
public ImmutableArray<string> BaseArtifactDigests { get; }
|
||||
|
||||
[JsonPropertyName("source_snapshot_id")]
|
||||
public string? SourceSnapshotId { get; }
|
||||
|
||||
public GraphSnapshotLineage(
|
||||
ImmutableArray<string> derivedFromSbomDigests,
|
||||
ImmutableArray<string> baseArtifactDigests,
|
||||
string? sourceSnapshotId)
|
||||
{
|
||||
DerivedFromSbomDigests = derivedFromSbomDigests;
|
||||
BaseArtifactDigests = baseArtifactDigests;
|
||||
SourceSnapshotId = sourceSnapshotId;
|
||||
}
|
||||
|
||||
public JsonObject ToJson()
|
||||
{
|
||||
var obj = new JsonObject
|
||||
{
|
||||
["derived_from_sbom_digests"] = CreateArray(DerivedFromSbomDigests),
|
||||
["base_artifact_digests"] = CreateArray(BaseArtifactDigests),
|
||||
["source_snapshot_id"] = SourceSnapshotId
|
||||
};
|
||||
|
||||
return obj;
|
||||
}
|
||||
|
||||
private static JsonArray CreateArray(ImmutableArray<string> values)
|
||||
{
|
||||
var array = new JsonArray();
|
||||
foreach (var value in values)
|
||||
{
|
||||
array.Add(value);
|
||||
}
|
||||
|
||||
return array;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class GraphSnapshotFiles
|
||||
{
|
||||
[JsonPropertyName("nodes")]
|
||||
public string Nodes { get; }
|
||||
|
||||
[JsonPropertyName("edges")]
|
||||
public string Edges { get; }
|
||||
|
||||
[JsonPropertyName("adjacency")]
|
||||
public string Adjacency { get; }
|
||||
|
||||
public GraphSnapshotFiles(string nodes, string edges, string adjacency)
|
||||
{
|
||||
Nodes = nodes;
|
||||
Edges = edges;
|
||||
Adjacency = adjacency;
|
||||
}
|
||||
|
||||
public JsonObject ToJson()
|
||||
{
|
||||
return new JsonObject
|
||||
{
|
||||
["nodes"] = Nodes,
|
||||
["edges"] = Edges,
|
||||
["adjacency"] = Adjacency
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class GraphAdjacencyManifest
|
||||
{
|
||||
[JsonPropertyName("tenant")]
|
||||
public string Tenant { get; }
|
||||
|
||||
[JsonPropertyName("artifact_node_id")]
|
||||
public string ArtifactNodeId { get; }
|
||||
|
||||
[JsonPropertyName("generated_at")]
|
||||
public DateTimeOffset GeneratedAt { get; }
|
||||
|
||||
[JsonPropertyName("snapshot_id")]
|
||||
public string SnapshotId { get; }
|
||||
|
||||
[JsonPropertyName("nodes")]
|
||||
public ImmutableArray<GraphAdjacencyNode> Nodes { get; }
|
||||
|
||||
public GraphAdjacencyManifest(
|
||||
string tenant,
|
||||
string artifactNodeId,
|
||||
DateTimeOffset generatedAt,
|
||||
string snapshotId,
|
||||
ImmutableArray<GraphAdjacencyNode> nodes)
|
||||
{
|
||||
Tenant = tenant;
|
||||
ArtifactNodeId = artifactNodeId;
|
||||
GeneratedAt = generatedAt;
|
||||
SnapshotId = snapshotId;
|
||||
Nodes = nodes;
|
||||
}
|
||||
|
||||
public JsonObject ToJson()
|
||||
{
|
||||
var obj = new JsonObject
|
||||
{
|
||||
["tenant"] = Tenant,
|
||||
["artifact_node_id"] = ArtifactNodeId,
|
||||
["generated_at"] = GraphTimestamp.Format(GeneratedAt),
|
||||
["snapshot_id"] = SnapshotId
|
||||
};
|
||||
|
||||
var nodesArray = new JsonArray();
|
||||
foreach (var node in Nodes)
|
||||
{
|
||||
nodesArray.Add(node.ToJson());
|
||||
}
|
||||
|
||||
obj["nodes"] = nodesArray;
|
||||
return obj;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class GraphAdjacencyNode
|
||||
{
|
||||
[JsonPropertyName("node_id")]
|
||||
public string NodeId { get; }
|
||||
|
||||
[JsonPropertyName("kind")]
|
||||
public string Kind { get; }
|
||||
|
||||
[JsonPropertyName("outgoing_edges")]
|
||||
public ImmutableArray<string> OutgoingEdges { get; }
|
||||
|
||||
[JsonPropertyName("incoming_edges")]
|
||||
public ImmutableArray<string> IncomingEdges { get; }
|
||||
|
||||
public GraphAdjacencyNode(
|
||||
string nodeId,
|
||||
string kind,
|
||||
ImmutableArray<string> outgoingEdges,
|
||||
ImmutableArray<string> incomingEdges)
|
||||
{
|
||||
NodeId = nodeId;
|
||||
Kind = kind;
|
||||
OutgoingEdges = outgoingEdges;
|
||||
IncomingEdges = incomingEdges;
|
||||
}
|
||||
|
||||
public JsonObject ToJson()
|
||||
{
|
||||
return new JsonObject
|
||||
{
|
||||
["node_id"] = NodeId,
|
||||
["kind"] = Kind,
|
||||
["outgoing_edges"] = CreateArray(OutgoingEdges),
|
||||
["incoming_edges"] = CreateArray(IncomingEdges)
|
||||
};
|
||||
}
|
||||
|
||||
private static JsonArray CreateArray(ImmutableArray<string> values)
|
||||
{
|
||||
var array = new JsonArray();
|
||||
foreach (var value in values)
|
||||
{
|
||||
array.Add(value);
|
||||
}
|
||||
|
||||
return array;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,449 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Nodes;
|
||||
using StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
using StellaOps.Graph.Indexer.Schema;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Documents;
|
||||
|
||||
public sealed class GraphSnapshotBuilder
|
||||
{
|
||||
private const string NodesFileName = "nodes.jsonl";
|
||||
private const string EdgesFileName = "edges.jsonl";
|
||||
private const string AdjacencyFileName = "adjacency.json";
|
||||
|
||||
public GraphSnapshot Build(SbomSnapshot sbomSnapshot, GraphBuildBatch batch, DateTimeOffset generatedAt)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(sbomSnapshot);
|
||||
ArgumentNullException.ThrowIfNull(batch);
|
||||
|
||||
var tenant = sbomSnapshot.Tenant ?? string.Empty;
|
||||
var nodes = batch.Nodes;
|
||||
var edges = batch.Edges;
|
||||
|
||||
var nodesById = nodes.ToImmutableDictionary(
|
||||
node => node["id"]!.GetValue<string>(),
|
||||
node => node,
|
||||
StringComparer.Ordinal);
|
||||
|
||||
var artifactNodeId = ResolveArtifactNodeId(sbomSnapshot, nodes);
|
||||
var snapshotId = ComputeSnapshotId(sbomSnapshot.Tenant, sbomSnapshot.ArtifactDigest, sbomSnapshot.SbomDigest);
|
||||
|
||||
var derivedSbomDigests = sbomSnapshot.BaseArtifacts
|
||||
.Select(baseArtifact => baseArtifact.SbomDigest)
|
||||
.Where(static digest => !string.IsNullOrWhiteSpace(digest))
|
||||
.Select(static digest => digest.Trim())
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.OrderBy(static digest => digest, StringComparer.Ordinal)
|
||||
.ToImmutableArray();
|
||||
|
||||
var baseArtifactDigests = sbomSnapshot.BaseArtifacts
|
||||
.Select(baseArtifact => baseArtifact.ArtifactDigest)
|
||||
.Where(static digest => !string.IsNullOrWhiteSpace(digest))
|
||||
.Select(static digest => digest.Trim())
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.OrderBy(static digest => digest, StringComparer.Ordinal)
|
||||
.ToImmutableArray();
|
||||
|
||||
var lineage = new GraphSnapshotLineage(derivedSbomDigests, baseArtifactDigests, null);
|
||||
var files = new GraphSnapshotFiles(NodesFileName, EdgesFileName, AdjacencyFileName);
|
||||
|
||||
var manifest = CreateManifest(
|
||||
tenant,
|
||||
sbomSnapshot.ArtifactDigest,
|
||||
sbomSnapshot.SbomDigest,
|
||||
snapshotId,
|
||||
generatedAt,
|
||||
nodes.Length,
|
||||
edges.Length,
|
||||
lineage,
|
||||
files);
|
||||
|
||||
var adjacency = BuildAdjacencyManifest(
|
||||
sbomSnapshot,
|
||||
snapshotId,
|
||||
generatedAt,
|
||||
artifactNodeId,
|
||||
nodes,
|
||||
edges,
|
||||
nodesById);
|
||||
|
||||
return new GraphSnapshot(manifest, adjacency);
|
||||
}
|
||||
|
||||
private static GraphSnapshotManifest CreateManifest(
|
||||
string tenant,
|
||||
string artifactDigest,
|
||||
string sbomDigest,
|
||||
string snapshotId,
|
||||
DateTimeOffset generatedAt,
|
||||
int nodeCount,
|
||||
int edgeCount,
|
||||
GraphSnapshotLineage lineage,
|
||||
GraphSnapshotFiles files)
|
||||
{
|
||||
var json = new JsonObject
|
||||
{
|
||||
["tenant"] = tenant,
|
||||
["artifact_digest"] = artifactDigest,
|
||||
["sbom_digest"] = sbomDigest,
|
||||
["snapshot_id"] = snapshotId,
|
||||
["generated_at"] = GraphTimestamp.Format(generatedAt),
|
||||
["node_count"] = nodeCount,
|
||||
["edge_count"] = edgeCount,
|
||||
["lineage"] = lineage.ToJson(),
|
||||
["files"] = files.ToJson()
|
||||
};
|
||||
|
||||
var hash = GraphIdentity.ComputeDocumentHash(json);
|
||||
json["hash"] = hash;
|
||||
|
||||
return new GraphSnapshotManifest(
|
||||
tenant,
|
||||
artifactDigest,
|
||||
sbomDigest,
|
||||
snapshotId,
|
||||
generatedAt,
|
||||
nodeCount,
|
||||
edgeCount,
|
||||
lineage,
|
||||
files,
|
||||
hash);
|
||||
}
|
||||
|
||||
private static GraphAdjacencyManifest BuildAdjacencyManifest(
|
||||
SbomSnapshot snapshot,
|
||||
string snapshotId,
|
||||
DateTimeOffset generatedAt,
|
||||
string artifactNodeId,
|
||||
ImmutableArray<JsonObject> nodes,
|
||||
ImmutableArray<JsonObject> edges,
|
||||
IReadOnlyDictionary<string, JsonObject> nodesById)
|
||||
{
|
||||
var nodeEntries = new Dictionary<string, AdjacencyNodeBuilder>(StringComparer.Ordinal);
|
||||
foreach (var node in nodes)
|
||||
{
|
||||
var nodeId = node["id"]!.GetValue<string>();
|
||||
var kind = node["kind"]!.GetValue<string>();
|
||||
nodeEntries[nodeId] = new AdjacencyNodeBuilder(nodeId, kind);
|
||||
}
|
||||
|
||||
var componentNodeByPurl = BuildComponentIndex(nodes);
|
||||
var artifactNodeByDigest = BuildArtifactIndex(nodes);
|
||||
|
||||
foreach (var edge in edges)
|
||||
{
|
||||
if (!TryResolveEdgeEndpoints(
|
||||
edge,
|
||||
nodesById,
|
||||
componentNodeByPurl,
|
||||
artifactNodeByDigest,
|
||||
out var sourceNodeId,
|
||||
out var targetNodeId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!nodeEntries.TryGetValue(sourceNodeId, out var source))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!nodeEntries.TryGetValue(targetNodeId, out var target))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var edgeId = edge["id"]!.GetValue<string>();
|
||||
source.AddOutgoing(edgeId);
|
||||
target.AddIncoming(edgeId);
|
||||
}
|
||||
|
||||
var nodesArray = nodeEntries.Values
|
||||
.Select(builder => builder.ToNode())
|
||||
.OrderBy(node => node.NodeId, StringComparer.Ordinal)
|
||||
.ToImmutableArray();
|
||||
|
||||
return new GraphAdjacencyManifest(
|
||||
snapshot.Tenant,
|
||||
artifactNodeId,
|
||||
generatedAt,
|
||||
snapshotId,
|
||||
nodesArray);
|
||||
}
|
||||
|
||||
private static string ResolveArtifactNodeId(SbomSnapshot snapshot, ImmutableArray<JsonObject> nodes)
|
||||
{
|
||||
foreach (var node in nodes)
|
||||
{
|
||||
if (!string.Equals(node["kind"]!.GetValue<string>(), "artifact", StringComparison.Ordinal))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (TryMatchArtifact(node, snapshot.ArtifactDigest, snapshot.SbomDigest))
|
||||
{
|
||||
return node["id"]!.GetValue<string>();
|
||||
}
|
||||
}
|
||||
|
||||
throw new InvalidOperationException($"Unable to locate artifact node for digest '{snapshot.ArtifactDigest}' and SBOM '{snapshot.SbomDigest}'.");
|
||||
}
|
||||
|
||||
private static bool TryMatchArtifact(JsonObject node, string artifactDigest, string sbomDigest)
|
||||
{
|
||||
if (!node.TryGetPropertyValue("attributes", out var attributesNode) || attributesNode is not JsonObject attributes)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var nodeArtifactDigest = attributes.TryGetPropertyValue("artifact_digest", out var digestValue)
|
||||
? digestValue?.GetValue<string>() ?? string.Empty
|
||||
: string.Empty;
|
||||
|
||||
var nodeSbomDigest = attributes.TryGetPropertyValue("sbom_digest", out var sbomValue)
|
||||
? sbomValue?.GetValue<string>() ?? string.Empty
|
||||
: string.Empty;
|
||||
|
||||
return string.Equals(nodeArtifactDigest, artifactDigest, StringComparison.Ordinal)
|
||||
&& string.Equals(nodeSbomDigest, sbomDigest, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
private static string ComputeSnapshotId(string tenant, string artifactDigest, string sbomDigest)
|
||||
{
|
||||
var identity = new JsonObject
|
||||
{
|
||||
["tenant"] = tenant.Trim().ToLowerInvariant(),
|
||||
["artifact_digest"] = artifactDigest.Trim().ToLowerInvariant(),
|
||||
["sbom_digest"] = sbomDigest.Trim().ToLowerInvariant()
|
||||
};
|
||||
|
||||
var hash = GraphIdentity.ComputeDocumentHash(identity);
|
||||
return $"gs:{tenant.Trim().ToLowerInvariant()}:{hash}";
|
||||
}
|
||||
|
||||
private static Dictionary<string, string> BuildComponentIndex(ImmutableArray<JsonObject> nodes)
|
||||
{
|
||||
var components = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var node in nodes)
|
||||
{
|
||||
if (!string.Equals(node["kind"]!.GetValue<string>(), "component", StringComparison.Ordinal))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!node.TryGetPropertyValue("attributes", out var attributesNode) || attributesNode is not JsonObject attributes)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!attributes.TryGetPropertyValue("purl", out var purlNode) || purlNode is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var purl = purlNode.GetValue<string>();
|
||||
if (!string.IsNullOrWhiteSpace(purl))
|
||||
{
|
||||
components.TryAdd(purl.Trim(), node["id"]!.GetValue<string>());
|
||||
}
|
||||
}
|
||||
|
||||
return components;
|
||||
}
|
||||
|
||||
private static Dictionary<string, string> BuildArtifactIndex(ImmutableArray<JsonObject> nodes)
|
||||
{
|
||||
var artifacts = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var node in nodes)
|
||||
{
|
||||
if (!string.Equals(node["kind"]!.GetValue<string>(), "artifact", StringComparison.Ordinal))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!node.TryGetPropertyValue("attributes", out var attributesNode) || attributesNode is not JsonObject attributes)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!attributes.TryGetPropertyValue("artifact_digest", out var digestNode) || digestNode is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var digest = digestNode.GetValue<string>();
|
||||
if (!string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
artifacts.TryAdd(digest.Trim(), node["id"]!.GetValue<string>());
|
||||
}
|
||||
}
|
||||
|
||||
return artifacts;
|
||||
}
|
||||
|
||||
private static bool TryResolveEdgeEndpoints(
|
||||
JsonObject edge,
|
||||
IReadOnlyDictionary<string, JsonObject> nodesById,
|
||||
IReadOnlyDictionary<string, string> componentNodeByPurl,
|
||||
IReadOnlyDictionary<string, string> artifactNodeByDigest,
|
||||
out string sourceNodeId,
|
||||
out string targetNodeId)
|
||||
{
|
||||
var kind = edge["kind"]!.GetValue<string>();
|
||||
var canonicalKey = edge["canonical_key"]!.AsObject();
|
||||
|
||||
string? source = null;
|
||||
string? target = null;
|
||||
|
||||
switch (kind)
|
||||
{
|
||||
case "CONTAINS":
|
||||
source = canonicalKey.TryGetPropertyValue("artifact_node_id", out var containsSource) ? containsSource?.GetValue<string>() : null;
|
||||
target = canonicalKey.TryGetPropertyValue("component_node_id", out var containsTarget) ? containsTarget?.GetValue<string>() : null;
|
||||
break;
|
||||
case "DECLARED_IN":
|
||||
source = canonicalKey.TryGetPropertyValue("component_node_id", out var declaredSource) ? declaredSource?.GetValue<string>() : null;
|
||||
target = canonicalKey.TryGetPropertyValue("file_node_id", out var declaredTarget) ? declaredTarget?.GetValue<string>() : null;
|
||||
break;
|
||||
case "AFFECTED_BY":
|
||||
source = canonicalKey.TryGetPropertyValue("component_node_id", out var affectedSource) ? affectedSource?.GetValue<string>() : null;
|
||||
target = canonicalKey.TryGetPropertyValue("advisory_node_id", out var affectedTarget) ? affectedTarget?.GetValue<string>() : null;
|
||||
break;
|
||||
case "VEX_EXEMPTS":
|
||||
source = canonicalKey.TryGetPropertyValue("component_node_id", out var vexSource) ? vexSource?.GetValue<string>() : null;
|
||||
target = canonicalKey.TryGetPropertyValue("vex_node_id", out var vexTarget) ? vexTarget?.GetValue<string>() : null;
|
||||
break;
|
||||
case "GOVERNS_WITH":
|
||||
source = canonicalKey.TryGetPropertyValue("policy_node_id", out var policySource) ? policySource?.GetValue<string>() : null;
|
||||
target = canonicalKey.TryGetPropertyValue("component_node_id", out var policyTarget) ? policyTarget?.GetValue<string>() : null;
|
||||
break;
|
||||
case "OBSERVED_RUNTIME":
|
||||
source = canonicalKey.TryGetPropertyValue("runtime_node_id", out var runtimeSource) ? runtimeSource?.GetValue<string>() : null;
|
||||
target = canonicalKey.TryGetPropertyValue("component_node_id", out var runtimeTarget) ? runtimeTarget?.GetValue<string>() : null;
|
||||
break;
|
||||
case "BUILT_FROM":
|
||||
source = canonicalKey.TryGetPropertyValue("parent_artifact_node_id", out var builtSource) ? builtSource?.GetValue<string>() : null;
|
||||
if (canonicalKey.TryGetPropertyValue("child_artifact_node_id", out var builtTargetNode) && builtTargetNode is not null)
|
||||
{
|
||||
target = builtTargetNode.GetValue<string>();
|
||||
}
|
||||
else if (canonicalKey.TryGetPropertyValue("child_artifact_digest", out var builtTargetDigest) && builtTargetDigest is not null)
|
||||
{
|
||||
artifactNodeByDigest.TryGetValue(builtTargetDigest.GetValue<string>(), out target);
|
||||
}
|
||||
break;
|
||||
case "DEPENDS_ON":
|
||||
source = canonicalKey.TryGetPropertyValue("component_node_id", out var dependsSource) ? dependsSource?.GetValue<string>() : null;
|
||||
if (canonicalKey.TryGetPropertyValue("dependency_node_id", out var dependsTargetNode) && dependsTargetNode is not null)
|
||||
{
|
||||
target = dependsTargetNode.GetValue<string>();
|
||||
}
|
||||
else if (canonicalKey.TryGetPropertyValue("dependency_purl", out var dependencyPurl) && dependencyPurl is not null)
|
||||
{
|
||||
componentNodeByPurl.TryGetValue(dependencyPurl.GetValue<string>(), out target);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
source = ExtractFirstNodeId(canonicalKey);
|
||||
target = ExtractSecondNodeId(canonicalKey);
|
||||
break;
|
||||
}
|
||||
|
||||
if (source is null || target is null)
|
||||
{
|
||||
sourceNodeId = string.Empty;
|
||||
targetNodeId = string.Empty;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!nodesById.ContainsKey(source) || !nodesById.ContainsKey(target))
|
||||
{
|
||||
sourceNodeId = string.Empty;
|
||||
targetNodeId = string.Empty;
|
||||
return false;
|
||||
}
|
||||
|
||||
sourceNodeId = source;
|
||||
targetNodeId = target;
|
||||
return true;
|
||||
}
|
||||
|
||||
private static string? ExtractFirstNodeId(JsonObject canonicalKey)
|
||||
{
|
||||
foreach (var property in canonicalKey)
|
||||
{
|
||||
if (property.Value is JsonValue value
|
||||
&& value.TryGetValue(out string? candidate)
|
||||
&& candidate is not null
|
||||
&& candidate.StartsWith("gn:", StringComparison.Ordinal))
|
||||
{
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string? ExtractSecondNodeId(JsonObject canonicalKey)
|
||||
{
|
||||
var encountered = false;
|
||||
foreach (var property in canonicalKey)
|
||||
{
|
||||
if (property.Value is JsonValue value
|
||||
&& value.TryGetValue(out string? candidate)
|
||||
&& candidate is not null
|
||||
&& candidate.StartsWith("gn:", StringComparison.Ordinal))
|
||||
{
|
||||
if (!encountered)
|
||||
{
|
||||
encountered = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private sealed class AdjacencyNodeBuilder
|
||||
{
|
||||
private readonly SortedSet<string> _outgoing = new(StringComparer.Ordinal);
|
||||
private readonly SortedSet<string> _incoming = new(StringComparer.Ordinal);
|
||||
|
||||
public AdjacencyNodeBuilder(string nodeId, string kind)
|
||||
{
|
||||
NodeId = nodeId;
|
||||
Kind = kind;
|
||||
}
|
||||
|
||||
public string NodeId { get; }
|
||||
public string Kind { get; }
|
||||
|
||||
public void AddOutgoing(string edgeId)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(edgeId))
|
||||
{
|
||||
_outgoing.Add(edgeId);
|
||||
}
|
||||
}
|
||||
|
||||
public void AddIncoming(string edgeId)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(edgeId))
|
||||
{
|
||||
_incoming.Add(edgeId);
|
||||
}
|
||||
}
|
||||
|
||||
public GraphAdjacencyNode ToNode()
|
||||
{
|
||||
return new GraphAdjacencyNode(
|
||||
NodeId,
|
||||
Kind,
|
||||
_outgoing.ToImmutableArray(),
|
||||
_incoming.ToImmutableArray());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
|
||||
|
||||
public sealed class AdvisoryLinksetMetrics : IAdvisoryLinksetMetrics, IDisposable
|
||||
{
|
||||
public const string MeterName = "StellaOps.Graph.Indexer";
|
||||
public const string MeterVersion = "1.0.0";
|
||||
|
||||
private readonly Meter _meter;
|
||||
private readonly bool _ownsMeter;
|
||||
private readonly Counter<long> _batchesTotal;
|
||||
private readonly Histogram<double> _batchDurationSeconds;
|
||||
private readonly Counter<long> _nodesTotal;
|
||||
private readonly Counter<long> _edgesTotal;
|
||||
private bool _disposed;
|
||||
|
||||
public AdvisoryLinksetMetrics()
|
||||
: this(null)
|
||||
{
|
||||
}
|
||||
|
||||
public AdvisoryLinksetMetrics(Meter? meter)
|
||||
{
|
||||
_meter = meter ?? new Meter(MeterName, MeterVersion);
|
||||
_ownsMeter = meter is null;
|
||||
|
||||
_batchesTotal = _meter.CreateCounter<long>(
|
||||
name: "graph_advisory_ingest_batches_total",
|
||||
unit: "count",
|
||||
description: "Advisory linkset ingest batches processed grouped by source, tenant, and result.");
|
||||
|
||||
_batchDurationSeconds = _meter.CreateHistogram<double>(
|
||||
name: "graph_advisory_ingest_duration_seconds",
|
||||
unit: "s",
|
||||
description: "Latency to transform and persist advisory linkset batches grouped by source, tenant, and result.");
|
||||
|
||||
_nodesTotal = _meter.CreateCounter<long>(
|
||||
name: "graph_advisory_ingest_nodes_total",
|
||||
unit: "count",
|
||||
description: "Advisory nodes produced by linkset ingest grouped by source and tenant.");
|
||||
|
||||
_edgesTotal = _meter.CreateCounter<long>(
|
||||
name: "graph_advisory_ingest_edges_total",
|
||||
unit: "count",
|
||||
description: "Affected_by edges produced by linkset ingest grouped by source and tenant.");
|
||||
}
|
||||
|
||||
public void RecordBatch(string source, string tenant, int nodeCount, int edgeCount, TimeSpan duration, bool success)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(source);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
|
||||
|
||||
var normalizedDurationSeconds = Math.Max(duration.TotalSeconds, 0d);
|
||||
var resultTag = success ? "success" : "failure";
|
||||
|
||||
var tags = new[]
|
||||
{
|
||||
new KeyValuePair<string, object?>("source", source),
|
||||
new KeyValuePair<string, object?>("tenant", tenant),
|
||||
new KeyValuePair<string, object?>("result", resultTag)
|
||||
};
|
||||
|
||||
_batchesTotal.Add(1, tags);
|
||||
_batchDurationSeconds.Record(normalizedDurationSeconds, tags);
|
||||
|
||||
if (!success)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var volumeTags = new[]
|
||||
{
|
||||
new KeyValuePair<string, object?>("source", source),
|
||||
new KeyValuePair<string, object?>("tenant", tenant)
|
||||
};
|
||||
|
||||
if (nodeCount > 0)
|
||||
{
|
||||
_nodesTotal.Add(nodeCount, volumeTags);
|
||||
}
|
||||
|
||||
if (edgeCount > 0)
|
||||
{
|
||||
_edgesTotal.Add(edgeCount, volumeTags);
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (_ownsMeter)
|
||||
{
|
||||
_meter.Dispose();
|
||||
}
|
||||
|
||||
_disposed = true;
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
using System.Diagnostics;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
|
||||
|
||||
public sealed class AdvisoryLinksetProcessor
|
||||
{
|
||||
private readonly AdvisoryLinksetTransformer _transformer;
|
||||
private readonly IGraphDocumentWriter _writer;
|
||||
private readonly IAdvisoryLinksetMetrics _metrics;
|
||||
private readonly ILogger<AdvisoryLinksetProcessor> _logger;
|
||||
|
||||
public AdvisoryLinksetProcessor(
|
||||
AdvisoryLinksetTransformer transformer,
|
||||
IGraphDocumentWriter writer,
|
||||
IAdvisoryLinksetMetrics metrics,
|
||||
ILogger<AdvisoryLinksetProcessor> logger)
|
||||
{
|
||||
_transformer = transformer ?? throw new ArgumentNullException(nameof(transformer));
|
||||
_writer = writer ?? throw new ArgumentNullException(nameof(writer));
|
||||
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task ProcessAsync(AdvisoryLinksetSnapshot snapshot, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(snapshot);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
GraphBuildBatch batch;
|
||||
|
||||
try
|
||||
{
|
||||
batch = _transformer.Transform(snapshot);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, 0, 0, stopwatch.Elapsed, success: false);
|
||||
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"graph-indexer: failed to transform advisory linkset {LinksetDigest} for tenant {Tenant}",
|
||||
snapshot.LinksetDigest,
|
||||
snapshot.Tenant);
|
||||
|
||||
throw;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
await _writer.WriteAsync(batch, cancellationToken).ConfigureAwait(false);
|
||||
stopwatch.Stop();
|
||||
|
||||
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, batch.Nodes.Length, batch.Edges.Length, stopwatch.Elapsed, success: true);
|
||||
|
||||
_logger.LogInformation(
|
||||
"graph-indexer: indexed advisory linkset {LinksetDigest} for tenant {Tenant} with {NodeCount} nodes and {EdgeCount} edges in {DurationMs:F2} ms",
|
||||
snapshot.LinksetDigest,
|
||||
snapshot.Tenant,
|
||||
batch.Nodes.Length,
|
||||
batch.Edges.Length,
|
||||
stopwatch.Elapsed.TotalMilliseconds);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, batch.Nodes.Length, batch.Edges.Length, stopwatch.Elapsed, success: false);
|
||||
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"graph-indexer: failed to persist advisory linkset {LinksetDigest} for tenant {Tenant}",
|
||||
snapshot.LinksetDigest,
|
||||
snapshot.Tenant);
|
||||
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
|
||||
|
||||
public sealed class AdvisoryLinksetSnapshot
|
||||
{
|
||||
[JsonPropertyName("tenant")]
|
||||
public string Tenant { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("linksetDigest")]
|
||||
public string LinksetDigest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long EventOffset { get; init; } = 0;
|
||||
|
||||
[JsonPropertyName("advisory")]
|
||||
public AdvisoryDetails Advisory { get; init; } = new();
|
||||
|
||||
[JsonPropertyName("components")]
|
||||
public IReadOnlyList<AdvisoryComponentImpact> Components { get; init; }
|
||||
= Array.Empty<AdvisoryComponentImpact>();
|
||||
}
|
||||
|
||||
public sealed class AdvisoryDetails
|
||||
{
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("advisorySource")]
|
||||
public string AdvisorySource { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("advisoryId")]
|
||||
public string AdvisoryId { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("severity")]
|
||||
public string Severity { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("publishedAt")]
|
||||
public DateTimeOffset PublishedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("contentHash")]
|
||||
public string ContentHash { get; init; } = string.Empty;
|
||||
}
|
||||
|
||||
public sealed class AdvisoryComponentImpact
|
||||
{
|
||||
[JsonPropertyName("purl")]
|
||||
public string ComponentPurl { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("sourceType")]
|
||||
public string ComponentSourceType { get; init; } = "inventory";
|
||||
|
||||
[JsonPropertyName("evidenceDigest")]
|
||||
public string EvidenceDigest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("matchedVersions")]
|
||||
public IReadOnlyList<string> MatchedVersions { get; init; }
|
||||
= Array.Empty<string>();
|
||||
|
||||
[JsonPropertyName("cvss")]
|
||||
public double? Cvss { get; init; }
|
||||
= null;
|
||||
|
||||
[JsonPropertyName("confidence")]
|
||||
public double? Confidence { get; init; }
|
||||
= null;
|
||||
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long EventOffset { get; init; } = 0;
|
||||
|
||||
[JsonPropertyName("sbomDigest")]
|
||||
public string? SbomDigest { get; init; }
|
||||
= null;
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Text.Json.Nodes;
|
||||
using StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
using StellaOps.Graph.Indexer.Schema;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
|
||||
|
||||
public sealed class AdvisoryLinksetTransformer
|
||||
{
|
||||
private const string AdvisoryNodeKind = "advisory";
|
||||
private const string ComponentNodeKind = "component";
|
||||
private const string AffectedByEdgeKind = "AFFECTED_BY";
|
||||
|
||||
public GraphBuildBatch Transform(AdvisoryLinksetSnapshot snapshot)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(snapshot);
|
||||
|
||||
var nodes = new List<JsonObject>();
|
||||
var edgesById = new Dictionary<string, JsonObject>(StringComparer.Ordinal);
|
||||
|
||||
var advisoryNode = CreateAdvisoryNode(snapshot);
|
||||
nodes.Add(advisoryNode);
|
||||
|
||||
foreach (var component in snapshot.Components ?? Array.Empty<AdvisoryComponentImpact>())
|
||||
{
|
||||
if (component is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var edge = CreateAffectedByEdge(snapshot, advisoryNode, component);
|
||||
edgesById[edge["id"]!.GetValue<string>()] = edge;
|
||||
}
|
||||
|
||||
var orderedNodes = nodes
|
||||
.OrderBy(node => node["kind"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ThenBy(node => node["id"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ToImmutableArray();
|
||||
|
||||
var orderedEdges = edgesById.Values
|
||||
.OrderBy(edge => edge["kind"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ThenBy(edge => edge["id"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ToImmutableArray();
|
||||
|
||||
return new GraphBuildBatch(orderedNodes, orderedEdges);
|
||||
}
|
||||
|
||||
private static JsonObject CreateAdvisoryNode(AdvisoryLinksetSnapshot snapshot)
|
||||
{
|
||||
var details = snapshot.Advisory ?? new AdvisoryDetails();
|
||||
var advisorySource = (details.AdvisorySource ?? string.Empty).Trim();
|
||||
var advisoryId = (details.AdvisoryId ?? string.Empty).Trim();
|
||||
var severity = (details.Severity ?? string.Empty).Trim();
|
||||
var contentHash = (details.ContentHash ?? string.Empty).Trim();
|
||||
var linksetDigest = (snapshot.LinksetDigest ?? string.Empty).Trim();
|
||||
var provenanceSource = ResolveSource(details.Source, snapshot.Source);
|
||||
|
||||
var validFrom = details.PublishedAt == DateTimeOffset.UnixEpoch
|
||||
? snapshot.CollectedAt
|
||||
: details.PublishedAt;
|
||||
|
||||
var attributes = new JsonObject
|
||||
{
|
||||
["advisory_source"] = advisorySource,
|
||||
["advisory_id"] = advisoryId,
|
||||
["severity"] = severity,
|
||||
["published_at"] = GraphTimestamp.Format(validFrom),
|
||||
["content_hash"] = contentHash,
|
||||
["linkset_digest"] = linksetDigest
|
||||
};
|
||||
|
||||
var canonicalKey = new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["advisory_source"] = advisorySource,
|
||||
["advisory_id"] = advisoryId,
|
||||
["content_hash"] = contentHash
|
||||
};
|
||||
|
||||
var node = GraphDocumentFactory.CreateNode(new GraphNodeSpec(
|
||||
Tenant: snapshot.Tenant,
|
||||
Kind: AdvisoryNodeKind,
|
||||
CanonicalKey: canonicalKey,
|
||||
Attributes: attributes,
|
||||
Provenance: new GraphProvenanceSpec(provenanceSource, snapshot.CollectedAt, null, snapshot.EventOffset),
|
||||
ValidFrom: validFrom,
|
||||
ValidTo: null));
|
||||
|
||||
var provenance = node["provenance"]!.AsObject();
|
||||
var sourceNode = provenance["source"]!.DeepClone();
|
||||
var collectedAtNode = provenance["collected_at"]!.DeepClone();
|
||||
var eventOffsetNode = provenance.ContainsKey("event_offset")
|
||||
? provenance["event_offset"]!.DeepClone()
|
||||
: null;
|
||||
|
||||
var reorderedProvenance = new JsonObject
|
||||
{
|
||||
["source"] = sourceNode,
|
||||
["collected_at"] = collectedAtNode,
|
||||
["sbom_digest"] = null
|
||||
};
|
||||
|
||||
if (eventOffsetNode is not null)
|
||||
{
|
||||
reorderedProvenance["event_offset"] = eventOffsetNode;
|
||||
}
|
||||
|
||||
node["provenance"] = reorderedProvenance;
|
||||
|
||||
node.Remove("hash");
|
||||
node["hash"] = GraphIdentity.ComputeDocumentHash(node);
|
||||
|
||||
return node;
|
||||
}
|
||||
|
||||
private static JsonObject CreateAffectedByEdge(
|
||||
AdvisoryLinksetSnapshot snapshot,
|
||||
JsonObject advisoryNode,
|
||||
AdvisoryComponentImpact component)
|
||||
{
|
||||
var advisoryNodeId = advisoryNode["id"]!.GetValue<string>();
|
||||
var componentSourceType = string.IsNullOrWhiteSpace(component.ComponentSourceType)
|
||||
? "inventory"
|
||||
: component.ComponentSourceType.Trim();
|
||||
var componentPurl = (component.ComponentPurl ?? string.Empty).Trim();
|
||||
var linksetDigest = (snapshot.LinksetDigest ?? string.Empty).Trim();
|
||||
|
||||
var componentIdentity = new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["purl"] = componentPurl,
|
||||
["source_type"] = componentSourceType
|
||||
};
|
||||
|
||||
var componentNodeId = GraphIdentity.ComputeNodeId(snapshot.Tenant, ComponentNodeKind, componentIdentity);
|
||||
|
||||
var canonicalKey = new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["component_node_id"] = componentNodeId,
|
||||
["advisory_node_id"] = advisoryNodeId,
|
||||
["linkset_digest"] = linksetDigest
|
||||
};
|
||||
|
||||
var attributes = new JsonObject
|
||||
{
|
||||
["evidence_digest"] = component.EvidenceDigest?.Trim() ?? string.Empty,
|
||||
["matched_versions"] = CreateJsonArray(component.MatchedVersions ?? Array.Empty<string>())
|
||||
};
|
||||
|
||||
if (component.Cvss is { } cvss)
|
||||
{
|
||||
attributes["cvss"] = cvss;
|
||||
}
|
||||
|
||||
if (component.Confidence is { } confidence)
|
||||
{
|
||||
attributes["confidence"] = confidence;
|
||||
}
|
||||
|
||||
var collectedAt = component.CollectedAt == DateTimeOffset.UnixEpoch
|
||||
? snapshot.CollectedAt
|
||||
: component.CollectedAt;
|
||||
|
||||
var eventOffset = component.EventOffset != 0 ? component.EventOffset : snapshot.EventOffset;
|
||||
var source = ResolveSource(component.Source, snapshot.Source);
|
||||
|
||||
return GraphDocumentFactory.CreateEdge(new GraphEdgeSpec(
|
||||
Tenant: snapshot.Tenant,
|
||||
Kind: AffectedByEdgeKind,
|
||||
CanonicalKey: canonicalKey,
|
||||
Attributes: attributes,
|
||||
Provenance: new GraphProvenanceSpec(source, collectedAt, component.SbomDigest, eventOffset),
|
||||
ValidFrom: collectedAt,
|
||||
ValidTo: null));
|
||||
}
|
||||
|
||||
private static JsonArray CreateJsonArray(IEnumerable<string> values)
|
||||
{
|
||||
var array = new JsonArray();
|
||||
foreach (var value in values
|
||||
.Where(v => !string.IsNullOrWhiteSpace(v))
|
||||
.Select(v => v.Trim())
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.OrderBy(v => v, StringComparer.Ordinal))
|
||||
{
|
||||
array.Add(value);
|
||||
}
|
||||
|
||||
return array;
|
||||
}
|
||||
|
||||
private static string ResolveSource(params string?[] candidates)
|
||||
{
|
||||
foreach (var candidate in candidates)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(candidate))
|
||||
{
|
||||
return candidate.Trim();
|
||||
}
|
||||
}
|
||||
|
||||
return string.Empty;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
|
||||
|
||||
public interface IAdvisoryLinksetMetrics
|
||||
{
|
||||
void RecordBatch(string source, string tenant, int nodeCount, int edgeCount, TimeSpan duration, bool success);
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Encodings.Web;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
|
||||
|
||||
public sealed class MongoGraphDocumentWriter : IGraphDocumentWriter
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new()
|
||||
{
|
||||
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
|
||||
WriteIndented = false
|
||||
};
|
||||
|
||||
private readonly IMongoCollection<BsonDocument> _nodes;
|
||||
private readonly IMongoCollection<BsonDocument> _edges;
|
||||
|
||||
public MongoGraphDocumentWriter(IMongoDatabase database, MongoGraphDocumentWriterOptions? options = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
|
||||
var resolved = options ?? new MongoGraphDocumentWriterOptions();
|
||||
_nodes = database.GetCollection<BsonDocument>(resolved.NodeCollectionName);
|
||||
_edges = database.GetCollection<BsonDocument>(resolved.EdgeCollectionName);
|
||||
}
|
||||
|
||||
public async Task WriteAsync(GraphBuildBatch batch, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(batch);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (batch.Nodes.Length > 0)
|
||||
{
|
||||
var nodeModels = CreateReplaceModels(_nodes, batch.Nodes);
|
||||
if (nodeModels.Count > 0)
|
||||
{
|
||||
await _nodes.BulkWriteAsync(nodeModels, new BulkWriteOptions { IsOrdered = false }, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
if (batch.Edges.Length > 0)
|
||||
{
|
||||
var edgeModels = CreateReplaceModels(_edges, batch.Edges);
|
||||
if (edgeModels.Count > 0)
|
||||
{
|
||||
await _edges.BulkWriteAsync(edgeModels, new BulkWriteOptions { IsOrdered = false }, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static List<WriteModel<BsonDocument>> CreateReplaceModels(IMongoCollection<BsonDocument> collection, IReadOnlyList<JsonObject> documents)
|
||||
{
|
||||
var models = new List<WriteModel<BsonDocument>>(documents.Count);
|
||||
foreach (var document in documents)
|
||||
{
|
||||
if (!document.TryGetPropertyValue("id", out var idNode) || idNode is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var id = idNode.GetValue<string>();
|
||||
var filter = Builders<BsonDocument>.Filter.Eq("id", id);
|
||||
var bsonDocument = ToBsonDocument(document);
|
||||
models.Add(new ReplaceOneModel<BsonDocument>(filter, bsonDocument) { IsUpsert = true });
|
||||
}
|
||||
|
||||
return models;
|
||||
}
|
||||
|
||||
private static BsonDocument ToBsonDocument(JsonObject json)
|
||||
{
|
||||
var jsonString = json.ToJsonString(SerializerOptions);
|
||||
return BsonDocument.Parse(jsonString);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Advisory;
|
||||
|
||||
public sealed class MongoGraphDocumentWriterOptions
|
||||
{
|
||||
public string NodeCollectionName { get; init; } = "graph_nodes";
|
||||
public string EdgeCollectionName { get; init; } = "graph_edges";
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Policy;
|
||||
|
||||
public interface IPolicyOverlayMetrics
|
||||
{
|
||||
void RecordBatch(string source, string tenant, int nodeCount, int edgeCount, TimeSpan duration, bool success);
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Policy;
|
||||
|
||||
public sealed class PolicyOverlayMetrics : IPolicyOverlayMetrics, IDisposable
|
||||
{
|
||||
public const string MeterName = "StellaOps.Graph.Indexer";
|
||||
public const string MeterVersion = "1.0.0";
|
||||
|
||||
private const string BatchesTotalName = "graph_policy_overlay_batches_total";
|
||||
private const string BatchDurationSecondsName = "graph_policy_overlay_duration_seconds";
|
||||
private const string NodesTotalName = "graph_policy_overlay_nodes_total";
|
||||
private const string EdgesTotalName = "graph_policy_overlay_edges_total";
|
||||
|
||||
private const string UnitCount = "count";
|
||||
private const string UnitSeconds = "s";
|
||||
|
||||
private readonly Meter _meter;
|
||||
private readonly bool _ownsMeter;
|
||||
private readonly Counter<long> _batchesTotal;
|
||||
private readonly Histogram<double> _batchDurationSeconds;
|
||||
private readonly Counter<long> _nodesTotal;
|
||||
private readonly Counter<long> _edgesTotal;
|
||||
private bool _disposed;
|
||||
|
||||
public PolicyOverlayMetrics()
|
||||
: this(null)
|
||||
{
|
||||
}
|
||||
|
||||
public PolicyOverlayMetrics(Meter? meter)
|
||||
{
|
||||
_meter = meter ?? new Meter(MeterName, MeterVersion);
|
||||
_ownsMeter = meter is null;
|
||||
|
||||
_batchesTotal = _meter.CreateCounter<long>(
|
||||
name: BatchesTotalName,
|
||||
unit: UnitCount,
|
||||
description: "Policy overlay batches processed grouped by source, tenant, and result.");
|
||||
|
||||
_batchDurationSeconds = _meter.CreateHistogram<double>(
|
||||
name: BatchDurationSecondsName,
|
||||
unit: UnitSeconds,
|
||||
description: "Latency to transform and persist policy overlay batches grouped by source, tenant, and result.");
|
||||
|
||||
_nodesTotal = _meter.CreateCounter<long>(
|
||||
name: NodesTotalName,
|
||||
unit: UnitCount,
|
||||
description: "Policy overlay nodes produced grouped by source and tenant.");
|
||||
|
||||
_edgesTotal = _meter.CreateCounter<long>(
|
||||
name: EdgesTotalName,
|
||||
unit: UnitCount,
|
||||
description: "GOVERNS_WITH edges produced grouped by source and tenant.");
|
||||
}
|
||||
|
||||
public void RecordBatch(string source, string tenant, int nodeCount, int edgeCount, TimeSpan duration, bool success)
|
||||
{
|
||||
ThrowIfDisposed();
|
||||
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(source);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
|
||||
|
||||
var normalizedDuration = Math.Max(duration.TotalSeconds, 0d);
|
||||
var resultTag = success ? "success" : "failure";
|
||||
|
||||
var tags = new[]
|
||||
{
|
||||
new KeyValuePair<string, object?>("source", source),
|
||||
new KeyValuePair<string, object?>("tenant", tenant),
|
||||
new KeyValuePair<string, object?>("result", resultTag)
|
||||
};
|
||||
|
||||
_batchesTotal.Add(1, tags);
|
||||
_batchDurationSeconds.Record(normalizedDuration, tags);
|
||||
|
||||
if (!success)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var volumeTags = new[]
|
||||
{
|
||||
new KeyValuePair<string, object?>("source", source),
|
||||
new KeyValuePair<string, object?>("tenant", tenant)
|
||||
};
|
||||
|
||||
if (nodeCount > 0)
|
||||
{
|
||||
_nodesTotal.Add(nodeCount, volumeTags);
|
||||
}
|
||||
|
||||
if (edgeCount > 0)
|
||||
{
|
||||
_edgesTotal.Add(edgeCount, volumeTags);
|
||||
}
|
||||
}
|
||||
|
||||
private void ThrowIfDisposed()
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
throw new ObjectDisposedException(nameof(PolicyOverlayMetrics));
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (_ownsMeter)
|
||||
{
|
||||
_meter.Dispose();
|
||||
}
|
||||
|
||||
_disposed = true;
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
using System.Diagnostics;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Policy;
|
||||
|
||||
public sealed class PolicyOverlayProcessor
|
||||
{
|
||||
private readonly PolicyOverlayTransformer _transformer;
|
||||
private readonly IGraphDocumentWriter _writer;
|
||||
private readonly IPolicyOverlayMetrics _metrics;
|
||||
private readonly ILogger<PolicyOverlayProcessor> _logger;
|
||||
|
||||
public PolicyOverlayProcessor(
|
||||
PolicyOverlayTransformer transformer,
|
||||
IGraphDocumentWriter writer,
|
||||
IPolicyOverlayMetrics metrics,
|
||||
ILogger<PolicyOverlayProcessor> logger)
|
||||
{
|
||||
_transformer = transformer ?? throw new ArgumentNullException(nameof(transformer));
|
||||
_writer = writer ?? throw new ArgumentNullException(nameof(writer));
|
||||
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task ProcessAsync(PolicyOverlaySnapshot snapshot, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(snapshot);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
GraphBuildBatch batch;
|
||||
|
||||
try
|
||||
{
|
||||
batch = _transformer.Transform(snapshot);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, 0, 0, stopwatch.Elapsed, success: false);
|
||||
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"graph-indexer: failed to transform policy overlay {PolicyPackDigest} for tenant {Tenant}",
|
||||
snapshot.Policy?.PolicyPackDigest ?? string.Empty,
|
||||
snapshot.Tenant);
|
||||
|
||||
throw;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
await _writer.WriteAsync(batch, cancellationToken).ConfigureAwait(false);
|
||||
stopwatch.Stop();
|
||||
|
||||
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, batch.Nodes.Length, batch.Edges.Length, stopwatch.Elapsed, success: true);
|
||||
|
||||
_logger.LogInformation(
|
||||
"graph-indexer: indexed policy overlay {PolicyPackDigest} (effective {EffectiveFrom}) for tenant {Tenant} with {NodeCount} nodes and {EdgeCount} edges in {DurationMs:F2} ms",
|
||||
snapshot.Policy?.PolicyPackDigest ?? string.Empty,
|
||||
(snapshot.Policy?.EffectiveFrom ?? snapshot.CollectedAt).ToUniversalTime(),
|
||||
snapshot.Tenant,
|
||||
batch.Nodes.Length,
|
||||
batch.Edges.Length,
|
||||
stopwatch.Elapsed.TotalMilliseconds);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, batch.Nodes.Length, batch.Edges.Length, stopwatch.Elapsed, success: false);
|
||||
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"graph-indexer: failed to persist policy overlay {PolicyPackDigest} for tenant {Tenant}",
|
||||
snapshot.Policy?.PolicyPackDigest ?? string.Empty,
|
||||
snapshot.Tenant);
|
||||
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Policy;
|
||||
|
||||
public sealed class PolicyOverlaySnapshot
|
||||
{
|
||||
[JsonPropertyName("tenant")]
|
||||
public string Tenant { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long EventOffset { get; init; }
|
||||
|
||||
[JsonPropertyName("policy")]
|
||||
public PolicyVersionDetails Policy { get; init; } = new();
|
||||
|
||||
[JsonPropertyName("evaluations")]
|
||||
public IReadOnlyList<PolicyEvaluation> Evaluations { get; init; }
|
||||
= Array.Empty<PolicyEvaluation>();
|
||||
}
|
||||
|
||||
public sealed class PolicyVersionDetails
|
||||
{
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("policyPackDigest")]
|
||||
public string PolicyPackDigest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("policyName")]
|
||||
public string PolicyName { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("effectiveFrom")]
|
||||
public DateTimeOffset EffectiveFrom { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("expiresAt")]
|
||||
public DateTimeOffset ExpiresAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("explainHash")]
|
||||
public string ExplainHash { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long EventOffset { get; init; }
|
||||
}
|
||||
|
||||
public sealed class PolicyEvaluation
|
||||
{
|
||||
[JsonPropertyName("componentPurl")]
|
||||
public string ComponentPurl { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("componentSourceType")]
|
||||
public string ComponentSourceType { get; init; } = "inventory";
|
||||
|
||||
[JsonPropertyName("findingExplainHash")]
|
||||
public string FindingExplainHash { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("explainHash")]
|
||||
public string? ExplainHash { get; init; }
|
||||
|
||||
[JsonPropertyName("policyRuleId")]
|
||||
public string PolicyRuleId { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("verdict")]
|
||||
public string Verdict { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("evaluationTimestamp")]
|
||||
public DateTimeOffset EvaluationTimestamp { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("sbomDigest")]
|
||||
public string? SbomDigest { get; init; }
|
||||
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long EventOffset { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Text.Json.Nodes;
|
||||
using StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
using StellaOps.Graph.Indexer.Schema;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Policy;
|
||||
|
||||
public sealed class PolicyOverlayTransformer
|
||||
{
|
||||
private const string PolicyNodeKind = "policy_version";
|
||||
private const string ComponentNodeKind = "component";
|
||||
private const string GovernsWithEdgeKind = "GOVERNS_WITH";
|
||||
|
||||
public GraphBuildBatch Transform(PolicyOverlaySnapshot snapshot)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(snapshot);
|
||||
|
||||
var tenant = snapshot.Tenant?.Trim() ?? string.Empty;
|
||||
var nodes = new List<JsonObject>();
|
||||
var edges = new List<JsonObject>();
|
||||
var seenEdgeIds = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
var policyDetails = snapshot.Policy ?? new PolicyVersionDetails();
|
||||
|
||||
var policyNode = CreatePolicyNode(tenant, snapshot, policyDetails);
|
||||
nodes.Add(policyNode);
|
||||
|
||||
var policyNodeId = policyNode["id"]!.GetValue<string>();
|
||||
|
||||
foreach (var evaluation in snapshot.Evaluations ?? Array.Empty<PolicyEvaluation>())
|
||||
{
|
||||
if (!IsEvaluationCandidate(evaluation))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var edge = CreateGovernsWithEdge(
|
||||
tenant,
|
||||
snapshot,
|
||||
policyDetails,
|
||||
policyNodeId,
|
||||
evaluation!);
|
||||
|
||||
var edgeId = edge["id"]!.GetValue<string>();
|
||||
if (seenEdgeIds.Add(edgeId))
|
||||
{
|
||||
edges.Add(edge);
|
||||
}
|
||||
}
|
||||
|
||||
return new GraphBuildBatch(
|
||||
nodes
|
||||
.OrderBy(node => node["kind"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ThenBy(node => node["id"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ToImmutableArray(),
|
||||
edges
|
||||
.OrderBy(edge => edge["kind"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ThenBy(edge => edge["id"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ToImmutableArray());
|
||||
}
|
||||
|
||||
private static bool IsEvaluationCandidate(PolicyEvaluation? evaluation)
|
||||
{
|
||||
return evaluation is not null
|
||||
&& !string.IsNullOrWhiteSpace(evaluation.ComponentPurl)
|
||||
&& !string.IsNullOrWhiteSpace(evaluation.FindingExplainHash);
|
||||
}
|
||||
|
||||
private static JsonObject CreatePolicyNode(
|
||||
string tenant,
|
||||
PolicyOverlaySnapshot snapshot,
|
||||
PolicyVersionDetails policy)
|
||||
{
|
||||
var policyPackDigest = policy.PolicyPackDigest?.Trim() ?? string.Empty;
|
||||
var policyName = policy.PolicyName?.Trim() ?? string.Empty;
|
||||
var explainHash = policy.ExplainHash?.Trim() ?? string.Empty;
|
||||
|
||||
var effectiveFrom = policy.EffectiveFrom == DateTimeOffset.UnixEpoch
|
||||
? snapshot.CollectedAt
|
||||
: policy.EffectiveFrom;
|
||||
|
||||
var expiresAt = policy.ExpiresAt == DateTimeOffset.UnixEpoch
|
||||
? (DateTimeOffset?)null
|
||||
: policy.ExpiresAt;
|
||||
|
||||
var policyCollectedAt = policy.CollectedAt == DateTimeOffset.UnixEpoch
|
||||
? snapshot.CollectedAt
|
||||
: policy.CollectedAt;
|
||||
|
||||
var eventOffset = policy.EventOffset != 0 ? policy.EventOffset : snapshot.EventOffset;
|
||||
var source = ResolveSource(policy.Source, snapshot.Source);
|
||||
|
||||
var canonicalKey = new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = tenant,
|
||||
["policy_pack_digest"] = policyPackDigest,
|
||||
["effective_from"] = GraphTimestamp.Format(effectiveFrom)
|
||||
};
|
||||
|
||||
var attributes = new JsonObject
|
||||
{
|
||||
["policy_pack_digest"] = policyPackDigest,
|
||||
["policy_name"] = policyName,
|
||||
["effective_from"] = GraphTimestamp.Format(effectiveFrom),
|
||||
["expires_at"] = expiresAt is null ? null : GraphTimestamp.Format(expiresAt.Value),
|
||||
["explain_hash"] = explainHash
|
||||
};
|
||||
|
||||
var node = GraphDocumentFactory.CreateNode(new GraphNodeSpec(
|
||||
Tenant: tenant,
|
||||
Kind: PolicyNodeKind,
|
||||
CanonicalKey: canonicalKey,
|
||||
Attributes: attributes,
|
||||
Provenance: new GraphProvenanceSpec(source, policyCollectedAt, SbomDigest: null, EventOffset: eventOffset),
|
||||
ValidFrom: effectiveFrom,
|
||||
ValidTo: expiresAt));
|
||||
|
||||
NormalizeOverlayProvenance(node);
|
||||
|
||||
return node;
|
||||
}
|
||||
|
||||
private static JsonObject CreateGovernsWithEdge(
|
||||
string tenant,
|
||||
PolicyOverlaySnapshot snapshot,
|
||||
PolicyVersionDetails policy,
|
||||
string policyNodeId,
|
||||
PolicyEvaluation evaluation)
|
||||
{
|
||||
var componentSourceType = string.IsNullOrWhiteSpace(evaluation.ComponentSourceType)
|
||||
? "inventory"
|
||||
: evaluation.ComponentSourceType.Trim();
|
||||
|
||||
var componentIdentity = new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = tenant,
|
||||
["purl"] = evaluation.ComponentPurl.Trim(),
|
||||
["source_type"] = componentSourceType
|
||||
};
|
||||
|
||||
var componentNodeId = GraphIdentity.ComputeNodeId(tenant, ComponentNodeKind, componentIdentity);
|
||||
|
||||
var findingExplainHash = evaluation.FindingExplainHash.Trim();
|
||||
var edgeCanonicalKey = new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = tenant,
|
||||
["policy_node_id"] = policyNodeId,
|
||||
["component_node_id"] = componentNodeId,
|
||||
["finding_explain_hash"] = findingExplainHash
|
||||
};
|
||||
|
||||
var evaluationTimestamp = evaluation.EvaluationTimestamp == DateTimeOffset.UnixEpoch
|
||||
? snapshot.CollectedAt
|
||||
: evaluation.EvaluationTimestamp;
|
||||
|
||||
var explainHash = !string.IsNullOrWhiteSpace(evaluation.ExplainHash)
|
||||
? evaluation.ExplainHash.Trim()
|
||||
: !string.IsNullOrWhiteSpace(policy.ExplainHash)
|
||||
? policy.ExplainHash.Trim()
|
||||
: findingExplainHash;
|
||||
|
||||
var attributes = new JsonObject
|
||||
{
|
||||
["verdict"] = evaluation.Verdict?.Trim() ?? string.Empty,
|
||||
["explain_hash"] = explainHash,
|
||||
["policy_rule_id"] = evaluation.PolicyRuleId?.Trim() ?? string.Empty,
|
||||
["evaluation_timestamp"] = GraphTimestamp.Format(evaluationTimestamp)
|
||||
};
|
||||
|
||||
var collectedAt = evaluation.CollectedAt == DateTimeOffset.UnixEpoch
|
||||
? snapshot.CollectedAt
|
||||
: evaluation.CollectedAt;
|
||||
|
||||
var eventOffset = evaluation.EventOffset != 0 ? evaluation.EventOffset : snapshot.EventOffset;
|
||||
var source = ResolveSource(evaluation.Source, policy.Source, snapshot.Source);
|
||||
|
||||
return GraphDocumentFactory.CreateEdge(new GraphEdgeSpec(
|
||||
Tenant: tenant,
|
||||
Kind: GovernsWithEdgeKind,
|
||||
CanonicalKey: edgeCanonicalKey,
|
||||
Attributes: attributes,
|
||||
Provenance: new GraphProvenanceSpec(
|
||||
source,
|
||||
collectedAt,
|
||||
NormalizeOptional(evaluation.SbomDigest),
|
||||
eventOffset),
|
||||
ValidFrom: evaluationTimestamp,
|
||||
ValidTo: null));
|
||||
}
|
||||
|
||||
private static void NormalizeOverlayProvenance(JsonObject node)
|
||||
{
|
||||
var provenance = node["provenance"]!.AsObject();
|
||||
var sourceNode = provenance["source"]!.DeepClone();
|
||||
var collectedAtNode = provenance["collected_at"]!.DeepClone();
|
||||
var eventOffsetNode = provenance.ContainsKey("event_offset")
|
||||
? provenance["event_offset"]!.DeepClone()
|
||||
: null;
|
||||
|
||||
var normalized = new JsonObject
|
||||
{
|
||||
["source"] = sourceNode,
|
||||
["collected_at"] = collectedAtNode,
|
||||
["sbom_digest"] = null
|
||||
};
|
||||
|
||||
if (eventOffsetNode is not null)
|
||||
{
|
||||
normalized["event_offset"] = eventOffsetNode;
|
||||
}
|
||||
|
||||
node["provenance"] = normalized;
|
||||
node.Remove("hash");
|
||||
node["hash"] = GraphIdentity.ComputeDocumentHash(node);
|
||||
}
|
||||
|
||||
private static string ResolveSource(params string?[] candidates)
|
||||
{
|
||||
foreach (var candidate in candidates)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(candidate))
|
||||
{
|
||||
return candidate.Trim();
|
||||
}
|
||||
}
|
||||
|
||||
return "policy.engine.v1";
|
||||
}
|
||||
|
||||
private static string? NormalizeOptional(string? value)
|
||||
{
|
||||
return string.IsNullOrWhiteSpace(value) ? null : value.Trim();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
using System.Text;
|
||||
using System.Text.Json.Nodes;
|
||||
using StellaOps.Graph.Indexer.Documents;
|
||||
using StellaOps.Graph.Indexer.Schema;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
public sealed class FileSystemSnapshotFileWriter : ISnapshotFileWriter
|
||||
{
|
||||
private readonly string _root;
|
||||
|
||||
public FileSystemSnapshotFileWriter(string rootDirectory)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(rootDirectory))
|
||||
{
|
||||
throw new ArgumentException("Snapshot root directory must be provided.", nameof(rootDirectory));
|
||||
}
|
||||
|
||||
_root = Path.GetFullPath(rootDirectory);
|
||||
}
|
||||
|
||||
public async Task WriteJsonAsync(string relativePath, JsonObject content, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(content);
|
||||
|
||||
var fullPath = ResolvePath(relativePath);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!);
|
||||
|
||||
var canonicalBytes = CanonicalJson.ToCanonicalUtf8Bytes(content);
|
||||
await File.WriteAllBytesAsync(fullPath, canonicalBytes, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task WriteJsonLinesAsync(string relativePath, IEnumerable<JsonObject> items, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(items);
|
||||
|
||||
var fullPath = ResolvePath(relativePath);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!);
|
||||
|
||||
await using var stream = new FileStream(fullPath, FileMode.Create, FileAccess.Write, FileShare.None, 4096, useAsync: true);
|
||||
await using var writer = new StreamWriter(stream, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false));
|
||||
|
||||
foreach (var item in items)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var canonicalBytes = CanonicalJson.ToCanonicalUtf8Bytes(item);
|
||||
await writer.WriteAsync(Encoding.UTF8.GetString(canonicalBytes)).ConfigureAwait(false);
|
||||
await writer.WriteLineAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private string ResolvePath(string relativePath)
|
||||
{
|
||||
var sanitized = relativePath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar);
|
||||
return Path.Combine(_root, sanitized);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Nodes;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
public sealed record GraphBuildBatch(
|
||||
ImmutableArray<JsonObject> Nodes,
|
||||
ImmutableArray<JsonObject> Edges);
|
||||
@@ -0,0 +1,9 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
public interface IGraphDocumentWriter
|
||||
{
|
||||
Task WriteAsync(GraphBuildBatch batch, CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
public interface ISbomIngestMetrics
|
||||
{
|
||||
void RecordBatch(string source, string tenant, int nodeCount, int edgeCount, TimeSpan duration, bool success);
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
public sealed class SbomIngestMetrics : ISbomIngestMetrics, IDisposable
|
||||
{
|
||||
public const string MeterName = "StellaOps.Graph.Indexer";
|
||||
public const string MeterVersion = "1.0.0";
|
||||
|
||||
private const string BatchesTotalName = "graph_sbom_ingest_batches_total";
|
||||
private const string BatchDurationSecondsName = "graph_sbom_ingest_duration_seconds";
|
||||
private const string NodesTotalName = "graph_sbom_ingest_nodes_total";
|
||||
private const string EdgesTotalName = "graph_sbom_ingest_edges_total";
|
||||
|
||||
private const string UnitCount = "count";
|
||||
private const string UnitSeconds = "s";
|
||||
|
||||
private readonly Meter _meter;
|
||||
private readonly bool _ownsMeter;
|
||||
private readonly Counter<long> _batchesTotal;
|
||||
private readonly Histogram<double> _batchDurationSeconds;
|
||||
private readonly Counter<long> _nodesTotal;
|
||||
private readonly Counter<long> _edgesTotal;
|
||||
private bool _disposed;
|
||||
|
||||
public SbomIngestMetrics()
|
||||
: this(null)
|
||||
{
|
||||
}
|
||||
|
||||
public SbomIngestMetrics(Meter? meter)
|
||||
{
|
||||
_meter = meter ?? new Meter(MeterName, MeterVersion);
|
||||
_ownsMeter = meter is null;
|
||||
|
||||
_batchesTotal = _meter.CreateCounter<long>(
|
||||
name: BatchesTotalName,
|
||||
unit: UnitCount,
|
||||
description: "Total SBOM ingest batches processed.");
|
||||
|
||||
_batchDurationSeconds = _meter.CreateHistogram<double>(
|
||||
name: BatchDurationSecondsName,
|
||||
unit: UnitSeconds,
|
||||
description: "Duration, in seconds, for SBOM ingest batches.");
|
||||
|
||||
_nodesTotal = _meter.CreateCounter<long>(
|
||||
name: NodesTotalName,
|
||||
unit: UnitCount,
|
||||
description: "Total graph nodes emitted from SBOM ingest.");
|
||||
|
||||
_edgesTotal = _meter.CreateCounter<long>(
|
||||
name: EdgesTotalName,
|
||||
unit: UnitCount,
|
||||
description: "Total graph edges emitted from SBOM ingest.");
|
||||
}
|
||||
|
||||
public void RecordBatch(string source, string tenant, int nodeCount, int edgeCount, TimeSpan duration, bool success)
|
||||
{
|
||||
ThrowIfDisposed();
|
||||
|
||||
var tags = new KeyValuePair<string, object?>[]
|
||||
{
|
||||
new("source", source ?? string.Empty),
|
||||
new("tenant", tenant ?? string.Empty),
|
||||
new("success", success)
|
||||
};
|
||||
|
||||
var tagSpan = tags.AsSpan();
|
||||
_batchesTotal.Add(1, tagSpan);
|
||||
_nodesTotal.Add(nodeCount, tagSpan);
|
||||
_edgesTotal.Add(edgeCount, tagSpan);
|
||||
_batchDurationSeconds.Record(duration.TotalSeconds, tagSpan);
|
||||
}
|
||||
|
||||
private void ThrowIfDisposed()
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
throw new ObjectDisposedException(nameof(SbomIngestMetrics));
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (_ownsMeter)
|
||||
{
|
||||
_meter.Dispose();
|
||||
}
|
||||
|
||||
_disposed = true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
public sealed class SbomIngestOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Optional override for the snapshot export root directory. When null or whitespace,
|
||||
/// <c>STELLAOPS_GRAPH_SNAPSHOT_DIR</c> or the default <c>artifacts/graph-snapshots</c>
|
||||
/// location will be used.
|
||||
/// </summary>
|
||||
public string? SnapshotRootDirectory { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
using System.Diagnostics;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
public sealed class SbomIngestProcessor
|
||||
{
|
||||
private readonly SbomIngestTransformer _transformer;
|
||||
private readonly IGraphDocumentWriter _writer;
|
||||
private readonly ISbomIngestMetrics _metrics;
|
||||
private readonly ISbomSnapshotExporter _snapshotExporter;
|
||||
private readonly ILogger<SbomIngestProcessor> _logger;
|
||||
|
||||
public SbomIngestProcessor(
|
||||
SbomIngestTransformer transformer,
|
||||
IGraphDocumentWriter writer,
|
||||
ISbomIngestMetrics metrics,
|
||||
ISbomSnapshotExporter snapshotExporter,
|
||||
ILogger<SbomIngestProcessor> logger)
|
||||
{
|
||||
_transformer = transformer ?? throw new ArgumentNullException(nameof(transformer));
|
||||
_writer = writer ?? throw new ArgumentNullException(nameof(writer));
|
||||
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
|
||||
_snapshotExporter = snapshotExporter ?? throw new ArgumentNullException(nameof(snapshotExporter));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task ProcessAsync(SbomSnapshot snapshot, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(snapshot);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
GraphBuildBatch batch;
|
||||
|
||||
try
|
||||
{
|
||||
batch = _transformer.Transform(snapshot);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, 0, 0, stopwatch.Elapsed, success: false);
|
||||
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"graph-indexer: failed to transform SBOM {SbomDigest} for tenant {Tenant}",
|
||||
snapshot.SbomDigest,
|
||||
snapshot.Tenant);
|
||||
|
||||
throw;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
await _writer.WriteAsync(batch, cancellationToken).ConfigureAwait(false);
|
||||
await _snapshotExporter.ExportAsync(snapshot, batch, cancellationToken).ConfigureAwait(false);
|
||||
stopwatch.Stop();
|
||||
|
||||
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, batch.Nodes.Length, batch.Edges.Length, stopwatch.Elapsed, success: true);
|
||||
|
||||
_logger.LogInformation(
|
||||
"graph-indexer: indexed SBOM {SbomDigest} for tenant {Tenant} with {NodeCount} nodes and {EdgeCount} edges in {DurationMs:F2} ms",
|
||||
snapshot.SbomDigest,
|
||||
snapshot.Tenant,
|
||||
batch.Nodes.Length,
|
||||
batch.Edges.Length,
|
||||
stopwatch.Elapsed.TotalMilliseconds);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, batch.Nodes.Length, batch.Edges.Length, stopwatch.Elapsed, success: false);
|
||||
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"graph-indexer: failed to persist SBOM {SbomDigest} for tenant {Tenant}",
|
||||
snapshot.SbomDigest,
|
||||
snapshot.Tenant);
|
||||
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Graph.Indexer.Documents;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
public static class SbomIngestProcessorFactory
|
||||
{
|
||||
private const string SnapshotDirEnv = "STELLAOPS_GRAPH_SNAPSHOT_DIR";
|
||||
|
||||
public static SbomIngestProcessor CreateDefault(
|
||||
SbomIngestTransformer transformer,
|
||||
IGraphDocumentWriter writer,
|
||||
ISbomIngestMetrics metrics,
|
||||
ILogger<SbomIngestProcessor> logger,
|
||||
string? snapshotRoot = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(transformer);
|
||||
ArgumentNullException.ThrowIfNull(writer);
|
||||
ArgumentNullException.ThrowIfNull(metrics);
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
|
||||
var root = ResolveSnapshotRoot(snapshotRoot);
|
||||
var exporter = new SbomSnapshotExporter(new GraphSnapshotBuilder(), new FileSystemSnapshotFileWriter(root));
|
||||
return new SbomIngestProcessor(transformer, writer, metrics, exporter, logger);
|
||||
}
|
||||
|
||||
private static string ResolveSnapshotRoot(string? snapshotRoot)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(snapshotRoot))
|
||||
{
|
||||
return snapshotRoot!;
|
||||
}
|
||||
|
||||
var envRoot = Environment.GetEnvironmentVariable(SnapshotDirEnv);
|
||||
if (!string.IsNullOrWhiteSpace(envRoot))
|
||||
{
|
||||
return envRoot!;
|
||||
}
|
||||
|
||||
return Path.Combine(Environment.CurrentDirectory, "artifacts", "graph-snapshots");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
public static class SbomIngestServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddSbomIngestPipeline(
|
||||
this IServiceCollection services,
|
||||
Action<SbomIngestOptions>? configure = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
services.AddOptions<SbomIngestOptions>();
|
||||
|
||||
if (configure is not null)
|
||||
{
|
||||
services.Configure(configure);
|
||||
}
|
||||
|
||||
services.TryAddSingleton<SbomIngestTransformer>();
|
||||
services.TryAddSingleton<ISbomIngestMetrics, SbomIngestMetrics>();
|
||||
|
||||
services.TryAddSingleton<SbomIngestProcessor>(provider =>
|
||||
{
|
||||
var transformer = provider.GetRequiredService<SbomIngestTransformer>();
|
||||
var writer = provider.GetRequiredService<IGraphDocumentWriter>();
|
||||
var metrics = provider.GetRequiredService<ISbomIngestMetrics>();
|
||||
var logger = provider.GetService<ILogger<SbomIngestProcessor>>() ?? NullLogger<SbomIngestProcessor>.Instance;
|
||||
var options = provider.GetService<IOptions<SbomIngestOptions>>();
|
||||
var snapshotRoot = options?.Value.SnapshotRootDirectory;
|
||||
|
||||
return SbomIngestProcessorFactory.CreateDefault(
|
||||
transformer,
|
||||
writer,
|
||||
metrics,
|
||||
logger,
|
||||
snapshotRoot);
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,453 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Nodes;
|
||||
using StellaOps.Graph.Indexer.Schema;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
public sealed class SbomIngestTransformer
|
||||
{
|
||||
private const string ContainsEdgeKind = "CONTAINS";
|
||||
private const string DependsOnEdgeKind = "DEPENDS_ON";
|
||||
private const string DeclaredInEdgeKind = "DECLARED_IN";
|
||||
private const string BuiltFromEdgeKind = "BUILT_FROM";
|
||||
|
||||
public GraphBuildBatch Transform(SbomSnapshot snapshot)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(snapshot);
|
||||
|
||||
var nodes = new List<JsonObject>();
|
||||
var edges = new List<JsonObject>();
|
||||
|
||||
var artifactNodes = new Dictionary<string, JsonObject>(StringComparer.OrdinalIgnoreCase);
|
||||
var componentNodes = new Dictionary<string, JsonObject>(StringComparer.OrdinalIgnoreCase);
|
||||
var fileNodes = new Dictionary<string, JsonObject>(StringComparer.OrdinalIgnoreCase);
|
||||
var licenseCandidates = new Dictionary<(string License, string SourceDigest), LicenseCandidate>(LicenseKeyComparer.Instance);
|
||||
|
||||
long nextEdgeOffset = snapshot.EventOffset + 918;
|
||||
long NextEdgeOffset() => nextEdgeOffset++;
|
||||
|
||||
var artifactNode = CreateArtifactNode(snapshot);
|
||||
nodes.Add(artifactNode);
|
||||
artifactNodes[GetArtifactKey(snapshot.ArtifactDigest, snapshot.SbomDigest)] = artifactNode;
|
||||
|
||||
foreach (var component in snapshot.Components)
|
||||
{
|
||||
var componentNode = CreateComponentNode(snapshot, component);
|
||||
nodes.Add(componentNode);
|
||||
componentNodes[GetComponentKey(component.Purl, component.SourceType)] = componentNode;
|
||||
|
||||
if (string.Equals(component.Usage, "direct", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
var containsEdge = CreateContainsEdge(snapshot, artifactNode, componentNode, component, NextEdgeOffset());
|
||||
edges.Add(containsEdge);
|
||||
}
|
||||
|
||||
foreach (var dependency in component.Dependencies)
|
||||
{
|
||||
var dependsOnEdge = CreateDependsOnEdge(snapshot, componentNode, dependency, NextEdgeOffset());
|
||||
edges.Add(dependsOnEdge);
|
||||
}
|
||||
|
||||
foreach (var file in component.Files)
|
||||
{
|
||||
var fileNodeKey = GetFileKey(snapshot.ArtifactDigest, file.Path, file.ContentSha256);
|
||||
if (!fileNodes.TryGetValue(fileNodeKey, out var fileNode))
|
||||
{
|
||||
fileNode = CreateFileNode(snapshot, file);
|
||||
nodes.Add(fileNode);
|
||||
fileNodes[fileNodeKey] = fileNode;
|
||||
}
|
||||
|
||||
var declaredInEdge = CreateDeclaredInEdge(snapshot, componentNode, component, fileNode, file, NextEdgeOffset());
|
||||
edges.Add(declaredInEdge);
|
||||
}
|
||||
|
||||
if (HasLicenseMetadata(component.License))
|
||||
{
|
||||
var licenseKey = (component.License.Spdx, component.License.SourceDigest);
|
||||
var candidate = CreateLicenseCandidate(snapshot, component);
|
||||
if (!licenseCandidates.TryGetValue(licenseKey, out var existing) || existing.EventOffset > candidate.EventOffset)
|
||||
{
|
||||
licenseCandidates[licenseKey] = candidate;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var candidate in licenseCandidates.Values)
|
||||
{
|
||||
nodes.Add(CreateLicenseNode(snapshot, candidate));
|
||||
}
|
||||
|
||||
foreach (var baseArtifact in snapshot.BaseArtifacts)
|
||||
{
|
||||
var node = CreateBaseArtifactNode(snapshot, baseArtifact);
|
||||
if (!artifactNodes.ContainsKey(GetArtifactKey(baseArtifact.ArtifactDigest, baseArtifact.SbomDigest)))
|
||||
{
|
||||
nodes.Add(node);
|
||||
artifactNodes[GetArtifactKey(baseArtifact.ArtifactDigest, baseArtifact.SbomDigest)] = node;
|
||||
}
|
||||
|
||||
var edge = CreateBuiltFromEdge(snapshot, artifactNode, baseArtifact, NextEdgeOffset());
|
||||
edges.Add(edge);
|
||||
}
|
||||
|
||||
var orderedNodes = nodes
|
||||
.OrderBy(node => node["kind"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ThenBy(node => node["id"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ToImmutableArray();
|
||||
|
||||
var orderedEdges = edges
|
||||
.OrderBy(edge => edge["kind"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ThenBy(edge => edge["id"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ToImmutableArray();
|
||||
|
||||
return new GraphBuildBatch(orderedNodes, orderedEdges);
|
||||
}
|
||||
|
||||
private static JsonObject CreateArtifactNode(SbomSnapshot snapshot)
|
||||
{
|
||||
var labels = snapshot.Artifact.Labels?.OrderBy(x => x, StringComparer.Ordinal).ToArray() ?? Array.Empty<string>();
|
||||
|
||||
var attributes = new JsonObject
|
||||
{
|
||||
["display_name"] = snapshot.Artifact.DisplayName,
|
||||
["artifact_digest"] = snapshot.ArtifactDigest,
|
||||
["sbom_digest"] = snapshot.SbomDigest,
|
||||
["environment"] = snapshot.Artifact.Environment,
|
||||
["labels"] = CreateJsonArray(labels),
|
||||
["origin_registry"] = snapshot.Artifact.OriginRegistry,
|
||||
["supply_chain_stage"] = snapshot.Artifact.SupplyChainStage
|
||||
};
|
||||
|
||||
return GraphDocumentFactory.CreateNode(new GraphNodeSpec(
|
||||
Tenant: snapshot.Tenant,
|
||||
Kind: "artifact",
|
||||
CanonicalKey: new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["artifact_digest"] = snapshot.ArtifactDigest,
|
||||
["sbom_digest"] = snapshot.SbomDigest
|
||||
},
|
||||
Attributes: attributes,
|
||||
Provenance: new GraphProvenanceSpec(snapshot.Source, snapshot.CollectedAt, snapshot.SbomDigest, snapshot.EventOffset),
|
||||
ValidFrom: snapshot.CollectedAt,
|
||||
ValidTo: null));
|
||||
}
|
||||
|
||||
private static JsonObject CreateBaseArtifactNode(SbomSnapshot snapshot, SbomBaseArtifact baseArtifact)
|
||||
{
|
||||
var labels = baseArtifact.Labels?.OrderBy(x => x, StringComparer.Ordinal).ToArray() ?? Array.Empty<string>();
|
||||
|
||||
var attributes = new JsonObject
|
||||
{
|
||||
["display_name"] = baseArtifact.DisplayName,
|
||||
["artifact_digest"] = baseArtifact.ArtifactDigest,
|
||||
["sbom_digest"] = baseArtifact.SbomDigest,
|
||||
["environment"] = baseArtifact.Environment,
|
||||
["labels"] = CreateJsonArray(labels),
|
||||
["origin_registry"] = baseArtifact.OriginRegistry,
|
||||
["supply_chain_stage"] = baseArtifact.SupplyChainStage
|
||||
};
|
||||
|
||||
return GraphDocumentFactory.CreateNode(new GraphNodeSpec(
|
||||
Tenant: snapshot.Tenant,
|
||||
Kind: "artifact",
|
||||
CanonicalKey: new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["artifact_digest"] = baseArtifact.ArtifactDigest,
|
||||
["sbom_digest"] = baseArtifact.SbomDigest
|
||||
},
|
||||
Attributes: attributes,
|
||||
Provenance: new GraphProvenanceSpec(
|
||||
ResolveSource(baseArtifact.Source, snapshot.Source),
|
||||
baseArtifact.CollectedAt,
|
||||
baseArtifact.SbomDigest,
|
||||
baseArtifact.EventOffset),
|
||||
ValidFrom: baseArtifact.CollectedAt,
|
||||
ValidTo: null));
|
||||
}
|
||||
|
||||
private static JsonObject CreateComponentNode(SbomSnapshot snapshot, SbomComponent component)
|
||||
{
|
||||
var attributes = new JsonObject
|
||||
{
|
||||
["purl"] = component.Purl,
|
||||
["version"] = component.Version,
|
||||
["ecosystem"] = component.Ecosystem,
|
||||
["scope"] = component.Scope,
|
||||
["license_spdx"] = component.License.Spdx,
|
||||
["usage"] = component.Usage
|
||||
};
|
||||
|
||||
return GraphDocumentFactory.CreateNode(new GraphNodeSpec(
|
||||
Tenant: snapshot.Tenant,
|
||||
Kind: "component",
|
||||
CanonicalKey: new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["purl"] = component.Purl,
|
||||
["source_type"] = component.SourceType
|
||||
},
|
||||
Attributes: attributes,
|
||||
Provenance: new GraphProvenanceSpec(
|
||||
ResolveSource(component.Source, snapshot.Source),
|
||||
component.CollectedAt,
|
||||
snapshot.SbomDigest,
|
||||
component.EventOffset),
|
||||
ValidFrom: component.CollectedAt,
|
||||
ValidTo: null));
|
||||
}
|
||||
|
||||
private static JsonObject CreateFileNode(SbomSnapshot snapshot, SbomComponentFile file)
|
||||
{
|
||||
var attributes = new JsonObject
|
||||
{
|
||||
["normalized_path"] = file.Path,
|
||||
["content_sha256"] = file.ContentSha256,
|
||||
["language_hint"] = file.LanguageHint,
|
||||
["size_bytes"] = file.SizeBytes,
|
||||
["scope"] = file.Scope
|
||||
};
|
||||
|
||||
return GraphDocumentFactory.CreateNode(new GraphNodeSpec(
|
||||
Tenant: snapshot.Tenant,
|
||||
Kind: "file",
|
||||
CanonicalKey: new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["artifact_digest"] = snapshot.ArtifactDigest,
|
||||
["normalized_path"] = file.Path,
|
||||
["content_sha256"] = file.ContentSha256
|
||||
},
|
||||
Attributes: attributes,
|
||||
Provenance: new GraphProvenanceSpec(
|
||||
ResolveSource(file.Source, file.DetectedBy, snapshot.Source),
|
||||
file.CollectedAt,
|
||||
snapshot.SbomDigest,
|
||||
file.EventOffset),
|
||||
ValidFrom: file.CollectedAt,
|
||||
ValidTo: null));
|
||||
}
|
||||
|
||||
private static JsonObject CreateContainsEdge(SbomSnapshot snapshot, JsonObject artifactNode, JsonObject componentNode, SbomComponent component, long eventOffset)
|
||||
{
|
||||
return GraphDocumentFactory.CreateEdge(new GraphEdgeSpec(
|
||||
Tenant: snapshot.Tenant,
|
||||
Kind: ContainsEdgeKind,
|
||||
CanonicalKey: new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["artifact_node_id"] = artifactNode["id"]!.GetValue<string>(),
|
||||
["component_node_id"] = componentNode["id"]!.GetValue<string>(),
|
||||
["sbom_digest"] = snapshot.SbomDigest
|
||||
},
|
||||
Attributes: new JsonObject
|
||||
{
|
||||
["detected_by"] = component.DetectedBy,
|
||||
["layer_digest"] = component.LayerDigest,
|
||||
["scope"] = component.Scope,
|
||||
["evidence_digest"] = component.EvidenceDigest
|
||||
},
|
||||
Provenance: new GraphProvenanceSpec(
|
||||
ResolveSource(component.Source, component.DetectedBy, snapshot.Source),
|
||||
component.CollectedAt.AddSeconds(1),
|
||||
snapshot.SbomDigest,
|
||||
eventOffset),
|
||||
ValidFrom: component.CollectedAt.AddSeconds(1),
|
||||
ValidTo: null));
|
||||
}
|
||||
|
||||
private static JsonObject CreateDependsOnEdge(SbomSnapshot snapshot, JsonObject componentNode, SbomDependency dependency, long eventOffset)
|
||||
{
|
||||
return GraphDocumentFactory.CreateEdge(new GraphEdgeSpec(
|
||||
Tenant: snapshot.Tenant,
|
||||
Kind: DependsOnEdgeKind,
|
||||
CanonicalKey: new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["component_node_id"] = componentNode["id"]!.GetValue<string>(),
|
||||
["dependency_purl"] = dependency.Purl,
|
||||
["sbom_digest"] = snapshot.SbomDigest
|
||||
},
|
||||
Attributes: new JsonObject
|
||||
{
|
||||
["dependency_purl"] = dependency.Purl,
|
||||
["dependency_version"] = dependency.Version,
|
||||
["relationship"] = dependency.Relationship,
|
||||
["evidence_digest"] = dependency.EvidenceDigest
|
||||
},
|
||||
Provenance: new GraphProvenanceSpec(
|
||||
ResolveSource(dependency.Source, snapshot.Source),
|
||||
dependency.CollectedAt.AddSeconds(1),
|
||||
snapshot.SbomDigest,
|
||||
eventOffset),
|
||||
ValidFrom: dependency.CollectedAt.AddSeconds(1),
|
||||
ValidTo: null));
|
||||
}
|
||||
|
||||
private static JsonObject CreateDeclaredInEdge(SbomSnapshot snapshot, JsonObject componentNode, SbomComponent component, JsonObject fileNode, SbomComponentFile file, long eventOffset)
|
||||
{
|
||||
return GraphDocumentFactory.CreateEdge(new GraphEdgeSpec(
|
||||
Tenant: snapshot.Tenant,
|
||||
Kind: DeclaredInEdgeKind,
|
||||
CanonicalKey: new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["component_node_id"] = componentNode["id"]!.GetValue<string>(),
|
||||
["file_node_id"] = fileNode["id"]!.GetValue<string>(),
|
||||
["sbom_digest"] = snapshot.SbomDigest
|
||||
},
|
||||
Attributes: new JsonObject
|
||||
{
|
||||
["detected_by"] = file.DetectedBy,
|
||||
["scope"] = ResolveScope(component.Scope, file.Scope),
|
||||
["evidence_digest"] = file.EvidenceDigest
|
||||
},
|
||||
Provenance: new GraphProvenanceSpec(
|
||||
ResolveSource(file.Source, file.DetectedBy, snapshot.Source),
|
||||
file.CollectedAt.AddSeconds(1),
|
||||
snapshot.SbomDigest,
|
||||
eventOffset),
|
||||
ValidFrom: file.CollectedAt.AddSeconds(1),
|
||||
ValidTo: null));
|
||||
}
|
||||
|
||||
private static JsonObject CreateBuiltFromEdge(SbomSnapshot snapshot, JsonObject parentNode, SbomBaseArtifact baseArtifact, long eventOffset)
|
||||
{
|
||||
return GraphDocumentFactory.CreateEdge(new GraphEdgeSpec(
|
||||
Tenant: snapshot.Tenant,
|
||||
Kind: BuiltFromEdgeKind,
|
||||
CanonicalKey: new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["parent_artifact_node_id"] = parentNode["id"]!.GetValue<string>(),
|
||||
["child_artifact_digest"] = baseArtifact.ArtifactDigest
|
||||
},
|
||||
Attributes: new JsonObject
|
||||
{
|
||||
["build_type"] = snapshot.Build.BuildType,
|
||||
["builder_id"] = snapshot.Build.BuilderId,
|
||||
["attestation_digest"] = snapshot.Build.AttestationDigest
|
||||
},
|
||||
Provenance: new GraphProvenanceSpec(
|
||||
ResolveSource(snapshot.Build.Source, snapshot.Source),
|
||||
snapshot.Build.CollectedAt,
|
||||
snapshot.SbomDigest,
|
||||
eventOffset),
|
||||
ValidFrom: snapshot.Build.CollectedAt,
|
||||
ValidTo: null));
|
||||
}
|
||||
|
||||
private static string GetComponentKey(string purl, string sourceType)
|
||||
{
|
||||
var normalizedPurl = (purl ?? string.Empty).Trim();
|
||||
var normalizedSourceType = (sourceType ?? string.Empty).Trim();
|
||||
return $"{normalizedPurl}|{normalizedSourceType}";
|
||||
}
|
||||
|
||||
private static bool HasLicenseMetadata(SbomLicense license)
|
||||
=> !string.IsNullOrWhiteSpace(license.Spdx)
|
||||
&& !string.IsNullOrWhiteSpace(license.SourceDigest);
|
||||
|
||||
private static string ResolveScope(string componentScope, string fileScope)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(componentScope))
|
||||
{
|
||||
return componentScope.Trim();
|
||||
}
|
||||
|
||||
return string.IsNullOrWhiteSpace(fileScope) ? string.Empty : fileScope.Trim();
|
||||
}
|
||||
|
||||
private static string ResolveSource(params string?[] sources)
|
||||
{
|
||||
foreach (var source in sources)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(source))
|
||||
{
|
||||
return source!.Trim();
|
||||
}
|
||||
}
|
||||
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
private static string GetArtifactKey(string artifactDigest, string sbomDigest)
|
||||
=> $"{(artifactDigest ?? string.Empty).Trim()}|{(sbomDigest ?? string.Empty).Trim()}";
|
||||
|
||||
private static string GetFileKey(string artifactDigest, string path, string contentSha)
|
||||
=> $"{(artifactDigest ?? string.Empty).Trim()}|{(path ?? string.Empty).Trim()}|{(contentSha ?? string.Empty).Trim()}";
|
||||
|
||||
private static JsonArray CreateJsonArray(IEnumerable<string> values)
|
||||
{
|
||||
var array = new JsonArray();
|
||||
foreach (var value in values)
|
||||
{
|
||||
array.Add(value);
|
||||
}
|
||||
|
||||
return array;
|
||||
}
|
||||
|
||||
private static LicenseCandidate CreateLicenseCandidate(SbomSnapshot snapshot, SbomComponent component)
|
||||
{
|
||||
var collectedAt = component.CollectedAt.AddSeconds(2);
|
||||
var eventOffset = component.EventOffset + 3;
|
||||
return new LicenseCandidate(
|
||||
License: component.License,
|
||||
CollectedAt: collectedAt,
|
||||
EventOffset: eventOffset,
|
||||
SbomDigest: snapshot.SbomDigest,
|
||||
Source: ResolveSource(component.Source, snapshot.Source));
|
||||
}
|
||||
|
||||
private static JsonObject CreateLicenseNode(SbomSnapshot snapshot, LicenseCandidate candidate)
|
||||
{
|
||||
var attributes = new JsonObject
|
||||
{
|
||||
["license_spdx"] = candidate.License.Spdx,
|
||||
["name"] = candidate.License.Name,
|
||||
["classification"] = candidate.License.Classification,
|
||||
["notice_uri"] = candidate.License.NoticeUri is null ? null : candidate.License.NoticeUri
|
||||
};
|
||||
|
||||
return GraphDocumentFactory.CreateNode(new GraphNodeSpec(
|
||||
Tenant: snapshot.Tenant,
|
||||
Kind: "license",
|
||||
CanonicalKey: new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["license_spdx"] = candidate.License.Spdx,
|
||||
["source_digest"] = candidate.License.SourceDigest
|
||||
},
|
||||
Attributes: attributes,
|
||||
Provenance: new GraphProvenanceSpec(candidate.Source, candidate.CollectedAt, candidate.SbomDigest, candidate.EventOffset),
|
||||
ValidFrom: candidate.CollectedAt,
|
||||
ValidTo: null));
|
||||
}
|
||||
|
||||
private sealed record LicenseCandidate(
|
||||
SbomLicense License,
|
||||
DateTimeOffset CollectedAt,
|
||||
long EventOffset,
|
||||
string SbomDigest,
|
||||
string Source);
|
||||
|
||||
private sealed class LicenseKeyComparer : IEqualityComparer<(string License, string SourceDigest)>
|
||||
{
|
||||
public static readonly LicenseKeyComparer Instance = new();
|
||||
|
||||
public bool Equals((string License, string SourceDigest) x, (string License, string SourceDigest) y)
|
||||
=> string.Equals(x.License, y.License, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(x.SourceDigest, y.SourceDigest, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
public int GetHashCode((string License, string SourceDigest) obj)
|
||||
{
|
||||
var hash = new HashCode();
|
||||
hash.Add(obj.License, StringComparer.OrdinalIgnoreCase);
|
||||
hash.Add(obj.SourceDigest, StringComparer.OrdinalIgnoreCase);
|
||||
return hash.ToHashCode();
|
||||
}
|
||||
}
|
||||
}
|
||||
231
src/Graph/StellaOps.Graph.Indexer/Ingestion/Sbom/SbomSnapshot.cs
Normal file
231
src/Graph/StellaOps.Graph.Indexer/Ingestion/Sbom/SbomSnapshot.cs
Normal file
@@ -0,0 +1,231 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
public sealed class SbomSnapshot
|
||||
{
|
||||
[JsonPropertyName("tenant")]
|
||||
public string Tenant { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("artifactDigest")]
|
||||
public string ArtifactDigest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("sbomDigest")]
|
||||
public string SbomDigest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long EventOffset { get; init; }
|
||||
|
||||
[JsonPropertyName("artifact")]
|
||||
public SbomArtifactMetadata Artifact { get; init; } = new();
|
||||
|
||||
[JsonPropertyName("build")]
|
||||
public SbomBuildMetadata Build { get; init; } = new();
|
||||
|
||||
[JsonPropertyName("components")]
|
||||
public IReadOnlyList<SbomComponent> Components { get; init; } = Array.Empty<SbomComponent>();
|
||||
|
||||
[JsonPropertyName("baseArtifacts")]
|
||||
public IReadOnlyList<SbomBaseArtifact> BaseArtifacts { get; init; } = Array.Empty<SbomBaseArtifact>();
|
||||
}
|
||||
|
||||
public sealed class SbomArtifactMetadata
|
||||
{
|
||||
[JsonPropertyName("displayName")]
|
||||
public string DisplayName { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("environment")]
|
||||
public string Environment { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("labels")]
|
||||
public IReadOnlyList<string> Labels { get; init; } = Array.Empty<string>();
|
||||
|
||||
[JsonPropertyName("originRegistry")]
|
||||
public string OriginRegistry { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("supplyChainStage")]
|
||||
public string SupplyChainStage { get; init; } = string.Empty;
|
||||
}
|
||||
|
||||
public sealed class SbomBuildMetadata
|
||||
{
|
||||
[JsonPropertyName("builderId")]
|
||||
public string BuilderId { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("buildType")]
|
||||
public string BuildType { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("attestationDigest")]
|
||||
public string AttestationDigest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long EventOffset { get; init; }
|
||||
}
|
||||
|
||||
public sealed class SbomComponent
|
||||
{
|
||||
[JsonPropertyName("purl")]
|
||||
public string Purl { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("version")]
|
||||
public string Version { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("ecosystem")]
|
||||
public string Ecosystem { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("scope")]
|
||||
public string Scope { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("license")]
|
||||
public SbomLicense License { get; init; } = new();
|
||||
|
||||
[JsonPropertyName("usage")]
|
||||
public string Usage { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("detectedBy")]
|
||||
public string DetectedBy { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("layerDigest")]
|
||||
public string LayerDigest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("evidenceDigest")]
|
||||
public string EvidenceDigest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long EventOffset { get; init; }
|
||||
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("files")]
|
||||
public IReadOnlyList<SbomComponentFile> Files { get; init; } = Array.Empty<SbomComponentFile>();
|
||||
|
||||
[JsonPropertyName("dependencies")]
|
||||
public IReadOnlyList<SbomDependency> Dependencies { get; init; } = Array.Empty<SbomDependency>();
|
||||
|
||||
[JsonPropertyName("sourceType")]
|
||||
public string SourceType { get; init; } = "inventory";
|
||||
}
|
||||
|
||||
public sealed class SbomLicense
|
||||
{
|
||||
[JsonPropertyName("spdx")]
|
||||
public string Spdx { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("name")]
|
||||
public string Name { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("classification")]
|
||||
public string Classification { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("noticeUri")]
|
||||
public string? NoticeUri { get; init; }
|
||||
|
||||
[JsonPropertyName("sourceDigest")]
|
||||
public string SourceDigest { get; init; } = string.Empty;
|
||||
}
|
||||
|
||||
public sealed class SbomComponentFile
|
||||
{
|
||||
[JsonPropertyName("path")]
|
||||
public string Path { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("contentSha256")]
|
||||
public string ContentSha256 { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("languageHint")]
|
||||
public string LanguageHint { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("sizeBytes")]
|
||||
public long SizeBytes { get; init; }
|
||||
|
||||
[JsonPropertyName("scope")]
|
||||
public string Scope { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("detectedBy")]
|
||||
public string DetectedBy { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("evidenceDigest")]
|
||||
public string EvidenceDigest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long EventOffset { get; init; }
|
||||
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; init; } = string.Empty;
|
||||
}
|
||||
|
||||
public sealed class SbomDependency
|
||||
{
|
||||
[JsonPropertyName("purl")]
|
||||
public string Purl { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("version")]
|
||||
public string Version { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("relationship")]
|
||||
public string Relationship { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("evidenceDigest")]
|
||||
public string EvidenceDigest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long EventOffset { get; init; }
|
||||
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; init; } = string.Empty;
|
||||
}
|
||||
|
||||
public sealed class SbomBaseArtifact
|
||||
{
|
||||
[JsonPropertyName("artifactDigest")]
|
||||
public string ArtifactDigest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("sbomDigest")]
|
||||
public string SbomDigest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("displayName")]
|
||||
public string DisplayName { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("environment")]
|
||||
public string Environment { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("labels")]
|
||||
public IReadOnlyList<string> Labels { get; init; } = Array.Empty<string>();
|
||||
|
||||
[JsonPropertyName("originRegistry")]
|
||||
public string OriginRegistry { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("supplyChainStage")]
|
||||
public string SupplyChainStage { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long EventOffset { get; init; }
|
||||
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; init; } = string.Empty;
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using StellaOps.Graph.Indexer.Documents;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
|
||||
public interface ISnapshotFileWriter
|
||||
{
|
||||
Task WriteJsonAsync(string relativePath, JsonObject content, CancellationToken cancellationToken);
|
||||
Task WriteJsonLinesAsync(string relativePath, IEnumerable<JsonObject> items, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public interface ISbomSnapshotExporter
|
||||
{
|
||||
Task ExportAsync(SbomSnapshot snapshot, GraphBuildBatch batch, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed class SbomSnapshotExporter : ISbomSnapshotExporter
|
||||
{
|
||||
private readonly GraphSnapshotBuilder _snapshotBuilder;
|
||||
private readonly ISnapshotFileWriter _fileWriter;
|
||||
|
||||
public SbomSnapshotExporter(GraphSnapshotBuilder snapshotBuilder, ISnapshotFileWriter fileWriter)
|
||||
{
|
||||
_snapshotBuilder = snapshotBuilder ?? throw new ArgumentNullException(nameof(snapshotBuilder));
|
||||
_fileWriter = fileWriter ?? throw new ArgumentNullException(nameof(fileWriter));
|
||||
}
|
||||
|
||||
public async Task ExportAsync(SbomSnapshot snapshot, GraphBuildBatch batch, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(snapshot);
|
||||
ArgumentNullException.ThrowIfNull(batch);
|
||||
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var graphSnapshot = _snapshotBuilder.Build(snapshot, batch, DateTimeOffset.UtcNow);
|
||||
|
||||
await _fileWriter.WriteJsonAsync("manifest.json", graphSnapshot.Manifest.ToJson(), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await _fileWriter.WriteJsonAsync("adjacency.json", graphSnapshot.Adjacency.ToJson(), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await _fileWriter.WriteJsonLinesAsync("nodes.jsonl", batch.Nodes, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await _fileWriter.WriteJsonLinesAsync("edges.jsonl", batch.Edges, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Vex;
|
||||
|
||||
public sealed class VexOverlaySnapshot
|
||||
{
|
||||
[JsonPropertyName("tenant")]
|
||||
public string Tenant { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long EventOffset { get; init; }
|
||||
|
||||
[JsonPropertyName("statement")]
|
||||
public VexStatementDetails Statement { get; init; } = new();
|
||||
|
||||
[JsonPropertyName("exemptions")]
|
||||
public IReadOnlyList<VexComponentExemption> Exemptions { get; init; } = Array.Empty<VexComponentExemption>();
|
||||
}
|
||||
|
||||
public sealed class VexStatementDetails
|
||||
{
|
||||
[JsonPropertyName("vexSource")]
|
||||
public string VexSource { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("statementId")]
|
||||
public string StatementId { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("status")]
|
||||
public string Status { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("justification")]
|
||||
public string Justification { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("impactStatement")]
|
||||
public string ImpactStatement { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("issuedAt")]
|
||||
public DateTimeOffset IssuedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("expiresAt")]
|
||||
public DateTimeOffset ExpiresAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("contentHash")]
|
||||
public string ContentHash { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("provenanceSource")]
|
||||
public string? ProvenanceSource { get; init; }
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset StatementCollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long StatementEventOffset { get; init; }
|
||||
}
|
||||
|
||||
public sealed class VexComponentExemption
|
||||
{
|
||||
[JsonPropertyName("componentPurl")]
|
||||
public string ComponentPurl { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("componentSourceType")]
|
||||
public string ComponentSourceType { get; init; } = "inventory";
|
||||
|
||||
[JsonPropertyName("sbomDigest")]
|
||||
public string? SbomDigest { get; init; }
|
||||
|
||||
[JsonPropertyName("statementHash")]
|
||||
public string? StatementHash { get; init; }
|
||||
|
||||
[JsonPropertyName("status")]
|
||||
public string Status { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("justification")]
|
||||
public string Justification { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("impactStatement")]
|
||||
public string ImpactStatement { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("evidenceDigest")]
|
||||
public string EvidenceDigest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("provenanceSource")]
|
||||
public string? ProvenanceSource { get; init; }
|
||||
|
||||
[JsonPropertyName("collectedAt")]
|
||||
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
[JsonPropertyName("eventOffset")]
|
||||
public long EventOffset { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Text.Json.Nodes;
|
||||
using StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
using StellaOps.Graph.Indexer.Schema;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Ingestion.Vex;
|
||||
|
||||
public sealed class VexOverlayTransformer
|
||||
{
|
||||
private const string VexNodeKind = "vex_statement";
|
||||
private const string ComponentNodeKind = "component";
|
||||
private const string VexExemptsEdgeKind = "VEX_EXEMPTS";
|
||||
|
||||
public GraphBuildBatch Transform(VexOverlaySnapshot snapshot)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(snapshot);
|
||||
|
||||
var nodes = new List<JsonObject>();
|
||||
var edges = new List<JsonObject>();
|
||||
var seenEdgeIds = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
var statement = snapshot.Statement ?? new VexStatementDetails();
|
||||
var canonicalStatementHash = statement.ContentHash?.Trim() ?? string.Empty;
|
||||
|
||||
var vexNode = CreateVexStatementNode(snapshot, statement, canonicalStatementHash);
|
||||
nodes.Add(vexNode);
|
||||
|
||||
var vexNodeId = vexNode["id"]!.GetValue<string>();
|
||||
foreach (var exemption in snapshot.Exemptions ?? Array.Empty<VexComponentExemption>())
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(exemption.ComponentPurl))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var edge = CreateVexExemptsEdge(snapshot, statement, canonicalStatementHash, vexNodeId, exemption);
|
||||
var edgeId = edge["id"]!.GetValue<string>();
|
||||
|
||||
if (seenEdgeIds.Add(edgeId))
|
||||
{
|
||||
edges.Add(edge);
|
||||
}
|
||||
}
|
||||
|
||||
return new GraphBuildBatch(
|
||||
nodes
|
||||
.OrderBy(node => node["kind"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ThenBy(node => node["id"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ToImmutableArray(),
|
||||
edges
|
||||
.OrderBy(edge => edge["kind"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ThenBy(edge => edge["id"]!.GetValue<string>(), StringComparer.Ordinal)
|
||||
.ToImmutableArray());
|
||||
}
|
||||
|
||||
private static JsonObject CreateVexStatementNode(
|
||||
VexOverlaySnapshot snapshot,
|
||||
VexStatementDetails statement,
|
||||
string canonicalStatementHash)
|
||||
{
|
||||
var vexSource = statement.VexSource?.Trim() ?? string.Empty;
|
||||
var statementId = statement.StatementId?.Trim() ?? string.Empty;
|
||||
var status = statement.Status?.Trim() ?? string.Empty;
|
||||
var justification = statement.Justification?.Trim() ?? string.Empty;
|
||||
|
||||
var issuedAt = statement.IssuedAt == DateTimeOffset.UnixEpoch
|
||||
? snapshot.CollectedAt
|
||||
: statement.IssuedAt;
|
||||
|
||||
var expiresAt = statement.ExpiresAt == DateTimeOffset.UnixEpoch
|
||||
? (DateTimeOffset?)null
|
||||
: statement.ExpiresAt;
|
||||
|
||||
var attributes = new JsonObject
|
||||
{
|
||||
["status"] = status,
|
||||
["statement_id"] = statementId,
|
||||
["justification"] = justification,
|
||||
["issued_at"] = GraphTimestamp.Format(issuedAt)
|
||||
};
|
||||
|
||||
if (expiresAt.HasValue)
|
||||
{
|
||||
attributes["expires_at"] = GraphTimestamp.Format(expiresAt.Value);
|
||||
}
|
||||
|
||||
attributes["content_hash"] = canonicalStatementHash;
|
||||
|
||||
var canonicalKey = new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["vex_source"] = vexSource,
|
||||
["statement_id"] = statementId,
|
||||
["content_hash"] = canonicalStatementHash
|
||||
};
|
||||
|
||||
var provenanceSource = string.IsNullOrWhiteSpace(statement.ProvenanceSource)
|
||||
? NormalizeVexSource(snapshot.Source)
|
||||
: statement.ProvenanceSource.Trim();
|
||||
|
||||
var collectedAt = statement.StatementCollectedAt == DateTimeOffset.UnixEpoch
|
||||
? snapshot.CollectedAt
|
||||
: statement.StatementCollectedAt;
|
||||
|
||||
var eventOffset = statement.StatementEventOffset != 0
|
||||
? statement.StatementEventOffset
|
||||
: snapshot.EventOffset;
|
||||
|
||||
var node = GraphDocumentFactory.CreateNode(new GraphNodeSpec(
|
||||
Tenant: snapshot.Tenant,
|
||||
Kind: VexNodeKind,
|
||||
CanonicalKey: canonicalKey,
|
||||
Attributes: attributes,
|
||||
Provenance: new GraphProvenanceSpec(provenanceSource, collectedAt, null, eventOffset),
|
||||
ValidFrom: issuedAt,
|
||||
ValidTo: null));
|
||||
|
||||
var provenance = node["provenance"]!.AsObject();
|
||||
var sourceNode = provenance["source"]!.DeepClone();
|
||||
var collectedAtNode = provenance["collected_at"]!.DeepClone();
|
||||
var eventOffsetNode = provenance.ContainsKey("event_offset")
|
||||
? provenance["event_offset"]!.DeepClone()
|
||||
: null;
|
||||
|
||||
var reorderedProvenance = new JsonObject
|
||||
{
|
||||
["source"] = sourceNode,
|
||||
["collected_at"] = collectedAtNode,
|
||||
["sbom_digest"] = null
|
||||
};
|
||||
|
||||
if (eventOffsetNode is not null)
|
||||
{
|
||||
reorderedProvenance["event_offset"] = eventOffsetNode;
|
||||
}
|
||||
|
||||
node["provenance"] = reorderedProvenance;
|
||||
|
||||
node.Remove("hash");
|
||||
node["hash"] = GraphIdentity.ComputeDocumentHash(node);
|
||||
|
||||
return node;
|
||||
}
|
||||
|
||||
private static JsonObject CreateVexExemptsEdge(
|
||||
VexOverlaySnapshot snapshot,
|
||||
VexStatementDetails statement,
|
||||
string canonicalStatementHash,
|
||||
string vexNodeId,
|
||||
VexComponentExemption exemption)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(exemption);
|
||||
|
||||
var normalizedSourceType = string.IsNullOrWhiteSpace(exemption.ComponentSourceType)
|
||||
? "inventory"
|
||||
: exemption.ComponentSourceType.Trim();
|
||||
var componentPurl = exemption.ComponentPurl.Trim();
|
||||
var statementHash = !string.IsNullOrWhiteSpace(exemption.StatementHash)
|
||||
? exemption.StatementHash.Trim()
|
||||
: canonicalStatementHash;
|
||||
|
||||
var componentIdentity = new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["purl"] = componentPurl,
|
||||
["source_type"] = normalizedSourceType
|
||||
};
|
||||
|
||||
var componentNodeId = GraphIdentity.ComputeNodeId(snapshot.Tenant, ComponentNodeKind, componentIdentity);
|
||||
|
||||
var canonicalKey = new Dictionary<string, string>
|
||||
{
|
||||
["tenant"] = snapshot.Tenant,
|
||||
["component_node_id"] = componentNodeId,
|
||||
["vex_node_id"] = vexNodeId,
|
||||
["statement_hash"] = statementHash
|
||||
};
|
||||
|
||||
var impactStatement = !string.IsNullOrWhiteSpace(exemption.ImpactStatement)
|
||||
? exemption.ImpactStatement.Trim()
|
||||
: statement.ImpactStatement?.Trim() ?? string.Empty;
|
||||
|
||||
var status = !string.IsNullOrWhiteSpace(exemption.Status)
|
||||
? exemption.Status.Trim()
|
||||
: statement.Status?.Trim() ?? string.Empty;
|
||||
|
||||
var justification = !string.IsNullOrWhiteSpace(exemption.Justification)
|
||||
? exemption.Justification.Trim()
|
||||
: statement.Justification?.Trim() ?? string.Empty;
|
||||
|
||||
var attributes = new JsonObject
|
||||
{
|
||||
["status"] = status,
|
||||
["justification"] = justification,
|
||||
["impact_statement"] = impactStatement,
|
||||
["evidence_digest"] = exemption.EvidenceDigest?.Trim() ?? string.Empty
|
||||
};
|
||||
|
||||
var collectedAt = exemption.CollectedAt == DateTimeOffset.UnixEpoch
|
||||
? snapshot.CollectedAt
|
||||
: exemption.CollectedAt;
|
||||
|
||||
var eventOffset = exemption.EventOffset != 0 ? exemption.EventOffset : snapshot.EventOffset;
|
||||
var source = ResolveSource(exemption.ProvenanceSource, snapshot.Source);
|
||||
|
||||
return GraphDocumentFactory.CreateEdge(new GraphEdgeSpec(
|
||||
Tenant: snapshot.Tenant,
|
||||
Kind: VexExemptsEdgeKind,
|
||||
CanonicalKey: canonicalKey,
|
||||
Attributes: attributes,
|
||||
Provenance: new GraphProvenanceSpec(source, collectedAt, exemption.SbomDigest?.Trim(), eventOffset),
|
||||
ValidFrom: collectedAt,
|
||||
ValidTo: null));
|
||||
}
|
||||
|
||||
private static string ResolveSource(string? candidate, string? fallback)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(candidate))
|
||||
{
|
||||
return candidate.Trim();
|
||||
}
|
||||
|
||||
return string.IsNullOrWhiteSpace(fallback)
|
||||
? "excititor.overlay.v1"
|
||||
: fallback.Trim();
|
||||
}
|
||||
|
||||
private static string NormalizeVexSource(string? source)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(source))
|
||||
{
|
||||
return "excititor.vex.v1";
|
||||
}
|
||||
|
||||
var trimmed = source.Trim();
|
||||
return trimmed.Contains(".overlay.", StringComparison.Ordinal)
|
||||
? trimmed.Replace(".overlay.", ".vex.", StringComparison.Ordinal)
|
||||
: trimmed;
|
||||
}
|
||||
}
|
||||
44
src/Graph/StellaOps.Graph.Indexer/Schema/Base32Crockford.cs
Normal file
44
src/Graph/StellaOps.Graph.Indexer/Schema/Base32Crockford.cs
Normal file
@@ -0,0 +1,44 @@
|
||||
using System.Text;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Schema;
|
||||
|
||||
/// <summary>
|
||||
/// Base32 encoder using the Crockford alphabet (0-9A-HJKMNPQRSTVWXYZ).
|
||||
/// </summary>
|
||||
internal static class Base32Crockford
|
||||
{
|
||||
private const string Alphabet = "0123456789ABCDEFGHJKMNPQRSTVWXYZ";
|
||||
|
||||
public static string Encode(ReadOnlySpan<byte> data)
|
||||
{
|
||||
if (data.IsEmpty)
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var output = new StringBuilder((data.Length * 8 + 4) / 5);
|
||||
var buffer = 0;
|
||||
var bitsLeft = 0;
|
||||
|
||||
foreach (var b in data)
|
||||
{
|
||||
buffer = (buffer << 8) | b;
|
||||
bitsLeft += 8;
|
||||
|
||||
while (bitsLeft >= 5)
|
||||
{
|
||||
bitsLeft -= 5;
|
||||
var index = (buffer >> bitsLeft) & 0x1F;
|
||||
output.Append(Alphabet[index]);
|
||||
}
|
||||
}
|
||||
|
||||
if (bitsLeft > 0)
|
||||
{
|
||||
var index = (buffer << (5 - bitsLeft)) & 0x1F;
|
||||
output.Append(Alphabet[index]);
|
||||
}
|
||||
|
||||
return output.ToString();
|
||||
}
|
||||
}
|
||||
134
src/Graph/StellaOps.Graph.Indexer/Schema/CanonicalJson.cs
Normal file
134
src/Graph/StellaOps.Graph.Indexer/Schema/CanonicalJson.cs
Normal file
@@ -0,0 +1,134 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Schema;
|
||||
|
||||
/// <summary>
|
||||
/// Canonical JSON serialiser used for deterministic hashing.
|
||||
/// </summary>
|
||||
public static class CanonicalJson
|
||||
{
|
||||
public static byte[] ToCanonicalUtf8Bytes(JsonNode node)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(node);
|
||||
|
||||
using var stream = new MemoryStream();
|
||||
using var writer = new Utf8JsonWriter(stream, new JsonWriterOptions
|
||||
{
|
||||
Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
|
||||
Indented = false
|
||||
});
|
||||
|
||||
WriteNode(node, writer);
|
||||
writer.Flush();
|
||||
return stream.ToArray();
|
||||
}
|
||||
|
||||
private static void WriteNode(JsonNode node, Utf8JsonWriter writer)
|
||||
{
|
||||
switch (node)
|
||||
{
|
||||
case JsonObject obj:
|
||||
writer.WriteStartObject();
|
||||
foreach (var property in obj.OrderBy(static p => p.Key, StringComparer.Ordinal))
|
||||
{
|
||||
writer.WritePropertyName(property.Key);
|
||||
WriteNode(property.Value!, writer);
|
||||
}
|
||||
|
||||
writer.WriteEndObject();
|
||||
break;
|
||||
|
||||
case JsonArray array:
|
||||
writer.WriteStartArray();
|
||||
foreach (var item in array)
|
||||
{
|
||||
if (item is null)
|
||||
{
|
||||
writer.WriteNullValue();
|
||||
}
|
||||
else
|
||||
{
|
||||
WriteNode(item, writer);
|
||||
}
|
||||
}
|
||||
|
||||
writer.WriteEndArray();
|
||||
break;
|
||||
|
||||
case JsonValue value:
|
||||
WriteValue(value, writer);
|
||||
break;
|
||||
|
||||
default:
|
||||
writer.WriteNullValue();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private static void WriteValue(JsonValue value, Utf8JsonWriter writer)
|
||||
{
|
||||
if (value.TryGetValue(out string? stringValue))
|
||||
{
|
||||
writer.WriteStringValue(stringValue);
|
||||
return;
|
||||
}
|
||||
|
||||
if (value.TryGetValue(out bool boolValue))
|
||||
{
|
||||
writer.WriteBooleanValue(boolValue);
|
||||
return;
|
||||
}
|
||||
|
||||
if (value.TryGetValue(out long longValue))
|
||||
{
|
||||
writer.WriteNumberValue(longValue);
|
||||
return;
|
||||
}
|
||||
|
||||
if (value.TryGetValue(out int intValue))
|
||||
{
|
||||
writer.WriteNumberValue(intValue);
|
||||
return;
|
||||
}
|
||||
|
||||
if (value.TryGetValue(out double doubleValue))
|
||||
{
|
||||
writer.WriteNumberValue(doubleValue);
|
||||
return;
|
||||
}
|
||||
|
||||
if (value.TryGetValue(out decimal decimalValue))
|
||||
{
|
||||
writer.WriteNumberValue(decimalValue);
|
||||
return;
|
||||
}
|
||||
|
||||
if (value.TryGetValue(out float floatValue))
|
||||
{
|
||||
writer.WriteNumberValue(floatValue);
|
||||
return;
|
||||
}
|
||||
|
||||
if (value.TryGetValue(out Guid guidValue))
|
||||
{
|
||||
writer.WriteStringValue(guidValue);
|
||||
return;
|
||||
}
|
||||
|
||||
if (value.TryGetValue(out DateTime dateTimeValue))
|
||||
{
|
||||
writer.WriteStringValue(dateTimeValue.ToUniversalTime());
|
||||
return;
|
||||
}
|
||||
|
||||
if (value.TryGetValue(out DateTimeOffset dateTimeOffsetValue))
|
||||
{
|
||||
writer.WriteStringValue(dateTimeOffsetValue.ToUniversalTime());
|
||||
return;
|
||||
}
|
||||
|
||||
// Fallback to raw text.
|
||||
writer.WriteStringValue(value.ToJsonString());
|
||||
}
|
||||
}
|
||||
132
src/Graph/StellaOps.Graph.Indexer/Schema/GraphDocumentFactory.cs
Normal file
132
src/Graph/StellaOps.Graph.Indexer/Schema/GraphDocumentFactory.cs
Normal file
@@ -0,0 +1,132 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Schema;
|
||||
|
||||
public readonly record struct GraphProvenanceSpec(
|
||||
string Source,
|
||||
DateTimeOffset CollectedAt,
|
||||
string? SbomDigest,
|
||||
long? EventOffset)
|
||||
{
|
||||
public JsonObject ToJson()
|
||||
{
|
||||
var obj = new JsonObject
|
||||
{
|
||||
["source"] = Source,
|
||||
["collected_at"] = GraphTimestamp.Format(CollectedAt)
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(SbomDigest))
|
||||
{
|
||||
obj["sbom_digest"] = SbomDigest;
|
||||
}
|
||||
|
||||
if (EventOffset.HasValue)
|
||||
{
|
||||
obj["event_offset"] = EventOffset.Value;
|
||||
}
|
||||
|
||||
return obj;
|
||||
}
|
||||
}
|
||||
|
||||
public readonly record struct GraphNodeSpec(
|
||||
string Tenant,
|
||||
string Kind,
|
||||
IReadOnlyDictionary<string, string> CanonicalKey,
|
||||
JsonObject Attributes,
|
||||
GraphProvenanceSpec Provenance,
|
||||
DateTimeOffset ValidFrom,
|
||||
DateTimeOffset? ValidTo);
|
||||
|
||||
public readonly record struct GraphEdgeSpec(
|
||||
string Tenant,
|
||||
string Kind,
|
||||
IReadOnlyDictionary<string, string> CanonicalKey,
|
||||
JsonObject Attributes,
|
||||
GraphProvenanceSpec Provenance,
|
||||
DateTimeOffset ValidFrom,
|
||||
DateTimeOffset? ValidTo);
|
||||
|
||||
public static class GraphDocumentFactory
|
||||
{
|
||||
public static JsonObject CreateNode(GraphNodeSpec spec)
|
||||
{
|
||||
var canonicalKey = CreateCanonicalKey(spec.CanonicalKey);
|
||||
|
||||
var node = new JsonObject
|
||||
{
|
||||
["tenant"] = spec.Tenant,
|
||||
["kind"] = spec.Kind,
|
||||
["canonical_key"] = canonicalKey,
|
||||
["attributes"] = (JsonObject)spec.Attributes.DeepClone(),
|
||||
["provenance"] = spec.Provenance.ToJson(),
|
||||
["valid_from"] = GraphTimestamp.Format(spec.ValidFrom),
|
||||
["valid_to"] = spec.ValidTo.HasValue ? GraphTimestamp.Format(spec.ValidTo.Value) : null
|
||||
};
|
||||
|
||||
var identityTuple = GraphIdentity.ExtractIdentityTuple(canonicalKey);
|
||||
node["id"] = GraphIdentity.ComputeNodeId(spec.Tenant, spec.Kind, identityTuple);
|
||||
|
||||
var hash = GraphIdentity.ComputeDocumentHash(node);
|
||||
node["hash"] = hash;
|
||||
|
||||
return node;
|
||||
}
|
||||
|
||||
public static JsonObject CreateEdge(GraphEdgeSpec spec)
|
||||
{
|
||||
var canonicalKey = CreateCanonicalKey(spec.CanonicalKey);
|
||||
|
||||
var edge = new JsonObject
|
||||
{
|
||||
["tenant"] = spec.Tenant,
|
||||
["kind"] = spec.Kind,
|
||||
["canonical_key"] = canonicalKey,
|
||||
["attributes"] = (JsonObject)spec.Attributes.DeepClone(),
|
||||
["provenance"] = spec.Provenance.ToJson(),
|
||||
["valid_from"] = GraphTimestamp.Format(spec.ValidFrom),
|
||||
["valid_to"] = spec.ValidTo.HasValue ? GraphTimestamp.Format(spec.ValidTo.Value) : null
|
||||
};
|
||||
|
||||
var identityTuple = GraphIdentity.ExtractIdentityTuple(canonicalKey);
|
||||
edge["id"] = GraphIdentity.ComputeEdgeId(spec.Tenant, spec.Kind, identityTuple);
|
||||
|
||||
var hash = GraphIdentity.ComputeDocumentHash(edge);
|
||||
edge["hash"] = hash;
|
||||
|
||||
return edge;
|
||||
}
|
||||
|
||||
private static JsonObject CreateCanonicalKey(IReadOnlyDictionary<string, string> entries)
|
||||
{
|
||||
var builder = ImmutableSortedDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
|
||||
foreach (var (key, value) in entries)
|
||||
{
|
||||
builder[key.Trim()] = value.Trim();
|
||||
}
|
||||
|
||||
var obj = new JsonObject();
|
||||
foreach (var (key, value) in builder)
|
||||
{
|
||||
obj[key] = value;
|
||||
}
|
||||
|
||||
return obj;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
internal static class GraphTimestamp
|
||||
{
|
||||
public static string Format(DateTimeOffset value)
|
||||
{
|
||||
var utc = value.UtcDateTime;
|
||||
return utc.Ticks % TimeSpan.TicksPerSecond == 0
|
||||
? utc.ToString("yyyy-MM-dd'T'HH:mm:ss'Z'", CultureInfo.InvariantCulture)
|
||||
: utc.ToString("yyyy-MM-dd'T'HH:mm:ss.fffffff'Z'", CultureInfo.InvariantCulture);
|
||||
}
|
||||
}
|
||||
152
src/Graph/StellaOps.Graph.Indexer/Schema/GraphIdentity.cs
Normal file
152
src/Graph/StellaOps.Graph.Indexer/Schema/GraphIdentity.cs
Normal file
@@ -0,0 +1,152 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Schema;
|
||||
|
||||
/// <summary>
|
||||
/// Helpers for computing deterministic identifiers and hashes for graph documents.
|
||||
/// </summary>
|
||||
public static class GraphIdentity
|
||||
{
|
||||
private const string NodePrefix = "gn:";
|
||||
private const StringComparison OrdinalIgnoreCase = StringComparison.OrdinalIgnoreCase;
|
||||
|
||||
private static readonly string[] CaseSensitiveHints = { "digest", "hash", "fingerprint", "id" };
|
||||
|
||||
/// <summary>
|
||||
/// Computes a deterministic node identifier using the canonical identity tuple.
|
||||
/// </summary>
|
||||
public static string ComputeNodeId(string tenant, string kind, IReadOnlyDictionary<string, string> identityTuple)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(kind);
|
||||
ArgumentNullException.ThrowIfNull(identityTuple);
|
||||
|
||||
var normalizedKind = kind.Trim().ToLowerInvariant();
|
||||
var normalizedTenant = tenant.Trim().ToLowerInvariant();
|
||||
var tuplePayload = JoinTuple(identityTuple);
|
||||
var hash = HashBase32(tuplePayload);
|
||||
|
||||
return $"{NodePrefix}{normalizedTenant}:{normalizedKind}:{hash}";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computes a deterministic edge identifier using the canonical identity tuple.
|
||||
/// </summary>
|
||||
public static string ComputeEdgeId(string tenant, string kind, IReadOnlyDictionary<string, string> identityTuple)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(kind);
|
||||
ArgumentNullException.ThrowIfNull(identityTuple);
|
||||
|
||||
var normalizedKind = kind.Trim().ToUpperInvariant();
|
||||
var normalizedTenant = tenant.Trim().ToLowerInvariant();
|
||||
var tuplePayload = JoinTuple(identityTuple);
|
||||
var hash = HashBase32(tuplePayload);
|
||||
|
||||
return $"ge:{normalizedTenant}:{normalizedKind}:{hash}";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computes the canonical SHA-256 hash for a JSON document. Canonicalisation sorts object keys
|
||||
/// and serialises using UTF-8 JSON without extra whitespace.
|
||||
/// </summary>
|
||||
public static string ComputeDocumentHash(JsonNode document)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
var canonicalBytes = CanonicalJson.ToCanonicalUtf8Bytes(document);
|
||||
var hash = SHA256.HashData(canonicalBytes);
|
||||
return Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Converts a JSON object into an ordered dictionary of string components used for identifiers.
|
||||
/// </summary>
|
||||
public static IReadOnlyDictionary<string, string> ExtractIdentityTuple(JsonObject source)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(source);
|
||||
|
||||
var builder = ImmutableSortedDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
|
||||
foreach (var kvp in source)
|
||||
{
|
||||
if (kvp.Value is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (kvp.Value is JsonValue value)
|
||||
{
|
||||
builder[kvp.Key.Trim()] = value.ToJsonString().Trim('"');
|
||||
}
|
||||
else
|
||||
{
|
||||
builder[kvp.Key.Trim()] = kvp.Value.ToJsonString();
|
||||
}
|
||||
}
|
||||
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
|
||||
private static string JoinTuple(IReadOnlyDictionary<string, string> tuple)
|
||||
{
|
||||
var builder = new StringBuilder();
|
||||
var sorted = tuple.OrderBy(static pair => pair.Key, StringComparer.Ordinal);
|
||||
var first = true;
|
||||
|
||||
foreach (var (key, value) in sorted)
|
||||
{
|
||||
if (!first)
|
||||
{
|
||||
builder.Append('|');
|
||||
}
|
||||
|
||||
first = false;
|
||||
|
||||
var normalizedKey = key.Trim().ToLowerInvariant();
|
||||
var normalizedValue = NormalizeValue(normalizedKey, value);
|
||||
builder.Append(normalizedKey);
|
||||
builder.Append('=');
|
||||
builder.Append(normalizedValue);
|
||||
}
|
||||
|
||||
return builder.ToString();
|
||||
}
|
||||
|
||||
private static string NormalizeValue(string key, string value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
if (IsCaseSensitiveKey(key))
|
||||
{
|
||||
return value.Trim();
|
||||
}
|
||||
|
||||
return value.Trim().ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static bool IsCaseSensitiveKey(string key)
|
||||
{
|
||||
foreach (var hint in CaseSensitiveHints)
|
||||
{
|
||||
if (key.Contains(hint, OrdinalIgnoreCase))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static string HashBase32(string payload)
|
||||
{
|
||||
var data = SHA256.HashData(Encoding.UTF8.GetBytes(payload));
|
||||
return Base32Crockford.Encode(data);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<AssemblyName>StellaOps.Graph.Indexer</AssemblyName>
|
||||
<RootNamespace>StellaOps.Graph.Indexer</RootNamespace>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0-rc.2.25502.107" />
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.0-rc.2.25502.107" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0-rc.2.25502.107" />
|
||||
<PackageReference Include="MongoDB.Driver" Version="3.5.0" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -1,13 +1,14 @@
|
||||
# Graph Indexer Task Board — Epic 5: SBOM Graph Explorer
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| GRAPH-INDEX-28-001 | TODO | Graph Indexer Guild | SBOM-SERVICE-21-001, CARTO-GRAPH-21-001 | Define canonical node/edge schemas, attribute dictionaries, identity rules, and seed fixtures; publish schema doc. | Schema doc merged; identity property tests pass; fixtures committed for CI usage. |
|
||||
| GRAPH-INDEX-28-002 | TODO | Graph Indexer Guild | GRAPH-INDEX-28-001, SBOM-SERVICE-21-002 | Implement SBOM ingest consumer producing artifact/package/file nodes and edges with `valid_from/valid_to`, scope metadata, and provenance links. | Ingest pipeline processes sample SBOMs deterministically; metrics recorded; unit tests cover identity stability. |
|
||||
| GRAPH-INDEX-28-003 | TODO | Graph Indexer Guild | GRAPH-INDEX-28-001, CONCELIER-CONSOLE-23-001 | Project Concelier linksets into overlay tiles (`affected_by` edges, evidence refs) without mutating source observations; keep advisory aggregates in overlay store only. | Overlay documents generated deterministically; raw node/edge collections remain immutable; tests cover overlay refresh and eviction. |
|
||||
| GRAPH-INDEX-28-004 | TODO | Graph Indexer Guild | GRAPH-INDEX-28-001, EXCITITOR-CONSOLE-23-001 | Integrate VEX statements (`vex_exempts` edges) with justification metadata and precedence markers for overlays. | VEX edges generated; conflicts resolved deterministically; tests cover status transitions. |
|
||||
| GRAPH-INDEX-28-005 | TODO | Graph Indexer Guild, Policy Guild | POLICY-ENGINE-27-001, POLICY-ENGINE-27-002 | Hydrate policy overlays into graph (`governs_with` nodes/edges) referencing effective findings and explain hashes for sampled nodes. | Overlay nodes stored with policy version id, severity, status; explain references captured; validation tests pass. |
|
||||
| GRAPH-INDEX-28-006 | TODO | Graph Indexer Guild | GRAPH-INDEX-28-002..005 | Generate graph snapshots per SBOM with lineage (`derived_from`), adjacency manifests, and metadata for diff jobs. | Snapshot documents produced; lineage recorded; tests assert diff readiness; metrics emitted. |
|
||||
| GRAPH-INDEX-28-007 | TODO | Graph Indexer Guild, Observability Guild | GRAPH-INDEX-28-002..006 | Implement clustering/centrality background jobs (Louvain/degree/betweenness approximations) with configurable schedules and store cluster ids on nodes. | Clustering jobs run on fixtures; metrics logged; cluster ids accessible via API; SLA documented. |
|
||||
| GRAPH-INDEX-28-008 | TODO | Graph Indexer Guild | GRAPH-INDEX-28-002..007 | Provide incremental update + backfill pipeline with change streams, retry/backoff, idempotent operations, and backlog metrics. | Incremental updates replay sample change logs; retries/backoff validated; backlog metrics exported. |
|
||||
| GRAPH-INDEX-28-009 | TODO | Graph Indexer Guild, QA Guild | GRAPH-INDEX-28-002..008 | Add unit/property/integration tests, synthetic large graph fixtures, chaos testing (missing overlays, cycles), and determinism checks across runs. | Test suite green; determinism harness passes across two runs; perf metrics recorded. |
|
||||
| GRAPH-INDEX-28-010 | TODO | Graph Indexer Guild, DevOps Guild | GRAPH-INDEX-28-008 | Package deployment artifacts (Helm/Compose), offline seed bundles, and configuration docs; integrate Offline Kit. | Deployment descriptors merged; offline seed bundle documented; smoke deploy tested. |
|
||||
# Graph Indexer Task Board — Epic 5: SBOM Graph Explorer
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| GRAPH-INDEX-28-001 | DONE (2025-11-03) | Graph Indexer Guild | SBOM-SERVICE-21-001, CARTO-GRAPH-21-001 | Define canonical node/edge schemas, attribute dictionaries, identity rules, and seed fixtures; publish schema doc.<br>2025-11-03: Schema doc v1 published, fixtures added (`nodes.json`, `edges.json`, `schema-matrix.json`), GraphIdentity determinism tests green. | Schema doc merged; identity property tests pass; fixtures committed for CI usage. |
|
||||
| GRAPH-INDEX-28-002 | DONE (2025-11-03) | Graph Indexer Guild | GRAPH-INDEX-28-001, SBOM-SERVICE-21-002 | Implement SBOM ingest consumer producing artifact/package/file nodes and edges with `valid_from/valid_to`, scope metadata, and provenance links.<br>2025-11-03: Snapshot models repaired, provenance resolution tightened, ingest processor/metrics surfaces added, and transformer/fixtures/tests expanded for license/base artifact determinism. | Ingest pipeline processes sample SBOMs deterministically; metrics recorded; unit tests cover identity stability. |
|
||||
| GRAPH-INDEX-28-003 | DONE (2025-11-03) | Graph Indexer Guild | GRAPH-INDEX-28-001, CONCELIER-CONSOLE-23-001 | Project Concelier linksets into overlay tiles (`affected_by` edges, evidence refs) without mutating source observations; keep advisory aggregates in overlay store only.<br>2025-11-03: Snapshot model repaired, transformer finalized with dedupe + provenance normalization, fixtures/tests refreshed, full graph suite green. | Overlay documents generated deterministically; raw node/edge collections remain immutable; tests cover overlay refresh and eviction. |
|
||||
| GRAPH-INDEX-28-004 | DONE (2025-11-03) | Graph Indexer Guild | GRAPH-INDEX-28-001, EXCITITOR-CONSOLE-23-001 | Integrate VEX statements (`vex_exempts` edges) with justification metadata and precedence markers for overlays.<br>2025-11-03: VEX snapshot/transformer emit deterministic VEX_EXEMPTS overlays with provenance hashes; fixtures and tests updated; full graph indexer suite green. | VEX edges generated; conflicts resolved deterministically; tests cover status transitions. |
|
||||
| GRAPH-INDEX-28-005 | DONE (2025-11-03) | Graph Indexer Guild, Policy Guild | POLICY-ENGINE-27-001, POLICY-ENGINE-27-002 | Hydrate policy overlays into graph (`governs_with` nodes/edges) referencing effective findings and explain hashes for sampled nodes.<br>2025-11-03: Policy overlay snapshot/transformer added with deterministic nodes/edges, fixtures + tests updated, targeted graph tests pass; Mongo writer tests now probe `STELLAOPS_TEST_MONGO_URI` or localhost before falling back to Mongo2Go and skip with guidance when neither path is available.<br>2025-11-03: Processor + metrics wired atop Mongo writer; unit tests cover success/failure paths. | Overlay nodes stored with policy version id, severity, status; explain references captured; validation tests pass. |
|
||||
| GRAPH-INDEX-28-006 | DONE (2025-11-03) | Graph Indexer Guild | GRAPH-INDEX-28-002..005 | Generate graph snapshots per SBOM with lineage (`derived_from`), adjacency manifests, and metadata for diff jobs.<br>2025-11-03: Snapshot builder emits hashed manifest + adjacency (incoming/outgoing edges), integration tests cover lineage/diff readiness, docs updated with required Mongo env.<br>2025-11-03: Snapshot exporter writes manifest/adjacency/nodes/edges to snapshot directory with deterministic ordering. | Snapshot documents produced; lineage recorded; tests assert diff readiness; metrics emitted. |
|
||||
| GRAPH-INDEX-28-007 | TODO | Graph Indexer Guild, Observability Guild | GRAPH-INDEX-28-002..006 | Implement clustering/centrality background jobs (Louvain/degree/betweenness approximations) with configurable schedules and store cluster ids on nodes. | Clustering jobs run on fixtures; metrics logged; cluster ids accessible via API; SLA documented. |
|
||||
| GRAPH-INDEX-28-008 | TODO | Graph Indexer Guild | GRAPH-INDEX-28-002..007 | Provide incremental update + backfill pipeline with change streams, retry/backoff, idempotent operations, and backlog metrics. | Incremental updates replay sample change logs; retries/backoff validated; backlog metrics exported. |
|
||||
| GRAPH-INDEX-28-009 | TODO | Graph Indexer Guild, QA Guild | GRAPH-INDEX-28-002..008 | Add unit/property/integration tests, synthetic large graph fixtures, chaos testing (missing overlays, cycles), and determinism checks across runs. | Test suite green; determinism harness passes across two runs; perf metrics recorded. |
|
||||
| GRAPH-INDEX-28-010 | TODO | Graph Indexer Guild, DevOps Guild | GRAPH-INDEX-28-008 | Package deployment artifacts (Helm/Compose), offline seed bundles, and configuration docs; integrate Offline Kit. | Deployment descriptors merged; offline seed bundle documented; smoke deploy tested. |
|
||||
| GRAPH-INDEX-28-011 | DONE (2025-11-04) | Graph Indexer Guild | GRAPH-INDEX-28-002..006 | Wire SBOM ingest runtime to emit graph snapshot artifacts and harden Mongo test configuration.<br>2025-11-04: Adopted `SbomIngestProcessorFactory.CreateDefault` inside a DI extension, added configurable snapshot root (`STELLAOPS_GRAPH_SNAPSHOT_DIR` or options), documented Mongo/snapshot env guidance, and verified Graph Indexer tests (Mongo writer skipped when no URI). | Composition root uses factory/exporter, snapshot files land in configured artifacts directory, and dev/CI guidance ensures Mongo availability without manual edits. |
|
||||
|
||||
Reference in New Issue
Block a user