Add unit tests for SBOM ingestion and transformation
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Implement `SbomIngestServiceCollectionExtensionsTests` to verify the SBOM ingestion pipeline exports snapshots correctly.
- Create `SbomIngestTransformerTests` to ensure the transformation produces expected nodes and edges, including deduplication of license nodes and normalization of timestamps.
- Add `SbomSnapshotExporterTests` to test the export functionality for manifest, adjacency, nodes, and edges.
- Introduce `VexOverlayTransformerTests` to validate the transformation of VEX nodes and edges.
- Set up project file for the test project with necessary dependencies and configurations.
- Include JSON fixture files for testing purposes.
This commit is contained in:
master
2025-11-04 07:49:39 +02:00
parent f72c5c513a
commit 2eb6852d34
491 changed files with 39445 additions and 3917 deletions

View File

@@ -0,0 +1,8 @@
using System;
namespace StellaOps.Graph.Indexer.Ingestion.Policy;
public interface IPolicyOverlayMetrics
{
void RecordBatch(string source, string tenant, int nodeCount, int edgeCount, TimeSpan duration, bool success);
}

View File

@@ -0,0 +1,124 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
namespace StellaOps.Graph.Indexer.Ingestion.Policy;
public sealed class PolicyOverlayMetrics : IPolicyOverlayMetrics, IDisposable
{
public const string MeterName = "StellaOps.Graph.Indexer";
public const string MeterVersion = "1.0.0";
private const string BatchesTotalName = "graph_policy_overlay_batches_total";
private const string BatchDurationSecondsName = "graph_policy_overlay_duration_seconds";
private const string NodesTotalName = "graph_policy_overlay_nodes_total";
private const string EdgesTotalName = "graph_policy_overlay_edges_total";
private const string UnitCount = "count";
private const string UnitSeconds = "s";
private readonly Meter _meter;
private readonly bool _ownsMeter;
private readonly Counter<long> _batchesTotal;
private readonly Histogram<double> _batchDurationSeconds;
private readonly Counter<long> _nodesTotal;
private readonly Counter<long> _edgesTotal;
private bool _disposed;
public PolicyOverlayMetrics()
: this(null)
{
}
public PolicyOverlayMetrics(Meter? meter)
{
_meter = meter ?? new Meter(MeterName, MeterVersion);
_ownsMeter = meter is null;
_batchesTotal = _meter.CreateCounter<long>(
name: BatchesTotalName,
unit: UnitCount,
description: "Policy overlay batches processed grouped by source, tenant, and result.");
_batchDurationSeconds = _meter.CreateHistogram<double>(
name: BatchDurationSecondsName,
unit: UnitSeconds,
description: "Latency to transform and persist policy overlay batches grouped by source, tenant, and result.");
_nodesTotal = _meter.CreateCounter<long>(
name: NodesTotalName,
unit: UnitCount,
description: "Policy overlay nodes produced grouped by source and tenant.");
_edgesTotal = _meter.CreateCounter<long>(
name: EdgesTotalName,
unit: UnitCount,
description: "GOVERNS_WITH edges produced grouped by source and tenant.");
}
public void RecordBatch(string source, string tenant, int nodeCount, int edgeCount, TimeSpan duration, bool success)
{
ThrowIfDisposed();
ArgumentException.ThrowIfNullOrWhiteSpace(source);
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
var normalizedDuration = Math.Max(duration.TotalSeconds, 0d);
var resultTag = success ? "success" : "failure";
var tags = new[]
{
new KeyValuePair<string, object?>("source", source),
new KeyValuePair<string, object?>("tenant", tenant),
new KeyValuePair<string, object?>("result", resultTag)
};
_batchesTotal.Add(1, tags);
_batchDurationSeconds.Record(normalizedDuration, tags);
if (!success)
{
return;
}
var volumeTags = new[]
{
new KeyValuePair<string, object?>("source", source),
new KeyValuePair<string, object?>("tenant", tenant)
};
if (nodeCount > 0)
{
_nodesTotal.Add(nodeCount, volumeTags);
}
if (edgeCount > 0)
{
_edgesTotal.Add(edgeCount, volumeTags);
}
}
private void ThrowIfDisposed()
{
if (_disposed)
{
throw new ObjectDisposedException(nameof(PolicyOverlayMetrics));
}
}
public void Dispose()
{
if (_disposed)
{
return;
}
if (_ownsMeter)
{
_meter.Dispose();
}
_disposed = true;
GC.SuppressFinalize(this);
}
}

View File

@@ -0,0 +1,85 @@
using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Graph.Indexer.Ingestion.Sbom;
namespace StellaOps.Graph.Indexer.Ingestion.Policy;
public sealed class PolicyOverlayProcessor
{
private readonly PolicyOverlayTransformer _transformer;
private readonly IGraphDocumentWriter _writer;
private readonly IPolicyOverlayMetrics _metrics;
private readonly ILogger<PolicyOverlayProcessor> _logger;
public PolicyOverlayProcessor(
PolicyOverlayTransformer transformer,
IGraphDocumentWriter writer,
IPolicyOverlayMetrics metrics,
ILogger<PolicyOverlayProcessor> logger)
{
_transformer = transformer ?? throw new ArgumentNullException(nameof(transformer));
_writer = writer ?? throw new ArgumentNullException(nameof(writer));
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task ProcessAsync(PolicyOverlaySnapshot snapshot, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(snapshot);
cancellationToken.ThrowIfCancellationRequested();
var stopwatch = Stopwatch.StartNew();
GraphBuildBatch batch;
try
{
batch = _transformer.Transform(snapshot);
}
catch (Exception ex)
{
stopwatch.Stop();
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, 0, 0, stopwatch.Elapsed, success: false);
_logger.LogError(
ex,
"graph-indexer: failed to transform policy overlay {PolicyPackDigest} for tenant {Tenant}",
snapshot.Policy?.PolicyPackDigest ?? string.Empty,
snapshot.Tenant);
throw;
}
try
{
cancellationToken.ThrowIfCancellationRequested();
await _writer.WriteAsync(batch, cancellationToken).ConfigureAwait(false);
stopwatch.Stop();
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, batch.Nodes.Length, batch.Edges.Length, stopwatch.Elapsed, success: true);
_logger.LogInformation(
"graph-indexer: indexed policy overlay {PolicyPackDigest} (effective {EffectiveFrom}) for tenant {Tenant} with {NodeCount} nodes and {EdgeCount} edges in {DurationMs:F2} ms",
snapshot.Policy?.PolicyPackDigest ?? string.Empty,
(snapshot.Policy?.EffectiveFrom ?? snapshot.CollectedAt).ToUniversalTime(),
snapshot.Tenant,
batch.Nodes.Length,
batch.Edges.Length,
stopwatch.Elapsed.TotalMilliseconds);
}
catch (Exception ex)
{
stopwatch.Stop();
_metrics.RecordBatch(snapshot.Source, snapshot.Tenant, batch.Nodes.Length, batch.Edges.Length, stopwatch.Elapsed, success: false);
_logger.LogError(
ex,
"graph-indexer: failed to persist policy overlay {PolicyPackDigest} for tenant {Tenant}",
snapshot.Policy?.PolicyPackDigest ?? string.Empty,
snapshot.Tenant);
throw;
}
}
}

View File

@@ -0,0 +1,90 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Graph.Indexer.Ingestion.Policy;
public sealed class PolicyOverlaySnapshot
{
[JsonPropertyName("tenant")]
public string Tenant { get; init; } = string.Empty;
[JsonPropertyName("source")]
public string Source { get; init; } = string.Empty;
[JsonPropertyName("collectedAt")]
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
[JsonPropertyName("eventOffset")]
public long EventOffset { get; init; }
[JsonPropertyName("policy")]
public PolicyVersionDetails Policy { get; init; } = new();
[JsonPropertyName("evaluations")]
public IReadOnlyList<PolicyEvaluation> Evaluations { get; init; }
= Array.Empty<PolicyEvaluation>();
}
public sealed class PolicyVersionDetails
{
[JsonPropertyName("source")]
public string Source { get; init; } = string.Empty;
[JsonPropertyName("policyPackDigest")]
public string PolicyPackDigest { get; init; } = string.Empty;
[JsonPropertyName("policyName")]
public string PolicyName { get; init; } = string.Empty;
[JsonPropertyName("effectiveFrom")]
public DateTimeOffset EffectiveFrom { get; init; } = DateTimeOffset.UnixEpoch;
[JsonPropertyName("expiresAt")]
public DateTimeOffset ExpiresAt { get; init; } = DateTimeOffset.UnixEpoch;
[JsonPropertyName("explainHash")]
public string ExplainHash { get; init; } = string.Empty;
[JsonPropertyName("collectedAt")]
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
[JsonPropertyName("eventOffset")]
public long EventOffset { get; init; }
}
public sealed class PolicyEvaluation
{
[JsonPropertyName("componentPurl")]
public string ComponentPurl { get; init; } = string.Empty;
[JsonPropertyName("componentSourceType")]
public string ComponentSourceType { get; init; } = "inventory";
[JsonPropertyName("findingExplainHash")]
public string FindingExplainHash { get; init; } = string.Empty;
[JsonPropertyName("explainHash")]
public string? ExplainHash { get; init; }
[JsonPropertyName("policyRuleId")]
public string PolicyRuleId { get; init; } = string.Empty;
[JsonPropertyName("verdict")]
public string Verdict { get; init; } = string.Empty;
[JsonPropertyName("evaluationTimestamp")]
public DateTimeOffset EvaluationTimestamp { get; init; } = DateTimeOffset.UnixEpoch;
[JsonPropertyName("sbomDigest")]
public string? SbomDigest { get; init; }
[JsonPropertyName("source")]
public string Source { get; init; } = string.Empty;
[JsonPropertyName("collectedAt")]
public DateTimeOffset CollectedAt { get; init; } = DateTimeOffset.UnixEpoch;
[JsonPropertyName("eventOffset")]
public long EventOffset { get; init; }
}

View File

@@ -0,0 +1,237 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text.Json.Nodes;
using StellaOps.Graph.Indexer.Ingestion.Sbom;
using StellaOps.Graph.Indexer.Schema;
namespace StellaOps.Graph.Indexer.Ingestion.Policy;
public sealed class PolicyOverlayTransformer
{
private const string PolicyNodeKind = "policy_version";
private const string ComponentNodeKind = "component";
private const string GovernsWithEdgeKind = "GOVERNS_WITH";
public GraphBuildBatch Transform(PolicyOverlaySnapshot snapshot)
{
ArgumentNullException.ThrowIfNull(snapshot);
var tenant = snapshot.Tenant?.Trim() ?? string.Empty;
var nodes = new List<JsonObject>();
var edges = new List<JsonObject>();
var seenEdgeIds = new HashSet<string>(StringComparer.Ordinal);
var policyDetails = snapshot.Policy ?? new PolicyVersionDetails();
var policyNode = CreatePolicyNode(tenant, snapshot, policyDetails);
nodes.Add(policyNode);
var policyNodeId = policyNode["id"]!.GetValue<string>();
foreach (var evaluation in snapshot.Evaluations ?? Array.Empty<PolicyEvaluation>())
{
if (!IsEvaluationCandidate(evaluation))
{
continue;
}
var edge = CreateGovernsWithEdge(
tenant,
snapshot,
policyDetails,
policyNodeId,
evaluation!);
var edgeId = edge["id"]!.GetValue<string>();
if (seenEdgeIds.Add(edgeId))
{
edges.Add(edge);
}
}
return new GraphBuildBatch(
nodes
.OrderBy(node => node["kind"]!.GetValue<string>(), StringComparer.Ordinal)
.ThenBy(node => node["id"]!.GetValue<string>(), StringComparer.Ordinal)
.ToImmutableArray(),
edges
.OrderBy(edge => edge["kind"]!.GetValue<string>(), StringComparer.Ordinal)
.ThenBy(edge => edge["id"]!.GetValue<string>(), StringComparer.Ordinal)
.ToImmutableArray());
}
private static bool IsEvaluationCandidate(PolicyEvaluation? evaluation)
{
return evaluation is not null
&& !string.IsNullOrWhiteSpace(evaluation.ComponentPurl)
&& !string.IsNullOrWhiteSpace(evaluation.FindingExplainHash);
}
private static JsonObject CreatePolicyNode(
string tenant,
PolicyOverlaySnapshot snapshot,
PolicyVersionDetails policy)
{
var policyPackDigest = policy.PolicyPackDigest?.Trim() ?? string.Empty;
var policyName = policy.PolicyName?.Trim() ?? string.Empty;
var explainHash = policy.ExplainHash?.Trim() ?? string.Empty;
var effectiveFrom = policy.EffectiveFrom == DateTimeOffset.UnixEpoch
? snapshot.CollectedAt
: policy.EffectiveFrom;
var expiresAt = policy.ExpiresAt == DateTimeOffset.UnixEpoch
? (DateTimeOffset?)null
: policy.ExpiresAt;
var policyCollectedAt = policy.CollectedAt == DateTimeOffset.UnixEpoch
? snapshot.CollectedAt
: policy.CollectedAt;
var eventOffset = policy.EventOffset != 0 ? policy.EventOffset : snapshot.EventOffset;
var source = ResolveSource(policy.Source, snapshot.Source);
var canonicalKey = new Dictionary<string, string>
{
["tenant"] = tenant,
["policy_pack_digest"] = policyPackDigest,
["effective_from"] = GraphTimestamp.Format(effectiveFrom)
};
var attributes = new JsonObject
{
["policy_pack_digest"] = policyPackDigest,
["policy_name"] = policyName,
["effective_from"] = GraphTimestamp.Format(effectiveFrom),
["expires_at"] = expiresAt is null ? null : GraphTimestamp.Format(expiresAt.Value),
["explain_hash"] = explainHash
};
var node = GraphDocumentFactory.CreateNode(new GraphNodeSpec(
Tenant: tenant,
Kind: PolicyNodeKind,
CanonicalKey: canonicalKey,
Attributes: attributes,
Provenance: new GraphProvenanceSpec(source, policyCollectedAt, SbomDigest: null, EventOffset: eventOffset),
ValidFrom: effectiveFrom,
ValidTo: expiresAt));
NormalizeOverlayProvenance(node);
return node;
}
private static JsonObject CreateGovernsWithEdge(
string tenant,
PolicyOverlaySnapshot snapshot,
PolicyVersionDetails policy,
string policyNodeId,
PolicyEvaluation evaluation)
{
var componentSourceType = string.IsNullOrWhiteSpace(evaluation.ComponentSourceType)
? "inventory"
: evaluation.ComponentSourceType.Trim();
var componentIdentity = new Dictionary<string, string>
{
["tenant"] = tenant,
["purl"] = evaluation.ComponentPurl.Trim(),
["source_type"] = componentSourceType
};
var componentNodeId = GraphIdentity.ComputeNodeId(tenant, ComponentNodeKind, componentIdentity);
var findingExplainHash = evaluation.FindingExplainHash.Trim();
var edgeCanonicalKey = new Dictionary<string, string>
{
["tenant"] = tenant,
["policy_node_id"] = policyNodeId,
["component_node_id"] = componentNodeId,
["finding_explain_hash"] = findingExplainHash
};
var evaluationTimestamp = evaluation.EvaluationTimestamp == DateTimeOffset.UnixEpoch
? snapshot.CollectedAt
: evaluation.EvaluationTimestamp;
var explainHash = !string.IsNullOrWhiteSpace(evaluation.ExplainHash)
? evaluation.ExplainHash.Trim()
: !string.IsNullOrWhiteSpace(policy.ExplainHash)
? policy.ExplainHash.Trim()
: findingExplainHash;
var attributes = new JsonObject
{
["verdict"] = evaluation.Verdict?.Trim() ?? string.Empty,
["explain_hash"] = explainHash,
["policy_rule_id"] = evaluation.PolicyRuleId?.Trim() ?? string.Empty,
["evaluation_timestamp"] = GraphTimestamp.Format(evaluationTimestamp)
};
var collectedAt = evaluation.CollectedAt == DateTimeOffset.UnixEpoch
? snapshot.CollectedAt
: evaluation.CollectedAt;
var eventOffset = evaluation.EventOffset != 0 ? evaluation.EventOffset : snapshot.EventOffset;
var source = ResolveSource(evaluation.Source, policy.Source, snapshot.Source);
return GraphDocumentFactory.CreateEdge(new GraphEdgeSpec(
Tenant: tenant,
Kind: GovernsWithEdgeKind,
CanonicalKey: edgeCanonicalKey,
Attributes: attributes,
Provenance: new GraphProvenanceSpec(
source,
collectedAt,
NormalizeOptional(evaluation.SbomDigest),
eventOffset),
ValidFrom: evaluationTimestamp,
ValidTo: null));
}
private static void NormalizeOverlayProvenance(JsonObject node)
{
var provenance = node["provenance"]!.AsObject();
var sourceNode = provenance["source"]!.DeepClone();
var collectedAtNode = provenance["collected_at"]!.DeepClone();
var eventOffsetNode = provenance.ContainsKey("event_offset")
? provenance["event_offset"]!.DeepClone()
: null;
var normalized = new JsonObject
{
["source"] = sourceNode,
["collected_at"] = collectedAtNode,
["sbom_digest"] = null
};
if (eventOffsetNode is not null)
{
normalized["event_offset"] = eventOffsetNode;
}
node["provenance"] = normalized;
node.Remove("hash");
node["hash"] = GraphIdentity.ComputeDocumentHash(node);
}
private static string ResolveSource(params string?[] candidates)
{
foreach (var candidate in candidates)
{
if (!string.IsNullOrWhiteSpace(candidate))
{
return candidate.Trim();
}
}
return "policy.engine.v1";
}
private static string? NormalizeOptional(string? value)
{
return string.IsNullOrWhiteSpace(value) ? null : value.Trim();
}
}