stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 3 Releases Wiki Activity

Add determinism tests for verdict artifact generation and update SHA256 sums script

Browse Source 
- Implemented comprehensive tests for verdict artifact generation to ensure deterministic outputs across various scenarios, including identical inputs, parallel execution, and change ordering.
- Created helper methods for generating sample verdict inputs and computing canonical hashes.
- Added tests to validate the stability of canonical hashes, proof spine ordering, and summary statistics.
- Introduced a new PowerShell script to update SHA256 sums for files, ensuring accurate hash generation and file integrity checks.
This commit is contained in:
StellaOps Bot
2025-12-24 02:17:34 +02:00
parent e59921374e
commit 7503c19b8f
390 changed files with 37389 additions and 5380 deletions
Download Patch File Download Diff File Expand all files Collapse all files

494
src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/Properties/ReachabilityGraphPropertyTests.cs Normal file
View File

@@ -0,0 +1,494 @@
// -----------------------------------------------------------------------------
// ReachabilityGraphPropertyTests.cs
// Sprint: SPRINT_5100_0009_0001 (Scanner Tests)
// Task: SCANNER-5100-002 - Property tests for graph invariants
// Description: Property-based tests for reachability graph verifying
// acyclicity detection, deterministic node IDs, stable ordering.
// -----------------------------------------------------------------------------
using FsCheck;
using FsCheck.Xunit;
using StellaOps.Scanner.Reachability;
using StellaOps.Scanner.Reachability.Ordering;
using Xunit;
using FluentAssertions;
namespace StellaOps.Scanner.Reachability.Tests.Properties;
/// <summary>
/// Property-based tests for reachability graph invariants.
/// Verifies:
/// - Deterministic node IDs (same inputs produce same IDs)
/// - Stable ordering (same graph produces same canonical order)
/// - Acyclicity detection (cycles are handled consistently)
/// </summary>
[Trait("Category", "Property")]
public class ReachabilityGraphPropertyTests
{
private readonly DeterministicGraphOrderer _orderer = new();
#region Determinism Tests
/// <summary>
/// Canonicalization produces same hash regardless of input ordering.
/// </summary>
[Property(MaxTest = 100)]
public Property Canonicalize_IsDeterministic_AcrossInputOrdering()
{
return Prop.ForAll(
GraphArb(),
graph =>
{
// Create shuffled version
var shuffled = ShuffleGraph(graph);
var canonical1 = _orderer.Canonicalize(graph, GraphOrderingStrategy.TopologicalLexicographic);
var canonical2 = _orderer.Canonicalize(shuffled, GraphOrderingStrategy.TopologicalLexicographic);
return canonical1.ContentHash == canonical2.ContentHash;
});
}
/// <summary>
/// Same graph canonicalized twice produces identical hash.
/// </summary>
[Property(MaxTest = 100)]
public Property Canonicalize_IsIdempotent()
{
return Prop.ForAll(
GraphArb(),
graph =>
{
var hash1 = _orderer.Canonicalize(graph, GraphOrderingStrategy.TopologicalLexicographic).ContentHash;
var hash2 = _orderer.Canonicalize(graph, GraphOrderingStrategy.TopologicalLexicographic).ContentHash;
return hash1 == hash2;
});
}
/// <summary>
/// Node ordering is deterministic for same input.
/// </summary>
[Property(MaxTest = 100)]
public Property OrderNodes_IsDeterministic()
{
return Prop.ForAll(
GraphArb(),
graph =>
{
var order1 = _orderer.OrderNodes(graph, GraphOrderingStrategy.TopologicalLexicographic);
var order2 = _orderer.OrderNodes(graph, GraphOrderingStrategy.TopologicalLexicographic);
return order1.SequenceEqual(order2);
});
}
/// <summary>
/// Edge ordering is deterministic for same node order.
/// </summary>
[Property(MaxTest = 100)]
public Property OrderEdges_IsDeterministic()
{
return Prop.ForAll(
GraphArb(),
graph =>
{
var nodeOrder = _orderer.OrderNodes(graph, GraphOrderingStrategy.TopologicalLexicographic);
var edges1 = _orderer.OrderEdges(graph, nodeOrder);
var edges2 = _orderer.OrderEdges(graph, nodeOrder);
return edges1.Count == edges2.Count &&
edges1.Zip(edges2, (e1, e2) => e1.From == e2.From && e1.To == e2.To && e1.Kind == e2.Kind).All(x => x);
});
}
#endregion
#region Ordering Strategy Tests
/// <summary>
/// All ordering strategies produce same set of nodes.
/// </summary>
[Property(MaxTest = 100)]
public Property AllStrategies_ContainAllNodes()
{
return Prop.ForAll(
GraphArb(),
graph =>
{
var strategies = new[]
{
GraphOrderingStrategy.TopologicalLexicographic,
GraphOrderingStrategy.BreadthFirstLexicographic,
GraphOrderingStrategy.DepthFirstLexicographic,
GraphOrderingStrategy.Lexicographic
};
var expectedNodes = graph.Nodes.Select(n => n.Id).OrderBy(x => x).ToHashSet();
return strategies.All(strategy =>
{
var ordered = _orderer.OrderNodes(graph, strategy).ToHashSet();
return ordered.SetEquals(expectedNodes);
});
});
}
/// <summary>
/// Lexicographic ordering produces alphabetically sorted nodes.
/// </summary>
[Property(MaxTest = 100)]
public Property LexicographicOrdering_IsSorted()
{
return Prop.ForAll(
GraphArb(),
graph =>
{
var order = _orderer.OrderNodes(graph, GraphOrderingStrategy.Lexicographic);
var sorted = order.OrderBy(x => x, StringComparer.Ordinal).ToList();
return order.SequenceEqual(sorted);
});
}
/// <summary>
/// BFS ordering starts from anchor nodes (roots).
/// </summary>
[Property(MaxTest = 50)]
public Property BfsOrdering_StartsFromAnchors()
{
return Prop.ForAll(
GraphWithRootsArb(),
graph =>
{
if (graph.Roots.Count == 0)
return true;
var order = _orderer.OrderNodes(graph, GraphOrderingStrategy.BreadthFirstLexicographic);
var firstNodes = order.Take(graph.Roots.Count).ToHashSet();
var rootIds = graph.Roots.Select(r => r.Id).ToHashSet();
// First nodes should be anchors (roots)
return firstNodes.Intersect(rootIds).Any();
});
}
#endregion
#region Graph Invariant Tests
/// <summary>
/// All edges reference existing nodes in the ordered output.
/// </summary>
[Property(MaxTest = 100)]
public Property OrderedEdges_ReferenceExistingNodes()
{
return Prop.ForAll(
GraphArb(),
graph =>
{
var nodeOrder = _orderer.OrderNodes(graph, GraphOrderingStrategy.TopologicalLexicographic);
var nodeSet = nodeOrder.ToHashSet();
var edges = _orderer.OrderEdges(graph, nodeOrder);
return edges.All(e => nodeSet.Contains(e.From) && nodeSet.Contains(e.To));
});
}
/// <summary>
/// Canonical graph has valid node indices.
/// </summary>
[Property(MaxTest = 100)]
public Property CanonicalGraph_HasValidIndices()
{
return Prop.ForAll(
GraphArb(),
graph =>
{
var canonical = _orderer.Canonicalize(graph, GraphOrderingStrategy.TopologicalLexicographic);
var nodeCount = canonical.Nodes.Count;
// All node indices are sequential from 0
var nodeIndicesValid = canonical.Nodes
.Select((n, i) => n.Index == i)
.All(x => x);
// All edge indices are within bounds
var edgeIndicesValid = canonical.Edges
.All(e => e.SourceIndex >= 0 && e.SourceIndex < nodeCount &&
e.TargetIndex >= 0 && e.TargetIndex < nodeCount);
return nodeIndicesValid && edgeIndicesValid;
});
}
/// <summary>
/// Adding a node doesn't change existing node order (stability).
/// </summary>
[Property(MaxTest = 50)]
public Property AddingNode_MaintainsRelativeOrder()
{
return Prop.ForAll(
GraphArb(),
Gen.Elements("Z1", "Z2", "Z3", "ZNew").ToArbitrary(),
(graph, newNodeId) =>
{
// Skip if node already exists
if (graph.Nodes.Any(n => n.Id == newNodeId))
return true;
var originalOrder = _orderer.OrderNodes(graph, GraphOrderingStrategy.TopologicalLexicographic);
var newGraph = graph with
{
Nodes = graph.Nodes.Append(CreateNode(newNodeId)).ToList()
};
var newOrder = _orderer.OrderNodes(newGraph, GraphOrderingStrategy.TopologicalLexicographic);
// Relative order of original nodes should be preserved
var originalFiltered = newOrder.Where(id => originalOrder.Contains(id)).ToList();
return originalFiltered.SequenceEqual(originalOrder);
});
}
/// <summary>
/// Empty graph produces empty canonical output.
/// </summary>
[Fact]
public void EmptyGraph_ProducesEmptyCanonical()
{
var graph = new RichGraph(
Nodes: Array.Empty<RichGraphNode>(),
Edges: Array.Empty<RichGraphEdge>(),
Roots: Array.Empty<RichGraphRoot>(),
Analyzer: new RichGraphAnalyzer("test", "1.0", null));
var canonical = _orderer.Canonicalize(graph, GraphOrderingStrategy.TopologicalLexicographic);
canonical.Nodes.Should().BeEmpty();
canonical.Edges.Should().BeEmpty();
}
/// <summary>
/// Single node graph produces single node canonical output.
/// </summary>
[Fact]
public void SingleNodeGraph_ProducesSingleNodeCanonical()
{
var graph = new RichGraph(
Nodes: new[] { CreateNode("A") },
Edges: Array.Empty<RichGraphEdge>(),
Roots: Array.Empty<RichGraphRoot>(),
Analyzer: new RichGraphAnalyzer("test", "1.0", null));
var canonical = _orderer.Canonicalize(graph, GraphOrderingStrategy.TopologicalLexicographic);
canonical.Nodes.Should().HaveCount(1);
canonical.Nodes[0].Id.Should().Be("A");
canonical.Edges.Should().BeEmpty();
}
#endregion
#region Cycle Detection Tests
/// <summary>
/// Graphs with cycles are still canonicalized (cycles handled gracefully).
/// </summary>
[Property(MaxTest = 50)]
public Property GraphWithCycles_StillCanonicalizes()
{
return Prop.ForAll(
CyclicGraphArb(),
graph =>
{
var canonical = _orderer.Canonicalize(graph, GraphOrderingStrategy.TopologicalLexicographic);
// Should still produce valid output
return canonical.Nodes.Count == graph.Nodes.Count &&
!string.IsNullOrEmpty(canonical.ContentHash);
});
}
/// <summary>
/// Cyclic graph ordering is still deterministic.
/// </summary>
[Property(MaxTest = 50)]
public Property CyclicGraph_OrderingIsDeterministic()
{
return Prop.ForAll(
CyclicGraphArb(),
graph =>
{
var order1 = _orderer.OrderNodes(graph, GraphOrderingStrategy.TopologicalLexicographic);
var order2 = _orderer.OrderNodes(graph, GraphOrderingStrategy.TopologicalLexicographic);
return order1.SequenceEqual(order2);
});
}
#endregion
#region RichGraph.Trimmed Tests
/// <summary>
/// Trimmed graph is idempotent.
/// </summary>
[Property(MaxTest = 50)]
public Property Trimmed_IsIdempotent()
{
return Prop.ForAll(
GraphArb(),
graph =>
{
var trimmed1 = graph.Trimmed();
var trimmed2 = trimmed1.Trimmed();
// Nodes and edges should be identical
return trimmed1.Nodes.Count == trimmed2.Nodes.Count &&
trimmed1.Edges.Count == trimmed2.Edges.Count;
});
}
/// <summary>
/// Trimmed graph has deterministic ordering.
/// </summary>
[Property(MaxTest = 50)]
public Property Trimmed_HasDeterministicOrdering()
{
return Prop.ForAll(
GraphArb(),
graph =>
{
var trimmed = graph.Trimmed();
// Nodes should be ordered by Id
var nodeIds = trimmed.Nodes.Select(n => n.Id).ToList();
var sortedNodeIds = nodeIds.OrderBy(x => x, StringComparer.Ordinal).ToList();
return nodeIds.SequenceEqual(sortedNodeIds);
});
}
#endregion
#region Generators and Helpers
private static Arbitrary<RichGraph> GraphArb()
{
var nodeIdsGen = Gen.ListOf(Gen.Elements("A", "B", "C", "D", "E", "F", "G", "H"))
.Select(ids => ids.Distinct().ToList());
return (from nodeIds in nodeIdsGen
let nodes = nodeIds.Select(id => CreateNode(id)).ToList()
let edges = GenerateEdges(nodeIds)
select new RichGraph(
Nodes: nodes,
Edges: edges,
Roots: Array.Empty<RichGraphRoot>(),
Analyzer: new RichGraphAnalyzer("test", "1.0", null))).ToArbitrary();
}
private static Arbitrary<RichGraph> GraphWithRootsArb()
{
var nodeIdsGen = Gen.ListOf(Gen.Elements("A", "B", "C", "D", "E"))
.Select(ids => ids.Distinct().ToList());
return (from nodeIds in nodeIdsGen
where nodeIds.Count > 0
let nodes = nodeIds.Select(id => CreateNode(id)).ToList()
let edges = GenerateEdges(nodeIds)
let roots = new[] { new RichGraphRoot(nodeIds.First(), "runtime", null) }
select new RichGraph(
Nodes: nodes,
Edges: edges,
Roots: roots,
Analyzer: new RichGraphAnalyzer("test", "1.0", null))).ToArbitrary();
}
private static Arbitrary<RichGraph> CyclicGraphArb()
{
return Gen.Elements(
CreateCyclicGraph("A", "B"),
CreateCyclicGraph("A", "B", "C"),
CreateCyclicGraph("A", "B", "C", "D")).ToArbitrary();
}
private static RichGraph CreateCyclicGraph(params string[] nodeIds)
{
var nodes = nodeIds.Select(id => CreateNode(id)).ToList();
var edges = new List<RichGraphEdge>();
// Create a cycle: A -> B -> C -> ... -> A
for (var i = 0; i < nodeIds.Length; i++)
{
var from = nodeIds[i];
var to = nodeIds[(i + 1) % nodeIds.Length];
edges.Add(CreateEdge(from, to));
}
return new RichGraph(
Nodes: nodes,
Edges: edges,
Roots: Array.Empty<RichGraphRoot>(),
Analyzer: new RichGraphAnalyzer("test", "1.0", null));
}
private static List<RichGraphEdge> GenerateEdges(List<string> nodeIds)
{
if (nodeIds.Count < 2)
return new List<RichGraphEdge>();
var edges = new List<RichGraphEdge>();
var random = new System.Random(42); // Fixed seed for determinism
// Generate some random edges
for (var i = 0; i < nodeIds.Count - 1; i++)
{
if (random.NextDouble() > 0.3) // 70% chance of edge
{
var to = nodeIds[random.Next(i + 1, nodeIds.Count)];
edges.Add(CreateEdge(nodeIds[i], to));
}
}
return edges;
}
private static RichGraphNode CreateNode(string id)
{
return new RichGraphNode(
Id: id,
SymbolId: $"sym_{id}",
CodeId: null,
Purl: null,
Lang: "csharp",
Kind: "method",
Display: $"Method {id}",
BuildId: null,
Evidence: null,
Attributes: null,
SymbolDigest: null);
}
private static RichGraphEdge CreateEdge(string from, string to, string kind = "call")
{
return new RichGraphEdge(
From: from,
To: to,
Kind: kind,
Purl: null,
SymbolDigest: null,
Evidence: null,
Confidence: 1.0,
Candidates: null);
}
private static RichGraph ShuffleGraph(RichGraph graph)
{
var random = new System.Random(12345); // Fixed seed
var nodes = graph.Nodes.OrderBy(_ => random.Next()).ToList();
var edges = graph.Edges.OrderBy(_ => random.Next()).ToList();
var roots = graph.Roots.OrderBy(_ => random.Next()).ToList();
return graph with { Nodes = nodes, Edges = edges, Roots = roots };
}
#endregion
}