house keeping work
This commit is contained in:
StellaOps Bot
@@ -0,0 +1,752 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// IncrementalCacheBenchmarkTests.cs
|
||||
// Sprint: SPRINT_3700_0006_0001_incremental_cache (CACHE-015)
|
||||
// Description: Performance benchmark tests for incremental reachability cache.
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Diagnostics;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using StellaOps.Scanner.Reachability.Cache;
|
||||
using Xunit;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace StellaOps.Scanner.Reachability.Tests.Benchmarks;
|
||||
|
||||
/// <summary>
|
||||
/// Performance benchmark tests for incremental reachability cache.
|
||||
/// Validates performance targets defined in SPRINT_3700_0006_0001_incremental_cache.md:
|
||||
/// - Cache lookup: <10ms
|
||||
/// - Delta computation: <100ms
|
||||
/// - Impact set calculation: <500ms
|
||||
/// - Full recompute: <30s (baseline for 50K node graph)
|
||||
/// - Incremental (cache hit): <1s (90th percentile)
|
||||
/// - Incremental (partial): <5s (10% of graph changed)
|
||||
/// </summary>
|
||||
public sealed class IncrementalCacheBenchmarkTests
|
||||
{
|
||||
private readonly ITestOutputHelper _output;
|
||||
|
||||
public IncrementalCacheBenchmarkTests(ITestOutputHelper output)
|
||||
{
|
||||
_output = output;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Benchmark: Cache lookup should complete in <10ms.
|
||||
/// Uses in-memory cache implementation.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task CacheLookup_ShouldCompleteInUnder10ms()
|
||||
{
|
||||
// Arrange
|
||||
var cache = new InMemoryReachabilityCache();
|
||||
var serviceId = "benchmark-service";
|
||||
var graphHash = "abc123";
|
||||
|
||||
// Pre-populate cache with entries
|
||||
var entry = CreateCacheEntry(serviceId, graphHash, 100);
|
||||
await cache.SetAsync(entry);
|
||||
|
||||
// Warm up
|
||||
_ = await cache.GetAsync(serviceId, graphHash);
|
||||
|
||||
// Act - measure multiple lookups
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
const int iterations = 100;
|
||||
|
||||
for (int i = 0; i < iterations; i++)
|
||||
{
|
||||
var result = await cache.GetAsync(serviceId, graphHash);
|
||||
result.Should().NotBeNull();
|
||||
}
|
||||
|
||||
stopwatch.Stop();
|
||||
var averageMs = stopwatch.Elapsed.TotalMilliseconds / iterations;
|
||||
_output.WriteLine($"Cache lookup average: {averageMs:F3}ms over {iterations} iterations");
|
||||
|
||||
// Assert
|
||||
averageMs.Should().BeLessThan(10, "cache lookup should complete in <10ms");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Benchmark: Delta computation should complete in <100ms for 50K nodes.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void DeltaComputation_ShouldCompleteInUnder100ms_For50KNodes()
|
||||
{
|
||||
// Arrange
|
||||
const int nodeCount = 50_000;
|
||||
const int edgeCount = 100_000;
|
||||
const int changedNodes = 100; // 0.2% of nodes changed
|
||||
|
||||
var previousGraph = CreateMockGraphSnapshot(nodeCount, edgeCount, seed: 42);
|
||||
var currentGraph = CreateModifiedGraphSnapshot(previousGraph, changedNodes, seed: 43);
|
||||
|
||||
// Warm up - simple delta computation
|
||||
_ = ComputeDelta(previousGraph, currentGraph);
|
||||
|
||||
// Act
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
var delta = ComputeDelta(previousGraph, currentGraph);
|
||||
stopwatch.Stop();
|
||||
|
||||
_output.WriteLine($"Delta computation for {nodeCount} nodes: {stopwatch.ElapsedMilliseconds}ms");
|
||||
_output.WriteLine($" Added nodes: {delta.AddedNodes.Count}");
|
||||
_output.WriteLine($" Removed nodes: {delta.RemovedNodes.Count}");
|
||||
_output.WriteLine($" Added edges: {delta.AddedEdges.Count}");
|
||||
_output.WriteLine($" Removed edges: {delta.RemovedEdges.Count}");
|
||||
|
||||
// Assert
|
||||
stopwatch.ElapsedMilliseconds.Should().BeLessThan(100,
|
||||
"delta computation should complete in <100ms for 50K node graph");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Benchmark: Impact set calculation should complete in <500ms.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void ImpactSetCalculation_ShouldCompleteInUnder500ms()
|
||||
{
|
||||
// Arrange
|
||||
const int nodeCount = 50_000;
|
||||
const int edgeCount = 100_000;
|
||||
|
||||
var graph = CreateMockGraphSnapshot(nodeCount, edgeCount, seed: 42);
|
||||
var addedNodes = new HashSet<string>(CreateNodeIds(100, "added"));
|
||||
var removedNodes = new HashSet<string>(CreateNodeIds(50, "removed"));
|
||||
|
||||
// Warm up
|
||||
_ = CalculateImpactSet(graph, addedNodes, removedNodes);
|
||||
|
||||
// Act
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
var impactSet = CalculateImpactSet(graph, addedNodes, removedNodes);
|
||||
stopwatch.Stop();
|
||||
|
||||
_output.WriteLine($"Impact set calculation for {nodeCount} nodes: {stopwatch.ElapsedMilliseconds}ms");
|
||||
_output.WriteLine($" Impact set size: {impactSet.Count}");
|
||||
|
||||
// Assert
|
||||
stopwatch.ElapsedMilliseconds.Should().BeLessThan(500,
|
||||
"impact set calculation should complete in <500ms");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Benchmark: State flip detection should complete quickly.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void StateFlipDetection_ShouldCompleteInUnder50ms()
|
||||
{
|
||||
// Arrange
|
||||
var previousResults = CreateReachablePairResults(1000, reachableRatio: 0.3);
|
||||
var currentResults = CreateReachablePairResultsWithChanges(previousResults, changeRatio: 0.1);
|
||||
|
||||
var detector = new StateFlipDetector(NullLogger<StateFlipDetector>.Instance);
|
||||
|
||||
// Warm up
|
||||
_ = detector.DetectFlips(previousResults, currentResults);
|
||||
|
||||
// Act
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
const int iterations = 100;
|
||||
|
||||
for (int i = 0; i < iterations; i++)
|
||||
{
|
||||
_ = detector.DetectFlips(previousResults, currentResults);
|
||||
}
|
||||
|
||||
stopwatch.Stop();
|
||||
var averageMs = stopwatch.Elapsed.TotalMilliseconds / iterations;
|
||||
_output.WriteLine($"State flip detection average: {averageMs:F3}ms over {iterations} iterations");
|
||||
_output.WriteLine($" Previous results: {previousResults.Count}");
|
||||
_output.WriteLine($" Current results: {currentResults.Count}");
|
||||
|
||||
// Assert
|
||||
averageMs.Should().BeLessThan(50, "state flip detection should be fast");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Benchmark: PR gate evaluation should be fast.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void PrGateEvaluation_ShouldCompleteInUnder10ms()
|
||||
{
|
||||
// Arrange
|
||||
var flips = new StateFlipResult
|
||||
{
|
||||
NewlyReachable = CreateStateFlips(50, newReachable: true),
|
||||
NewlyUnreachable = CreateStateFlips(30, newReachable: false),
|
||||
};
|
||||
|
||||
var incrementalResult = new IncrementalReachabilityResult
|
||||
{
|
||||
ServiceId = "test-service",
|
||||
Results = CreateIncrementalResults(100, reachableRatio: 0.3),
|
||||
StateFlips = flips,
|
||||
FromCache = false,
|
||||
WasIncremental = true,
|
||||
SavingsRatio = 0.7,
|
||||
Duration = TimeSpan.FromMilliseconds(500),
|
||||
};
|
||||
|
||||
var gate = CreatePrReachabilityGate();
|
||||
|
||||
// Warm up
|
||||
_ = gate.Evaluate(incrementalResult);
|
||||
|
||||
// Act
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
const int iterations = 1000;
|
||||
|
||||
for (int i = 0; i < iterations; i++)
|
||||
{
|
||||
_ = gate.Evaluate(incrementalResult);
|
||||
}
|
||||
|
||||
stopwatch.Stop();
|
||||
var averageMs = stopwatch.Elapsed.TotalMilliseconds / iterations;
|
||||
_output.WriteLine($"PR gate evaluation average: {averageMs:F4}ms over {iterations} iterations");
|
||||
|
||||
// Assert
|
||||
averageMs.Should().BeLessThan(10, "PR gate evaluation should be very fast");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Benchmark: Memory pressure test for large caches.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task LargeCache_ShouldHandleMemoryEfficiently()
|
||||
{
|
||||
// Arrange
|
||||
var cache = new InMemoryReachabilityCache();
|
||||
const int serviceCount = 10;
|
||||
const int entriesPerService = 1000;
|
||||
|
||||
var beforeMemory = GC.GetTotalMemory(true);
|
||||
|
||||
// Act - populate cache with many entries
|
||||
for (int s = 0; s < serviceCount; s++)
|
||||
{
|
||||
var serviceId = $"service-{s}";
|
||||
var graphHash = $"hash-{s}";
|
||||
var entry = CreateCacheEntry(serviceId, graphHash, entriesPerService);
|
||||
await cache.SetAsync(entry);
|
||||
}
|
||||
|
||||
var afterMemory = GC.GetTotalMemory(true);
|
||||
var memoryUsedMB = (afterMemory - beforeMemory) / (1024.0 * 1024.0);
|
||||
|
||||
_output.WriteLine($"Cache memory usage: {memoryUsedMB:F2}MB for {serviceCount * entriesPerService} entries");
|
||||
|
||||
// Assert - ensure memory usage is reasonable (< 100MB for 10K entries)
|
||||
memoryUsedMB.Should().BeLessThan(100,
|
||||
"cache should be memory efficient");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Benchmark: Hash computation for graph snapshots should be fast.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void GraphHashComputation_ShouldCompleteQuickly()
|
||||
{
|
||||
// Arrange
|
||||
const int nodeCount = 50_000;
|
||||
const int edgeCount = 100_000;
|
||||
var graph = CreateMockGraphSnapshot(nodeCount, edgeCount, seed: 42);
|
||||
|
||||
// Warm up
|
||||
_ = graph.Hash;
|
||||
|
||||
// Act
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
const int iterations = 100;
|
||||
|
||||
for (int i = 0; i < iterations; i++)
|
||||
{
|
||||
_ = graph.Hash;
|
||||
}
|
||||
|
||||
stopwatch.Stop();
|
||||
var averageMs = stopwatch.Elapsed.TotalMilliseconds / iterations;
|
||||
_output.WriteLine($"Graph hash computation average: {averageMs:F4}ms over {iterations} iterations");
|
||||
|
||||
// Assert - hash should be precomputed or very fast
|
||||
averageMs.Should().BeLessThan(1, "graph hash should be precomputed or very fast");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Benchmark: Concurrent cache access should be thread-safe and performant.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task ConcurrentCacheAccess_ShouldBePerformant()
|
||||
{
|
||||
// Arrange
|
||||
var cache = new InMemoryReachabilityCache();
|
||||
var serviceId = "concurrent-service";
|
||||
var graphHash = "concurrent-hash";
|
||||
|
||||
// Pre-populate cache
|
||||
var entry = CreateCacheEntry(serviceId, graphHash, 500);
|
||||
await cache.SetAsync(entry);
|
||||
|
||||
// Act - concurrent reads
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
const int concurrency = 10;
|
||||
const int iterationsPerTask = 100;
|
||||
|
||||
var tasks = Enumerable.Range(0, concurrency)
|
||||
.Select(_ => Task.Run(async () =>
|
||||
{
|
||||
for (int i = 0; i < iterationsPerTask; i++)
|
||||
{
|
||||
var result = await cache.GetAsync(serviceId, graphHash);
|
||||
result.Should().NotBeNull();
|
||||
}
|
||||
}))
|
||||
.ToList();
|
||||
|
||||
await Task.WhenAll(tasks);
|
||||
stopwatch.Stop();
|
||||
|
||||
var totalOperations = concurrency * iterationsPerTask;
|
||||
var opsPerSecond = totalOperations / stopwatch.Elapsed.TotalSeconds;
|
||||
_output.WriteLine($"Concurrent cache access: {opsPerSecond:F0} ops/sec ({totalOperations} total in {stopwatch.ElapsedMilliseconds}ms)");
|
||||
|
||||
// Assert
|
||||
opsPerSecond.Should().BeGreaterThan(1000, "cache should handle >1000 ops/sec concurrent access");
|
||||
}
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private static ReachabilityCacheEntry CreateCacheEntry(
|
||||
string serviceId,
|
||||
string graphHash,
|
||||
int pairCount)
|
||||
{
|
||||
var pairs = new List<ReachablePairResult>();
|
||||
for (int i = 0; i < pairCount; i++)
|
||||
{
|
||||
pairs.Add(new ReachablePairResult
|
||||
{
|
||||
EntryMethodKey = $"entry-{i}",
|
||||
SinkMethodKey = $"sink-{i % 50}",
|
||||
IsReachable = i % 3 == 0,
|
||||
PathLength = i % 3 == 0 ? 3 : null,
|
||||
Confidence = 0.9,
|
||||
ComputedAt = DateTimeOffset.UtcNow,
|
||||
});
|
||||
}
|
||||
|
||||
return new ReachabilityCacheEntry
|
||||
{
|
||||
ServiceId = serviceId,
|
||||
GraphHash = graphHash,
|
||||
ReachablePairs = pairs,
|
||||
EntryPointCount = pairCount / 10,
|
||||
SinkCount = 50,
|
||||
};
|
||||
}
|
||||
|
||||
private static MockGraphSnapshot CreateMockGraphSnapshot(int nodeCount, int edgeCount, int seed)
|
||||
{
|
||||
var random = new Random(seed);
|
||||
var nodeKeys = new HashSet<string>(
|
||||
Enumerable.Range(0, nodeCount).Select(i => $"node-{i}"));
|
||||
|
||||
var edges = new List<GraphEdge>();
|
||||
var nodeList = nodeKeys.ToList();
|
||||
|
||||
for (int i = 0; i < edgeCount; i++)
|
||||
{
|
||||
var from = nodeList[random.Next(nodeCount)];
|
||||
var to = nodeList[random.Next(nodeCount)];
|
||||
edges.Add(new GraphEdge(from, to));
|
||||
}
|
||||
|
||||
var entryPoints = new HashSet<string>(
|
||||
Enumerable.Range(0, nodeCount / 100).Select(i => $"node-{i}"));
|
||||
|
||||
return new MockGraphSnapshot(nodeKeys, edges, entryPoints, seed);
|
||||
}
|
||||
|
||||
private static MockGraphSnapshot CreateModifiedGraphSnapshot(
|
||||
MockGraphSnapshot previous,
|
||||
int changedNodes,
|
||||
int seed)
|
||||
{
|
||||
var random = new Random(seed);
|
||||
var nodeKeys = new HashSet<string>(previous.NodeKeys);
|
||||
var edges = previous.Edges.ToList();
|
||||
|
||||
// Remove some nodes
|
||||
var toRemove = nodeKeys.Take(changedNodes / 2).ToList();
|
||||
foreach (var node in toRemove)
|
||||
{
|
||||
nodeKeys.Remove(node);
|
||||
edges.RemoveAll(e => e.CallerKey == node || e.CalleeKey == node);
|
||||
}
|
||||
|
||||
// Add some new nodes
|
||||
for (int i = 0; i < changedNodes / 2; i++)
|
||||
{
|
||||
nodeKeys.Add($"new-node-{seed}-{i}");
|
||||
}
|
||||
|
||||
// Add some new edges
|
||||
var nodeList = nodeKeys.ToList();
|
||||
var newEdgeCount = Math.Min(changedNodes * 2, nodeList.Count);
|
||||
for (int i = 0; i < newEdgeCount; i++)
|
||||
{
|
||||
var from = nodeList[random.Next(nodeList.Count)];
|
||||
var to = nodeList[random.Next(nodeList.Count)];
|
||||
edges.Add(new GraphEdge(from, to));
|
||||
}
|
||||
|
||||
var entryPoints = new HashSet<string>(
|
||||
nodeKeys.Take(nodeKeys.Count / 100));
|
||||
|
||||
return new MockGraphSnapshot(nodeKeys, edges, entryPoints, seed);
|
||||
}
|
||||
|
||||
private static GraphDelta ComputeDelta(MockGraphSnapshot previous, MockGraphSnapshot current)
|
||||
{
|
||||
var addedNodes = new HashSet<string>(current.NodeKeys.Except(previous.NodeKeys));
|
||||
var removedNodes = new HashSet<string>(previous.NodeKeys.Except(current.NodeKeys));
|
||||
|
||||
var prevEdgeSet = new HashSet<GraphEdge>(previous.Edges);
|
||||
var currEdgeSet = new HashSet<GraphEdge>(current.Edges);
|
||||
|
||||
var addedEdges = currEdgeSet.Except(prevEdgeSet).ToList();
|
||||
var removedEdges = prevEdgeSet.Except(currEdgeSet).ToList();
|
||||
|
||||
var affected = new HashSet<string>(addedNodes);
|
||||
affected.UnionWith(removedNodes);
|
||||
foreach (var e in addedEdges)
|
||||
{
|
||||
affected.Add(e.CallerKey);
|
||||
affected.Add(e.CalleeKey);
|
||||
}
|
||||
foreach (var e in removedEdges)
|
||||
{
|
||||
affected.Add(e.CallerKey);
|
||||
affected.Add(e.CalleeKey);
|
||||
}
|
||||
|
||||
return new GraphDelta
|
||||
{
|
||||
AddedNodes = addedNodes,
|
||||
RemovedNodes = removedNodes,
|
||||
AddedEdges = addedEdges,
|
||||
RemovedEdges = removedEdges,
|
||||
AffectedMethodKeys = affected,
|
||||
PreviousHash = previous.Hash,
|
||||
CurrentHash = current.Hash,
|
||||
};
|
||||
}
|
||||
|
||||
private static HashSet<string> CalculateImpactSet(
|
||||
MockGraphSnapshot graph,
|
||||
HashSet<string> addedNodes,
|
||||
HashSet<string> removedNodes)
|
||||
{
|
||||
var impactSet = new HashSet<string>(addedNodes);
|
||||
impactSet.UnionWith(removedNodes);
|
||||
|
||||
// BFS to find affected neighbors
|
||||
var visited = new HashSet<string>(impactSet);
|
||||
var queue = new Queue<string>(impactSet);
|
||||
const int maxDepth = 2;
|
||||
|
||||
for (int depth = 0; depth < maxDepth && queue.Count > 0; depth++)
|
||||
{
|
||||
var levelSize = queue.Count;
|
||||
for (int i = 0; i < levelSize; i++)
|
||||
{
|
||||
var node = queue.Dequeue();
|
||||
foreach (var edge in graph.Edges)
|
||||
{
|
||||
if (edge.CallerKey == node && !visited.Contains(edge.CalleeKey))
|
||||
{
|
||||
visited.Add(edge.CalleeKey);
|
||||
impactSet.Add(edge.CalleeKey);
|
||||
queue.Enqueue(edge.CalleeKey);
|
||||
}
|
||||
else if (edge.CalleeKey == node && !visited.Contains(edge.CallerKey))
|
||||
{
|
||||
visited.Add(edge.CallerKey);
|
||||
impactSet.Add(edge.CallerKey);
|
||||
queue.Enqueue(edge.CallerKey);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return impactSet;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> CreateNodeIds(int count, string prefix)
|
||||
{
|
||||
return Enumerable.Range(0, count).Select(i => $"{prefix}-{i}").ToList();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<ReachablePairResult> CreateReachablePairResults(
|
||||
int count,
|
||||
double reachableRatio)
|
||||
{
|
||||
var random = new Random(42);
|
||||
var results = new List<ReachablePairResult>();
|
||||
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
var isReachable = random.NextDouble() < reachableRatio;
|
||||
results.Add(new ReachablePairResult
|
||||
{
|
||||
EntryMethodKey = $"entry-{i}",
|
||||
SinkMethodKey = $"sink-{i % 50}",
|
||||
IsReachable = isReachable,
|
||||
PathLength = isReachable ? 3 : null,
|
||||
Confidence = 0.9,
|
||||
ComputedAt = DateTimeOffset.UtcNow,
|
||||
});
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<ReachablePairResult> CreateReachablePairResultsWithChanges(
|
||||
IReadOnlyList<ReachablePairResult> previous,
|
||||
double changeRatio)
|
||||
{
|
||||
var random = new Random(43);
|
||||
var results = new List<ReachablePairResult>();
|
||||
|
||||
foreach (var prev in previous)
|
||||
{
|
||||
var shouldFlip = random.NextDouble() < changeRatio;
|
||||
var newReachable = shouldFlip ? !prev.IsReachable : prev.IsReachable;
|
||||
|
||||
results.Add(new ReachablePairResult
|
||||
{
|
||||
EntryMethodKey = prev.EntryMethodKey,
|
||||
SinkMethodKey = prev.SinkMethodKey,
|
||||
IsReachable = newReachable,
|
||||
PathLength = newReachable ? prev.PathLength : null,
|
||||
Confidence = prev.Confidence,
|
||||
ComputedAt = DateTimeOffset.UtcNow,
|
||||
});
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<ReachablePairResult> CreateIncrementalResults(
|
||||
int count,
|
||||
double reachableRatio)
|
||||
{
|
||||
return CreateReachablePairResults(count, reachableRatio);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<StateFlip> CreateStateFlips(int count, bool newReachable)
|
||||
{
|
||||
return Enumerable.Range(0, count)
|
||||
.Select(i => new StateFlip
|
||||
{
|
||||
EntryMethodKey = $"entry-{i}",
|
||||
SinkMethodKey = $"sink-{i % 10}",
|
||||
CveId = $"CVE-2024-{1000 + i}",
|
||||
WasReachable = !newReachable,
|
||||
IsReachable = newReachable,
|
||||
Confidence = 0.9,
|
||||
})
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static IPrReachabilityGate CreatePrReachabilityGate()
|
||||
{
|
||||
var options = new PrReachabilityGateOptions
|
||||
{
|
||||
Enabled = true,
|
||||
BlockOnNewReachable = true,
|
||||
MinConfidenceThreshold = 0.8,
|
||||
MaxNewReachableCount = 10,
|
||||
IncludeAnnotations = true,
|
||||
};
|
||||
|
||||
return new PrReachabilityGate(
|
||||
Microsoft.Extensions.Options.Options.Create(options),
|
||||
NullLogger<PrReachabilityGate>.Instance);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
#region Mock/Test Implementations
|
||||
|
||||
/// <summary>
|
||||
/// In-memory implementation of reachability cache for benchmarking.
|
||||
/// </summary>
|
||||
file sealed class InMemoryReachabilityCache : IReachabilityCache
|
||||
{
|
||||
private readonly Dictionary<string, CachedReachabilityResult> _cache = new();
|
||||
private readonly object _lock = new();
|
||||
|
||||
public Task<CachedReachabilityResult?> GetAsync(
|
||||
string serviceId,
|
||||
string graphHash,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
var key = $"{serviceId}:{graphHash}";
|
||||
return Task.FromResult(_cache.TryGetValue(key, out var result) ? result : null);
|
||||
}
|
||||
}
|
||||
|
||||
public Task SetAsync(
|
||||
ReachabilityCacheEntry entry,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
var key = $"{entry.ServiceId}:{entry.GraphHash}";
|
||||
_cache[key] = new CachedReachabilityResult
|
||||
{
|
||||
ServiceId = entry.ServiceId,
|
||||
GraphHash = entry.GraphHash,
|
||||
ReachablePairs = entry.ReachablePairs,
|
||||
CachedAt = DateTimeOffset.UtcNow,
|
||||
EntryPointCount = entry.EntryPointCount,
|
||||
SinkCount = entry.SinkCount,
|
||||
};
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<ReachablePairResult?> GetReachablePairAsync(
|
||||
string serviceId,
|
||||
string entryMethodKey,
|
||||
string sinkMethodKey,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
foreach (var cached in _cache.Values)
|
||||
{
|
||||
if (cached.ServiceId == serviceId)
|
||||
{
|
||||
var result = cached.ReachablePairs.FirstOrDefault(r =>
|
||||
r.EntryMethodKey == entryMethodKey && r.SinkMethodKey == sinkMethodKey);
|
||||
if (result is not null)
|
||||
{
|
||||
return Task.FromResult<ReachablePairResult?>(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Task.FromResult<ReachablePairResult?>(null);
|
||||
}
|
||||
|
||||
public Task<int> InvalidateAsync(
|
||||
string serviceId,
|
||||
IEnumerable<string> affectedMethodKeys,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var keys = affectedMethodKeys.ToHashSet();
|
||||
var invalidated = 0;
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
var toRemove = new List<string>();
|
||||
foreach (var (cacheKey, cached) in _cache)
|
||||
{
|
||||
if (cached.ServiceId == serviceId)
|
||||
{
|
||||
var affected = cached.ReachablePairs.Any(r =>
|
||||
keys.Contains(r.EntryMethodKey) ||
|
||||
keys.Contains(r.SinkMethodKey));
|
||||
|
||||
if (affected)
|
||||
{
|
||||
toRemove.Add(cacheKey);
|
||||
invalidated++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var key in toRemove)
|
||||
{
|
||||
_cache.Remove(key);
|
||||
}
|
||||
}
|
||||
|
||||
return Task.FromResult(invalidated);
|
||||
}
|
||||
|
||||
public Task InvalidateAllAsync(
|
||||
string serviceId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
var toRemove = _cache.Keys
|
||||
.Where(k => k.StartsWith($"{serviceId}:"))
|
||||
.ToList();
|
||||
|
||||
foreach (var key in toRemove)
|
||||
{
|
||||
_cache.Remove(key);
|
||||
}
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<CacheStatistics> GetStatisticsAsync(
|
||||
string serviceId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
var entries = _cache.Values
|
||||
.Where(c => c.ServiceId == serviceId)
|
||||
.ToList();
|
||||
|
||||
return Task.FromResult(new CacheStatistics
|
||||
{
|
||||
ServiceId = serviceId,
|
||||
CachedPairCount = entries.Sum(e => e.ReachablePairs.Count),
|
||||
HitCount = 0,
|
||||
MissCount = 0,
|
||||
LastPopulatedAt = entries.MaxBy(e => e.CachedAt)?.CachedAt,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Mock graph snapshot for benchmarking.
|
||||
/// </summary>
|
||||
file sealed class MockGraphSnapshot : IGraphSnapshot
|
||||
{
|
||||
public IReadOnlySet<string> NodeKeys { get; }
|
||||
public IReadOnlyList<GraphEdge> Edges { get; }
|
||||
public IReadOnlySet<string> EntryPoints { get; }
|
||||
public string Hash { get; }
|
||||
|
||||
public MockGraphSnapshot(
|
||||
IReadOnlySet<string> nodeKeys,
|
||||
IReadOnlyList<GraphEdge> edges,
|
||||
IReadOnlySet<string> entryPoints,
|
||||
int seed)
|
||||
{
|
||||
NodeKeys = nodeKeys;
|
||||
Edges = edges;
|
||||
EntryPoints = entryPoints;
|
||||
Hash = $"hash-{seed}-{nodeKeys.Count}-{edges.Count}";
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
@@ -0,0 +1,400 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// PrReachabilityGateTests.cs
|
||||
// Sprint: SPRINT_3700_0006_0001_incremental_cache (CACHE-014)
|
||||
// Description: Unit tests for PR reachability gate.
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Reachability.Cache;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Reachability.Tests;
|
||||
|
||||
public sealed class PrReachabilityGateTests
|
||||
{
|
||||
private readonly PrReachabilityGate _gate;
|
||||
private readonly PrReachabilityGateOptions _options;
|
||||
|
||||
public PrReachabilityGateTests()
|
||||
{
|
||||
_options = new PrReachabilityGateOptions();
|
||||
var optionsMonitor = new TestOptionsMonitor<PrReachabilityGateOptions>(_options);
|
||||
_gate = new PrReachabilityGate(optionsMonitor, NullLogger<PrReachabilityGate>.Instance);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EvaluateFlips_NoFlips_ReturnsPass()
|
||||
{
|
||||
// Arrange
|
||||
var stateFlips = StateFlipResult.Empty;
|
||||
|
||||
// Act
|
||||
var result = _gate.EvaluateFlips(stateFlips, wasIncremental: true, savingsRatio: 0.8, TimeSpan.FromMilliseconds(100));
|
||||
|
||||
// Assert
|
||||
result.Passed.Should().BeTrue();
|
||||
result.Reason.Should().Be("No reachability changes");
|
||||
result.Decision.NewReachableCount.Should().Be(0);
|
||||
result.Decision.MitigatedCount.Should().Be(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EvaluateFlips_NewReachable_ReturnsBlock()
|
||||
{
|
||||
// Arrange
|
||||
var stateFlips = new StateFlipResult
|
||||
{
|
||||
NewlyReachable = new List<StateFlip>
|
||||
{
|
||||
new StateFlip
|
||||
{
|
||||
EntryMethodKey = "Controller.Get",
|
||||
SinkMethodKey = "Vulnerable.Execute",
|
||||
IsReachable = true,
|
||||
Confidence = 0.9
|
||||
}
|
||||
},
|
||||
NewlyUnreachable = []
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _gate.EvaluateFlips(stateFlips, wasIncremental: true, savingsRatio: 0.7, TimeSpan.FromMilliseconds(150));
|
||||
|
||||
// Assert
|
||||
result.Passed.Should().BeFalse();
|
||||
result.Reason.Should().Contain("1 vulnerabilities became reachable");
|
||||
result.Decision.NewReachableCount.Should().Be(1);
|
||||
result.Decision.BlockingFlips.Should().HaveCount(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EvaluateFlips_OnlyMitigated_ReturnsPass()
|
||||
{
|
||||
// Arrange
|
||||
var stateFlips = new StateFlipResult
|
||||
{
|
||||
NewlyReachable = [],
|
||||
NewlyUnreachable = new List<StateFlip>
|
||||
{
|
||||
new StateFlip
|
||||
{
|
||||
EntryMethodKey = "Controller.Get",
|
||||
SinkMethodKey = "Vulnerable.Execute",
|
||||
IsReachable = false,
|
||||
WasReachable = true
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _gate.EvaluateFlips(stateFlips, wasIncremental: true, savingsRatio: 0.9, TimeSpan.FromMilliseconds(50));
|
||||
|
||||
// Assert
|
||||
result.Passed.Should().BeTrue();
|
||||
result.Reason.Should().Contain("mitigated");
|
||||
result.Decision.MitigatedCount.Should().Be(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EvaluateFlips_GateDisabled_AlwaysPasses()
|
||||
{
|
||||
// Arrange
|
||||
_options.Enabled = false;
|
||||
|
||||
var stateFlips = new StateFlipResult
|
||||
{
|
||||
NewlyReachable = new List<StateFlip>
|
||||
{
|
||||
new StateFlip
|
||||
{
|
||||
EntryMethodKey = "Controller.Get",
|
||||
SinkMethodKey = "Vulnerable.Execute",
|
||||
IsReachable = true
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _gate.EvaluateFlips(stateFlips, wasIncremental: true, savingsRatio: 0.5, TimeSpan.FromMilliseconds(100));
|
||||
|
||||
// Assert
|
||||
result.Passed.Should().BeTrue();
|
||||
result.Reason.Should().Be("PR gate is disabled");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EvaluateFlips_LowConfidence_Excluded()
|
||||
{
|
||||
// Arrange
|
||||
_options.RequireMinimumConfidence = true;
|
||||
_options.MinimumConfidenceThreshold = 0.8;
|
||||
|
||||
var stateFlips = new StateFlipResult
|
||||
{
|
||||
NewlyReachable = new List<StateFlip>
|
||||
{
|
||||
new StateFlip
|
||||
{
|
||||
EntryMethodKey = "Controller.Get",
|
||||
SinkMethodKey = "Vulnerable.Execute",
|
||||
IsReachable = true,
|
||||
Confidence = 0.5 // Below threshold
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _gate.EvaluateFlips(stateFlips, wasIncremental: true, savingsRatio: 0.8, TimeSpan.FromMilliseconds(100));
|
||||
|
||||
// Assert
|
||||
result.Passed.Should().BeTrue(); // Should pass because low confidence path is excluded
|
||||
result.Decision.BlockingFlips.Should().BeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EvaluateFlips_MaxNewReachableThreshold_AllowsUnderThreshold()
|
||||
{
|
||||
// Arrange
|
||||
_options.MaxNewReachablePaths = 2;
|
||||
|
||||
var stateFlips = new StateFlipResult
|
||||
{
|
||||
NewlyReachable = new List<StateFlip>
|
||||
{
|
||||
new StateFlip
|
||||
{
|
||||
EntryMethodKey = "A.Method",
|
||||
SinkMethodKey = "Vuln1",
|
||||
IsReachable = true,
|
||||
Confidence = 1.0
|
||||
},
|
||||
new StateFlip
|
||||
{
|
||||
EntryMethodKey = "B.Method",
|
||||
SinkMethodKey = "Vuln2",
|
||||
IsReachable = true,
|
||||
Confidence = 1.0
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _gate.EvaluateFlips(stateFlips, wasIncremental: true, savingsRatio: 0.7, TimeSpan.FromMilliseconds(200));
|
||||
|
||||
// Assert
|
||||
result.Passed.Should().BeTrue(); // 2 == threshold, so should pass
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EvaluateFlips_MaxNewReachableThreshold_BlocksOverThreshold()
|
||||
{
|
||||
// Arrange
|
||||
_options.MaxNewReachablePaths = 1;
|
||||
|
||||
var stateFlips = new StateFlipResult
|
||||
{
|
||||
NewlyReachable = new List<StateFlip>
|
||||
{
|
||||
new StateFlip
|
||||
{
|
||||
EntryMethodKey = "A.Method",
|
||||
SinkMethodKey = "Vuln1",
|
||||
IsReachable = true,
|
||||
Confidence = 1.0
|
||||
},
|
||||
new StateFlip
|
||||
{
|
||||
EntryMethodKey = "B.Method",
|
||||
SinkMethodKey = "Vuln2",
|
||||
IsReachable = true,
|
||||
Confidence = 1.0
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _gate.EvaluateFlips(stateFlips, wasIncremental: true, savingsRatio: 0.6, TimeSpan.FromMilliseconds(200));
|
||||
|
||||
// Assert
|
||||
result.Passed.Should().BeFalse(); // 2 > 1, so should block
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EvaluateFlips_Annotations_GeneratedForBlockingFlips()
|
||||
{
|
||||
// Arrange
|
||||
_options.AddAnnotations = true;
|
||||
_options.MaxAnnotations = 5;
|
||||
|
||||
var stateFlips = new StateFlipResult
|
||||
{
|
||||
NewlyReachable = new List<StateFlip>
|
||||
{
|
||||
new StateFlip
|
||||
{
|
||||
EntryMethodKey = "Controller.Get",
|
||||
SinkMethodKey = "Vulnerable.Execute",
|
||||
IsReachable = true,
|
||||
Confidence = 1.0,
|
||||
SourceFile = "Controllers/MyController.cs",
|
||||
StartLine = 42,
|
||||
EndLine = 45
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _gate.EvaluateFlips(stateFlips, wasIncremental: true, savingsRatio: 0.8, TimeSpan.FromMilliseconds(100));
|
||||
|
||||
// Assert
|
||||
result.Annotations.Should().HaveCount(1);
|
||||
result.Annotations[0].Level.Should().Be(PrAnnotationLevel.Error);
|
||||
result.Annotations[0].FilePath.Should().Be("Controllers/MyController.cs");
|
||||
result.Annotations[0].StartLine.Should().Be(42);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EvaluateFlips_AnnotationsDisabled_NoAnnotations()
|
||||
{
|
||||
// Arrange
|
||||
_options.AddAnnotations = false;
|
||||
|
||||
var stateFlips = new StateFlipResult
|
||||
{
|
||||
NewlyReachable = new List<StateFlip>
|
||||
{
|
||||
new StateFlip
|
||||
{
|
||||
EntryMethodKey = "Controller.Get",
|
||||
SinkMethodKey = "Vulnerable.Execute",
|
||||
IsReachable = true
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _gate.EvaluateFlips(stateFlips, wasIncremental: true, savingsRatio: 0.8, TimeSpan.FromMilliseconds(100));
|
||||
|
||||
// Assert
|
||||
result.Annotations.Should().BeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EvaluateFlips_SummaryMarkdown_Generated()
|
||||
{
|
||||
// Arrange
|
||||
var stateFlips = new StateFlipResult
|
||||
{
|
||||
NewlyReachable = new List<StateFlip>
|
||||
{
|
||||
new StateFlip
|
||||
{
|
||||
EntryMethodKey = "Controller.Get",
|
||||
SinkMethodKey = "Vulnerable.Execute",
|
||||
IsReachable = true,
|
||||
Confidence = 0.95
|
||||
}
|
||||
},
|
||||
NewlyUnreachable = new List<StateFlip>
|
||||
{
|
||||
new StateFlip
|
||||
{
|
||||
EntryMethodKey = "Old.Entry",
|
||||
SinkMethodKey = "Fixed.Sink",
|
||||
IsReachable = false,
|
||||
WasReachable = true
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _gate.EvaluateFlips(stateFlips, wasIncremental: true, savingsRatio: 0.75, TimeSpan.FromMilliseconds(150));
|
||||
|
||||
// Assert
|
||||
result.SummaryMarkdown.Should().NotBeNullOrEmpty();
|
||||
result.SummaryMarkdown.Should().Contain("Reachability Gate");
|
||||
result.SummaryMarkdown.Should().Contain("New reachable paths");
|
||||
result.SummaryMarkdown.Should().Contain("Mitigated paths");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Evaluate_NullStateFlips_ReturnsPass()
|
||||
{
|
||||
// Arrange
|
||||
var result = new IncrementalReachabilityResult
|
||||
{
|
||||
ServiceId = "test-service",
|
||||
Results = [],
|
||||
StateFlips = null,
|
||||
FromCache = false,
|
||||
WasIncremental = true,
|
||||
SavingsRatio = 1.0,
|
||||
Duration = TimeSpan.FromMilliseconds(50)
|
||||
};
|
||||
|
||||
// Act
|
||||
var gateResult = _gate.Evaluate(result);
|
||||
|
||||
// Assert
|
||||
gateResult.Passed.Should().BeTrue();
|
||||
gateResult.Reason.Should().Be("No state flip detection performed");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Evaluate_WithStateFlips_DelegatesCorrectly()
|
||||
{
|
||||
// Arrange
|
||||
var stateFlips = new StateFlipResult
|
||||
{
|
||||
NewlyReachable = new List<StateFlip>
|
||||
{
|
||||
new StateFlip
|
||||
{
|
||||
EntryMethodKey = "A",
|
||||
SinkMethodKey = "B",
|
||||
IsReachable = true,
|
||||
Confidence = 1.0
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
var analysisResult = new IncrementalReachabilityResult
|
||||
{
|
||||
ServiceId = "test-service",
|
||||
Results = [],
|
||||
StateFlips = stateFlips,
|
||||
FromCache = false,
|
||||
WasIncremental = true,
|
||||
SavingsRatio = 0.9,
|
||||
Duration = TimeSpan.FromMilliseconds(100)
|
||||
};
|
||||
|
||||
// Act
|
||||
var gateResult = _gate.Evaluate(analysisResult);
|
||||
|
||||
// Assert
|
||||
gateResult.Passed.Should().BeFalse();
|
||||
gateResult.Decision.WasIncremental.Should().BeTrue();
|
||||
gateResult.Decision.SavingsRatio.Should().Be(0.9);
|
||||
}
|
||||
|
||||
private sealed class TestOptionsMonitor<T> : IOptionsMonitor<T>
|
||||
{
|
||||
private readonly T _currentValue;
|
||||
|
||||
public TestOptionsMonitor(T value)
|
||||
{
|
||||
_currentValue = value;
|
||||
}
|
||||
|
||||
public T CurrentValue => _currentValue;
|
||||
|
||||
public T Get(string? name) => _currentValue;
|
||||
|
||||
public IDisposable? OnChange(Action<T, string?> listener) => null;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user