using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using FluentAssertions; using Microsoft.Extensions.Logging.Abstractions; using MongoDB.Bson; using StellaOps.Signals.Models; using StellaOps.Signals.Parsing; using StellaOps.Signals.Persistence; using StellaOps.Signals.Services; using Xunit; namespace StellaOps.Signals.Reachability.Tests; public sealed class ReachabilityScoringTests { private static readonly string RepoRoot = LocateRepoRoot(); private static readonly string FixtureRoot = Path.Combine(RepoRoot, "tests", "reachability", "fixtures", "reachbench-2025-expanded", "cases"); private static readonly (string CaseId, string Variant)[] SampleCases = { ("java-log4j-CVE-2021-44228-log4shell", "reachable"), ("java-log4j-CVE-2021-44228-log4shell", "unreachable"), ("redis-CVE-2022-0543-lua-sandbox-escape", "reachable") }; public static IEnumerable CaseVariants() { foreach (var (caseId, variant) in SampleCases) { var path = Path.Combine(FixtureRoot, caseId, "images", variant); if (Directory.Exists(path)) { yield return new object[] { caseId, variant }; } } } [Theory] [MemberData(nameof(CaseVariants))] public async Task RecomputedFactsMatchTruthFixtures(string caseId, string variant) { var casePath = Path.Combine(FixtureRoot, caseId); var variantPath = Path.Combine(casePath, "images", variant); var truth = JsonDocument.Parse(File.ReadAllText(Path.Combine(variantPath, "reachgraph.truth.json"))).RootElement; var sinks = truth.GetProperty("sinks").EnumerateArray().Select(x => x.GetProperty("sid").GetString()!).ToList(); var entryPoints = truth.GetProperty("paths").EnumerateArray() .Select(path => path[0].GetString()!) .Distinct(StringComparer.Ordinal) .ToList(); var callgraph = await LoadCallgraphAsync(caseId, variant, variantPath); var callgraphRepo = new InMemoryCallgraphRepository(callgraph); var factRepo = new InMemoryReachabilityFactRepository(); var scoringService = new ReachabilityScoringService(callgraphRepo, factRepo, TimeProvider.System, NullLogger.Instance); var request = BuildRequest(casePath, variant, sinks, entryPoints); request.CallgraphId = callgraph.Id; var fact = await scoringService.RecomputeAsync(request, CancellationToken.None); fact.States.Should().HaveCount(sinks.Count); var expectedReachable = variant == "reachable"; foreach (var sink in sinks) { var state = fact.States.Single(s => s.Target == sink); state.Reachable.Should().Be(expectedReachable, $"{caseId}:{variant} expected reachable={expectedReachable}"); if (expectedReachable) { state.Path.Should().NotBeEmpty(); state.Evidence.RuntimeHits.Should().NotBeEmpty(); } else { state.Path.Should().BeEmpty(); state.Evidence.BlockedEdges.Should().NotBeNull(); } } } private static ReachabilityRecomputeRequest BuildRequest(string casePath, string variant, List targets, List entryPoints) { var caseJson = JsonDocument.Parse(File.ReadAllText(Path.Combine(casePath, "case.json"))).RootElement; var variantKey = variant == "reachable" ? "reachable_variant" : "unreachable_variant"; var variantNode = caseJson.GetProperty("ground_truth").GetProperty(variantKey); var blockedEdges = new List(); if (variantNode.TryGetProperty("evidence", out var evidence) && evidence.TryGetProperty("blocked_edges", out var blockedArray)) { foreach (var item in blockedArray.EnumerateArray()) { var parts = item.GetString()?.Split("->", StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); if (parts is { Length: 2 }) { blockedEdges.Add(new ReachabilityBlockedEdge { From = parts[0], To = parts[1] }); } } } var runtimeHits = new List(); var tracePath = Path.Combine(casePath, "images", variant, "traces.runtime.jsonl"); if (File.Exists(tracePath)) { foreach (var line in File.ReadLines(tracePath)) { if (string.IsNullOrWhiteSpace(line)) { continue; } using var doc = JsonDocument.Parse(line); if (doc.RootElement.TryGetProperty("sid", out var sidProp)) { runtimeHits.Add(sidProp.GetString()!); } } } return new ReachabilityRecomputeRequest { Subject = new ReachabilitySubject { ScanId = $"{Path.GetFileName(casePath)}:{variant}", Component = Path.GetFileName(casePath), Version = variant }, EntryPoints = entryPoints, Targets = targets, RuntimeHits = runtimeHits, BlockedEdges = blockedEdges }; } private static async Task LoadCallgraphAsync(string caseId, string variant, string variantPath) { var parser = new SimpleJsonCallgraphParser("fixture"); var nodes = new Dictionary(StringComparer.Ordinal); var edges = new List(); foreach (var fileName in new[] { "callgraph.static.json", "callgraph.framework.json" }) { var path = Path.Combine(variantPath, fileName); if (!File.Exists(path)) { continue; } await using var stream = File.OpenRead(path); var result = await parser.ParseAsync(stream, CancellationToken.None); foreach (var node in result.Nodes) { nodes[node.Id] = node; } edges.AddRange(result.Edges); } return new CallgraphDocument { Id = ObjectId.GenerateNewId().ToString(), Language = "fixture", Component = caseId, Version = variant, Nodes = nodes.Values.ToList(), Edges = edges, Artifact = new CallgraphArtifactMetadata { Path = $"cas://fixtures/{caseId}/{variant}", Hash = "stub", ContentType = "application/json", Length = 0 } }; } private sealed class InMemoryCallgraphRepository : ICallgraphRepository { private readonly Dictionary storage; public InMemoryCallgraphRepository(CallgraphDocument document) { storage = new Dictionary(StringComparer.Ordinal) { [document.Id] = document }; } public Task UpsertAsync(CallgraphDocument document, CancellationToken cancellationToken) { storage[document.Id] = document; return Task.FromResult(document); } public Task GetByIdAsync(string id, CancellationToken cancellationToken) { storage.TryGetValue(id, out var document); return Task.FromResult(document); } } private sealed class InMemoryReachabilityFactRepository : IReachabilityFactRepository { private readonly Dictionary storage = new(StringComparer.Ordinal); public Task GetBySubjectAsync(string subjectKey, CancellationToken cancellationToken) { storage.TryGetValue(subjectKey, out var document); return Task.FromResult(document); } public Task UpsertAsync(ReachabilityFactDocument document, CancellationToken cancellationToken) { storage[document.SubjectKey] = document; return Task.FromResult(document); } } private static string LocateRepoRoot() { var current = new DirectoryInfo(AppContext.BaseDirectory); while (current != null) { if (File.Exists(Path.Combine(current.FullName, "Directory.Build.props"))) { return current.FullName; } current = current.Parent; } throw new InvalidOperationException("Cannot locate repository root (missing Directory.Build.props)."); } }