git.stella-ops.org/tests/reachability/StellaOps.Signals.Reachability.Tests/ReachabilityScoringTests.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FluentAssertions;
using Microsoft.Extensions.Logging.Abstractions;
using MongoDB.Bson;
using StellaOps.Signals.Models;
using StellaOps.Signals.Parsing;
using StellaOps.Signals.Persistence;
using StellaOps.Signals.Services;
using Xunit;

namespace StellaOps.Signals.Reachability.Tests;

public sealed class ReachabilityScoringTests
{
    private static readonly string RepoRoot = LocateRepoRoot();
    private static readonly string FixtureRoot = Path.Combine(RepoRoot, "tests", "reachability", "fixtures", "reachbench-2025-expanded", "cases");

    private static readonly (string CaseId, string Variant)[] SampleCases =
    {
        ("java-log4j-CVE-2021-44228-log4shell", "reachable"),
        ("java-log4j-CVE-2021-44228-log4shell", "unreachable"),
        ("redis-CVE-2022-0543-lua-sandbox-escape", "reachable")
    };

    public static IEnumerable<object[]> CaseVariants()
    {
        foreach (var (caseId, variant) in SampleCases)
        {
            var path = Path.Combine(FixtureRoot, caseId, "images", variant);
            if (Directory.Exists(path))
            {
                yield return new object[] { caseId, variant };
            }
        }
    }

    [Theory]
    [MemberData(nameof(CaseVariants))]
    public async Task RecomputedFactsMatchTruthFixtures(string caseId, string variant)
    {
        var casePath = Path.Combine(FixtureRoot, caseId);
        var variantPath = Path.Combine(casePath, "images", variant);
        var truth = JsonDocument.Parse(File.ReadAllText(Path.Combine(variantPath, "reachgraph.truth.json"))).RootElement;
        var sinks = truth.GetProperty("sinks").EnumerateArray().Select(x => x.GetProperty("sid").GetString()!).ToList();
        var entryPoints = truth.GetProperty("paths").EnumerateArray()
            .Select(path => path[0].GetString()!)
            .Distinct(StringComparer.Ordinal)
            .ToList();

        var callgraph = await LoadCallgraphAsync(caseId, variant, variantPath);
        var callgraphRepo = new InMemoryCallgraphRepository(callgraph);
        var factRepo = new InMemoryReachabilityFactRepository();
        var scoringService = new ReachabilityScoringService(callgraphRepo, factRepo, TimeProvider.System, NullLogger<ReachabilityScoringService>.Instance);

        var request = BuildRequest(casePath, variant, sinks, entryPoints);
        request.CallgraphId = callgraph.Id;

        var fact = await scoringService.RecomputeAsync(request, CancellationToken.None);
        fact.States.Should().HaveCount(sinks.Count);

        var expectedReachable = variant == "reachable";
        foreach (var sink in sinks)
        {
            var state = fact.States.Single(s => s.Target == sink);
            state.Reachable.Should().Be(expectedReachable, $"{caseId}:{variant} expected reachable={expectedReachable}");
            if (expectedReachable)
            {
                state.Path.Should().NotBeEmpty();
                state.Evidence.RuntimeHits.Should().NotBeEmpty();
            }
            else
            {
                state.Path.Should().BeEmpty();
                state.Evidence.BlockedEdges.Should().NotBeNull();
            }
        }
    }

    private static ReachabilityRecomputeRequest BuildRequest(string casePath, string variant, List<string> targets, List<string> entryPoints)
    {
        var caseJson = JsonDocument.Parse(File.ReadAllText(Path.Combine(casePath, "case.json"))).RootElement;
        var variantKey = variant == "reachable" ? "reachable_variant" : "unreachable_variant";
        var variantNode = caseJson.GetProperty("ground_truth").GetProperty(variantKey);

        var blockedEdges = new List<ReachabilityBlockedEdge>();
        if (variantNode.TryGetProperty("evidence", out var evidence) && evidence.TryGetProperty("blocked_edges", out var blockedArray))
        {
            foreach (var item in blockedArray.EnumerateArray())
            {
                var parts = item.GetString()?.Split("->", StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);
                if (parts is { Length: 2 })
                {
                    blockedEdges.Add(new ReachabilityBlockedEdge { From = parts[0], To = parts[1] });
                }
            }
        }

        var runtimeHits = new List<string>();
        var tracePath = Path.Combine(casePath, "images", variant, "traces.runtime.jsonl");
        if (File.Exists(tracePath))
        {
            foreach (var line in File.ReadLines(tracePath))
            {
                if (string.IsNullOrWhiteSpace(line))
                {
                    continue;
                }

                using var doc = JsonDocument.Parse(line);
                if (doc.RootElement.TryGetProperty("sid", out var sidProp))
                {
                    runtimeHits.Add(sidProp.GetString()!);
                }
            }
        }

        return new ReachabilityRecomputeRequest
        {
            Subject = new ReachabilitySubject
            {
                ScanId = $"{Path.GetFileName(casePath)}:{variant}",
                Component = Path.GetFileName(casePath),
                Version = variant
            },
            EntryPoints = entryPoints,
            Targets = targets,
            RuntimeHits = runtimeHits,
            BlockedEdges = blockedEdges
        };
    }

    private static async Task<CallgraphDocument> LoadCallgraphAsync(string caseId, string variant, string variantPath)
    {
        var parser = new SimpleJsonCallgraphParser("fixture");
        var nodes = new Dictionary<string, CallgraphNode>(StringComparer.Ordinal);
        var edges = new List<CallgraphEdge>();

        foreach (var fileName in new[] { "callgraph.static.json", "callgraph.framework.json" })
        {
            var path = Path.Combine(variantPath, fileName);
            if (!File.Exists(path))
            {
                continue;
            }

            await using var stream = File.OpenRead(path);
            var result = await parser.ParseAsync(stream, CancellationToken.None);
            foreach (var node in result.Nodes)
            {
                nodes[node.Id] = node;
            }

            edges.AddRange(result.Edges);
        }

        return new CallgraphDocument
        {
            Id = ObjectId.GenerateNewId().ToString(),
            Language = "fixture",
            Component = caseId,
            Version = variant,
            Nodes = nodes.Values.ToList(),
            Edges = edges,
            Artifact = new CallgraphArtifactMetadata
            {
                Path = $"cas://fixtures/{caseId}/{variant}",
                Hash = "stub",
                ContentType = "application/json",
                Length = 0
            }
        };
    }

    private sealed class InMemoryCallgraphRepository : ICallgraphRepository
    {
        private readonly Dictionary<string, CallgraphDocument> storage;

        public InMemoryCallgraphRepository(CallgraphDocument document)
        {
            storage = new Dictionary<string, CallgraphDocument>(StringComparer.Ordinal)
            {
                [document.Id] = document
            };
        }

        public Task<CallgraphDocument> UpsertAsync(CallgraphDocument document, CancellationToken cancellationToken)
        {
            storage[document.Id] = document;
            return Task.FromResult(document);
        }

        public Task<CallgraphDocument?> GetByIdAsync(string id, CancellationToken cancellationToken)
        {
            storage.TryGetValue(id, out var document);
            return Task.FromResult(document);
        }
    }

    private sealed class InMemoryReachabilityFactRepository : IReachabilityFactRepository
    {
        private readonly Dictionary<string, ReachabilityFactDocument> storage = new(StringComparer.Ordinal);

        public Task<ReachabilityFactDocument?> GetBySubjectAsync(string subjectKey, CancellationToken cancellationToken)
        {
            storage.TryGetValue(subjectKey, out var document);
            return Task.FromResult(document);
        }

        public Task<ReachabilityFactDocument> UpsertAsync(ReachabilityFactDocument document, CancellationToken cancellationToken)
        {
            storage[document.SubjectKey] = document;
            return Task.FromResult(document);
        }
    }
    private static string LocateRepoRoot()
    {
        var current = new DirectoryInfo(AppContext.BaseDirectory);
        while (current != null)
        {
            if (File.Exists(Path.Combine(current.FullName, "Directory.Build.props")))
            {
                return current.FullName;
            }

            current = current.Parent;
        }

        throw new InvalidOperationException("Cannot locate repository root (missing Directory.Build.props).");
    }
}