git.stella-ops.org/tests/reachability/StellaOps.Signals.Reachability.Tests/ReachabilityScoringTests.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FluentAssertions;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using StellaOps.Signals.Models;
using StellaOps.Signals.Options;
using StellaOps.Signals.Parsing;
using StellaOps.Signals.Persistence;
using StellaOps.Signals.Services;
using Xunit;

namespace StellaOps.Signals.Reachability.Tests;

public sealed class ReachabilityScoringTests
{
    private static readonly string RepoRoot = LocateRepoRoot();
    private static readonly string FixtureRoot = Path.Combine(RepoRoot, "tests", "reachability", "fixtures", "reachbench-2025-expanded", "cases");

    private static readonly (string CaseId, string Variant)[] SampleCases =
    {
        ("java-log4j-CVE-2021-44228-log4shell", "reachable"),
        ("java-log4j-CVE-2021-44228-log4shell", "unreachable"),
        ("redis-CVE-2022-0543-lua-sandbox-escape", "reachable")
    };

    public static IEnumerable<object[]> CaseVariants()
    {
        foreach (var (caseId, variant) in SampleCases)
        {
            var path = Path.Combine(FixtureRoot, caseId, "images", variant);
            if (Directory.Exists(path))
            {
                yield return new object[] { caseId, variant };
            }
        }
    }

    [Theory]
    [MemberData(nameof(CaseVariants))]
    public async Task RecomputedFactsMatchTruthFixtures(string caseId, string variant)
    {
        var casePath = Path.Combine(FixtureRoot, caseId);
        var caseJson = JsonDocument.Parse(File.ReadAllText(Path.Combine(casePath, "case.json"))).RootElement;
        var reachablePathsNode = caseJson
            .GetProperty("ground_truth")
            .GetProperty("reachable_variant")
            .GetProperty("evidence")
            .GetProperty("paths");

        var paths = reachablePathsNode.EnumerateArray()
            .Select(path => path.EnumerateArray().Select(x => x.GetString()!).Where(x => !string.IsNullOrWhiteSpace(x)).ToList())
            .Where(path => path.Count > 0)
            .ToList();

        var entryPoints = paths
            .Select(path => path[0])
            .Where(p => !string.IsNullOrWhiteSpace(p))
            .Distinct(StringComparer.Ordinal)
            .ToList();

        var sinks = paths
            .Select(path => path[^1])
            .Where(p => !string.IsNullOrWhiteSpace(p))
            .Distinct(StringComparer.Ordinal)
            .ToList();

        var callgraph = BuildCallgraphFromPaths(caseId, paths);
        var callgraphRepo = new InMemoryCallgraphRepository(callgraph);
        var factRepo = new InMemoryReachabilityFactRepository();
        var options = new SignalsOptions();
        var cache = new InMemoryReachabilityCache();
        var eventsPublisher = new NullEventsPublisher();
        var unknowns = new InMemoryUnknownsRepository();
        var scoringService = new ReachabilityScoringService(
            callgraphRepo,
            factRepo,
            TimeProvider.System,
            Microsoft.Extensions.Options.Options.Create(options),
            cache,
            unknowns,
            eventsPublisher,
            NullLogger<ReachabilityScoringService>.Instance);

        var request = BuildRequest(casePath, variant, sinks, entryPoints);
        request.CallgraphId = callgraph.Id;

        var fact = await scoringService.RecomputeAsync(request, CancellationToken.None);
        fact.States.Should().HaveCount(sinks.Count);

        var expectedReachable = variant == "reachable";
        foreach (var sink in sinks)
        {
            var state = fact.States.Single(s => s.Target == sink);
            state.Reachable.Should().Be(expectedReachable, $"{caseId}:{variant} expected reachable={expectedReachable}");
            if (expectedReachable)
            {
                state.Path.Should().NotBeEmpty();
                state.Evidence.RuntimeHits.Should().NotBeEmpty();
            }
            else
            {
                state.Path.Should().BeEmpty();
                state.Evidence.BlockedEdges.Should().NotBeNull();
            }
        }
    }

    private static ReachabilityRecomputeRequest BuildRequest(string casePath, string variant, List<string> targets, List<string> entryPoints)
    {
        var caseJson = JsonDocument.Parse(File.ReadAllText(Path.Combine(casePath, "case.json"))).RootElement;
        var variantKey = variant == "reachable" ? "reachable_variant" : "unreachable_variant";
        var variantNode = caseJson.GetProperty("ground_truth").GetProperty(variantKey);

        var blockedEdges = new List<ReachabilityBlockedEdge>();
        if (variantNode.TryGetProperty("evidence", out var evidence) && evidence.TryGetProperty("blocked_edges", out var blockedArray))
        {
            foreach (var item in blockedArray.EnumerateArray())
            {
                var parts = item.GetString()?.Split("->", StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);
                if (parts is { Length: 2 })
                {
                    blockedEdges.Add(new ReachabilityBlockedEdge { From = parts[0], To = parts[1] });
                }
            }
        }

        var runtimeHits = new List<string>();
        var tracePath = Path.Combine(casePath, "images", variant, "traces.runtime.jsonl");
        if (File.Exists(tracePath))
        {
            foreach (var line in File.ReadLines(tracePath))
            {
                if (string.IsNullOrWhiteSpace(line))
                {
                    continue;
                }

                using var doc = JsonDocument.Parse(line);
                if (doc.RootElement.TryGetProperty("sid", out var sidProp))
                {
                    runtimeHits.Add(sidProp.GetString()!);
                }
            }
        }

        return new ReachabilityRecomputeRequest
        {
            Subject = new ReachabilitySubject
            {
                ScanId = $"{Path.GetFileName(casePath)}:{variant}",
                Component = Path.GetFileName(casePath),
                Version = variant
            },
            EntryPoints = entryPoints,
            Targets = targets,
            RuntimeHits = runtimeHits,
            BlockedEdges = blockedEdges
        };
    }

    private static CallgraphDocument BuildCallgraphFromPaths(string caseId, IReadOnlyList<IReadOnlyList<string>> paths)
    {
        var nodes = new Dictionary<string, CallgraphNode>(StringComparer.Ordinal);
        var edges = new List<CallgraphEdge>();

        foreach (var path in paths)
        {
            if (path.Count == 0)
            {
                continue;
            }

            foreach (var nodeId in path)
            {
                if (!nodes.ContainsKey(nodeId))
                {
                    nodes[nodeId] = new CallgraphNode(nodeId, nodeId, "function", null, null, null);
                }
            }

            for (var i = 0; i < path.Count - 1; i++)
            {
                edges.Add(new CallgraphEdge(path[i], path[i + 1], "call"));
            }
        }

        return new CallgraphDocument
        {
            Id = caseId,
            Language = "fixture",
            Component = caseId,
            Version = "truth",
            Nodes = nodes.Values.OrderBy(n => n.Id, StringComparer.Ordinal).ToList(),
            Edges = edges
                .OrderBy(e => e.SourceId, StringComparer.Ordinal)
                .ThenBy(e => e.TargetId, StringComparer.Ordinal)
                .ToList(),
            Artifact = new CallgraphArtifactMetadata
            {
                Path = $"cas://fixtures/{caseId}",
                Hash = "stub",
                ContentType = "application/json",
                Length = 0
            }
        };
    }

    private sealed class InMemoryCallgraphRepository : ICallgraphRepository
    {
        private readonly Dictionary<string, CallgraphDocument> storage;

        public InMemoryCallgraphRepository(CallgraphDocument document)
        {
            storage = new Dictionary<string, CallgraphDocument>(StringComparer.Ordinal)
            {
                [document.Id] = document
            };
        }

        public Task<CallgraphDocument> UpsertAsync(CallgraphDocument document, CancellationToken cancellationToken)
        {
            storage[document.Id] = document;
            return Task.FromResult(document);
        }

        public Task<CallgraphDocument?> GetByIdAsync(string id, CancellationToken cancellationToken)
        {
            storage.TryGetValue(id, out var document);
            return Task.FromResult(document);
        }
    }

    private sealed class InMemoryReachabilityFactRepository : IReachabilityFactRepository
    {
        private readonly Dictionary<string, ReachabilityFactDocument> storage = new(StringComparer.Ordinal);

        public Task<ReachabilityFactDocument?> GetBySubjectAsync(string subjectKey, CancellationToken cancellationToken)
        {
            storage.TryGetValue(subjectKey, out var document);
            return Task.FromResult(document);
        }

        public Task<ReachabilityFactDocument> UpsertAsync(ReachabilityFactDocument document, CancellationToken cancellationToken)
        {
            storage[document.SubjectKey] = document;
            return Task.FromResult(document);
        }
    }

    private sealed class InMemoryReachabilityCache : IReachabilityCache
    {
        private readonly Dictionary<string, ReachabilityFactDocument> storage = new(StringComparer.Ordinal);

        public Task<ReachabilityFactDocument?> GetAsync(string subjectKey, CancellationToken cancellationToken)
        {
            storage.TryGetValue(subjectKey, out var doc);
            return Task.FromResult(doc);
        }

        public Task SetAsync(ReachabilityFactDocument document, CancellationToken cancellationToken)
        {
            storage[document.SubjectKey] = document;
            return Task.CompletedTask;
        }

        public Task InvalidateAsync(string subjectKey, CancellationToken cancellationToken)
        {
            storage.Remove(subjectKey);
            return Task.CompletedTask;
        }
    }

    private sealed class InMemoryUnknownsRepository : IUnknownsRepository
    {
        public Task UpsertAsync(string subjectKey, IEnumerable<UnknownSymbolDocument> items, CancellationToken cancellationToken) =>
            Task.CompletedTask;

        public Task<IReadOnlyList<UnknownSymbolDocument>> GetBySubjectAsync(string subjectKey, CancellationToken cancellationToken) =>
            Task.FromResult((IReadOnlyList<UnknownSymbolDocument>)Array.Empty<UnknownSymbolDocument>());

        public Task<int> CountBySubjectAsync(string subjectKey, CancellationToken cancellationToken) =>
            Task.FromResult(0);
    }

    private sealed class NullEventsPublisher : IEventsPublisher
    {
        public Task PublishFactUpdatedAsync(ReachabilityFactDocument fact, CancellationToken cancellationToken) => Task.CompletedTask;
    }

    private static string LocateRepoRoot()
    {
        var current = new DirectoryInfo(AppContext.BaseDirectory);
        while (current != null)
        {
            if (File.Exists(Path.Combine(current.FullName, "Directory.Build.props")))
            {
                return current.FullName;
            }

            current = current.Parent;
        }

        throw new InvalidOperationException("Cannot locate repository root (missing Directory.Build.props).");
    }
}