using System.Text.Json; using FluentAssertions; using Xunit; namespace StellaOps.Reachability.FixtureTests; public class ReachbenchEvaluationHarnessTests { private static readonly string RepoRoot = LocateRepoRoot(); private static readonly string CasesRoot = Path.Combine( RepoRoot, "tests", "reachability", "fixtures", "reachbench-2025-expanded", "cases"); public static IEnumerable CaseIds() { return Directory.EnumerateDirectories(CasesRoot) .OrderBy(path => path, StringComparer.Ordinal) .Select(path => new object[] { Path.GetFileName(path)! }); } [Theory] [MemberData(nameof(CaseIds))] public void GroundTruthStatusesMatchVariantIntent(string caseId) { var caseJsonPath = Path.Combine(CasesRoot, caseId, "case.json"); File.Exists(caseJsonPath).Should().BeTrue(); using var caseDoc = JsonDocument.Parse(File.ReadAllBytes(caseJsonPath)); var groundTruth = caseDoc.RootElement.GetProperty("ground_truth"); groundTruth.GetProperty("reachable_variant") .GetProperty("status") .GetString() .Should() .Be("affected", $"{caseId} reachable variant should be marked affected for evaluation harness"); groundTruth.GetProperty("unreachable_variant") .GetProperty("status") .GetString() .Should() .Be("not_affected", $"{caseId} unreachable variant should be marked not_affected for evaluation harness"); } [Theory] [MemberData(nameof(CaseIds))] public void TruthGraphsAlignWithExpectedReachability(string caseId) { var reachablePaths = CountTruthPaths(caseId, "reachable"); reachablePaths.Should().BeGreaterThan(0, $"{caseId} reachable variant should expose at least one execution path"); var unreachablePaths = CountTruthPaths(caseId, "unreachable"); unreachablePaths.Should().Be(0, $"{caseId} unreachable variant should have no execution paths"); } private static int CountTruthPaths(string caseId, string variant) { var truthPath = Path.Combine(CasesRoot, caseId, "images", variant, "reachgraph.truth.json"); File.Exists(truthPath).Should().BeTrue(); using var truthDoc = JsonDocument.Parse(File.ReadAllBytes(truthPath)); var paths = truthDoc.RootElement.GetProperty("paths"); paths.ValueKind.Should().Be(JsonValueKind.Array, $"{caseId}:{variant} should list truth paths as an array"); return paths.GetArrayLength(); } private static string LocateRepoRoot() { var current = new DirectoryInfo(AppContext.BaseDirectory); while (current != null) { if (File.Exists(Path.Combine(current.FullName, "Directory.Build.props"))) { return current.FullName; } current = current.Parent; } throw new InvalidOperationException("Cannot locate repository root (missing Directory.Build.props)."); } }