86 lines
3.0 KiB
C#
86 lines
3.0 KiB
C#
using System.Text.Json;
|
|
using FluentAssertions;
|
|
using Xunit;
|
|
|
|
namespace StellaOps.Reachability.FixtureTests;
|
|
|
|
public class ReachbenchEvaluationHarnessTests
|
|
{
|
|
private static readonly string RepoRoot = LocateRepoRoot();
|
|
private static readonly string CasesRoot = Path.Combine(
|
|
RepoRoot,
|
|
"tests",
|
|
"reachability",
|
|
"fixtures",
|
|
"reachbench-2025-expanded",
|
|
"cases");
|
|
|
|
public static IEnumerable<object[]> CaseIds()
|
|
{
|
|
return Directory.EnumerateDirectories(CasesRoot)
|
|
.OrderBy(path => path, StringComparer.Ordinal)
|
|
.Select(path => new object[] { Path.GetFileName(path)! });
|
|
}
|
|
|
|
[Theory]
|
|
[MemberData(nameof(CaseIds))]
|
|
public void GroundTruthStatusesMatchVariantIntent(string caseId)
|
|
{
|
|
var caseJsonPath = Path.Combine(CasesRoot, caseId, "case.json");
|
|
File.Exists(caseJsonPath).Should().BeTrue();
|
|
|
|
using var caseDoc = JsonDocument.Parse(File.ReadAllBytes(caseJsonPath));
|
|
var groundTruth = caseDoc.RootElement.GetProperty("ground_truth");
|
|
|
|
groundTruth.GetProperty("reachable_variant")
|
|
.GetProperty("status")
|
|
.GetString()
|
|
.Should()
|
|
.Be("affected", $"{caseId} reachable variant should be marked affected for evaluation harness");
|
|
|
|
groundTruth.GetProperty("unreachable_variant")
|
|
.GetProperty("status")
|
|
.GetString()
|
|
.Should()
|
|
.Be("not_affected", $"{caseId} unreachable variant should be marked not_affected for evaluation harness");
|
|
}
|
|
|
|
[Theory]
|
|
[MemberData(nameof(CaseIds))]
|
|
public void TruthGraphsAlignWithExpectedReachability(string caseId)
|
|
{
|
|
var reachablePaths = CountTruthPaths(caseId, "reachable");
|
|
reachablePaths.Should().BeGreaterThan(0, $"{caseId} reachable variant should expose at least one execution path");
|
|
|
|
var unreachablePaths = CountTruthPaths(caseId, "unreachable");
|
|
unreachablePaths.Should().Be(0, $"{caseId} unreachable variant should have no execution paths");
|
|
}
|
|
|
|
private static int CountTruthPaths(string caseId, string variant)
|
|
{
|
|
var truthPath = Path.Combine(CasesRoot, caseId, "images", variant, "reachgraph.truth.json");
|
|
File.Exists(truthPath).Should().BeTrue();
|
|
|
|
using var truthDoc = JsonDocument.Parse(File.ReadAllBytes(truthPath));
|
|
var paths = truthDoc.RootElement.GetProperty("paths");
|
|
paths.ValueKind.Should().Be(JsonValueKind.Array, $"{caseId}:{variant} should list truth paths as an array");
|
|
return paths.GetArrayLength();
|
|
}
|
|
|
|
private static string LocateRepoRoot()
|
|
{
|
|
var current = new DirectoryInfo(AppContext.BaseDirectory);
|
|
while (current != null)
|
|
{
|
|
if (File.Exists(Path.Combine(current.FullName, "Directory.Build.props")))
|
|
{
|
|
return current.FullName;
|
|
}
|
|
|
|
current = current.Parent;
|
|
}
|
|
|
|
throw new InvalidOperationException("Cannot locate repository root (missing Directory.Build.props).");
|
|
}
|
|
}
|