using System.Text.Json; using FluentAssertions; using Xunit; namespace StellaOps.Reachability.FixtureTests; public class ReachbenchFixtureTests { private static readonly string RepoRoot = LocateRepoRoot(); private static readonly string FixtureRoot = Path.Combine( RepoRoot, "tests", "reachability", "fixtures", "reachbench-2025-expanded"); private static readonly string CasesRoot = Path.Combine(FixtureRoot, "cases"); [Fact] public void IndexListsAllCases() { Directory.Exists(FixtureRoot).Should().BeTrue("reachbench fixtures should exist under tests/reachability/fixtures"); File.Exists(Path.Combine(FixtureRoot, "INDEX.json")).Should().BeTrue("the reachbench index must be present"); using var indexStream = File.OpenRead(Path.Combine(FixtureRoot, "INDEX.json")); using var document = JsonDocument.Parse(indexStream); var names = new List(); var found = false; JsonElement casesElement = default; foreach (var property in document.RootElement.EnumerateObject()) { names.Add(property.Name); if (property.NameEquals("cases")) { casesElement = property.Value; found = true; } } found.Should().BeTrue($"INDEX.json should contain 'cases'. Properties present: {string.Join(",", names)}"); casesElement.ValueKind.Should().Be(JsonValueKind.Array); casesElement.GetArrayLength().Should().BeGreaterOrEqualTo(20, "expanded pack should carry broad coverage"); foreach (var entry in casesElement.EnumerateArray()) { var id = entry.GetProperty("id").GetString(); id.Should().NotBeNullOrEmpty(); var rel = entry.TryGetProperty("path", out var relProp) ? relProp.GetString() : Path.Combine("cases", id!); rel.Should().NotBeNullOrEmpty(); var path = Path.Combine(FixtureRoot, rel!); Directory.Exists(path).Should().BeTrue($"case '{id}' folder '{rel}' should exist"); } } public static IEnumerable CaseVariantData() { foreach (var caseDir in Directory.EnumerateDirectories(CasesRoot)) { var caseId = Path.GetFileName(caseDir); yield return new object[] { caseId!, Path.Combine(caseDir, "images", "reachable") }; yield return new object[] { caseId!, Path.Combine(caseDir, "images", "unreachable") }; } } [Theory] [MemberData(nameof(CaseVariantData))] public void CaseVariantContainsExpectedArtifacts(string caseId, string variantPath) { Directory.Exists(variantPath).Should().BeTrue(); var requiredFiles = new[] { "manifest.json", "sbom.cdx.json", "sbom.spdx.json", "symbols.json", "callgraph.static.json", "callgraph.framework.json", "reachgraph.truth.json", "vex.openvex.json", "attestation.dsse.json" }; foreach (var file in requiredFiles) { File.Exists(Path.Combine(variantPath, file)).Should().BeTrue($"{caseId}:{Path.GetFileName(variantPath)} missing {file}"); } var truthPath = Path.Combine(variantPath, "reachgraph.truth.json"); using var truthStream = File.OpenRead(truthPath); using var truthDoc = JsonDocument.Parse(truthStream); truthDoc.RootElement.GetProperty("schema_version").GetString().Should().NotBeNullOrEmpty(); truthDoc.RootElement.GetProperty("paths").ValueKind.Should().Be(JsonValueKind.Array); } [Theory] [MemberData(nameof(CaseVariantData))] public void CaseGroundTruthMatchesVariants(string caseId, string variantPath) { var caseJsonPath = Path.Combine(Path.GetDirectoryName(Path.GetDirectoryName(variantPath))!, "case.json"); File.Exists(caseJsonPath).Should().BeTrue(); using var caseStream = File.OpenRead(caseJsonPath); using var caseDoc = JsonDocument.Parse(caseStream); var groundTruth = caseDoc.RootElement.GetProperty("ground_truth"); var variantKey = variantPath.EndsWith("reachable", StringComparison.OrdinalIgnoreCase) ? "reachable_variant" : "unreachable_variant"; var variant = groundTruth.GetProperty(variantKey); variant.GetProperty("status").GetString().Should().NotBeNullOrEmpty($"{caseId}:{variantKey} should set status"); variant.TryGetProperty("evidence", out var evidence).Should().BeTrue($"{caseId}:{variantKey} should define evidence"); evidence.TryGetProperty("paths", out var pathsProp).Should().BeTrue(); pathsProp.ValueKind.Should().Be(JsonValueKind.Array); } private static string LocateRepoRoot() { var current = new DirectoryInfo(AppContext.BaseDirectory); while (current != null) { if (File.Exists(Path.Combine(current.FullName, "Directory.Build.props"))) { return current.FullName; } current = current.Parent; } throw new InvalidOperationException("Cannot locate repository root (missing Directory.Build.props)."); } }