using System.Text.Json; using FluentAssertions; using Xunit; using System.Security.Cryptography; using System.Linq; namespace StellaOps.Reachability.FixtureTests; public class ReachbenchFixtureTests { private static readonly string RepoRoot = LocateRepoRoot(); private static readonly string FixtureRoot = Path.Combine( RepoRoot, "tests", "reachability", "fixtures", "reachbench-2025-expanded"); private static readonly string CasesRoot = Path.Combine(FixtureRoot, "cases"); [Fact] public void IndexListsAllCases() { Directory.Exists(FixtureRoot).Should().BeTrue("reachbench fixtures should exist under tests/reachability/fixtures"); File.Exists(Path.Combine(FixtureRoot, "INDEX.json")).Should().BeTrue("the reachbench index must be present"); using var indexStream = File.OpenRead(Path.Combine(FixtureRoot, "INDEX.json")); using var document = JsonDocument.Parse(indexStream); var names = new List(); var found = false; JsonElement casesElement = default; foreach (var property in document.RootElement.EnumerateObject()) { names.Add(property.Name); if (property.NameEquals("cases")) { casesElement = property.Value; found = true; } } found.Should().BeTrue($"INDEX.json should contain 'cases'. Properties present: {string.Join(",", names)}"); casesElement.ValueKind.Should().Be(JsonValueKind.Array); casesElement.GetArrayLength().Should().BeGreaterOrEqualTo(20, "expanded pack should carry broad coverage"); foreach (var entry in casesElement.EnumerateArray()) { var id = entry.GetProperty("id").GetString(); id.Should().NotBeNullOrEmpty(); var rel = entry.TryGetProperty("path", out var relProp) ? relProp.GetString() : Path.Combine("cases", id!); rel.Should().NotBeNullOrEmpty(); var path = Path.Combine(FixtureRoot, rel!); Directory.Exists(path).Should().BeTrue($"case '{id}' folder '{rel}' should exist"); } } public static IEnumerable CaseVariantData() { foreach (var caseDir in Directory.EnumerateDirectories(CasesRoot)) { var caseId = Path.GetFileName(caseDir); yield return new object[] { caseId!, Path.Combine(caseDir, "images", "reachable") }; yield return new object[] { caseId!, Path.Combine(caseDir, "images", "unreachable") }; } } [Theory] [MemberData(nameof(CaseVariantData))] public void CaseVariantContainsExpectedArtifacts(string caseId, string variantPath) { Directory.Exists(variantPath).Should().BeTrue(); var requiredFiles = new[] { "manifest.json", "sbom.cdx.json", "sbom.spdx.json", "symbols.json", "callgraph.static.json", "callgraph.framework.json", "reachgraph.truth.json", "vex.openvex.json", "attestation.dsse.json" }; foreach (var file in requiredFiles) { File.Exists(Path.Combine(variantPath, file)).Should().BeTrue($"{caseId}:{Path.GetFileName(variantPath)} missing {file}"); } var truthPath = Path.Combine(variantPath, "reachgraph.truth.json"); using var truthStream = File.OpenRead(truthPath); using var truthDoc = JsonDocument.Parse(truthStream); truthDoc.RootElement.GetProperty("schema_version").GetString().Should().NotBeNullOrEmpty(); truthDoc.RootElement.GetProperty("paths").ValueKind.Should().Be(JsonValueKind.Array); VerifyManifestHashes(caseId, variantPath, requiredFiles); } [Theory] [MemberData(nameof(CaseVariantData))] public void CaseGroundTruthMatchesVariants(string caseId, string variantPath) { var caseJsonPath = Path.Combine(Path.GetDirectoryName(Path.GetDirectoryName(variantPath))!, "case.json"); File.Exists(caseJsonPath).Should().BeTrue(); using var caseStream = File.OpenRead(caseJsonPath); using var caseDoc = JsonDocument.Parse(caseStream); var groundTruth = caseDoc.RootElement.GetProperty("ground_truth"); var variantKey = variantPath.EndsWith("reachable", StringComparison.OrdinalIgnoreCase) ? "reachable_variant" : "unreachable_variant"; var variant = groundTruth.GetProperty(variantKey); variant.GetProperty("status").GetString().Should().NotBeNullOrEmpty($"{caseId}:{variantKey} should set status"); variant.TryGetProperty("evidence", out var evidence).Should().BeTrue($"{caseId}:{variantKey} should define evidence"); evidence.TryGetProperty("paths", out var pathsProp).Should().BeTrue(); pathsProp.ValueKind.Should().Be(JsonValueKind.Array); var truthPath = Path.Combine(variantPath, "reachgraph.truth.json"); using var truthStream = File.OpenRead(truthPath); using var truthDoc = JsonDocument.Parse(truthStream); var paths = truthDoc.RootElement.GetProperty("paths"); paths.ValueKind.Should().Be(JsonValueKind.Array); } private static string LocateRepoRoot() { var current = new DirectoryInfo(AppContext.BaseDirectory); while (current != null) { if (File.Exists(Path.Combine(current.FullName, "Directory.Build.props"))) { return current.FullName; } current = current.Parent; } throw new InvalidOperationException("Cannot locate repository root (missing Directory.Build.props)."); } private static void VerifyManifestHashes(string caseId, string variantPath, IEnumerable requiredFiles) { var manifestPath = Path.Combine(variantPath, "manifest.json"); using var manifestStream = File.OpenRead(manifestPath); using var manifestDoc = JsonDocument.Parse(manifestStream); var files = manifestDoc.RootElement.GetProperty("files"); foreach (var file in requiredFiles.Where(f => f != "manifest.json")) { files.TryGetProperty(file, out var hashProp).Should().BeTrue($"{caseId}:{variantPath} manifest missing hash for {file}"); var expectedHash = hashProp.GetString(); expectedHash.Should().NotBeNullOrEmpty($"{caseId}:{variantPath} hash missing for {file}"); var path = Path.Combine(variantPath, file); var actualHash = BitConverter.ToString(SHA256.HashData(File.ReadAllBytes(path))).Replace("-", "").ToLowerInvariant(); actualHash.Should().Be(expectedHash, $"{caseId}:{variantPath} hash mismatch for {file}"); } } }