git.stella-ops.org/tests/reachability/StellaOps.Reachability.FixtureTests/ReachbenchFixtureTests.cs

using System.Text.Json;
using FluentAssertions;
using Xunit;
using System.Security.Cryptography;
using System.Linq;

namespace StellaOps.Reachability.FixtureTests;

public class ReachbenchFixtureTests
{
    private static readonly string RepoRoot = LocateRepoRoot();
    private static readonly string FixtureRoot = Path.Combine(
        RepoRoot, "tests", "reachability", "fixtures", "reachbench-2025-expanded");
    private static readonly string CasesRoot = Path.Combine(FixtureRoot, "cases");

    [Fact]
    public void IndexListsAllCases()
    {
        Directory.Exists(FixtureRoot).Should().BeTrue("reachbench fixtures should exist under tests/reachability/fixtures");
        File.Exists(Path.Combine(FixtureRoot, "INDEX.json")).Should().BeTrue("the reachbench index must be present");

        using var indexStream = File.OpenRead(Path.Combine(FixtureRoot, "INDEX.json"));
        using var document = JsonDocument.Parse(indexStream);
        var names = new List<string>();
        var found = false;
        JsonElement casesElement = default;
        foreach (var property in document.RootElement.EnumerateObject())
        {
            names.Add(property.Name);
            if (property.NameEquals("cases"))
            {
                casesElement = property.Value;
                found = true;
            }
        }

        found.Should().BeTrue($"INDEX.json should contain 'cases'. Properties present: {string.Join(",", names)}");
        casesElement.ValueKind.Should().Be(JsonValueKind.Array);
        casesElement.GetArrayLength().Should().BeGreaterOrEqualTo(20, "expanded pack should carry broad coverage");

        foreach (var entry in casesElement.EnumerateArray())
        {
            var id = entry.GetProperty("id").GetString();
            id.Should().NotBeNullOrEmpty();
            var rel = entry.TryGetProperty("path", out var relProp)
                ? relProp.GetString()
                : Path.Combine("cases", id!);
            rel.Should().NotBeNullOrEmpty();
            var path = Path.Combine(FixtureRoot, rel!);
            Directory.Exists(path).Should().BeTrue($"case '{id}' folder '{rel}' should exist");
        }
    }

    public static IEnumerable<object[]> CaseVariantData()
    {
        foreach (var caseDir in Directory.EnumerateDirectories(CasesRoot))
        {
            var caseId = Path.GetFileName(caseDir);
            yield return new object[] { caseId!, Path.Combine(caseDir, "images", "reachable") };
            yield return new object[] { caseId!, Path.Combine(caseDir, "images", "unreachable") };
        }
    }

    [Theory]
    [MemberData(nameof(CaseVariantData))]
    public void CaseVariantContainsExpectedArtifacts(string caseId, string variantPath)
    {
        Directory.Exists(variantPath).Should().BeTrue();

        var requiredFiles = new[]
        {
            "manifest.json",
            "sbom.cdx.json",
            "sbom.spdx.json",
            "symbols.json",
            "callgraph.static.json",
            "callgraph.framework.json",
            "reachgraph.truth.json",
            "vex.openvex.json",
            "attestation.dsse.json"
        };

        foreach (var file in requiredFiles)
        {
            File.Exists(Path.Combine(variantPath, file)).Should().BeTrue($"{caseId}:{Path.GetFileName(variantPath)} missing {file}");
        }

        var truthPath = Path.Combine(variantPath, "reachgraph.truth.json");
        using var truthStream = File.OpenRead(truthPath);
        using var truthDoc = JsonDocument.Parse(truthStream);
        truthDoc.RootElement.GetProperty("schema_version").GetString().Should().NotBeNullOrEmpty();
        truthDoc.RootElement.GetProperty("paths").ValueKind.Should().Be(JsonValueKind.Array);

        VerifyManifestHashes(caseId, variantPath, requiredFiles);
    }

    [Theory]
    [MemberData(nameof(CaseVariantData))]
    public void CaseGroundTruthMatchesVariants(string caseId, string variantPath)
    {
        var caseJsonPath = Path.Combine(Path.GetDirectoryName(Path.GetDirectoryName(variantPath))!, "case.json");
        File.Exists(caseJsonPath).Should().BeTrue();

        using var caseStream = File.OpenRead(caseJsonPath);
        using var caseDoc = JsonDocument.Parse(caseStream);
        var groundTruth = caseDoc.RootElement.GetProperty("ground_truth");
        var variantKey = variantPath.EndsWith("reachable", StringComparison.OrdinalIgnoreCase)
            ? "reachable_variant"
            : "unreachable_variant";

        var variant = groundTruth.GetProperty(variantKey);
        variant.GetProperty("status").GetString().Should().NotBeNullOrEmpty($"{caseId}:{variantKey} should set status");
        variant.TryGetProperty("evidence", out var evidence).Should().BeTrue($"{caseId}:{variantKey} should define evidence");
        evidence.TryGetProperty("paths", out var pathsProp).Should().BeTrue();
        pathsProp.ValueKind.Should().Be(JsonValueKind.Array);

        var truthPath = Path.Combine(variantPath, "reachgraph.truth.json");
        using var truthStream = File.OpenRead(truthPath);
        using var truthDoc = JsonDocument.Parse(truthStream);
        var paths = truthDoc.RootElement.GetProperty("paths");

        paths.ValueKind.Should().Be(JsonValueKind.Array);
    }

    private static string LocateRepoRoot()
    {
        var current = new DirectoryInfo(AppContext.BaseDirectory);
        while (current != null)
        {
            if (File.Exists(Path.Combine(current.FullName, "Directory.Build.props")))
            {
                return current.FullName;
            }

            current = current.Parent;
        }

        throw new InvalidOperationException("Cannot locate repository root (missing Directory.Build.props).");
    }

    private static void VerifyManifestHashes(string caseId, string variantPath, IEnumerable<string> requiredFiles)
    {
        var manifestPath = Path.Combine(variantPath, "manifest.json");
        using var manifestStream = File.OpenRead(manifestPath);
        using var manifestDoc = JsonDocument.Parse(manifestStream);
        var files = manifestDoc.RootElement.GetProperty("files");

        foreach (var file in requiredFiles.Where(f => f != "manifest.json"))
        {
            files.TryGetProperty(file, out var hashProp).Should().BeTrue($"{caseId}:{variantPath} manifest missing hash for {file}");
            var expectedHash = hashProp.GetString();
            expectedHash.Should().NotBeNullOrEmpty($"{caseId}:{variantPath} hash missing for {file}");

            var path = Path.Combine(variantPath, file);
            var actualHash = BitConverter.ToString(SHA256.HashData(File.ReadAllBytes(path))).Replace("-", "").ToLowerInvariant();
            actualHash.Should().Be(expectedHash, $"{caseId}:{variantPath} hash mismatch for {file}");
        }
    }
}