git.stella-ops.org/tests/reachability/StellaOps.Reachability.FixtureTests/PatchOracleHarnessTests.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using FluentAssertions;
using StellaOps.Reachability.FixtureTests.PatchOracle;
using StellaOps.Scanner.Reachability;
using Xunit;

namespace StellaOps.Reachability.FixtureTests;

/// <summary>
/// Tests for the patch-oracle harness infrastructure.
/// Validates that the oracle comparison logic correctly identifies missing and forbidden elements.
/// </summary>
public class PatchOracleHarnessTests
{
    private static readonly string RepoRoot = ReachbenchFixtureTests.LocateRepoRoot();
    private static readonly string PatchOracleRoot = Path.Combine(
        RepoRoot, "tests", "reachability", "fixtures", "patch-oracles");

    #region Oracle Loading Tests

    [Fact]
    public void Loader_IndexExists()
    {
        var loader = new PatchOracleLoader(PatchOracleRoot);
        loader.IndexExists().Should().BeTrue("patch-oracle INDEX.json should exist");
    }

    [Fact]
    public void Loader_IndexLoadsSuccessfully()
    {
        var loader = new PatchOracleLoader(PatchOracleRoot);
        var index = loader.LoadIndex();

        index.Should().NotBeNull();
        index.Version.Should().Be("1.0");
        index.Schema.Should().Be("patch-oracle/v1");
        index.Oracles.Should().NotBeEmpty("should have at least one oracle defined");
    }

    [Fact]
    public void Loader_AllOraclesLoadSuccessfully()
    {
        var loader = new PatchOracleLoader(PatchOracleRoot);
        var oracles = loader.LoadAllOracles().ToList();

        oracles.Should().NotBeEmpty();
        foreach (var oracle in oracles)
        {
            oracle.SchemaVersion.Should().Be("patch-oracle/v1");
            oracle.Id.Should().NotBeNullOrEmpty();
            oracle.CaseRef.Should().NotBeNullOrEmpty();
            oracle.Variant.Should().BeOneOf("reachable", "unreachable");
        }
    }

    [Fact]
    public void Loader_LoadOracleById()
    {
        var loader = new PatchOracleLoader(PatchOracleRoot);
        var oracle = loader.LoadOracle("curl-CVE-2023-38545-socks5-heap-reachable");

        oracle.Should().NotBeNull();
        oracle.Id.Should().Be("curl-CVE-2023-38545-socks5-heap-reachable");
        oracle.CaseRef.Should().Be("curl-CVE-2023-38545-socks5-heap");
        oracle.Variant.Should().Be("reachable");
    }

    #endregion

    #region Comparer Tests - Pass Cases

    [Fact]
    public void Comparer_PassesWhenAllExpectedElementsPresent()
    {
        var oracle = new PatchOracleDefinition
        {
            Id = "test-pass",
            CaseRef = "test-case",
            Variant = "reachable",
            ExpectedFunctions = new[]
            {
                new ExpectedFunction { SymbolId = "sym://test#func1", Required = true },
                new ExpectedFunction { SymbolId = "sym://test#func2", Required = true }
            },
            ExpectedEdges = new[]
            {
                new ExpectedEdge { From = "sym://test#func1", To = "sym://test#func2", Required = true }
            }
        };

        var graph = new RichGraph(
            Nodes: new[]
            {
                new RichGraphNode("sym://test#func1", "sym://test#func1", null, null, "test", "function", null, null, null, null, null),
                new RichGraphNode("sym://test#func2", "sym://test#func2", null, null, "test", "function", null, null, null, null, null)
            },
            Edges: new[]
            {
                new RichGraphEdge("sym://test#func1", "sym://test#func2", "call", null, null, null, 0.9, null)
            },
            Roots: Array.Empty<RichGraphRoot>(),
            Analyzer: new RichGraphAnalyzer("test", "1.0", null)
        );

        var comparer = new PatchOracleComparer(oracle);
        var result = comparer.Compare(graph);

        result.Success.Should().BeTrue();
        result.Violations.Should().BeEmpty();
    }

    [Fact]
    public void Comparer_PassesWithWildcardPatterns()
    {
        var oracle = new PatchOracleDefinition
        {
            Id = "test-wildcard",
            CaseRef = "test-case",
            Variant = "reachable",
            ExpectedFunctions = new[]
            {
                new ExpectedFunction { SymbolId = "sym://test#*", Required = true }
            }
        };

        var graph = new RichGraph(
            Nodes: new[]
            {
                new RichGraphNode("sym://test#anything", "sym://test#anything", null, null, "test", "function", null, null, null, null, null)
            },
            Edges: Array.Empty<RichGraphEdge>(),
            Roots: Array.Empty<RichGraphRoot>(),
            Analyzer: new RichGraphAnalyzer("test", "1.0", null)
        );

        var comparer = new PatchOracleComparer(oracle);
        var result = comparer.Compare(graph);

        result.Success.Should().BeTrue();
    }

    #endregion

    #region Comparer Tests - Fail Cases

    [Fact]
    public void Comparer_FailsWhenExpectedFunctionMissing()
    {
        var oracle = new PatchOracleDefinition
        {
            Id = "test-missing-func",
            CaseRef = "test-case",
            Variant = "reachable",
            ExpectedFunctions = new[]
            {
                new ExpectedFunction { SymbolId = "sym://test#missing", Required = true, Reason = "This function is critical" }
            }
        };

        var graph = new RichGraph(
            Nodes: Array.Empty<RichGraphNode>(),
            Edges: Array.Empty<RichGraphEdge>(),
            Roots: Array.Empty<RichGraphRoot>(),
            Analyzer: new RichGraphAnalyzer("test", "1.0", null)
        );

        var comparer = new PatchOracleComparer(oracle);
        var result = comparer.Compare(graph);

        result.Success.Should().BeFalse();
        result.Violations.Should().HaveCount(1);
        result.Violations[0].Type.Should().Be(ViolationType.MissingFunction);
        result.Violations[0].From.Should().Be("sym://test#missing");
        result.Summary.MissingFunctions.Should().Be(1);
    }

    [Fact]
    public void Comparer_FailsWhenExpectedEdgeMissing()
    {
        var oracle = new PatchOracleDefinition
        {
            Id = "test-missing-edge",
            CaseRef = "test-case",
            Variant = "reachable",
            ExpectedEdges = new[]
            {
                new ExpectedEdge { From = "sym://a", To = "sym://b", Required = true }
            }
        };

        var graph = new RichGraph(
            Nodes: new[]
            {
                new RichGraphNode("sym://a", "sym://a", null, null, "test", "function", null, null, null, null, null),
                new RichGraphNode("sym://b", "sym://b", null, null, "test", "function", null, null, null, null, null)
            },
            Edges: Array.Empty<RichGraphEdge>(),
            Roots: Array.Empty<RichGraphRoot>(),
            Analyzer: new RichGraphAnalyzer("test", "1.0", null)
        );

        var comparer = new PatchOracleComparer(oracle);
        var result = comparer.Compare(graph);

        result.Success.Should().BeFalse();
        result.Violations.Should().HaveCount(1);
        result.Violations[0].Type.Should().Be(ViolationType.MissingEdge);
        result.Summary.MissingEdges.Should().Be(1);
    }

    [Fact]
    public void Comparer_FailsWhenExpectedRootMissing()
    {
        var oracle = new PatchOracleDefinition
        {
            Id = "test-missing-root",
            CaseRef = "test-case",
            Variant = "reachable",
            ExpectedRoots = new[]
            {
                new ExpectedRoot { Id = "sym://root#main", Phase = "main", Required = true }
            }
        };

        var graph = new RichGraph(
            Nodes: Array.Empty<RichGraphNode>(),
            Edges: Array.Empty<RichGraphEdge>(),
            Roots: Array.Empty<RichGraphRoot>(),
            Analyzer: new RichGraphAnalyzer("test", "1.0", null)
        );

        var comparer = new PatchOracleComparer(oracle);
        var result = comparer.Compare(graph);

        result.Success.Should().BeFalse();
        result.Violations.Should().HaveCount(1);
        result.Violations[0].Type.Should().Be(ViolationType.MissingRoot);
        result.Summary.MissingRoots.Should().Be(1);
    }

    [Fact]
    public void Comparer_FailsWhenForbiddenFunctionPresent()
    {
        var oracle = new PatchOracleDefinition
        {
            Id = "test-forbidden-func",
            CaseRef = "test-case",
            Variant = "unreachable",
            ForbiddenFunctions = new[]
            {
                new ExpectedFunction { SymbolId = "sym://dangerous#sink", Reason = "Should not be reachable" }
            }
        };

        var graph = new RichGraph(
            Nodes: new[]
            {
                new RichGraphNode("sym://dangerous#sink", "sym://dangerous#sink", null, null, "test", "function", null, null, null, null, null)
            },
            Edges: Array.Empty<RichGraphEdge>(),
            Roots: Array.Empty<RichGraphRoot>(),
            Analyzer: new RichGraphAnalyzer("test", "1.0", null)
        );

        var comparer = new PatchOracleComparer(oracle);
        var result = comparer.Compare(graph);

        result.Success.Should().BeFalse();
        result.Violations.Should().HaveCount(1);
        result.Violations[0].Type.Should().Be(ViolationType.ForbiddenFunctionPresent);
        result.Summary.ForbiddenFunctionsPresent.Should().Be(1);
    }

    [Fact]
    public void Comparer_FailsWhenForbiddenEdgePresent()
    {
        var oracle = new PatchOracleDefinition
        {
            Id = "test-forbidden-edge",
            CaseRef = "test-case",
            Variant = "unreachable",
            ForbiddenEdges = new[]
            {
                new ExpectedEdge { From = "sym://entry", To = "sym://sink", Reason = "Path should be blocked" }
            }
        };

        var graph = new RichGraph(
            Nodes: Array.Empty<RichGraphNode>(),
            Edges: new[]
            {
                new RichGraphEdge("sym://entry", "sym://sink", "call", null, null, null, 1.0, null)
            },
            Roots: Array.Empty<RichGraphRoot>(),
            Analyzer: new RichGraphAnalyzer("test", "1.0", null)
        );

        var comparer = new PatchOracleComparer(oracle);
        var result = comparer.Compare(graph);

        result.Success.Should().BeFalse();
        result.Violations.Should().HaveCount(1);
        result.Violations[0].Type.Should().Be(ViolationType.ForbiddenEdgePresent);
        result.Summary.ForbiddenEdgesPresent.Should().Be(1);
    }

    #endregion

    #region Confidence Threshold Tests

    [Fact]
    public void Comparer_RespectsMinConfidenceThreshold()
    {
        var oracle = new PatchOracleDefinition
        {
            Id = "test-confidence",
            CaseRef = "test-case",
            Variant = "reachable",
            MinConfidence = 0.8,
            ExpectedEdges = new[]
            {
                new ExpectedEdge { From = "sym://a", To = "sym://b", Required = true }
            }
        };

        var lowConfidenceGraph = new RichGraph(
            Nodes: Array.Empty<RichGraphNode>(),
            Edges: new[]
            {
                new RichGraphEdge("sym://a", "sym://b", "call", null, null, null, 0.5, null)
            },
            Roots: Array.Empty<RichGraphRoot>(),
            Analyzer: new RichGraphAnalyzer("test", "1.0", null)
        );

        var comparer = new PatchOracleComparer(oracle);
        var result = comparer.Compare(lowConfidenceGraph);

        result.Success.Should().BeFalse("edge confidence 0.5 is below threshold 0.8");
        result.Summary.MissingEdges.Should().Be(1);
    }

    [Fact]
    public void Comparer_EdgeSpecificConfidenceOverridesDefault()
    {
        var oracle = new PatchOracleDefinition
        {
            Id = "test-edge-confidence",
            CaseRef = "test-case",
            Variant = "reachable",
            MinConfidence = 0.8,
            ExpectedEdges = new[]
            {
                new ExpectedEdge { From = "sym://a", To = "sym://b", MinConfidence = 0.3, Required = true }
            }
        };

        var lowConfidenceGraph = new RichGraph(
            Nodes: Array.Empty<RichGraphNode>(),
            Edges: new[]
            {
                new RichGraphEdge("sym://a", "sym://b", "call", null, null, null, 0.5, null)
            },
            Roots: Array.Empty<RichGraphRoot>(),
            Analyzer: new RichGraphAnalyzer("test", "1.0", null)
        );

        var comparer = new PatchOracleComparer(oracle);
        var result = comparer.Compare(lowConfidenceGraph);

        result.Success.Should().BeTrue("edge-specific threshold 0.3 allows confidence 0.5");
    }

    #endregion

    #region Strict Mode Tests

    [Fact]
    public void Comparer_StrictModeRejectsUnexpectedNodes()
    {
        var oracle = new PatchOracleDefinition
        {
            Id = "test-strict",
            CaseRef = "test-case",
            Variant = "reachable",
            StrictMode = true,
            ExpectedFunctions = new[]
            {
                new ExpectedFunction { SymbolId = "sym://expected", Required = true }
            }
        };

        var graph = new RichGraph(
            Nodes: new[]
            {
                new RichGraphNode("sym://expected", "sym://expected", null, null, "test", "function", null, null, null, null, null),
                new RichGraphNode("sym://unexpected", "sym://unexpected", null, null, "test", "function", null, null, null, null, null)
            },
            Edges: Array.Empty<RichGraphEdge>(),
            Roots: Array.Empty<RichGraphRoot>(),
            Analyzer: new RichGraphAnalyzer("test", "1.0", null)
        );

        var comparer = new PatchOracleComparer(oracle);
        var result = comparer.Compare(graph);

        result.Success.Should().BeFalse();
        result.Violations.Should().Contain(v => v.Type == ViolationType.UnexpectedFunction);
        result.Summary.UnexpectedFunctions.Should().Be(1);
    }

    #endregion

    #region Report Generation Tests

    [Fact]
    public void Result_GeneratesReadableReport()
    {
        var oracle = new PatchOracleDefinition
        {
            Id = "test-report",
            CaseRef = "test-case",
            Variant = "reachable",
            ExpectedFunctions = new[]
            {
                new ExpectedFunction { SymbolId = "sym://missing", Required = true, Reason = "Critical sink" }
            }
        };

        var graph = new RichGraph(
            Nodes: new[]
            {
                new RichGraphNode("sym://other", "sym://other", null, null, "test", "function", null, null, null, null, null)
            },
            Edges: Array.Empty<RichGraphEdge>(),
            Roots: Array.Empty<RichGraphRoot>(),
            Analyzer: new RichGraphAnalyzer("test", "1.0", null)
        );

        var comparer = new PatchOracleComparer(oracle);
        var result = comparer.Compare(graph);

        var report = result.ToReport();

        report.Should().Contain("FAIL");
        report.Should().Contain("test-report");
        report.Should().Contain("MissingFunction");
        report.Should().Contain("sym://missing");
    }

    #endregion

    #region Integration with Fixture Data

    public static IEnumerable<object[]> AllOracleData()
    {
        var loader = new PatchOracleLoader(PatchOracleRoot);
        if (!loader.IndexExists())
        {
            yield break;
        }

        foreach (var entry in loader.EnumerateOracles())
        {
            yield return new object[] { entry.Id, entry.CaseRef, entry.Variant };
        }
    }

    [Theory]
    [MemberData(nameof(AllOracleData))]
    public void AllOracles_HaveValidStructure(string oracleId, string caseRef, string variant)
    {
        var loader = new PatchOracleLoader(PatchOracleRoot);
        var oracle = loader.LoadOracle(oracleId);

        oracle.Id.Should().Be(oracleId);
        oracle.CaseRef.Should().Be(caseRef);
        oracle.Variant.Should().Be(variant);
        oracle.SchemaVersion.Should().Be("patch-oracle/v1");

        // At least one expectation should be defined
        var hasExpectations = oracle.ExpectedFunctions.Count > 0
            || oracle.ExpectedEdges.Count > 0
            || oracle.ExpectedRoots.Count > 0
            || oracle.ForbiddenFunctions.Count > 0
            || oracle.ForbiddenEdges.Count > 0;
        hasExpectations.Should().BeTrue($"Oracle '{oracleId}' should define at least one expectation");
    }

    #endregion
}