up

tests/reachability/StellaOps.Reachability.FixtureTests/PatchOracleHarnessTests.cs

@@ -0,0 +1,494 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using FluentAssertions;
+using StellaOps.Reachability.FixtureTests.PatchOracle;
+using StellaOps.Scanner.Reachability;
+using Xunit;
+
+namespace StellaOps.Reachability.FixtureTests;
+
+/// <summary>
+/// Tests for the patch-oracle harness infrastructure.
+/// Validates that the oracle comparison logic correctly identifies missing and forbidden elements.
+/// </summary>
+public class PatchOracleHarnessTests
+{
+    private static readonly string RepoRoot = ReachbenchFixtureTests.LocateRepoRoot();
+    private static readonly string PatchOracleRoot = Path.Combine(
+        RepoRoot, "tests", "reachability", "fixtures", "patch-oracles");
+
+    #region Oracle Loading Tests
+
+    [Fact]
+    public void Loader_IndexExists()
+    {
+        var loader = new PatchOracleLoader(PatchOracleRoot);
+        loader.IndexExists().Should().BeTrue("patch-oracle INDEX.json should exist");
+    }
+
+    [Fact]
+    public void Loader_IndexLoadsSuccessfully()
+    {
+        var loader = new PatchOracleLoader(PatchOracleRoot);
+        var index = loader.LoadIndex();
+
+        index.Should().NotBeNull();
+        index.Version.Should().Be("1.0");
+        index.Schema.Should().Be("patch-oracle/v1");
+        index.Oracles.Should().NotBeEmpty("should have at least one oracle defined");
+    }
+
+    [Fact]
+    public void Loader_AllOraclesLoadSuccessfully()
+    {
+        var loader = new PatchOracleLoader(PatchOracleRoot);
+        var oracles = loader.LoadAllOracles().ToList();
+
+        oracles.Should().NotBeEmpty();
+        foreach (var oracle in oracles)
+        {
+            oracle.SchemaVersion.Should().Be("patch-oracle/v1");
+            oracle.Id.Should().NotBeNullOrEmpty();
+            oracle.CaseRef.Should().NotBeNullOrEmpty();
+            oracle.Variant.Should().BeOneOf("reachable", "unreachable");
+        }
+    }
+
+    [Fact]
+    public void Loader_LoadOracleById()
+    {
+        var loader = new PatchOracleLoader(PatchOracleRoot);
+        var oracle = loader.LoadOracle("curl-CVE-2023-38545-socks5-heap-reachable");
+
+        oracle.Should().NotBeNull();
+        oracle.Id.Should().Be("curl-CVE-2023-38545-socks5-heap-reachable");
+        oracle.CaseRef.Should().Be("curl-CVE-2023-38545-socks5-heap");
+        oracle.Variant.Should().Be("reachable");
+    }
+
+    #endregion
+
+    #region Comparer Tests - Pass Cases
+
+    [Fact]
+    public void Comparer_PassesWhenAllExpectedElementsPresent()
+    {
+        var oracle = new PatchOracleDefinition
+        {
+            Id = "test-pass",
+            CaseRef = "test-case",
+            Variant = "reachable",
+            ExpectedFunctions = new[]
+            {
+                new ExpectedFunction { SymbolId = "sym://test#func1", Required = true },
+                new ExpectedFunction { SymbolId = "sym://test#func2", Required = true }
+            },
+            ExpectedEdges = new[]
+            {
+                new ExpectedEdge { From = "sym://test#func1", To = "sym://test#func2", Required = true }
+            }
+        };
+
+        var graph = new RichGraph(
+            Nodes: new[]
+            {
+                new RichGraphNode("sym://test#func1", "sym://test#func1", null, null, "test", "function", null, null, null, null, null),
+                new RichGraphNode("sym://test#func2", "sym://test#func2", null, null, "test", "function", null, null, null, null, null)
+            },
+            Edges: new[]
+            {
+                new RichGraphEdge("sym://test#func1", "sym://test#func2", "call", null, null, null, 0.9, null)
+            },
+            Roots: Array.Empty<RichGraphRoot>(),
+            Analyzer: new RichGraphAnalyzer("test", "1.0", null)
+        );
+
+        var comparer = new PatchOracleComparer(oracle);
+        var result = comparer.Compare(graph);
+
+        result.Success.Should().BeTrue();
+        result.Violations.Should().BeEmpty();
+    }
+
+    [Fact]
+    public void Comparer_PassesWithWildcardPatterns()
+    {
+        var oracle = new PatchOracleDefinition
+        {
+            Id = "test-wildcard",
+            CaseRef = "test-case",
+            Variant = "reachable",
+            ExpectedFunctions = new[]
+            {
+                new ExpectedFunction { SymbolId = "sym://test#*", Required = true }
+            }
+        };
+
+        var graph = new RichGraph(
+            Nodes: new[]
+            {
+                new RichGraphNode("sym://test#anything", "sym://test#anything", null, null, "test", "function", null, null, null, null, null)
+            },
+            Edges: Array.Empty<RichGraphEdge>(),
+            Roots: Array.Empty<RichGraphRoot>(),
+            Analyzer: new RichGraphAnalyzer("test", "1.0", null)
+        );
+
+        var comparer = new PatchOracleComparer(oracle);
+        var result = comparer.Compare(graph);
+
+        result.Success.Should().BeTrue();
+    }
+
+    #endregion
+
+    #region Comparer Tests - Fail Cases
+
+    [Fact]
+    public void Comparer_FailsWhenExpectedFunctionMissing()
+    {
+        var oracle = new PatchOracleDefinition
+        {
+            Id = "test-missing-func",
+            CaseRef = "test-case",
+            Variant = "reachable",
+            ExpectedFunctions = new[]
+            {
+                new ExpectedFunction { SymbolId = "sym://test#missing", Required = true, Reason = "This function is critical" }
+            }
+        };
+
+        var graph = new RichGraph(
+            Nodes: Array.Empty<RichGraphNode>(),
+            Edges: Array.Empty<RichGraphEdge>(),
+            Roots: Array.Empty<RichGraphRoot>(),
+            Analyzer: new RichGraphAnalyzer("test", "1.0", null)
+        );
+
+        var comparer = new PatchOracleComparer(oracle);
+        var result = comparer.Compare(graph);
+
+        result.Success.Should().BeFalse();
+        result.Violations.Should().HaveCount(1);
+        result.Violations[0].Type.Should().Be(ViolationType.MissingFunction);
+        result.Violations[0].From.Should().Be("sym://test#missing");
+        result.Summary.MissingFunctions.Should().Be(1);
+    }
+
+    [Fact]
+    public void Comparer_FailsWhenExpectedEdgeMissing()
+    {
+        var oracle = new PatchOracleDefinition
+        {
+            Id = "test-missing-edge",
+            CaseRef = "test-case",
+            Variant = "reachable",
+            ExpectedEdges = new[]
+            {
+                new ExpectedEdge { From = "sym://a", To = "sym://b", Required = true }
+            }
+        };
+
+        var graph = new RichGraph(
+            Nodes: new[]
+            {
+                new RichGraphNode("sym://a", "sym://a", null, null, "test", "function", null, null, null, null, null),
+                new RichGraphNode("sym://b", "sym://b", null, null, "test", "function", null, null, null, null, null)
+            },
+            Edges: Array.Empty<RichGraphEdge>(),
+            Roots: Array.Empty<RichGraphRoot>(),
+            Analyzer: new RichGraphAnalyzer("test", "1.0", null)
+        );
+
+        var comparer = new PatchOracleComparer(oracle);
+        var result = comparer.Compare(graph);
+
+        result.Success.Should().BeFalse();
+        result.Violations.Should().HaveCount(1);
+        result.Violations[0].Type.Should().Be(ViolationType.MissingEdge);
+        result.Summary.MissingEdges.Should().Be(1);
+    }
+
+    [Fact]
+    public void Comparer_FailsWhenExpectedRootMissing()
+    {
+        var oracle = new PatchOracleDefinition
+        {
+            Id = "test-missing-root",
+            CaseRef = "test-case",
+            Variant = "reachable",
+            ExpectedRoots = new[]
+            {
+                new ExpectedRoot { Id = "sym://root#main", Phase = "main", Required = true }
+            }
+        };
+
+        var graph = new RichGraph(
+            Nodes: Array.Empty<RichGraphNode>(),
+            Edges: Array.Empty<RichGraphEdge>(),
+            Roots: Array.Empty<RichGraphRoot>(),
+            Analyzer: new RichGraphAnalyzer("test", "1.0", null)
+        );
+
+        var comparer = new PatchOracleComparer(oracle);
+        var result = comparer.Compare(graph);
+
+        result.Success.Should().BeFalse();
+        result.Violations.Should().HaveCount(1);
+        result.Violations[0].Type.Should().Be(ViolationType.MissingRoot);
+        result.Summary.MissingRoots.Should().Be(1);
+    }
+
+    [Fact]
+    public void Comparer_FailsWhenForbiddenFunctionPresent()
+    {
+        var oracle = new PatchOracleDefinition
+        {
+            Id = "test-forbidden-func",
+            CaseRef = "test-case",
+            Variant = "unreachable",
+            ForbiddenFunctions = new[]
+            {
+                new ExpectedFunction { SymbolId = "sym://dangerous#sink", Reason = "Should not be reachable" }
+            }
+        };
+
+        var graph = new RichGraph(
+            Nodes: new[]
+            {
+                new RichGraphNode("sym://dangerous#sink", "sym://dangerous#sink", null, null, "test", "function", null, null, null, null, null)
+            },
+            Edges: Array.Empty<RichGraphEdge>(),
+            Roots: Array.Empty<RichGraphRoot>(),
+            Analyzer: new RichGraphAnalyzer("test", "1.0", null)
+        );
+
+        var comparer = new PatchOracleComparer(oracle);
+        var result = comparer.Compare(graph);
+
+        result.Success.Should().BeFalse();
+        result.Violations.Should().HaveCount(1);
+        result.Violations[0].Type.Should().Be(ViolationType.ForbiddenFunctionPresent);
+        result.Summary.ForbiddenFunctionsPresent.Should().Be(1);
+    }
+
+    [Fact]
+    public void Comparer_FailsWhenForbiddenEdgePresent()
+    {
+        var oracle = new PatchOracleDefinition
+        {
+            Id = "test-forbidden-edge",
+            CaseRef = "test-case",
+            Variant = "unreachable",
+            ForbiddenEdges = new[]
+            {
+                new ExpectedEdge { From = "sym://entry", To = "sym://sink", Reason = "Path should be blocked" }
+            }
+        };
+
+        var graph = new RichGraph(
+            Nodes: Array.Empty<RichGraphNode>(),
+            Edges: new[]
+            {
+                new RichGraphEdge("sym://entry", "sym://sink", "call", null, null, null, 1.0, null)
+            },
+            Roots: Array.Empty<RichGraphRoot>(),
+            Analyzer: new RichGraphAnalyzer("test", "1.0", null)
+        );
+
+        var comparer = new PatchOracleComparer(oracle);
+        var result = comparer.Compare(graph);
+
+        result.Success.Should().BeFalse();
+        result.Violations.Should().HaveCount(1);
+        result.Violations[0].Type.Should().Be(ViolationType.ForbiddenEdgePresent);
+        result.Summary.ForbiddenEdgesPresent.Should().Be(1);
+    }
+
+    #endregion
+
+    #region Confidence Threshold Tests
+
+    [Fact]
+    public void Comparer_RespectsMinConfidenceThreshold()
+    {
+        var oracle = new PatchOracleDefinition
+        {
+            Id = "test-confidence",
+            CaseRef = "test-case",
+            Variant = "reachable",
+            MinConfidence = 0.8,
+            ExpectedEdges = new[]
+            {
+                new ExpectedEdge { From = "sym://a", To = "sym://b", Required = true }
+            }
+        };
+
+        var lowConfidenceGraph = new RichGraph(
+            Nodes: Array.Empty<RichGraphNode>(),
+            Edges: new[]
+            {
+                new RichGraphEdge("sym://a", "sym://b", "call", null, null, null, 0.5, null)
+            },
+            Roots: Array.Empty<RichGraphRoot>(),
+            Analyzer: new RichGraphAnalyzer("test", "1.0", null)
+        );
+
+        var comparer = new PatchOracleComparer(oracle);
+        var result = comparer.Compare(lowConfidenceGraph);
+
+        result.Success.Should().BeFalse("edge confidence 0.5 is below threshold 0.8");
+        result.Summary.MissingEdges.Should().Be(1);
+    }
+
+    [Fact]
+    public void Comparer_EdgeSpecificConfidenceOverridesDefault()
+    {
+        var oracle = new PatchOracleDefinition
+        {
+            Id = "test-edge-confidence",
+            CaseRef = "test-case",
+            Variant = "reachable",
+            MinConfidence = 0.8,
+            ExpectedEdges = new[]
+            {
+                new ExpectedEdge { From = "sym://a", To = "sym://b", MinConfidence = 0.3, Required = true }
+            }
+        };
+
+        var lowConfidenceGraph = new RichGraph(
+            Nodes: Array.Empty<RichGraphNode>(),
+            Edges: new[]
+            {
+                new RichGraphEdge("sym://a", "sym://b", "call", null, null, null, 0.5, null)
+            },
+            Roots: Array.Empty<RichGraphRoot>(),
+            Analyzer: new RichGraphAnalyzer("test", "1.0", null)
+        );
+
+        var comparer = new PatchOracleComparer(oracle);
+        var result = comparer.Compare(lowConfidenceGraph);
+
+        result.Success.Should().BeTrue("edge-specific threshold 0.3 allows confidence 0.5");
+    }
+
+    #endregion
+
+    #region Strict Mode Tests
+
+    [Fact]
+    public void Comparer_StrictModeRejectsUnexpectedNodes()
+    {
+        var oracle = new PatchOracleDefinition
+        {
+            Id = "test-strict",
+            CaseRef = "test-case",
+            Variant = "reachable",
+            StrictMode = true,
+            ExpectedFunctions = new[]
+            {
+                new ExpectedFunction { SymbolId = "sym://expected", Required = true }
+            }
+        };
+
+        var graph = new RichGraph(
+            Nodes: new[]
+            {
+                new RichGraphNode("sym://expected", "sym://expected", null, null, "test", "function", null, null, null, null, null),
+                new RichGraphNode("sym://unexpected", "sym://unexpected", null, null, "test", "function", null, null, null, null, null)
+            },
+            Edges: Array.Empty<RichGraphEdge>(),
+            Roots: Array.Empty<RichGraphRoot>(),
+            Analyzer: new RichGraphAnalyzer("test", "1.0", null)
+        );
+
+        var comparer = new PatchOracleComparer(oracle);
+        var result = comparer.Compare(graph);
+
+        result.Success.Should().BeFalse();
+        result.Violations.Should().Contain(v => v.Type == ViolationType.UnexpectedFunction);
+        result.Summary.UnexpectedFunctions.Should().Be(1);
+    }
+
+    #endregion
+
+    #region Report Generation Tests
+
+    [Fact]
+    public void Result_GeneratesReadableReport()
+    {
+        var oracle = new PatchOracleDefinition
+        {
+            Id = "test-report",
+            CaseRef = "test-case",
+            Variant = "reachable",
+            ExpectedFunctions = new[]
+            {
+                new ExpectedFunction { SymbolId = "sym://missing", Required = true, Reason = "Critical sink" }
+            }
+        };
+
+        var graph = new RichGraph(
+            Nodes: new[]
+            {
+                new RichGraphNode("sym://other", "sym://other", null, null, "test", "function", null, null, null, null, null)
+            },
+            Edges: Array.Empty<RichGraphEdge>(),
+            Roots: Array.Empty<RichGraphRoot>(),
+            Analyzer: new RichGraphAnalyzer("test", "1.0", null)
+        );
+
+        var comparer = new PatchOracleComparer(oracle);
+        var result = comparer.Compare(graph);
+
+        var report = result.ToReport();
+
+        report.Should().Contain("FAIL");
+        report.Should().Contain("test-report");
+        report.Should().Contain("MissingFunction");
+        report.Should().Contain("sym://missing");
+    }
+
+    #endregion
+
+    #region Integration with Fixture Data
+
+    public static IEnumerable<object[]> AllOracleData()
+    {
+        var loader = new PatchOracleLoader(PatchOracleRoot);
+        if (!loader.IndexExists())
+        {
+            yield break;
+        }
+
+        foreach (var entry in loader.EnumerateOracles())
+        {
+            yield return new object[] { entry.Id, entry.CaseRef, entry.Variant };
+        }
+    }
+
+    [Theory]
+    [MemberData(nameof(AllOracleData))]
+    public void AllOracles_HaveValidStructure(string oracleId, string caseRef, string variant)
+    {
+        var loader = new PatchOracleLoader(PatchOracleRoot);
+        var oracle = loader.LoadOracle(oracleId);
+
+        oracle.Id.Should().Be(oracleId);
+        oracle.CaseRef.Should().Be(caseRef);
+        oracle.Variant.Should().Be(variant);
+        oracle.SchemaVersion.Should().Be("patch-oracle/v1");
+
+        // At least one expectation should be defined
+        var hasExpectations = oracle.ExpectedFunctions.Count > 0
+            || oracle.ExpectedEdges.Count > 0
+            || oracle.ExpectedRoots.Count > 0
+            || oracle.ForbiddenFunctions.Count > 0
+            || oracle.ForbiddenEdges.Count > 0;
+        hasExpectations.Should().BeTrue($"Oracle '{oracleId}' should define at least one expectation");
+    }
+
+    #endregion
+}