using System; using System.Collections.Generic; using System.IO; using System.Linq; using FluentAssertions; using StellaOps.Reachability.FixtureTests.PatchOracle; using StellaOps.Scanner.Reachability; using Xunit; namespace StellaOps.Reachability.FixtureTests; /// /// Tests for the patch-oracle harness infrastructure. /// Validates that the oracle comparison logic correctly identifies missing and forbidden elements. /// public class PatchOracleHarnessTests { private static readonly string RepoRoot = ReachbenchFixtureTests.LocateRepoRoot(); private static readonly string PatchOracleRoot = Path.Combine( RepoRoot, "tests", "reachability", "fixtures", "patch-oracles"); #region Oracle Loading Tests [Fact] public void Loader_IndexExists() { var loader = new PatchOracleLoader(PatchOracleRoot); loader.IndexExists().Should().BeTrue("patch-oracle INDEX.json should exist"); } [Fact] public void Loader_IndexLoadsSuccessfully() { var loader = new PatchOracleLoader(PatchOracleRoot); var index = loader.LoadIndex(); index.Should().NotBeNull(); index.Version.Should().Be("1.0"); index.Schema.Should().Be("patch-oracle/v1"); index.Oracles.Should().NotBeEmpty("should have at least one oracle defined"); } [Fact] public void Loader_AllOraclesLoadSuccessfully() { var loader = new PatchOracleLoader(PatchOracleRoot); var oracles = loader.LoadAllOracles().ToList(); oracles.Should().NotBeEmpty(); foreach (var oracle in oracles) { oracle.SchemaVersion.Should().Be("patch-oracle/v1"); oracle.Id.Should().NotBeNullOrEmpty(); oracle.CaseRef.Should().NotBeNullOrEmpty(); oracle.Variant.Should().BeOneOf("reachable", "unreachable"); } } [Fact] public void Loader_LoadOracleById() { var loader = new PatchOracleLoader(PatchOracleRoot); var oracle = loader.LoadOracle("curl-CVE-2023-38545-socks5-heap-reachable"); oracle.Should().NotBeNull(); oracle.Id.Should().Be("curl-CVE-2023-38545-socks5-heap-reachable"); oracle.CaseRef.Should().Be("curl-CVE-2023-38545-socks5-heap"); oracle.Variant.Should().Be("reachable"); } #endregion #region Comparer Tests - Pass Cases [Fact] public void Comparer_PassesWhenAllExpectedElementsPresent() { var oracle = new PatchOracleDefinition { Id = "test-pass", CaseRef = "test-case", Variant = "reachable", ExpectedFunctions = new[] { new ExpectedFunction { SymbolId = "sym://test#func1", Required = true }, new ExpectedFunction { SymbolId = "sym://test#func2", Required = true } }, ExpectedEdges = new[] { new ExpectedEdge { From = "sym://test#func1", To = "sym://test#func2", Required = true } } }; var graph = new RichGraph( Nodes: new[] { new RichGraphNode("sym://test#func1", "sym://test#func1", null, null, "test", "function", null, null, null, null, null), new RichGraphNode("sym://test#func2", "sym://test#func2", null, null, "test", "function", null, null, null, null, null) }, Edges: new[] { new RichGraphEdge("sym://test#func1", "sym://test#func2", "call", null, null, null, 0.9, null) }, Roots: Array.Empty(), Analyzer: new RichGraphAnalyzer("test", "1.0", null) ); var comparer = new PatchOracleComparer(oracle); var result = comparer.Compare(graph); result.Success.Should().BeTrue(); result.Violations.Should().BeEmpty(); } [Fact] public void Comparer_PassesWithWildcardPatterns() { var oracle = new PatchOracleDefinition { Id = "test-wildcard", CaseRef = "test-case", Variant = "reachable", ExpectedFunctions = new[] { new ExpectedFunction { SymbolId = "sym://test#*", Required = true } } }; var graph = new RichGraph( Nodes: new[] { new RichGraphNode("sym://test#anything", "sym://test#anything", null, null, "test", "function", null, null, null, null, null) }, Edges: Array.Empty(), Roots: Array.Empty(), Analyzer: new RichGraphAnalyzer("test", "1.0", null) ); var comparer = new PatchOracleComparer(oracle); var result = comparer.Compare(graph); result.Success.Should().BeTrue(); } #endregion #region Comparer Tests - Fail Cases [Fact] public void Comparer_FailsWhenExpectedFunctionMissing() { var oracle = new PatchOracleDefinition { Id = "test-missing-func", CaseRef = "test-case", Variant = "reachable", ExpectedFunctions = new[] { new ExpectedFunction { SymbolId = "sym://test#missing", Required = true, Reason = "This function is critical" } } }; var graph = new RichGraph( Nodes: Array.Empty(), Edges: Array.Empty(), Roots: Array.Empty(), Analyzer: new RichGraphAnalyzer("test", "1.0", null) ); var comparer = new PatchOracleComparer(oracle); var result = comparer.Compare(graph); result.Success.Should().BeFalse(); result.Violations.Should().HaveCount(1); result.Violations[0].Type.Should().Be(ViolationType.MissingFunction); result.Violations[0].From.Should().Be("sym://test#missing"); result.Summary.MissingFunctions.Should().Be(1); } [Fact] public void Comparer_FailsWhenExpectedEdgeMissing() { var oracle = new PatchOracleDefinition { Id = "test-missing-edge", CaseRef = "test-case", Variant = "reachable", ExpectedEdges = new[] { new ExpectedEdge { From = "sym://a", To = "sym://b", Required = true } } }; var graph = new RichGraph( Nodes: new[] { new RichGraphNode("sym://a", "sym://a", null, null, "test", "function", null, null, null, null, null), new RichGraphNode("sym://b", "sym://b", null, null, "test", "function", null, null, null, null, null) }, Edges: Array.Empty(), Roots: Array.Empty(), Analyzer: new RichGraphAnalyzer("test", "1.0", null) ); var comparer = new PatchOracleComparer(oracle); var result = comparer.Compare(graph); result.Success.Should().BeFalse(); result.Violations.Should().HaveCount(1); result.Violations[0].Type.Should().Be(ViolationType.MissingEdge); result.Summary.MissingEdges.Should().Be(1); } [Fact] public void Comparer_FailsWhenExpectedRootMissing() { var oracle = new PatchOracleDefinition { Id = "test-missing-root", CaseRef = "test-case", Variant = "reachable", ExpectedRoots = new[] { new ExpectedRoot { Id = "sym://root#main", Phase = "main", Required = true } } }; var graph = new RichGraph( Nodes: Array.Empty(), Edges: Array.Empty(), Roots: Array.Empty(), Analyzer: new RichGraphAnalyzer("test", "1.0", null) ); var comparer = new PatchOracleComparer(oracle); var result = comparer.Compare(graph); result.Success.Should().BeFalse(); result.Violations.Should().HaveCount(1); result.Violations[0].Type.Should().Be(ViolationType.MissingRoot); result.Summary.MissingRoots.Should().Be(1); } [Fact] public void Comparer_FailsWhenForbiddenFunctionPresent() { var oracle = new PatchOracleDefinition { Id = "test-forbidden-func", CaseRef = "test-case", Variant = "unreachable", ForbiddenFunctions = new[] { new ExpectedFunction { SymbolId = "sym://dangerous#sink", Reason = "Should not be reachable" } } }; var graph = new RichGraph( Nodes: new[] { new RichGraphNode("sym://dangerous#sink", "sym://dangerous#sink", null, null, "test", "function", null, null, null, null, null) }, Edges: Array.Empty(), Roots: Array.Empty(), Analyzer: new RichGraphAnalyzer("test", "1.0", null) ); var comparer = new PatchOracleComparer(oracle); var result = comparer.Compare(graph); result.Success.Should().BeFalse(); result.Violations.Should().HaveCount(1); result.Violations[0].Type.Should().Be(ViolationType.ForbiddenFunctionPresent); result.Summary.ForbiddenFunctionsPresent.Should().Be(1); } [Fact] public void Comparer_FailsWhenForbiddenEdgePresent() { var oracle = new PatchOracleDefinition { Id = "test-forbidden-edge", CaseRef = "test-case", Variant = "unreachable", ForbiddenEdges = new[] { new ExpectedEdge { From = "sym://entry", To = "sym://sink", Reason = "Path should be blocked" } } }; var graph = new RichGraph( Nodes: Array.Empty(), Edges: new[] { new RichGraphEdge("sym://entry", "sym://sink", "call", null, null, null, 1.0, null) }, Roots: Array.Empty(), Analyzer: new RichGraphAnalyzer("test", "1.0", null) ); var comparer = new PatchOracleComparer(oracle); var result = comparer.Compare(graph); result.Success.Should().BeFalse(); result.Violations.Should().HaveCount(1); result.Violations[0].Type.Should().Be(ViolationType.ForbiddenEdgePresent); result.Summary.ForbiddenEdgesPresent.Should().Be(1); } #endregion #region Confidence Threshold Tests [Fact] public void Comparer_RespectsMinConfidenceThreshold() { var oracle = new PatchOracleDefinition { Id = "test-confidence", CaseRef = "test-case", Variant = "reachable", MinConfidence = 0.8, ExpectedEdges = new[] { new ExpectedEdge { From = "sym://a", To = "sym://b", Required = true } } }; var lowConfidenceGraph = new RichGraph( Nodes: Array.Empty(), Edges: new[] { new RichGraphEdge("sym://a", "sym://b", "call", null, null, null, 0.5, null) }, Roots: Array.Empty(), Analyzer: new RichGraphAnalyzer("test", "1.0", null) ); var comparer = new PatchOracleComparer(oracle); var result = comparer.Compare(lowConfidenceGraph); result.Success.Should().BeFalse("edge confidence 0.5 is below threshold 0.8"); result.Summary.MissingEdges.Should().Be(1); } [Fact] public void Comparer_EdgeSpecificConfidenceOverridesDefault() { var oracle = new PatchOracleDefinition { Id = "test-edge-confidence", CaseRef = "test-case", Variant = "reachable", MinConfidence = 0.8, ExpectedEdges = new[] { new ExpectedEdge { From = "sym://a", To = "sym://b", MinConfidence = 0.3, Required = true } } }; var lowConfidenceGraph = new RichGraph( Nodes: Array.Empty(), Edges: new[] { new RichGraphEdge("sym://a", "sym://b", "call", null, null, null, 0.5, null) }, Roots: Array.Empty(), Analyzer: new RichGraphAnalyzer("test", "1.0", null) ); var comparer = new PatchOracleComparer(oracle); var result = comparer.Compare(lowConfidenceGraph); result.Success.Should().BeTrue("edge-specific threshold 0.3 allows confidence 0.5"); } #endregion #region Strict Mode Tests [Fact] public void Comparer_StrictModeRejectsUnexpectedNodes() { var oracle = new PatchOracleDefinition { Id = "test-strict", CaseRef = "test-case", Variant = "reachable", StrictMode = true, ExpectedFunctions = new[] { new ExpectedFunction { SymbolId = "sym://expected", Required = true } } }; var graph = new RichGraph( Nodes: new[] { new RichGraphNode("sym://expected", "sym://expected", null, null, "test", "function", null, null, null, null, null), new RichGraphNode("sym://unexpected", "sym://unexpected", null, null, "test", "function", null, null, null, null, null) }, Edges: Array.Empty(), Roots: Array.Empty(), Analyzer: new RichGraphAnalyzer("test", "1.0", null) ); var comparer = new PatchOracleComparer(oracle); var result = comparer.Compare(graph); result.Success.Should().BeFalse(); result.Violations.Should().Contain(v => v.Type == ViolationType.UnexpectedFunction); result.Summary.UnexpectedFunctions.Should().Be(1); } #endregion #region Report Generation Tests [Fact] public void Result_GeneratesReadableReport() { var oracle = new PatchOracleDefinition { Id = "test-report", CaseRef = "test-case", Variant = "reachable", ExpectedFunctions = new[] { new ExpectedFunction { SymbolId = "sym://missing", Required = true, Reason = "Critical sink" } } }; var graph = new RichGraph( Nodes: new[] { new RichGraphNode("sym://other", "sym://other", null, null, "test", "function", null, null, null, null, null) }, Edges: Array.Empty(), Roots: Array.Empty(), Analyzer: new RichGraphAnalyzer("test", "1.0", null) ); var comparer = new PatchOracleComparer(oracle); var result = comparer.Compare(graph); var report = result.ToReport(); report.Should().Contain("FAIL"); report.Should().Contain("test-report"); report.Should().Contain("MissingFunction"); report.Should().Contain("sym://missing"); } #endregion #region Integration with Fixture Data public static IEnumerable AllOracleData() { var loader = new PatchOracleLoader(PatchOracleRoot); if (!loader.IndexExists()) { yield break; } foreach (var entry in loader.EnumerateOracles()) { yield return new object[] { entry.Id, entry.CaseRef, entry.Variant }; } } [Theory] [MemberData(nameof(AllOracleData))] public void AllOracles_HaveValidStructure(string oracleId, string caseRef, string variant) { var loader = new PatchOracleLoader(PatchOracleRoot); var oracle = loader.LoadOracle(oracleId); oracle.Id.Should().Be(oracleId); oracle.CaseRef.Should().Be(caseRef); oracle.Variant.Should().Be(variant); oracle.SchemaVersion.Should().Be("patch-oracle/v1"); // At least one expectation should be defined var hasExpectations = oracle.ExpectedFunctions.Count > 0 || oracle.ExpectedEdges.Count > 0 || oracle.ExpectedRoots.Count > 0 || oracle.ForbiddenFunctions.Count > 0 || oracle.ForbiddenEdges.Count > 0; hasExpectations.Should().BeTrue($"Oracle '{oracleId}' should define at least one expectation"); } #endregion }