using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using FluentAssertions;
using StellaOps.Reachability.FixtureTests.PatchOracle;
using StellaOps.Scanner.Reachability;
using Xunit;
namespace StellaOps.Reachability.FixtureTests;
///
/// Tests for the patch-oracle harness infrastructure.
/// Validates that the oracle comparison logic correctly identifies missing and forbidden elements.
///
public class PatchOracleHarnessTests
{
private static readonly string RepoRoot = ReachbenchFixtureTests.LocateRepoRoot();
private static readonly string PatchOracleRoot = Path.Combine(
RepoRoot, "tests", "reachability", "fixtures", "patch-oracles");
#region Oracle Loading Tests
[Fact]
public void Loader_IndexExists()
{
var loader = new PatchOracleLoader(PatchOracleRoot);
loader.IndexExists().Should().BeTrue("patch-oracle INDEX.json should exist");
}
[Fact]
public void Loader_IndexLoadsSuccessfully()
{
var loader = new PatchOracleLoader(PatchOracleRoot);
var index = loader.LoadIndex();
index.Should().NotBeNull();
index.Version.Should().Be("1.0");
index.Schema.Should().Be("patch-oracle/v1");
index.Oracles.Should().NotBeEmpty("should have at least one oracle defined");
}
[Fact]
public void Loader_AllOraclesLoadSuccessfully()
{
var loader = new PatchOracleLoader(PatchOracleRoot);
var oracles = loader.LoadAllOracles().ToList();
oracles.Should().NotBeEmpty();
foreach (var oracle in oracles)
{
oracle.SchemaVersion.Should().Be("patch-oracle/v1");
oracle.Id.Should().NotBeNullOrEmpty();
oracle.CaseRef.Should().NotBeNullOrEmpty();
oracle.Variant.Should().BeOneOf("reachable", "unreachable");
}
}
[Fact]
public void Loader_LoadOracleById()
{
var loader = new PatchOracleLoader(PatchOracleRoot);
var oracle = loader.LoadOracle("curl-CVE-2023-38545-socks5-heap-reachable");
oracle.Should().NotBeNull();
oracle.Id.Should().Be("curl-CVE-2023-38545-socks5-heap-reachable");
oracle.CaseRef.Should().Be("curl-CVE-2023-38545-socks5-heap");
oracle.Variant.Should().Be("reachable");
}
#endregion
#region Comparer Tests - Pass Cases
[Fact]
public void Comparer_PassesWhenAllExpectedElementsPresent()
{
var oracle = new PatchOracleDefinition
{
Id = "test-pass",
CaseRef = "test-case",
Variant = "reachable",
ExpectedFunctions = new[]
{
new ExpectedFunction { SymbolId = "sym://test#func1", Required = true },
new ExpectedFunction { SymbolId = "sym://test#func2", Required = true }
},
ExpectedEdges = new[]
{
new ExpectedEdge { From = "sym://test#func1", To = "sym://test#func2", Required = true }
}
};
var graph = new RichGraph(
Nodes: new[]
{
new RichGraphNode("sym://test#func1", "sym://test#func1", null, null, "test", "function", null, null, null, null, null),
new RichGraphNode("sym://test#func2", "sym://test#func2", null, null, "test", "function", null, null, null, null, null)
},
Edges: new[]
{
new RichGraphEdge("sym://test#func1", "sym://test#func2", "call", null, null, null, 0.9, null)
},
Roots: Array.Empty(),
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
);
var comparer = new PatchOracleComparer(oracle);
var result = comparer.Compare(graph);
result.Success.Should().BeTrue();
result.Violations.Should().BeEmpty();
}
[Fact]
public void Comparer_PassesWithWildcardPatterns()
{
var oracle = new PatchOracleDefinition
{
Id = "test-wildcard",
CaseRef = "test-case",
Variant = "reachable",
ExpectedFunctions = new[]
{
new ExpectedFunction { SymbolId = "sym://test#*", Required = true }
}
};
var graph = new RichGraph(
Nodes: new[]
{
new RichGraphNode("sym://test#anything", "sym://test#anything", null, null, "test", "function", null, null, null, null, null)
},
Edges: Array.Empty(),
Roots: Array.Empty(),
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
);
var comparer = new PatchOracleComparer(oracle);
var result = comparer.Compare(graph);
result.Success.Should().BeTrue();
}
#endregion
#region Comparer Tests - Fail Cases
[Fact]
public void Comparer_FailsWhenExpectedFunctionMissing()
{
var oracle = new PatchOracleDefinition
{
Id = "test-missing-func",
CaseRef = "test-case",
Variant = "reachable",
ExpectedFunctions = new[]
{
new ExpectedFunction { SymbolId = "sym://test#missing", Required = true, Reason = "This function is critical" }
}
};
var graph = new RichGraph(
Nodes: Array.Empty(),
Edges: Array.Empty(),
Roots: Array.Empty(),
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
);
var comparer = new PatchOracleComparer(oracle);
var result = comparer.Compare(graph);
result.Success.Should().BeFalse();
result.Violations.Should().HaveCount(1);
result.Violations[0].Type.Should().Be(ViolationType.MissingFunction);
result.Violations[0].From.Should().Be("sym://test#missing");
result.Summary.MissingFunctions.Should().Be(1);
}
[Fact]
public void Comparer_FailsWhenExpectedEdgeMissing()
{
var oracle = new PatchOracleDefinition
{
Id = "test-missing-edge",
CaseRef = "test-case",
Variant = "reachable",
ExpectedEdges = new[]
{
new ExpectedEdge { From = "sym://a", To = "sym://b", Required = true }
}
};
var graph = new RichGraph(
Nodes: new[]
{
new RichGraphNode("sym://a", "sym://a", null, null, "test", "function", null, null, null, null, null),
new RichGraphNode("sym://b", "sym://b", null, null, "test", "function", null, null, null, null, null)
},
Edges: Array.Empty(),
Roots: Array.Empty(),
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
);
var comparer = new PatchOracleComparer(oracle);
var result = comparer.Compare(graph);
result.Success.Should().BeFalse();
result.Violations.Should().HaveCount(1);
result.Violations[0].Type.Should().Be(ViolationType.MissingEdge);
result.Summary.MissingEdges.Should().Be(1);
}
[Fact]
public void Comparer_FailsWhenExpectedRootMissing()
{
var oracle = new PatchOracleDefinition
{
Id = "test-missing-root",
CaseRef = "test-case",
Variant = "reachable",
ExpectedRoots = new[]
{
new ExpectedRoot { Id = "sym://root#main", Phase = "main", Required = true }
}
};
var graph = new RichGraph(
Nodes: Array.Empty(),
Edges: Array.Empty(),
Roots: Array.Empty(),
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
);
var comparer = new PatchOracleComparer(oracle);
var result = comparer.Compare(graph);
result.Success.Should().BeFalse();
result.Violations.Should().HaveCount(1);
result.Violations[0].Type.Should().Be(ViolationType.MissingRoot);
result.Summary.MissingRoots.Should().Be(1);
}
[Fact]
public void Comparer_FailsWhenForbiddenFunctionPresent()
{
var oracle = new PatchOracleDefinition
{
Id = "test-forbidden-func",
CaseRef = "test-case",
Variant = "unreachable",
ForbiddenFunctions = new[]
{
new ExpectedFunction { SymbolId = "sym://dangerous#sink", Reason = "Should not be reachable" }
}
};
var graph = new RichGraph(
Nodes: new[]
{
new RichGraphNode("sym://dangerous#sink", "sym://dangerous#sink", null, null, "test", "function", null, null, null, null, null)
},
Edges: Array.Empty(),
Roots: Array.Empty(),
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
);
var comparer = new PatchOracleComparer(oracle);
var result = comparer.Compare(graph);
result.Success.Should().BeFalse();
result.Violations.Should().HaveCount(1);
result.Violations[0].Type.Should().Be(ViolationType.ForbiddenFunctionPresent);
result.Summary.ForbiddenFunctionsPresent.Should().Be(1);
}
[Fact]
public void Comparer_FailsWhenForbiddenEdgePresent()
{
var oracle = new PatchOracleDefinition
{
Id = "test-forbidden-edge",
CaseRef = "test-case",
Variant = "unreachable",
ForbiddenEdges = new[]
{
new ExpectedEdge { From = "sym://entry", To = "sym://sink", Reason = "Path should be blocked" }
}
};
var graph = new RichGraph(
Nodes: Array.Empty(),
Edges: new[]
{
new RichGraphEdge("sym://entry", "sym://sink", "call", null, null, null, 1.0, null)
},
Roots: Array.Empty(),
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
);
var comparer = new PatchOracleComparer(oracle);
var result = comparer.Compare(graph);
result.Success.Should().BeFalse();
result.Violations.Should().HaveCount(1);
result.Violations[0].Type.Should().Be(ViolationType.ForbiddenEdgePresent);
result.Summary.ForbiddenEdgesPresent.Should().Be(1);
}
#endregion
#region Confidence Threshold Tests
[Fact]
public void Comparer_RespectsMinConfidenceThreshold()
{
var oracle = new PatchOracleDefinition
{
Id = "test-confidence",
CaseRef = "test-case",
Variant = "reachable",
MinConfidence = 0.8,
ExpectedEdges = new[]
{
new ExpectedEdge { From = "sym://a", To = "sym://b", Required = true }
}
};
var lowConfidenceGraph = new RichGraph(
Nodes: Array.Empty(),
Edges: new[]
{
new RichGraphEdge("sym://a", "sym://b", "call", null, null, null, 0.5, null)
},
Roots: Array.Empty(),
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
);
var comparer = new PatchOracleComparer(oracle);
var result = comparer.Compare(lowConfidenceGraph);
result.Success.Should().BeFalse("edge confidence 0.5 is below threshold 0.8");
result.Summary.MissingEdges.Should().Be(1);
}
[Fact]
public void Comparer_EdgeSpecificConfidenceOverridesDefault()
{
var oracle = new PatchOracleDefinition
{
Id = "test-edge-confidence",
CaseRef = "test-case",
Variant = "reachable",
MinConfidence = 0.8,
ExpectedEdges = new[]
{
new ExpectedEdge { From = "sym://a", To = "sym://b", MinConfidence = 0.3, Required = true }
}
};
var lowConfidenceGraph = new RichGraph(
Nodes: Array.Empty(),
Edges: new[]
{
new RichGraphEdge("sym://a", "sym://b", "call", null, null, null, 0.5, null)
},
Roots: Array.Empty(),
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
);
var comparer = new PatchOracleComparer(oracle);
var result = comparer.Compare(lowConfidenceGraph);
result.Success.Should().BeTrue("edge-specific threshold 0.3 allows confidence 0.5");
}
#endregion
#region Strict Mode Tests
[Fact]
public void Comparer_StrictModeRejectsUnexpectedNodes()
{
var oracle = new PatchOracleDefinition
{
Id = "test-strict",
CaseRef = "test-case",
Variant = "reachable",
StrictMode = true,
ExpectedFunctions = new[]
{
new ExpectedFunction { SymbolId = "sym://expected", Required = true }
}
};
var graph = new RichGraph(
Nodes: new[]
{
new RichGraphNode("sym://expected", "sym://expected", null, null, "test", "function", null, null, null, null, null),
new RichGraphNode("sym://unexpected", "sym://unexpected", null, null, "test", "function", null, null, null, null, null)
},
Edges: Array.Empty(),
Roots: Array.Empty(),
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
);
var comparer = new PatchOracleComparer(oracle);
var result = comparer.Compare(graph);
result.Success.Should().BeFalse();
result.Violations.Should().Contain(v => v.Type == ViolationType.UnexpectedFunction);
result.Summary.UnexpectedFunctions.Should().Be(1);
}
#endregion
#region Report Generation Tests
[Fact]
public void Result_GeneratesReadableReport()
{
var oracle = new PatchOracleDefinition
{
Id = "test-report",
CaseRef = "test-case",
Variant = "reachable",
ExpectedFunctions = new[]
{
new ExpectedFunction { SymbolId = "sym://missing", Required = true, Reason = "Critical sink" }
}
};
var graph = new RichGraph(
Nodes: new[]
{
new RichGraphNode("sym://other", "sym://other", null, null, "test", "function", null, null, null, null, null)
},
Edges: Array.Empty(),
Roots: Array.Empty(),
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
);
var comparer = new PatchOracleComparer(oracle);
var result = comparer.Compare(graph);
var report = result.ToReport();
report.Should().Contain("FAIL");
report.Should().Contain("test-report");
report.Should().Contain("MissingFunction");
report.Should().Contain("sym://missing");
}
#endregion
#region Integration with Fixture Data
public static IEnumerable