up
Some checks failed
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Manifest Integrity / Validate Schema Integrity (push) Has been cancelled
Manifest Integrity / Validate Contract Documents (push) Has been cancelled
Manifest Integrity / Validate Pack Fixtures (push) Has been cancelled
Manifest Integrity / Audit SHA256SUMS Files (push) Has been cancelled
Manifest Integrity / Verify Merkle Roots (push) Has been cancelled
devportal-offline / build-offline (push) Has been cancelled
Mirror Thin Bundle Sign & Verify / mirror-sign (push) Has been cancelled
Some checks failed
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Manifest Integrity / Validate Schema Integrity (push) Has been cancelled
Manifest Integrity / Validate Contract Documents (push) Has been cancelled
Manifest Integrity / Validate Pack Fixtures (push) Has been cancelled
Manifest Integrity / Audit SHA256SUMS Files (push) Has been cancelled
Manifest Integrity / Verify Merkle Roots (push) Has been cancelled
devportal-offline / build-offline (push) Has been cancelled
Mirror Thin Bundle Sign & Verify / mirror-sign (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,375 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using StellaOps.Scanner.Reachability;
|
||||
|
||||
namespace StellaOps.Reachability.FixtureTests.PatchOracle;
|
||||
|
||||
/// <summary>
|
||||
/// Compares a RichGraph against a patch-oracle definition.
|
||||
/// Reports missing expected elements and present forbidden elements.
|
||||
/// </summary>
|
||||
public sealed class PatchOracleComparer
|
||||
{
|
||||
private readonly PatchOracleDefinition _oracle;
|
||||
private readonly double _defaultMinConfidence;
|
||||
|
||||
public PatchOracleComparer(PatchOracleDefinition oracle)
|
||||
{
|
||||
_oracle = oracle ?? throw new ArgumentNullException(nameof(oracle));
|
||||
_defaultMinConfidence = oracle.MinConfidence;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Compares the graph against the oracle and returns a result.
|
||||
/// </summary>
|
||||
public PatchOracleResult Compare(RichGraph graph)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(graph);
|
||||
|
||||
var violations = new List<PatchOracleViolation>();
|
||||
|
||||
// Check expected functions
|
||||
foreach (var expected in _oracle.ExpectedFunctions.Where(f => f.Required))
|
||||
{
|
||||
if (!HasMatchingNode(graph, expected))
|
||||
{
|
||||
violations.Add(new PatchOracleViolation(
|
||||
ViolationType.MissingFunction,
|
||||
expected.SymbolId,
|
||||
null,
|
||||
expected.Reason ?? $"Expected function '{expected.SymbolId}' not found in graph"));
|
||||
}
|
||||
}
|
||||
|
||||
// Check expected edges
|
||||
foreach (var expected in _oracle.ExpectedEdges.Where(e => e.Required))
|
||||
{
|
||||
var minConf = expected.MinConfidence ?? _defaultMinConfidence;
|
||||
if (!HasMatchingEdge(graph, expected, minConf))
|
||||
{
|
||||
violations.Add(new PatchOracleViolation(
|
||||
ViolationType.MissingEdge,
|
||||
expected.From,
|
||||
expected.To,
|
||||
expected.Reason ?? $"Expected edge '{expected.From}' -> '{expected.To}' not found in graph"));
|
||||
}
|
||||
}
|
||||
|
||||
// Check expected roots
|
||||
foreach (var expected in _oracle.ExpectedRoots.Where(r => r.Required))
|
||||
{
|
||||
if (!HasMatchingRoot(graph, expected))
|
||||
{
|
||||
violations.Add(new PatchOracleViolation(
|
||||
ViolationType.MissingRoot,
|
||||
expected.Id,
|
||||
null,
|
||||
expected.Reason ?? $"Expected root '{expected.Id}' not found in graph"));
|
||||
}
|
||||
}
|
||||
|
||||
// Check forbidden functions
|
||||
foreach (var forbidden in _oracle.ForbiddenFunctions)
|
||||
{
|
||||
if (HasMatchingNode(graph, forbidden))
|
||||
{
|
||||
violations.Add(new PatchOracleViolation(
|
||||
ViolationType.ForbiddenFunctionPresent,
|
||||
forbidden.SymbolId,
|
||||
null,
|
||||
forbidden.Reason ?? $"Forbidden function '{forbidden.SymbolId}' is present in graph"));
|
||||
}
|
||||
}
|
||||
|
||||
// Check forbidden edges
|
||||
foreach (var forbidden in _oracle.ForbiddenEdges)
|
||||
{
|
||||
if (HasMatchingEdge(graph, forbidden, 0.0))
|
||||
{
|
||||
violations.Add(new PatchOracleViolation(
|
||||
ViolationType.ForbiddenEdgePresent,
|
||||
forbidden.From,
|
||||
forbidden.To,
|
||||
forbidden.Reason ?? $"Forbidden edge '{forbidden.From}' -> '{forbidden.To}' is present in graph"));
|
||||
}
|
||||
}
|
||||
|
||||
// Strict mode: check for unexpected elements
|
||||
if (_oracle.StrictMode)
|
||||
{
|
||||
var unexpectedNodes = FindUnexpectedNodes(graph);
|
||||
foreach (var node in unexpectedNodes)
|
||||
{
|
||||
violations.Add(new PatchOracleViolation(
|
||||
ViolationType.UnexpectedFunction,
|
||||
node.Id,
|
||||
null,
|
||||
$"Strict mode: unexpected function '{node.Id}' found in graph"));
|
||||
}
|
||||
|
||||
var unexpectedEdges = FindUnexpectedEdges(graph);
|
||||
foreach (var edge in unexpectedEdges)
|
||||
{
|
||||
violations.Add(new PatchOracleViolation(
|
||||
ViolationType.UnexpectedEdge,
|
||||
edge.From,
|
||||
edge.To,
|
||||
$"Strict mode: unexpected edge '{edge.From}' -> '{edge.To}' found in graph"));
|
||||
}
|
||||
}
|
||||
|
||||
return new PatchOracleResult(
|
||||
OracleId: _oracle.Id,
|
||||
CaseRef: _oracle.CaseRef,
|
||||
Variant: _oracle.Variant,
|
||||
Success: violations.Count == 0,
|
||||
Violations: violations,
|
||||
Summary: GenerateSummary(graph, violations));
|
||||
}
|
||||
|
||||
private bool HasMatchingNode(RichGraph graph, ExpectedFunction expected)
|
||||
{
|
||||
foreach (var node in graph.Nodes)
|
||||
{
|
||||
if (!MatchesPattern(node.Id, expected.SymbolId) &&
|
||||
!MatchesPattern(node.SymbolId, expected.SymbolId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(expected.Lang) &&
|
||||
!string.Equals(node.Lang, expected.Lang, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(expected.Kind) &&
|
||||
!string.Equals(node.Kind, expected.Kind, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(expected.PurlPattern) &&
|
||||
!MatchesPattern(node.Purl ?? string.Empty, expected.PurlPattern))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private bool HasMatchingEdge(RichGraph graph, ExpectedEdge expected, double minConfidence)
|
||||
{
|
||||
foreach (var edge in graph.Edges)
|
||||
{
|
||||
if (!MatchesPattern(edge.From, expected.From))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!MatchesPattern(edge.To, expected.To))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(expected.Kind) &&
|
||||
!string.Equals(edge.Kind, expected.Kind, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (edge.Confidence < minConfidence)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private bool HasMatchingRoot(RichGraph graph, ExpectedRoot expected)
|
||||
{
|
||||
foreach (var root in graph.Roots)
|
||||
{
|
||||
if (!MatchesPattern(root.Id, expected.Id))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(expected.Phase) &&
|
||||
!string.Equals(root.Phase, expected.Phase, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private IEnumerable<RichGraphNode> FindUnexpectedNodes(RichGraph graph)
|
||||
{
|
||||
var allExpected = _oracle.ExpectedFunctions
|
||||
.Select(f => f.SymbolId)
|
||||
.ToHashSet(StringComparer.Ordinal);
|
||||
|
||||
foreach (var node in graph.Nodes)
|
||||
{
|
||||
var isExpected = allExpected.Any(pattern => MatchesPattern(node.Id, pattern) || MatchesPattern(node.SymbolId, pattern));
|
||||
if (!isExpected)
|
||||
{
|
||||
yield return node;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private IEnumerable<RichGraphEdge> FindUnexpectedEdges(RichGraph graph)
|
||||
{
|
||||
foreach (var edge in graph.Edges)
|
||||
{
|
||||
var isExpected = _oracle.ExpectedEdges.Any(e =>
|
||||
MatchesPattern(edge.From, e.From) && MatchesPattern(edge.To, e.To));
|
||||
if (!isExpected)
|
||||
{
|
||||
yield return edge;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Matches a value against a pattern supporting '*' wildcards.
|
||||
/// </summary>
|
||||
private static bool MatchesPattern(string value, string pattern)
|
||||
{
|
||||
if (string.IsNullOrEmpty(pattern))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (string.IsNullOrEmpty(value))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// Exact match
|
||||
if (!pattern.Contains('*'))
|
||||
{
|
||||
return string.Equals(value, pattern, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
// Convert wildcard pattern to regex
|
||||
var regexPattern = "^" + Regex.Escape(pattern).Replace("\\*", ".*") + "$";
|
||||
return Regex.IsMatch(value, regexPattern, RegexOptions.None, TimeSpan.FromMilliseconds(100));
|
||||
}
|
||||
|
||||
private static PatchOracleSummary GenerateSummary(RichGraph graph, List<PatchOracleViolation> violations)
|
||||
{
|
||||
return new PatchOracleSummary(
|
||||
TotalNodes: graph.Nodes.Count,
|
||||
TotalEdges: graph.Edges.Count,
|
||||
TotalRoots: graph.Roots.Count,
|
||||
MissingFunctions: violations.Count(v => v.Type == ViolationType.MissingFunction),
|
||||
MissingEdges: violations.Count(v => v.Type == ViolationType.MissingEdge),
|
||||
MissingRoots: violations.Count(v => v.Type == ViolationType.MissingRoot),
|
||||
ForbiddenFunctionsPresent: violations.Count(v => v.Type == ViolationType.ForbiddenFunctionPresent),
|
||||
ForbiddenEdgesPresent: violations.Count(v => v.Type == ViolationType.ForbiddenEdgePresent),
|
||||
UnexpectedFunctions: violations.Count(v => v.Type == ViolationType.UnexpectedFunction),
|
||||
UnexpectedEdges: violations.Count(v => v.Type == ViolationType.UnexpectedEdge));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of comparing a graph against a patch-oracle.
|
||||
/// </summary>
|
||||
public sealed record PatchOracleResult(
|
||||
string OracleId,
|
||||
string CaseRef,
|
||||
string Variant,
|
||||
bool Success,
|
||||
IReadOnlyList<PatchOracleViolation> Violations,
|
||||
PatchOracleSummary Summary)
|
||||
{
|
||||
/// <summary>
|
||||
/// Generates a human-readable report.
|
||||
/// </summary>
|
||||
public string ToReport()
|
||||
{
|
||||
var lines = new List<string>
|
||||
{
|
||||
$"Patch-Oracle Validation Report",
|
||||
$"==============================",
|
||||
$"Oracle: {OracleId}",
|
||||
$"Case: {CaseRef} ({Variant})",
|
||||
$"Status: {(Success ? "PASS" : "FAIL")}",
|
||||
string.Empty,
|
||||
$"Graph Statistics:",
|
||||
$" Nodes: {Summary.TotalNodes}",
|
||||
$" Edges: {Summary.TotalEdges}",
|
||||
$" Roots: {Summary.TotalRoots}",
|
||||
string.Empty
|
||||
};
|
||||
|
||||
if (Violations.Count > 0)
|
||||
{
|
||||
lines.Add($"Violations ({Violations.Count}):");
|
||||
foreach (var v in Violations)
|
||||
{
|
||||
var target = v.To is not null ? $" -> {v.To}" : string.Empty;
|
||||
lines.Add($" [{v.Type}] {v.From}{target}");
|
||||
lines.Add($" Reason: {v.Message}");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
lines.Add("No violations found.");
|
||||
}
|
||||
|
||||
return string.Join(Environment.NewLine, lines);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A single violation found during oracle comparison.
|
||||
/// </summary>
|
||||
public sealed record PatchOracleViolation(
|
||||
ViolationType Type,
|
||||
string From,
|
||||
string? To,
|
||||
string Message);
|
||||
|
||||
/// <summary>
|
||||
/// Type of oracle violation.
|
||||
/// </summary>
|
||||
public enum ViolationType
|
||||
{
|
||||
MissingFunction,
|
||||
MissingEdge,
|
||||
MissingRoot,
|
||||
ForbiddenFunctionPresent,
|
||||
ForbiddenEdgePresent,
|
||||
UnexpectedFunction,
|
||||
UnexpectedEdge
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Summary statistics for oracle comparison.
|
||||
/// </summary>
|
||||
public sealed record PatchOracleSummary(
|
||||
int TotalNodes,
|
||||
int TotalEdges,
|
||||
int TotalRoots,
|
||||
int MissingFunctions,
|
||||
int MissingEdges,
|
||||
int MissingRoots,
|
||||
int ForbiddenFunctionsPresent,
|
||||
int ForbiddenEdgesPresent,
|
||||
int UnexpectedFunctions,
|
||||
int UnexpectedEdges);
|
||||
@@ -0,0 +1,112 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace StellaOps.Reachability.FixtureTests.PatchOracle;
|
||||
|
||||
/// <summary>
|
||||
/// Loads patch-oracle definitions from fixture files.
|
||||
/// </summary>
|
||||
public sealed class PatchOracleLoader
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
ReadCommentHandling = JsonCommentHandling.Skip,
|
||||
AllowTrailingCommas = true
|
||||
};
|
||||
|
||||
private readonly string _fixtureRoot;
|
||||
|
||||
public PatchOracleLoader(string fixtureRoot)
|
||||
{
|
||||
_fixtureRoot = fixtureRoot ?? throw new ArgumentNullException(nameof(fixtureRoot));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Loads the oracle index from INDEX.json.
|
||||
/// </summary>
|
||||
public PatchOracleIndex LoadIndex()
|
||||
{
|
||||
var indexPath = Path.Combine(_fixtureRoot, "INDEX.json");
|
||||
if (!File.Exists(indexPath))
|
||||
{
|
||||
throw new FileNotFoundException($"Patch-oracle INDEX.json not found at {indexPath}");
|
||||
}
|
||||
|
||||
var json = File.ReadAllText(indexPath);
|
||||
return JsonSerializer.Deserialize<PatchOracleIndex>(json, JsonOptions)
|
||||
?? throw new InvalidOperationException("Failed to deserialize patch-oracle index");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Loads an oracle definition by its ID.
|
||||
/// </summary>
|
||||
public PatchOracleDefinition LoadOracle(string oracleId)
|
||||
{
|
||||
var index = LoadIndex();
|
||||
var entry = index.Oracles
|
||||
.FirstOrDefault(o => string.Equals(o.Id, oracleId, StringComparison.Ordinal))
|
||||
?? throw new KeyNotFoundException($"Oracle '{oracleId}' not found in index");
|
||||
|
||||
return LoadOracleFromPath(entry.Path);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Loads an oracle definition from a relative path.
|
||||
/// </summary>
|
||||
public PatchOracleDefinition LoadOracleFromPath(string relativePath)
|
||||
{
|
||||
var fullPath = Path.Combine(_fixtureRoot, relativePath);
|
||||
if (!File.Exists(fullPath))
|
||||
{
|
||||
throw new FileNotFoundException($"Oracle file not found at {fullPath}");
|
||||
}
|
||||
|
||||
var json = File.ReadAllText(fullPath);
|
||||
return JsonSerializer.Deserialize<PatchOracleDefinition>(json, JsonOptions)
|
||||
?? throw new InvalidOperationException($"Failed to deserialize oracle from {fullPath}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Loads all oracles for a specific case.
|
||||
/// </summary>
|
||||
public IEnumerable<PatchOracleDefinition> LoadOraclesForCase(string caseRef)
|
||||
{
|
||||
var index = LoadIndex();
|
||||
foreach (var entry in index.Oracles.Where(o => string.Equals(o.CaseRef, caseRef, StringComparison.Ordinal)))
|
||||
{
|
||||
yield return LoadOracleFromPath(entry.Path);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Loads all available oracles.
|
||||
/// </summary>
|
||||
public IEnumerable<PatchOracleDefinition> LoadAllOracles()
|
||||
{
|
||||
var index = LoadIndex();
|
||||
foreach (var entry in index.Oracles)
|
||||
{
|
||||
yield return LoadOracleFromPath(entry.Path);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Enumerates all oracle entries without loading full definitions.
|
||||
/// </summary>
|
||||
public IEnumerable<PatchOracleIndexEntry> EnumerateOracles()
|
||||
{
|
||||
var index = LoadIndex();
|
||||
return index.Oracles;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if the oracle index exists.
|
||||
/// </summary>
|
||||
public bool IndexExists()
|
||||
{
|
||||
return File.Exists(Path.Combine(_fixtureRoot, "INDEX.json"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Reachability.FixtureTests.PatchOracle;
|
||||
|
||||
/// <summary>
|
||||
/// Root model for patch-oracle fixture files.
|
||||
/// </summary>
|
||||
public sealed record PatchOracleDefinition
|
||||
{
|
||||
[JsonPropertyName("schema_version")]
|
||||
public string SchemaVersion { get; init; } = "patch-oracle/v1";
|
||||
|
||||
[JsonPropertyName("id")]
|
||||
public required string Id { get; init; }
|
||||
|
||||
[JsonPropertyName("case_ref")]
|
||||
public required string CaseRef { get; init; }
|
||||
|
||||
[JsonPropertyName("variant")]
|
||||
public required string Variant { get; init; }
|
||||
|
||||
[JsonPropertyName("description")]
|
||||
public string? Description { get; init; }
|
||||
|
||||
[JsonPropertyName("expected_functions")]
|
||||
public IReadOnlyList<ExpectedFunction> ExpectedFunctions { get; init; } = Array.Empty<ExpectedFunction>();
|
||||
|
||||
[JsonPropertyName("expected_edges")]
|
||||
public IReadOnlyList<ExpectedEdge> ExpectedEdges { get; init; } = Array.Empty<ExpectedEdge>();
|
||||
|
||||
[JsonPropertyName("expected_roots")]
|
||||
public IReadOnlyList<ExpectedRoot> ExpectedRoots { get; init; } = Array.Empty<ExpectedRoot>();
|
||||
|
||||
[JsonPropertyName("forbidden_functions")]
|
||||
public IReadOnlyList<ExpectedFunction> ForbiddenFunctions { get; init; } = Array.Empty<ExpectedFunction>();
|
||||
|
||||
[JsonPropertyName("forbidden_edges")]
|
||||
public IReadOnlyList<ExpectedEdge> ForbiddenEdges { get; init; } = Array.Empty<ExpectedEdge>();
|
||||
|
||||
[JsonPropertyName("min_confidence")]
|
||||
public double MinConfidence { get; init; } = 0.5;
|
||||
|
||||
[JsonPropertyName("strict_mode")]
|
||||
public bool StrictMode { get; init; } = false;
|
||||
|
||||
[JsonPropertyName("created_at")]
|
||||
public DateTimeOffset? CreatedAt { get; init; }
|
||||
|
||||
[JsonPropertyName("updated_at")]
|
||||
public DateTimeOffset? UpdatedAt { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Expected function/node in the graph.
|
||||
/// </summary>
|
||||
public sealed record ExpectedFunction
|
||||
{
|
||||
[JsonPropertyName("symbol_id")]
|
||||
public required string SymbolId { get; init; }
|
||||
|
||||
[JsonPropertyName("lang")]
|
||||
public string? Lang { get; init; }
|
||||
|
||||
[JsonPropertyName("kind")]
|
||||
public string? Kind { get; init; }
|
||||
|
||||
[JsonPropertyName("purl_pattern")]
|
||||
public string? PurlPattern { get; init; }
|
||||
|
||||
[JsonPropertyName("required")]
|
||||
public bool Required { get; init; } = true;
|
||||
|
||||
[JsonPropertyName("reason")]
|
||||
public string? Reason { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Expected edge in the graph.
|
||||
/// </summary>
|
||||
public sealed record ExpectedEdge
|
||||
{
|
||||
[JsonPropertyName("from")]
|
||||
public required string From { get; init; }
|
||||
|
||||
[JsonPropertyName("to")]
|
||||
public required string To { get; init; }
|
||||
|
||||
[JsonPropertyName("kind")]
|
||||
public string? Kind { get; init; }
|
||||
|
||||
[JsonPropertyName("min_confidence")]
|
||||
public double? MinConfidence { get; init; }
|
||||
|
||||
[JsonPropertyName("required")]
|
||||
public bool Required { get; init; } = true;
|
||||
|
||||
[JsonPropertyName("reason")]
|
||||
public string? Reason { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Expected root node in the graph.
|
||||
/// </summary>
|
||||
public sealed record ExpectedRoot
|
||||
{
|
||||
[JsonPropertyName("id")]
|
||||
public required string Id { get; init; }
|
||||
|
||||
[JsonPropertyName("phase")]
|
||||
public string? Phase { get; init; }
|
||||
|
||||
[JsonPropertyName("required")]
|
||||
public bool Required { get; init; } = true;
|
||||
|
||||
[JsonPropertyName("reason")]
|
||||
public string? Reason { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Index entry for an oracle.
|
||||
/// </summary>
|
||||
public sealed record PatchOracleIndexEntry
|
||||
{
|
||||
[JsonPropertyName("id")]
|
||||
public required string Id { get; init; }
|
||||
|
||||
[JsonPropertyName("case_ref")]
|
||||
public required string CaseRef { get; init; }
|
||||
|
||||
[JsonPropertyName("variant")]
|
||||
public required string Variant { get; init; }
|
||||
|
||||
[JsonPropertyName("path")]
|
||||
public required string Path { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Root model for patch-oracle INDEX.json.
|
||||
/// </summary>
|
||||
public sealed record PatchOracleIndex
|
||||
{
|
||||
[JsonPropertyName("version")]
|
||||
public string Version { get; init; } = "1.0";
|
||||
|
||||
[JsonPropertyName("schema")]
|
||||
public string Schema { get; init; } = "patch-oracle/v1";
|
||||
|
||||
[JsonPropertyName("generated_at")]
|
||||
public DateTimeOffset? GeneratedAt { get; init; }
|
||||
|
||||
[JsonPropertyName("description")]
|
||||
public string? Description { get; init; }
|
||||
|
||||
[JsonPropertyName("oracles")]
|
||||
public IReadOnlyList<PatchOracleIndexEntry> Oracles { get; init; } = Array.Empty<PatchOracleIndexEntry>();
|
||||
}
|
||||
@@ -0,0 +1,494 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using FluentAssertions;
|
||||
using StellaOps.Reachability.FixtureTests.PatchOracle;
|
||||
using StellaOps.Scanner.Reachability;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Reachability.FixtureTests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for the patch-oracle harness infrastructure.
|
||||
/// Validates that the oracle comparison logic correctly identifies missing and forbidden elements.
|
||||
/// </summary>
|
||||
public class PatchOracleHarnessTests
|
||||
{
|
||||
private static readonly string RepoRoot = ReachbenchFixtureTests.LocateRepoRoot();
|
||||
private static readonly string PatchOracleRoot = Path.Combine(
|
||||
RepoRoot, "tests", "reachability", "fixtures", "patch-oracles");
|
||||
|
||||
#region Oracle Loading Tests
|
||||
|
||||
[Fact]
|
||||
public void Loader_IndexExists()
|
||||
{
|
||||
var loader = new PatchOracleLoader(PatchOracleRoot);
|
||||
loader.IndexExists().Should().BeTrue("patch-oracle INDEX.json should exist");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Loader_IndexLoadsSuccessfully()
|
||||
{
|
||||
var loader = new PatchOracleLoader(PatchOracleRoot);
|
||||
var index = loader.LoadIndex();
|
||||
|
||||
index.Should().NotBeNull();
|
||||
index.Version.Should().Be("1.0");
|
||||
index.Schema.Should().Be("patch-oracle/v1");
|
||||
index.Oracles.Should().NotBeEmpty("should have at least one oracle defined");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Loader_AllOraclesLoadSuccessfully()
|
||||
{
|
||||
var loader = new PatchOracleLoader(PatchOracleRoot);
|
||||
var oracles = loader.LoadAllOracles().ToList();
|
||||
|
||||
oracles.Should().NotBeEmpty();
|
||||
foreach (var oracle in oracles)
|
||||
{
|
||||
oracle.SchemaVersion.Should().Be("patch-oracle/v1");
|
||||
oracle.Id.Should().NotBeNullOrEmpty();
|
||||
oracle.CaseRef.Should().NotBeNullOrEmpty();
|
||||
oracle.Variant.Should().BeOneOf("reachable", "unreachable");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Loader_LoadOracleById()
|
||||
{
|
||||
var loader = new PatchOracleLoader(PatchOracleRoot);
|
||||
var oracle = loader.LoadOracle("curl-CVE-2023-38545-socks5-heap-reachable");
|
||||
|
||||
oracle.Should().NotBeNull();
|
||||
oracle.Id.Should().Be("curl-CVE-2023-38545-socks5-heap-reachable");
|
||||
oracle.CaseRef.Should().Be("curl-CVE-2023-38545-socks5-heap");
|
||||
oracle.Variant.Should().Be("reachable");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Comparer Tests - Pass Cases
|
||||
|
||||
[Fact]
|
||||
public void Comparer_PassesWhenAllExpectedElementsPresent()
|
||||
{
|
||||
var oracle = new PatchOracleDefinition
|
||||
{
|
||||
Id = "test-pass",
|
||||
CaseRef = "test-case",
|
||||
Variant = "reachable",
|
||||
ExpectedFunctions = new[]
|
||||
{
|
||||
new ExpectedFunction { SymbolId = "sym://test#func1", Required = true },
|
||||
new ExpectedFunction { SymbolId = "sym://test#func2", Required = true }
|
||||
},
|
||||
ExpectedEdges = new[]
|
||||
{
|
||||
new ExpectedEdge { From = "sym://test#func1", To = "sym://test#func2", Required = true }
|
||||
}
|
||||
};
|
||||
|
||||
var graph = new RichGraph(
|
||||
Nodes: new[]
|
||||
{
|
||||
new RichGraphNode("sym://test#func1", "sym://test#func1", null, null, "test", "function", null, null, null, null, null),
|
||||
new RichGraphNode("sym://test#func2", "sym://test#func2", null, null, "test", "function", null, null, null, null, null)
|
||||
},
|
||||
Edges: new[]
|
||||
{
|
||||
new RichGraphEdge("sym://test#func1", "sym://test#func2", "call", null, null, null, 0.9, null)
|
||||
},
|
||||
Roots: Array.Empty<RichGraphRoot>(),
|
||||
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
|
||||
);
|
||||
|
||||
var comparer = new PatchOracleComparer(oracle);
|
||||
var result = comparer.Compare(graph);
|
||||
|
||||
result.Success.Should().BeTrue();
|
||||
result.Violations.Should().BeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Comparer_PassesWithWildcardPatterns()
|
||||
{
|
||||
var oracle = new PatchOracleDefinition
|
||||
{
|
||||
Id = "test-wildcard",
|
||||
CaseRef = "test-case",
|
||||
Variant = "reachable",
|
||||
ExpectedFunctions = new[]
|
||||
{
|
||||
new ExpectedFunction { SymbolId = "sym://test#*", Required = true }
|
||||
}
|
||||
};
|
||||
|
||||
var graph = new RichGraph(
|
||||
Nodes: new[]
|
||||
{
|
||||
new RichGraphNode("sym://test#anything", "sym://test#anything", null, null, "test", "function", null, null, null, null, null)
|
||||
},
|
||||
Edges: Array.Empty<RichGraphEdge>(),
|
||||
Roots: Array.Empty<RichGraphRoot>(),
|
||||
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
|
||||
);
|
||||
|
||||
var comparer = new PatchOracleComparer(oracle);
|
||||
var result = comparer.Compare(graph);
|
||||
|
||||
result.Success.Should().BeTrue();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Comparer Tests - Fail Cases
|
||||
|
||||
[Fact]
|
||||
public void Comparer_FailsWhenExpectedFunctionMissing()
|
||||
{
|
||||
var oracle = new PatchOracleDefinition
|
||||
{
|
||||
Id = "test-missing-func",
|
||||
CaseRef = "test-case",
|
||||
Variant = "reachable",
|
||||
ExpectedFunctions = new[]
|
||||
{
|
||||
new ExpectedFunction { SymbolId = "sym://test#missing", Required = true, Reason = "This function is critical" }
|
||||
}
|
||||
};
|
||||
|
||||
var graph = new RichGraph(
|
||||
Nodes: Array.Empty<RichGraphNode>(),
|
||||
Edges: Array.Empty<RichGraphEdge>(),
|
||||
Roots: Array.Empty<RichGraphRoot>(),
|
||||
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
|
||||
);
|
||||
|
||||
var comparer = new PatchOracleComparer(oracle);
|
||||
var result = comparer.Compare(graph);
|
||||
|
||||
result.Success.Should().BeFalse();
|
||||
result.Violations.Should().HaveCount(1);
|
||||
result.Violations[0].Type.Should().Be(ViolationType.MissingFunction);
|
||||
result.Violations[0].From.Should().Be("sym://test#missing");
|
||||
result.Summary.MissingFunctions.Should().Be(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Comparer_FailsWhenExpectedEdgeMissing()
|
||||
{
|
||||
var oracle = new PatchOracleDefinition
|
||||
{
|
||||
Id = "test-missing-edge",
|
||||
CaseRef = "test-case",
|
||||
Variant = "reachable",
|
||||
ExpectedEdges = new[]
|
||||
{
|
||||
new ExpectedEdge { From = "sym://a", To = "sym://b", Required = true }
|
||||
}
|
||||
};
|
||||
|
||||
var graph = new RichGraph(
|
||||
Nodes: new[]
|
||||
{
|
||||
new RichGraphNode("sym://a", "sym://a", null, null, "test", "function", null, null, null, null, null),
|
||||
new RichGraphNode("sym://b", "sym://b", null, null, "test", "function", null, null, null, null, null)
|
||||
},
|
||||
Edges: Array.Empty<RichGraphEdge>(),
|
||||
Roots: Array.Empty<RichGraphRoot>(),
|
||||
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
|
||||
);
|
||||
|
||||
var comparer = new PatchOracleComparer(oracle);
|
||||
var result = comparer.Compare(graph);
|
||||
|
||||
result.Success.Should().BeFalse();
|
||||
result.Violations.Should().HaveCount(1);
|
||||
result.Violations[0].Type.Should().Be(ViolationType.MissingEdge);
|
||||
result.Summary.MissingEdges.Should().Be(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Comparer_FailsWhenExpectedRootMissing()
|
||||
{
|
||||
var oracle = new PatchOracleDefinition
|
||||
{
|
||||
Id = "test-missing-root",
|
||||
CaseRef = "test-case",
|
||||
Variant = "reachable",
|
||||
ExpectedRoots = new[]
|
||||
{
|
||||
new ExpectedRoot { Id = "sym://root#main", Phase = "main", Required = true }
|
||||
}
|
||||
};
|
||||
|
||||
var graph = new RichGraph(
|
||||
Nodes: Array.Empty<RichGraphNode>(),
|
||||
Edges: Array.Empty<RichGraphEdge>(),
|
||||
Roots: Array.Empty<RichGraphRoot>(),
|
||||
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
|
||||
);
|
||||
|
||||
var comparer = new PatchOracleComparer(oracle);
|
||||
var result = comparer.Compare(graph);
|
||||
|
||||
result.Success.Should().BeFalse();
|
||||
result.Violations.Should().HaveCount(1);
|
||||
result.Violations[0].Type.Should().Be(ViolationType.MissingRoot);
|
||||
result.Summary.MissingRoots.Should().Be(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Comparer_FailsWhenForbiddenFunctionPresent()
|
||||
{
|
||||
var oracle = new PatchOracleDefinition
|
||||
{
|
||||
Id = "test-forbidden-func",
|
||||
CaseRef = "test-case",
|
||||
Variant = "unreachable",
|
||||
ForbiddenFunctions = new[]
|
||||
{
|
||||
new ExpectedFunction { SymbolId = "sym://dangerous#sink", Reason = "Should not be reachable" }
|
||||
}
|
||||
};
|
||||
|
||||
var graph = new RichGraph(
|
||||
Nodes: new[]
|
||||
{
|
||||
new RichGraphNode("sym://dangerous#sink", "sym://dangerous#sink", null, null, "test", "function", null, null, null, null, null)
|
||||
},
|
||||
Edges: Array.Empty<RichGraphEdge>(),
|
||||
Roots: Array.Empty<RichGraphRoot>(),
|
||||
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
|
||||
);
|
||||
|
||||
var comparer = new PatchOracleComparer(oracle);
|
||||
var result = comparer.Compare(graph);
|
||||
|
||||
result.Success.Should().BeFalse();
|
||||
result.Violations.Should().HaveCount(1);
|
||||
result.Violations[0].Type.Should().Be(ViolationType.ForbiddenFunctionPresent);
|
||||
result.Summary.ForbiddenFunctionsPresent.Should().Be(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Comparer_FailsWhenForbiddenEdgePresent()
|
||||
{
|
||||
var oracle = new PatchOracleDefinition
|
||||
{
|
||||
Id = "test-forbidden-edge",
|
||||
CaseRef = "test-case",
|
||||
Variant = "unreachable",
|
||||
ForbiddenEdges = new[]
|
||||
{
|
||||
new ExpectedEdge { From = "sym://entry", To = "sym://sink", Reason = "Path should be blocked" }
|
||||
}
|
||||
};
|
||||
|
||||
var graph = new RichGraph(
|
||||
Nodes: Array.Empty<RichGraphNode>(),
|
||||
Edges: new[]
|
||||
{
|
||||
new RichGraphEdge("sym://entry", "sym://sink", "call", null, null, null, 1.0, null)
|
||||
},
|
||||
Roots: Array.Empty<RichGraphRoot>(),
|
||||
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
|
||||
);
|
||||
|
||||
var comparer = new PatchOracleComparer(oracle);
|
||||
var result = comparer.Compare(graph);
|
||||
|
||||
result.Success.Should().BeFalse();
|
||||
result.Violations.Should().HaveCount(1);
|
||||
result.Violations[0].Type.Should().Be(ViolationType.ForbiddenEdgePresent);
|
||||
result.Summary.ForbiddenEdgesPresent.Should().Be(1);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Confidence Threshold Tests
|
||||
|
||||
[Fact]
|
||||
public void Comparer_RespectsMinConfidenceThreshold()
|
||||
{
|
||||
var oracle = new PatchOracleDefinition
|
||||
{
|
||||
Id = "test-confidence",
|
||||
CaseRef = "test-case",
|
||||
Variant = "reachable",
|
||||
MinConfidence = 0.8,
|
||||
ExpectedEdges = new[]
|
||||
{
|
||||
new ExpectedEdge { From = "sym://a", To = "sym://b", Required = true }
|
||||
}
|
||||
};
|
||||
|
||||
var lowConfidenceGraph = new RichGraph(
|
||||
Nodes: Array.Empty<RichGraphNode>(),
|
||||
Edges: new[]
|
||||
{
|
||||
new RichGraphEdge("sym://a", "sym://b", "call", null, null, null, 0.5, null)
|
||||
},
|
||||
Roots: Array.Empty<RichGraphRoot>(),
|
||||
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
|
||||
);
|
||||
|
||||
var comparer = new PatchOracleComparer(oracle);
|
||||
var result = comparer.Compare(lowConfidenceGraph);
|
||||
|
||||
result.Success.Should().BeFalse("edge confidence 0.5 is below threshold 0.8");
|
||||
result.Summary.MissingEdges.Should().Be(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Comparer_EdgeSpecificConfidenceOverridesDefault()
|
||||
{
|
||||
var oracle = new PatchOracleDefinition
|
||||
{
|
||||
Id = "test-edge-confidence",
|
||||
CaseRef = "test-case",
|
||||
Variant = "reachable",
|
||||
MinConfidence = 0.8,
|
||||
ExpectedEdges = new[]
|
||||
{
|
||||
new ExpectedEdge { From = "sym://a", To = "sym://b", MinConfidence = 0.3, Required = true }
|
||||
}
|
||||
};
|
||||
|
||||
var lowConfidenceGraph = new RichGraph(
|
||||
Nodes: Array.Empty<RichGraphNode>(),
|
||||
Edges: new[]
|
||||
{
|
||||
new RichGraphEdge("sym://a", "sym://b", "call", null, null, null, 0.5, null)
|
||||
},
|
||||
Roots: Array.Empty<RichGraphRoot>(),
|
||||
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
|
||||
);
|
||||
|
||||
var comparer = new PatchOracleComparer(oracle);
|
||||
var result = comparer.Compare(lowConfidenceGraph);
|
||||
|
||||
result.Success.Should().BeTrue("edge-specific threshold 0.3 allows confidence 0.5");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Strict Mode Tests
|
||||
|
||||
[Fact]
|
||||
public void Comparer_StrictModeRejectsUnexpectedNodes()
|
||||
{
|
||||
var oracle = new PatchOracleDefinition
|
||||
{
|
||||
Id = "test-strict",
|
||||
CaseRef = "test-case",
|
||||
Variant = "reachable",
|
||||
StrictMode = true,
|
||||
ExpectedFunctions = new[]
|
||||
{
|
||||
new ExpectedFunction { SymbolId = "sym://expected", Required = true }
|
||||
}
|
||||
};
|
||||
|
||||
var graph = new RichGraph(
|
||||
Nodes: new[]
|
||||
{
|
||||
new RichGraphNode("sym://expected", "sym://expected", null, null, "test", "function", null, null, null, null, null),
|
||||
new RichGraphNode("sym://unexpected", "sym://unexpected", null, null, "test", "function", null, null, null, null, null)
|
||||
},
|
||||
Edges: Array.Empty<RichGraphEdge>(),
|
||||
Roots: Array.Empty<RichGraphRoot>(),
|
||||
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
|
||||
);
|
||||
|
||||
var comparer = new PatchOracleComparer(oracle);
|
||||
var result = comparer.Compare(graph);
|
||||
|
||||
result.Success.Should().BeFalse();
|
||||
result.Violations.Should().Contain(v => v.Type == ViolationType.UnexpectedFunction);
|
||||
result.Summary.UnexpectedFunctions.Should().Be(1);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Report Generation Tests
|
||||
|
||||
[Fact]
|
||||
public void Result_GeneratesReadableReport()
|
||||
{
|
||||
var oracle = new PatchOracleDefinition
|
||||
{
|
||||
Id = "test-report",
|
||||
CaseRef = "test-case",
|
||||
Variant = "reachable",
|
||||
ExpectedFunctions = new[]
|
||||
{
|
||||
new ExpectedFunction { SymbolId = "sym://missing", Required = true, Reason = "Critical sink" }
|
||||
}
|
||||
};
|
||||
|
||||
var graph = new RichGraph(
|
||||
Nodes: new[]
|
||||
{
|
||||
new RichGraphNode("sym://other", "sym://other", null, null, "test", "function", null, null, null, null, null)
|
||||
},
|
||||
Edges: Array.Empty<RichGraphEdge>(),
|
||||
Roots: Array.Empty<RichGraphRoot>(),
|
||||
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
|
||||
);
|
||||
|
||||
var comparer = new PatchOracleComparer(oracle);
|
||||
var result = comparer.Compare(graph);
|
||||
|
||||
var report = result.ToReport();
|
||||
|
||||
report.Should().Contain("FAIL");
|
||||
report.Should().Contain("test-report");
|
||||
report.Should().Contain("MissingFunction");
|
||||
report.Should().Contain("sym://missing");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Integration with Fixture Data
|
||||
|
||||
public static IEnumerable<object[]> AllOracleData()
|
||||
{
|
||||
var loader = new PatchOracleLoader(PatchOracleRoot);
|
||||
if (!loader.IndexExists())
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
foreach (var entry in loader.EnumerateOracles())
|
||||
{
|
||||
yield return new object[] { entry.Id, entry.CaseRef, entry.Variant };
|
||||
}
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(AllOracleData))]
|
||||
public void AllOracles_HaveValidStructure(string oracleId, string caseRef, string variant)
|
||||
{
|
||||
var loader = new PatchOracleLoader(PatchOracleRoot);
|
||||
var oracle = loader.LoadOracle(oracleId);
|
||||
|
||||
oracle.Id.Should().Be(oracleId);
|
||||
oracle.CaseRef.Should().Be(caseRef);
|
||||
oracle.Variant.Should().Be(variant);
|
||||
oracle.SchemaVersion.Should().Be("patch-oracle/v1");
|
||||
|
||||
// At least one expectation should be defined
|
||||
var hasExpectations = oracle.ExpectedFunctions.Count > 0
|
||||
|| oracle.ExpectedEdges.Count > 0
|
||||
|| oracle.ExpectedRoots.Count > 0
|
||||
|| oracle.ForbiddenFunctions.Count > 0
|
||||
|| oracle.ForbiddenEdges.Count > 0;
|
||||
hasExpectations.Should().BeTrue($"Oracle '{oracleId}' should define at least one expectation");
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
32
tests/reachability/fixtures/patch-oracles/INDEX.json
Normal file
32
tests/reachability/fixtures/patch-oracles/INDEX.json
Normal file
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"version": "1.0",
|
||||
"schema": "patch-oracle/v1",
|
||||
"generated_at": "2025-12-13T00:00:00Z",
|
||||
"description": "Patch-oracle fixtures for CI graph validation. Each oracle defines expected functions/edges that must be present (or absent) in generated reachability graphs.",
|
||||
"oracles": [
|
||||
{
|
||||
"id": "curl-CVE-2023-38545-socks5-heap-reachable",
|
||||
"case_ref": "curl-CVE-2023-38545-socks5-heap",
|
||||
"variant": "reachable",
|
||||
"path": "cases/curl-CVE-2023-38545-socks5-heap/reachable.oracle.json"
|
||||
},
|
||||
{
|
||||
"id": "curl-CVE-2023-38545-socks5-heap-unreachable",
|
||||
"case_ref": "curl-CVE-2023-38545-socks5-heap",
|
||||
"variant": "unreachable",
|
||||
"path": "cases/curl-CVE-2023-38545-socks5-heap/unreachable.oracle.json"
|
||||
},
|
||||
{
|
||||
"id": "java-log4j-CVE-2021-44228-log4shell-reachable",
|
||||
"case_ref": "java-log4j-CVE-2021-44228-log4shell",
|
||||
"variant": "reachable",
|
||||
"path": "cases/java-log4j-CVE-2021-44228-log4shell/reachable.oracle.json"
|
||||
},
|
||||
{
|
||||
"id": "dotnet-kestrel-CVE-2023-44487-http2-rapid-reset-reachable",
|
||||
"case_ref": "dotnet-kestrel-CVE-2023-44487-http2-rapid-reset",
|
||||
"variant": "reachable",
|
||||
"path": "cases/dotnet-kestrel-CVE-2023-44487-http2-rapid-reset/reachable.oracle.json"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"schema_version": "patch-oracle/v1",
|
||||
"id": "curl-CVE-2023-38545-socks5-heap-reachable",
|
||||
"case_ref": "curl-CVE-2023-38545-socks5-heap",
|
||||
"variant": "reachable",
|
||||
"description": "Validates that the SOCKS5 heap overflow vulnerability path is reachable from network handler to vulnerable sink",
|
||||
"expected_functions": [
|
||||
{
|
||||
"symbol_id": "sym://net:handler#read",
|
||||
"kind": "entrypoint",
|
||||
"required": true,
|
||||
"reason": "Network read handler is the entry point for external data"
|
||||
},
|
||||
{
|
||||
"symbol_id": "sym://curl:curl.c#entry",
|
||||
"kind": "function",
|
||||
"required": true,
|
||||
"reason": "SOCKS5 protocol handling entry point"
|
||||
},
|
||||
{
|
||||
"symbol_id": "sym://curl:curl.c#sink",
|
||||
"kind": "function",
|
||||
"required": true,
|
||||
"reason": "Vulnerable buffer handling function"
|
||||
}
|
||||
],
|
||||
"expected_edges": [
|
||||
{
|
||||
"from": "sym://net:handler#read",
|
||||
"to": "sym://curl:curl.c#entry",
|
||||
"kind": "call",
|
||||
"min_confidence": 0.8,
|
||||
"required": true,
|
||||
"reason": "Data flows from network handler to SOCKS5 handler"
|
||||
},
|
||||
{
|
||||
"from": "sym://curl:curl.c#entry",
|
||||
"to": "sym://curl:curl.c#sink",
|
||||
"kind": "call",
|
||||
"min_confidence": 0.8,
|
||||
"required": true,
|
||||
"reason": "SOCKS5 handler invokes vulnerable buffer function"
|
||||
}
|
||||
],
|
||||
"expected_roots": [
|
||||
{
|
||||
"id": "sym://net:handler#read",
|
||||
"phase": "runtime",
|
||||
"required": true,
|
||||
"reason": "Network handler is the runtime entry point"
|
||||
}
|
||||
],
|
||||
"min_confidence": 0.5,
|
||||
"strict_mode": false,
|
||||
"created_at": "2025-12-13T00:00:00Z"
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"schema_version": "patch-oracle/v1",
|
||||
"id": "curl-CVE-2023-38545-socks5-heap-unreachable",
|
||||
"case_ref": "curl-CVE-2023-38545-socks5-heap",
|
||||
"variant": "unreachable",
|
||||
"description": "Validates that the SOCKS5 heap overflow vulnerability path is NOT reachable when SOCKS5 is disabled",
|
||||
"expected_functions": [
|
||||
{
|
||||
"symbol_id": "sym://net:handler#read",
|
||||
"kind": "entrypoint",
|
||||
"required": true,
|
||||
"reason": "Network read handler still exists but cannot reach vulnerable code"
|
||||
}
|
||||
],
|
||||
"expected_edges": [],
|
||||
"forbidden_functions": [
|
||||
{
|
||||
"symbol_id": "sym://curl:curl.c#sink",
|
||||
"reason": "Vulnerable sink should not be in call graph when SOCKS5 disabled"
|
||||
}
|
||||
],
|
||||
"forbidden_edges": [
|
||||
{
|
||||
"from": "sym://curl:curl.c#entry",
|
||||
"to": "sym://curl:curl.c#sink",
|
||||
"reason": "This edge should not exist when SOCKS5 is disabled"
|
||||
}
|
||||
],
|
||||
"min_confidence": 0.5,
|
||||
"strict_mode": false,
|
||||
"created_at": "2025-12-13T00:00:00Z"
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"schema_version": "patch-oracle/v1",
|
||||
"id": "dotnet-kestrel-CVE-2023-44487-http2-rapid-reset-reachable",
|
||||
"case_ref": "dotnet-kestrel-CVE-2023-44487-http2-rapid-reset",
|
||||
"variant": "reachable",
|
||||
"description": "Validates that the HTTP/2 Rapid Reset DoS vulnerability path is reachable",
|
||||
"expected_functions": [
|
||||
{
|
||||
"symbol_id": "sym://dotnet:Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2.Http2Connection#ProcessRequestsAsync",
|
||||
"lang": "dotnet",
|
||||
"kind": "method",
|
||||
"required": true,
|
||||
"reason": "HTTP/2 connection handler entry point"
|
||||
},
|
||||
{
|
||||
"symbol_id": "sym://dotnet:Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2.Http2Stream#*",
|
||||
"lang": "dotnet",
|
||||
"kind": "method",
|
||||
"required": true,
|
||||
"reason": "HTTP/2 stream management affected by rapid reset"
|
||||
}
|
||||
],
|
||||
"expected_edges": [
|
||||
{
|
||||
"from": "sym://dotnet:Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2.Http2Connection#ProcessRequestsAsync",
|
||||
"to": "sym://dotnet:Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2.Http2Stream#*",
|
||||
"kind": "call",
|
||||
"min_confidence": 0.7,
|
||||
"required": true,
|
||||
"reason": "Connection handler creates/manages streams"
|
||||
}
|
||||
],
|
||||
"expected_roots": [
|
||||
{
|
||||
"id": "sym://dotnet:Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2.Http2Connection#ProcessRequestsAsync",
|
||||
"phase": "runtime",
|
||||
"required": true,
|
||||
"reason": "HTTP/2 processing is a runtime entry point"
|
||||
}
|
||||
],
|
||||
"min_confidence": 0.5,
|
||||
"strict_mode": false,
|
||||
"created_at": "2025-12-13T00:00:00Z"
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
{
|
||||
"schema_version": "patch-oracle/v1",
|
||||
"id": "java-log4j-CVE-2021-44228-log4shell-reachable",
|
||||
"case_ref": "java-log4j-CVE-2021-44228-log4shell",
|
||||
"variant": "reachable",
|
||||
"description": "Validates that the Log4Shell JNDI injection path is reachable from logger to JNDI lookup",
|
||||
"expected_functions": [
|
||||
{
|
||||
"symbol_id": "sym://java:org.apache.logging.log4j.core.Logger#logMessage",
|
||||
"lang": "java",
|
||||
"kind": "method",
|
||||
"required": true,
|
||||
"reason": "Logger entry point that processes user-controlled format strings"
|
||||
},
|
||||
{
|
||||
"symbol_id": "sym://java:org.apache.logging.log4j.core.pattern.MessagePatternConverter#format",
|
||||
"lang": "java",
|
||||
"kind": "method",
|
||||
"required": true,
|
||||
"reason": "Pattern converter that triggers lookup substitution"
|
||||
},
|
||||
{
|
||||
"symbol_id": "sym://java:org.apache.logging.log4j.core.lookup.StrSubstitutor#replace",
|
||||
"lang": "java",
|
||||
"kind": "method",
|
||||
"required": true,
|
||||
"reason": "String substitution that invokes lookups"
|
||||
},
|
||||
{
|
||||
"symbol_id": "sym://java:org.apache.logging.log4j.core.lookup.JndiLookup#lookup",
|
||||
"lang": "java",
|
||||
"kind": "method",
|
||||
"required": true,
|
||||
"reason": "Vulnerable JNDI lookup method"
|
||||
}
|
||||
],
|
||||
"expected_edges": [
|
||||
{
|
||||
"from": "sym://java:org.apache.logging.log4j.core.Logger#logMessage",
|
||||
"to": "sym://java:org.apache.logging.log4j.core.pattern.MessagePatternConverter#format",
|
||||
"kind": "call",
|
||||
"required": true,
|
||||
"reason": "Logger delegates to pattern converter"
|
||||
},
|
||||
{
|
||||
"from": "sym://java:org.apache.logging.log4j.core.lookup.StrSubstitutor#replace",
|
||||
"to": "sym://java:org.apache.logging.log4j.core.lookup.JndiLookup#lookup",
|
||||
"kind": "call",
|
||||
"required": true,
|
||||
"reason": "String substitution invokes JNDI lookup"
|
||||
}
|
||||
],
|
||||
"expected_roots": [
|
||||
{
|
||||
"id": "sym://java:org.apache.logging.log4j.core.Logger#*",
|
||||
"phase": "runtime",
|
||||
"required": true,
|
||||
"reason": "Logger methods are runtime entry points"
|
||||
}
|
||||
],
|
||||
"min_confidence": 0.6,
|
||||
"strict_mode": false,
|
||||
"created_at": "2025-12-13T00:00:00Z"
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"$id": "stellaops:patch-oracle/v1",
|
||||
"title": "Patch Oracle Schema v1",
|
||||
"description": "Defines expected functions/edges for reachability graph validation. CI fails when expected elements are missing.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"schema_version": {
|
||||
"type": "string",
|
||||
"const": "patch-oracle/v1",
|
||||
"description": "Schema version identifier"
|
||||
},
|
||||
"id": {
|
||||
"type": "string",
|
||||
"description": "Unique oracle identifier (e.g., 'curl-CVE-2023-38545-socks5-heap-reachable')"
|
||||
},
|
||||
"case_ref": {
|
||||
"type": "string",
|
||||
"description": "Reference to parent reachbench case (e.g., 'curl-CVE-2023-38545-socks5-heap')"
|
||||
},
|
||||
"variant": {
|
||||
"type": "string",
|
||||
"enum": ["reachable", "unreachable"],
|
||||
"description": "Which variant this oracle applies to"
|
||||
},
|
||||
"description": {
|
||||
"type": "string",
|
||||
"description": "Human-readable description of what this oracle validates"
|
||||
},
|
||||
"expected_functions": {
|
||||
"type": "array",
|
||||
"description": "Functions that MUST be present in the generated graph",
|
||||
"items": {
|
||||
"$ref": "#/definitions/expected_function"
|
||||
}
|
||||
},
|
||||
"expected_edges": {
|
||||
"type": "array",
|
||||
"description": "Edges that MUST be present in the generated graph",
|
||||
"items": {
|
||||
"$ref": "#/definitions/expected_edge"
|
||||
}
|
||||
},
|
||||
"expected_roots": {
|
||||
"type": "array",
|
||||
"description": "Root nodes that MUST be present in the generated graph",
|
||||
"items": {
|
||||
"$ref": "#/definitions/expected_root"
|
||||
}
|
||||
},
|
||||
"forbidden_functions": {
|
||||
"type": "array",
|
||||
"description": "Functions that MUST NOT be present (for unreachable variants)",
|
||||
"items": {
|
||||
"$ref": "#/definitions/expected_function"
|
||||
}
|
||||
},
|
||||
"forbidden_edges": {
|
||||
"type": "array",
|
||||
"description": "Edges that MUST NOT be present (for unreachable variants)",
|
||||
"items": {
|
||||
"$ref": "#/definitions/expected_edge"
|
||||
}
|
||||
},
|
||||
"min_confidence": {
|
||||
"type": "number",
|
||||
"minimum": 0.0,
|
||||
"maximum": 1.0,
|
||||
"default": 0.5,
|
||||
"description": "Minimum confidence threshold for edge matching"
|
||||
},
|
||||
"strict_mode": {
|
||||
"type": "boolean",
|
||||
"default": false,
|
||||
"description": "If true, extra functions/edges not in oracle cause failure"
|
||||
},
|
||||
"created_at": {
|
||||
"type": "string",
|
||||
"format": "date-time",
|
||||
"description": "When this oracle was created"
|
||||
},
|
||||
"updated_at": {
|
||||
"type": "string",
|
||||
"format": "date-time",
|
||||
"description": "When this oracle was last updated"
|
||||
}
|
||||
},
|
||||
"required": ["schema_version", "id", "case_ref", "variant"],
|
||||
"definitions": {
|
||||
"expected_function": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"symbol_id": {
|
||||
"type": "string",
|
||||
"description": "Expected symbol ID (exact match or pattern with '*' wildcards)"
|
||||
},
|
||||
"lang": {
|
||||
"type": "string",
|
||||
"description": "Expected language (optional, for filtering)"
|
||||
},
|
||||
"kind": {
|
||||
"type": "string",
|
||||
"description": "Expected node kind (e.g., 'function', 'method', 'entrypoint')"
|
||||
},
|
||||
"purl_pattern": {
|
||||
"type": "string",
|
||||
"description": "Expected purl pattern (optional, supports wildcards)"
|
||||
},
|
||||
"required": {
|
||||
"type": "boolean",
|
||||
"default": true,
|
||||
"description": "If true, missing this function fails CI"
|
||||
},
|
||||
"reason": {
|
||||
"type": "string",
|
||||
"description": "Why this function is expected (for documentation)"
|
||||
}
|
||||
},
|
||||
"required": ["symbol_id"]
|
||||
},
|
||||
"expected_edge": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"from": {
|
||||
"type": "string",
|
||||
"description": "Source node symbol ID (exact match or pattern)"
|
||||
},
|
||||
"to": {
|
||||
"type": "string",
|
||||
"description": "Target node symbol ID (exact match or pattern)"
|
||||
},
|
||||
"kind": {
|
||||
"type": "string",
|
||||
"description": "Expected edge kind (e.g., 'call', 'plt', 'indirect')"
|
||||
},
|
||||
"min_confidence": {
|
||||
"type": "number",
|
||||
"minimum": 0.0,
|
||||
"maximum": 1.0,
|
||||
"description": "Minimum confidence for this specific edge"
|
||||
},
|
||||
"required": {
|
||||
"type": "boolean",
|
||||
"default": true,
|
||||
"description": "If true, missing this edge fails CI"
|
||||
},
|
||||
"reason": {
|
||||
"type": "string",
|
||||
"description": "Why this edge is expected (for documentation)"
|
||||
}
|
||||
},
|
||||
"required": ["from", "to"]
|
||||
},
|
||||
"expected_root": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string",
|
||||
"description": "Root node ID (exact match or pattern)"
|
||||
},
|
||||
"phase": {
|
||||
"type": "string",
|
||||
"enum": ["load", "init", "main", "runtime", "fini"],
|
||||
"description": "Expected execution phase"
|
||||
},
|
||||
"required": {
|
||||
"type": "boolean",
|
||||
"default": true,
|
||||
"description": "If true, missing this root fails CI"
|
||||
},
|
||||
"reason": {
|
||||
"type": "string",
|
||||
"description": "Why this root is expected"
|
||||
}
|
||||
},
|
||||
"required": ["id"]
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user