up
Some checks failed
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Manifest Integrity / Validate Schema Integrity (push) Has been cancelled
Manifest Integrity / Validate Contract Documents (push) Has been cancelled
Manifest Integrity / Validate Pack Fixtures (push) Has been cancelled
Manifest Integrity / Audit SHA256SUMS Files (push) Has been cancelled
Manifest Integrity / Verify Merkle Roots (push) Has been cancelled
devportal-offline / build-offline (push) Has been cancelled
Mirror Thin Bundle Sign & Verify / mirror-sign (push) Has been cancelled

This commit is contained in:
StellaOps Bot
2025-12-13 18:08:55 +02:00
parent 6e45066e37
commit f1a39c4ce3
234 changed files with 24038 additions and 6910 deletions

View File

@@ -0,0 +1,375 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using StellaOps.Scanner.Reachability;
namespace StellaOps.Reachability.FixtureTests.PatchOracle;
/// <summary>
/// Compares a RichGraph against a patch-oracle definition.
/// Reports missing expected elements and present forbidden elements.
/// </summary>
public sealed class PatchOracleComparer
{
private readonly PatchOracleDefinition _oracle;
private readonly double _defaultMinConfidence;
public PatchOracleComparer(PatchOracleDefinition oracle)
{
_oracle = oracle ?? throw new ArgumentNullException(nameof(oracle));
_defaultMinConfidence = oracle.MinConfidence;
}
/// <summary>
/// Compares the graph against the oracle and returns a result.
/// </summary>
public PatchOracleResult Compare(RichGraph graph)
{
ArgumentNullException.ThrowIfNull(graph);
var violations = new List<PatchOracleViolation>();
// Check expected functions
foreach (var expected in _oracle.ExpectedFunctions.Where(f => f.Required))
{
if (!HasMatchingNode(graph, expected))
{
violations.Add(new PatchOracleViolation(
ViolationType.MissingFunction,
expected.SymbolId,
null,
expected.Reason ?? $"Expected function '{expected.SymbolId}' not found in graph"));
}
}
// Check expected edges
foreach (var expected in _oracle.ExpectedEdges.Where(e => e.Required))
{
var minConf = expected.MinConfidence ?? _defaultMinConfidence;
if (!HasMatchingEdge(graph, expected, minConf))
{
violations.Add(new PatchOracleViolation(
ViolationType.MissingEdge,
expected.From,
expected.To,
expected.Reason ?? $"Expected edge '{expected.From}' -> '{expected.To}' not found in graph"));
}
}
// Check expected roots
foreach (var expected in _oracle.ExpectedRoots.Where(r => r.Required))
{
if (!HasMatchingRoot(graph, expected))
{
violations.Add(new PatchOracleViolation(
ViolationType.MissingRoot,
expected.Id,
null,
expected.Reason ?? $"Expected root '{expected.Id}' not found in graph"));
}
}
// Check forbidden functions
foreach (var forbidden in _oracle.ForbiddenFunctions)
{
if (HasMatchingNode(graph, forbidden))
{
violations.Add(new PatchOracleViolation(
ViolationType.ForbiddenFunctionPresent,
forbidden.SymbolId,
null,
forbidden.Reason ?? $"Forbidden function '{forbidden.SymbolId}' is present in graph"));
}
}
// Check forbidden edges
foreach (var forbidden in _oracle.ForbiddenEdges)
{
if (HasMatchingEdge(graph, forbidden, 0.0))
{
violations.Add(new PatchOracleViolation(
ViolationType.ForbiddenEdgePresent,
forbidden.From,
forbidden.To,
forbidden.Reason ?? $"Forbidden edge '{forbidden.From}' -> '{forbidden.To}' is present in graph"));
}
}
// Strict mode: check for unexpected elements
if (_oracle.StrictMode)
{
var unexpectedNodes = FindUnexpectedNodes(graph);
foreach (var node in unexpectedNodes)
{
violations.Add(new PatchOracleViolation(
ViolationType.UnexpectedFunction,
node.Id,
null,
$"Strict mode: unexpected function '{node.Id}' found in graph"));
}
var unexpectedEdges = FindUnexpectedEdges(graph);
foreach (var edge in unexpectedEdges)
{
violations.Add(new PatchOracleViolation(
ViolationType.UnexpectedEdge,
edge.From,
edge.To,
$"Strict mode: unexpected edge '{edge.From}' -> '{edge.To}' found in graph"));
}
}
return new PatchOracleResult(
OracleId: _oracle.Id,
CaseRef: _oracle.CaseRef,
Variant: _oracle.Variant,
Success: violations.Count == 0,
Violations: violations,
Summary: GenerateSummary(graph, violations));
}
private bool HasMatchingNode(RichGraph graph, ExpectedFunction expected)
{
foreach (var node in graph.Nodes)
{
if (!MatchesPattern(node.Id, expected.SymbolId) &&
!MatchesPattern(node.SymbolId, expected.SymbolId))
{
continue;
}
if (!string.IsNullOrEmpty(expected.Lang) &&
!string.Equals(node.Lang, expected.Lang, StringComparison.OrdinalIgnoreCase))
{
continue;
}
if (!string.IsNullOrEmpty(expected.Kind) &&
!string.Equals(node.Kind, expected.Kind, StringComparison.OrdinalIgnoreCase))
{
continue;
}
if (!string.IsNullOrEmpty(expected.PurlPattern) &&
!MatchesPattern(node.Purl ?? string.Empty, expected.PurlPattern))
{
continue;
}
return true;
}
return false;
}
private bool HasMatchingEdge(RichGraph graph, ExpectedEdge expected, double minConfidence)
{
foreach (var edge in graph.Edges)
{
if (!MatchesPattern(edge.From, expected.From))
{
continue;
}
if (!MatchesPattern(edge.To, expected.To))
{
continue;
}
if (!string.IsNullOrEmpty(expected.Kind) &&
!string.Equals(edge.Kind, expected.Kind, StringComparison.OrdinalIgnoreCase))
{
continue;
}
if (edge.Confidence < minConfidence)
{
continue;
}
return true;
}
return false;
}
private bool HasMatchingRoot(RichGraph graph, ExpectedRoot expected)
{
foreach (var root in graph.Roots)
{
if (!MatchesPattern(root.Id, expected.Id))
{
continue;
}
if (!string.IsNullOrEmpty(expected.Phase) &&
!string.Equals(root.Phase, expected.Phase, StringComparison.OrdinalIgnoreCase))
{
continue;
}
return true;
}
return false;
}
private IEnumerable<RichGraphNode> FindUnexpectedNodes(RichGraph graph)
{
var allExpected = _oracle.ExpectedFunctions
.Select(f => f.SymbolId)
.ToHashSet(StringComparer.Ordinal);
foreach (var node in graph.Nodes)
{
var isExpected = allExpected.Any(pattern => MatchesPattern(node.Id, pattern) || MatchesPattern(node.SymbolId, pattern));
if (!isExpected)
{
yield return node;
}
}
}
private IEnumerable<RichGraphEdge> FindUnexpectedEdges(RichGraph graph)
{
foreach (var edge in graph.Edges)
{
var isExpected = _oracle.ExpectedEdges.Any(e =>
MatchesPattern(edge.From, e.From) && MatchesPattern(edge.To, e.To));
if (!isExpected)
{
yield return edge;
}
}
}
/// <summary>
/// Matches a value against a pattern supporting '*' wildcards.
/// </summary>
private static bool MatchesPattern(string value, string pattern)
{
if (string.IsNullOrEmpty(pattern))
{
return true;
}
if (string.IsNullOrEmpty(value))
{
return false;
}
// Exact match
if (!pattern.Contains('*'))
{
return string.Equals(value, pattern, StringComparison.Ordinal);
}
// Convert wildcard pattern to regex
var regexPattern = "^" + Regex.Escape(pattern).Replace("\\*", ".*") + "$";
return Regex.IsMatch(value, regexPattern, RegexOptions.None, TimeSpan.FromMilliseconds(100));
}
private static PatchOracleSummary GenerateSummary(RichGraph graph, List<PatchOracleViolation> violations)
{
return new PatchOracleSummary(
TotalNodes: graph.Nodes.Count,
TotalEdges: graph.Edges.Count,
TotalRoots: graph.Roots.Count,
MissingFunctions: violations.Count(v => v.Type == ViolationType.MissingFunction),
MissingEdges: violations.Count(v => v.Type == ViolationType.MissingEdge),
MissingRoots: violations.Count(v => v.Type == ViolationType.MissingRoot),
ForbiddenFunctionsPresent: violations.Count(v => v.Type == ViolationType.ForbiddenFunctionPresent),
ForbiddenEdgesPresent: violations.Count(v => v.Type == ViolationType.ForbiddenEdgePresent),
UnexpectedFunctions: violations.Count(v => v.Type == ViolationType.UnexpectedFunction),
UnexpectedEdges: violations.Count(v => v.Type == ViolationType.UnexpectedEdge));
}
}
/// <summary>
/// Result of comparing a graph against a patch-oracle.
/// </summary>
public sealed record PatchOracleResult(
string OracleId,
string CaseRef,
string Variant,
bool Success,
IReadOnlyList<PatchOracleViolation> Violations,
PatchOracleSummary Summary)
{
/// <summary>
/// Generates a human-readable report.
/// </summary>
public string ToReport()
{
var lines = new List<string>
{
$"Patch-Oracle Validation Report",
$"==============================",
$"Oracle: {OracleId}",
$"Case: {CaseRef} ({Variant})",
$"Status: {(Success ? "PASS" : "FAIL")}",
string.Empty,
$"Graph Statistics:",
$" Nodes: {Summary.TotalNodes}",
$" Edges: {Summary.TotalEdges}",
$" Roots: {Summary.TotalRoots}",
string.Empty
};
if (Violations.Count > 0)
{
lines.Add($"Violations ({Violations.Count}):");
foreach (var v in Violations)
{
var target = v.To is not null ? $" -> {v.To}" : string.Empty;
lines.Add($" [{v.Type}] {v.From}{target}");
lines.Add($" Reason: {v.Message}");
}
}
else
{
lines.Add("No violations found.");
}
return string.Join(Environment.NewLine, lines);
}
}
/// <summary>
/// A single violation found during oracle comparison.
/// </summary>
public sealed record PatchOracleViolation(
ViolationType Type,
string From,
string? To,
string Message);
/// <summary>
/// Type of oracle violation.
/// </summary>
public enum ViolationType
{
MissingFunction,
MissingEdge,
MissingRoot,
ForbiddenFunctionPresent,
ForbiddenEdgePresent,
UnexpectedFunction,
UnexpectedEdge
}
/// <summary>
/// Summary statistics for oracle comparison.
/// </summary>
public sealed record PatchOracleSummary(
int TotalNodes,
int TotalEdges,
int TotalRoots,
int MissingFunctions,
int MissingEdges,
int MissingRoots,
int ForbiddenFunctionsPresent,
int ForbiddenEdgesPresent,
int UnexpectedFunctions,
int UnexpectedEdges);

View File

@@ -0,0 +1,112 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text.Json;
namespace StellaOps.Reachability.FixtureTests.PatchOracle;
/// <summary>
/// Loads patch-oracle definitions from fixture files.
/// </summary>
public sealed class PatchOracleLoader
{
private static readonly JsonSerializerOptions JsonOptions = new()
{
PropertyNameCaseInsensitive = true,
ReadCommentHandling = JsonCommentHandling.Skip,
AllowTrailingCommas = true
};
private readonly string _fixtureRoot;
public PatchOracleLoader(string fixtureRoot)
{
_fixtureRoot = fixtureRoot ?? throw new ArgumentNullException(nameof(fixtureRoot));
}
/// <summary>
/// Loads the oracle index from INDEX.json.
/// </summary>
public PatchOracleIndex LoadIndex()
{
var indexPath = Path.Combine(_fixtureRoot, "INDEX.json");
if (!File.Exists(indexPath))
{
throw new FileNotFoundException($"Patch-oracle INDEX.json not found at {indexPath}");
}
var json = File.ReadAllText(indexPath);
return JsonSerializer.Deserialize<PatchOracleIndex>(json, JsonOptions)
?? throw new InvalidOperationException("Failed to deserialize patch-oracle index");
}
/// <summary>
/// Loads an oracle definition by its ID.
/// </summary>
public PatchOracleDefinition LoadOracle(string oracleId)
{
var index = LoadIndex();
var entry = index.Oracles
.FirstOrDefault(o => string.Equals(o.Id, oracleId, StringComparison.Ordinal))
?? throw new KeyNotFoundException($"Oracle '{oracleId}' not found in index");
return LoadOracleFromPath(entry.Path);
}
/// <summary>
/// Loads an oracle definition from a relative path.
/// </summary>
public PatchOracleDefinition LoadOracleFromPath(string relativePath)
{
var fullPath = Path.Combine(_fixtureRoot, relativePath);
if (!File.Exists(fullPath))
{
throw new FileNotFoundException($"Oracle file not found at {fullPath}");
}
var json = File.ReadAllText(fullPath);
return JsonSerializer.Deserialize<PatchOracleDefinition>(json, JsonOptions)
?? throw new InvalidOperationException($"Failed to deserialize oracle from {fullPath}");
}
/// <summary>
/// Loads all oracles for a specific case.
/// </summary>
public IEnumerable<PatchOracleDefinition> LoadOraclesForCase(string caseRef)
{
var index = LoadIndex();
foreach (var entry in index.Oracles.Where(o => string.Equals(o.CaseRef, caseRef, StringComparison.Ordinal)))
{
yield return LoadOracleFromPath(entry.Path);
}
}
/// <summary>
/// Loads all available oracles.
/// </summary>
public IEnumerable<PatchOracleDefinition> LoadAllOracles()
{
var index = LoadIndex();
foreach (var entry in index.Oracles)
{
yield return LoadOracleFromPath(entry.Path);
}
}
/// <summary>
/// Enumerates all oracle entries without loading full definitions.
/// </summary>
public IEnumerable<PatchOracleIndexEntry> EnumerateOracles()
{
var index = LoadIndex();
return index.Oracles;
}
/// <summary>
/// Checks if the oracle index exists.
/// </summary>
public bool IndexExists()
{
return File.Exists(Path.Combine(_fixtureRoot, "INDEX.json"));
}
}

View File

@@ -0,0 +1,158 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Reachability.FixtureTests.PatchOracle;
/// <summary>
/// Root model for patch-oracle fixture files.
/// </summary>
public sealed record PatchOracleDefinition
{
[JsonPropertyName("schema_version")]
public string SchemaVersion { get; init; } = "patch-oracle/v1";
[JsonPropertyName("id")]
public required string Id { get; init; }
[JsonPropertyName("case_ref")]
public required string CaseRef { get; init; }
[JsonPropertyName("variant")]
public required string Variant { get; init; }
[JsonPropertyName("description")]
public string? Description { get; init; }
[JsonPropertyName("expected_functions")]
public IReadOnlyList<ExpectedFunction> ExpectedFunctions { get; init; } = Array.Empty<ExpectedFunction>();
[JsonPropertyName("expected_edges")]
public IReadOnlyList<ExpectedEdge> ExpectedEdges { get; init; } = Array.Empty<ExpectedEdge>();
[JsonPropertyName("expected_roots")]
public IReadOnlyList<ExpectedRoot> ExpectedRoots { get; init; } = Array.Empty<ExpectedRoot>();
[JsonPropertyName("forbidden_functions")]
public IReadOnlyList<ExpectedFunction> ForbiddenFunctions { get; init; } = Array.Empty<ExpectedFunction>();
[JsonPropertyName("forbidden_edges")]
public IReadOnlyList<ExpectedEdge> ForbiddenEdges { get; init; } = Array.Empty<ExpectedEdge>();
[JsonPropertyName("min_confidence")]
public double MinConfidence { get; init; } = 0.5;
[JsonPropertyName("strict_mode")]
public bool StrictMode { get; init; } = false;
[JsonPropertyName("created_at")]
public DateTimeOffset? CreatedAt { get; init; }
[JsonPropertyName("updated_at")]
public DateTimeOffset? UpdatedAt { get; init; }
}
/// <summary>
/// Expected function/node in the graph.
/// </summary>
public sealed record ExpectedFunction
{
[JsonPropertyName("symbol_id")]
public required string SymbolId { get; init; }
[JsonPropertyName("lang")]
public string? Lang { get; init; }
[JsonPropertyName("kind")]
public string? Kind { get; init; }
[JsonPropertyName("purl_pattern")]
public string? PurlPattern { get; init; }
[JsonPropertyName("required")]
public bool Required { get; init; } = true;
[JsonPropertyName("reason")]
public string? Reason { get; init; }
}
/// <summary>
/// Expected edge in the graph.
/// </summary>
public sealed record ExpectedEdge
{
[JsonPropertyName("from")]
public required string From { get; init; }
[JsonPropertyName("to")]
public required string To { get; init; }
[JsonPropertyName("kind")]
public string? Kind { get; init; }
[JsonPropertyName("min_confidence")]
public double? MinConfidence { get; init; }
[JsonPropertyName("required")]
public bool Required { get; init; } = true;
[JsonPropertyName("reason")]
public string? Reason { get; init; }
}
/// <summary>
/// Expected root node in the graph.
/// </summary>
public sealed record ExpectedRoot
{
[JsonPropertyName("id")]
public required string Id { get; init; }
[JsonPropertyName("phase")]
public string? Phase { get; init; }
[JsonPropertyName("required")]
public bool Required { get; init; } = true;
[JsonPropertyName("reason")]
public string? Reason { get; init; }
}
/// <summary>
/// Index entry for an oracle.
/// </summary>
public sealed record PatchOracleIndexEntry
{
[JsonPropertyName("id")]
public required string Id { get; init; }
[JsonPropertyName("case_ref")]
public required string CaseRef { get; init; }
[JsonPropertyName("variant")]
public required string Variant { get; init; }
[JsonPropertyName("path")]
public required string Path { get; init; }
}
/// <summary>
/// Root model for patch-oracle INDEX.json.
/// </summary>
public sealed record PatchOracleIndex
{
[JsonPropertyName("version")]
public string Version { get; init; } = "1.0";
[JsonPropertyName("schema")]
public string Schema { get; init; } = "patch-oracle/v1";
[JsonPropertyName("generated_at")]
public DateTimeOffset? GeneratedAt { get; init; }
[JsonPropertyName("description")]
public string? Description { get; init; }
[JsonPropertyName("oracles")]
public IReadOnlyList<PatchOracleIndexEntry> Oracles { get; init; } = Array.Empty<PatchOracleIndexEntry>();
}

View File

@@ -0,0 +1,494 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using FluentAssertions;
using StellaOps.Reachability.FixtureTests.PatchOracle;
using StellaOps.Scanner.Reachability;
using Xunit;
namespace StellaOps.Reachability.FixtureTests;
/// <summary>
/// Tests for the patch-oracle harness infrastructure.
/// Validates that the oracle comparison logic correctly identifies missing and forbidden elements.
/// </summary>
public class PatchOracleHarnessTests
{
private static readonly string RepoRoot = ReachbenchFixtureTests.LocateRepoRoot();
private static readonly string PatchOracleRoot = Path.Combine(
RepoRoot, "tests", "reachability", "fixtures", "patch-oracles");
#region Oracle Loading Tests
[Fact]
public void Loader_IndexExists()
{
var loader = new PatchOracleLoader(PatchOracleRoot);
loader.IndexExists().Should().BeTrue("patch-oracle INDEX.json should exist");
}
[Fact]
public void Loader_IndexLoadsSuccessfully()
{
var loader = new PatchOracleLoader(PatchOracleRoot);
var index = loader.LoadIndex();
index.Should().NotBeNull();
index.Version.Should().Be("1.0");
index.Schema.Should().Be("patch-oracle/v1");
index.Oracles.Should().NotBeEmpty("should have at least one oracle defined");
}
[Fact]
public void Loader_AllOraclesLoadSuccessfully()
{
var loader = new PatchOracleLoader(PatchOracleRoot);
var oracles = loader.LoadAllOracles().ToList();
oracles.Should().NotBeEmpty();
foreach (var oracle in oracles)
{
oracle.SchemaVersion.Should().Be("patch-oracle/v1");
oracle.Id.Should().NotBeNullOrEmpty();
oracle.CaseRef.Should().NotBeNullOrEmpty();
oracle.Variant.Should().BeOneOf("reachable", "unreachable");
}
}
[Fact]
public void Loader_LoadOracleById()
{
var loader = new PatchOracleLoader(PatchOracleRoot);
var oracle = loader.LoadOracle("curl-CVE-2023-38545-socks5-heap-reachable");
oracle.Should().NotBeNull();
oracle.Id.Should().Be("curl-CVE-2023-38545-socks5-heap-reachable");
oracle.CaseRef.Should().Be("curl-CVE-2023-38545-socks5-heap");
oracle.Variant.Should().Be("reachable");
}
#endregion
#region Comparer Tests - Pass Cases
[Fact]
public void Comparer_PassesWhenAllExpectedElementsPresent()
{
var oracle = new PatchOracleDefinition
{
Id = "test-pass",
CaseRef = "test-case",
Variant = "reachable",
ExpectedFunctions = new[]
{
new ExpectedFunction { SymbolId = "sym://test#func1", Required = true },
new ExpectedFunction { SymbolId = "sym://test#func2", Required = true }
},
ExpectedEdges = new[]
{
new ExpectedEdge { From = "sym://test#func1", To = "sym://test#func2", Required = true }
}
};
var graph = new RichGraph(
Nodes: new[]
{
new RichGraphNode("sym://test#func1", "sym://test#func1", null, null, "test", "function", null, null, null, null, null),
new RichGraphNode("sym://test#func2", "sym://test#func2", null, null, "test", "function", null, null, null, null, null)
},
Edges: new[]
{
new RichGraphEdge("sym://test#func1", "sym://test#func2", "call", null, null, null, 0.9, null)
},
Roots: Array.Empty<RichGraphRoot>(),
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
);
var comparer = new PatchOracleComparer(oracle);
var result = comparer.Compare(graph);
result.Success.Should().BeTrue();
result.Violations.Should().BeEmpty();
}
[Fact]
public void Comparer_PassesWithWildcardPatterns()
{
var oracle = new PatchOracleDefinition
{
Id = "test-wildcard",
CaseRef = "test-case",
Variant = "reachable",
ExpectedFunctions = new[]
{
new ExpectedFunction { SymbolId = "sym://test#*", Required = true }
}
};
var graph = new RichGraph(
Nodes: new[]
{
new RichGraphNode("sym://test#anything", "sym://test#anything", null, null, "test", "function", null, null, null, null, null)
},
Edges: Array.Empty<RichGraphEdge>(),
Roots: Array.Empty<RichGraphRoot>(),
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
);
var comparer = new PatchOracleComparer(oracle);
var result = comparer.Compare(graph);
result.Success.Should().BeTrue();
}
#endregion
#region Comparer Tests - Fail Cases
[Fact]
public void Comparer_FailsWhenExpectedFunctionMissing()
{
var oracle = new PatchOracleDefinition
{
Id = "test-missing-func",
CaseRef = "test-case",
Variant = "reachable",
ExpectedFunctions = new[]
{
new ExpectedFunction { SymbolId = "sym://test#missing", Required = true, Reason = "This function is critical" }
}
};
var graph = new RichGraph(
Nodes: Array.Empty<RichGraphNode>(),
Edges: Array.Empty<RichGraphEdge>(),
Roots: Array.Empty<RichGraphRoot>(),
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
);
var comparer = new PatchOracleComparer(oracle);
var result = comparer.Compare(graph);
result.Success.Should().BeFalse();
result.Violations.Should().HaveCount(1);
result.Violations[0].Type.Should().Be(ViolationType.MissingFunction);
result.Violations[0].From.Should().Be("sym://test#missing");
result.Summary.MissingFunctions.Should().Be(1);
}
[Fact]
public void Comparer_FailsWhenExpectedEdgeMissing()
{
var oracle = new PatchOracleDefinition
{
Id = "test-missing-edge",
CaseRef = "test-case",
Variant = "reachable",
ExpectedEdges = new[]
{
new ExpectedEdge { From = "sym://a", To = "sym://b", Required = true }
}
};
var graph = new RichGraph(
Nodes: new[]
{
new RichGraphNode("sym://a", "sym://a", null, null, "test", "function", null, null, null, null, null),
new RichGraphNode("sym://b", "sym://b", null, null, "test", "function", null, null, null, null, null)
},
Edges: Array.Empty<RichGraphEdge>(),
Roots: Array.Empty<RichGraphRoot>(),
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
);
var comparer = new PatchOracleComparer(oracle);
var result = comparer.Compare(graph);
result.Success.Should().BeFalse();
result.Violations.Should().HaveCount(1);
result.Violations[0].Type.Should().Be(ViolationType.MissingEdge);
result.Summary.MissingEdges.Should().Be(1);
}
[Fact]
public void Comparer_FailsWhenExpectedRootMissing()
{
var oracle = new PatchOracleDefinition
{
Id = "test-missing-root",
CaseRef = "test-case",
Variant = "reachable",
ExpectedRoots = new[]
{
new ExpectedRoot { Id = "sym://root#main", Phase = "main", Required = true }
}
};
var graph = new RichGraph(
Nodes: Array.Empty<RichGraphNode>(),
Edges: Array.Empty<RichGraphEdge>(),
Roots: Array.Empty<RichGraphRoot>(),
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
);
var comparer = new PatchOracleComparer(oracle);
var result = comparer.Compare(graph);
result.Success.Should().BeFalse();
result.Violations.Should().HaveCount(1);
result.Violations[0].Type.Should().Be(ViolationType.MissingRoot);
result.Summary.MissingRoots.Should().Be(1);
}
[Fact]
public void Comparer_FailsWhenForbiddenFunctionPresent()
{
var oracle = new PatchOracleDefinition
{
Id = "test-forbidden-func",
CaseRef = "test-case",
Variant = "unreachable",
ForbiddenFunctions = new[]
{
new ExpectedFunction { SymbolId = "sym://dangerous#sink", Reason = "Should not be reachable" }
}
};
var graph = new RichGraph(
Nodes: new[]
{
new RichGraphNode("sym://dangerous#sink", "sym://dangerous#sink", null, null, "test", "function", null, null, null, null, null)
},
Edges: Array.Empty<RichGraphEdge>(),
Roots: Array.Empty<RichGraphRoot>(),
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
);
var comparer = new PatchOracleComparer(oracle);
var result = comparer.Compare(graph);
result.Success.Should().BeFalse();
result.Violations.Should().HaveCount(1);
result.Violations[0].Type.Should().Be(ViolationType.ForbiddenFunctionPresent);
result.Summary.ForbiddenFunctionsPresent.Should().Be(1);
}
[Fact]
public void Comparer_FailsWhenForbiddenEdgePresent()
{
var oracle = new PatchOracleDefinition
{
Id = "test-forbidden-edge",
CaseRef = "test-case",
Variant = "unreachable",
ForbiddenEdges = new[]
{
new ExpectedEdge { From = "sym://entry", To = "sym://sink", Reason = "Path should be blocked" }
}
};
var graph = new RichGraph(
Nodes: Array.Empty<RichGraphNode>(),
Edges: new[]
{
new RichGraphEdge("sym://entry", "sym://sink", "call", null, null, null, 1.0, null)
},
Roots: Array.Empty<RichGraphRoot>(),
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
);
var comparer = new PatchOracleComparer(oracle);
var result = comparer.Compare(graph);
result.Success.Should().BeFalse();
result.Violations.Should().HaveCount(1);
result.Violations[0].Type.Should().Be(ViolationType.ForbiddenEdgePresent);
result.Summary.ForbiddenEdgesPresent.Should().Be(1);
}
#endregion
#region Confidence Threshold Tests
[Fact]
public void Comparer_RespectsMinConfidenceThreshold()
{
var oracle = new PatchOracleDefinition
{
Id = "test-confidence",
CaseRef = "test-case",
Variant = "reachable",
MinConfidence = 0.8,
ExpectedEdges = new[]
{
new ExpectedEdge { From = "sym://a", To = "sym://b", Required = true }
}
};
var lowConfidenceGraph = new RichGraph(
Nodes: Array.Empty<RichGraphNode>(),
Edges: new[]
{
new RichGraphEdge("sym://a", "sym://b", "call", null, null, null, 0.5, null)
},
Roots: Array.Empty<RichGraphRoot>(),
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
);
var comparer = new PatchOracleComparer(oracle);
var result = comparer.Compare(lowConfidenceGraph);
result.Success.Should().BeFalse("edge confidence 0.5 is below threshold 0.8");
result.Summary.MissingEdges.Should().Be(1);
}
[Fact]
public void Comparer_EdgeSpecificConfidenceOverridesDefault()
{
var oracle = new PatchOracleDefinition
{
Id = "test-edge-confidence",
CaseRef = "test-case",
Variant = "reachable",
MinConfidence = 0.8,
ExpectedEdges = new[]
{
new ExpectedEdge { From = "sym://a", To = "sym://b", MinConfidence = 0.3, Required = true }
}
};
var lowConfidenceGraph = new RichGraph(
Nodes: Array.Empty<RichGraphNode>(),
Edges: new[]
{
new RichGraphEdge("sym://a", "sym://b", "call", null, null, null, 0.5, null)
},
Roots: Array.Empty<RichGraphRoot>(),
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
);
var comparer = new PatchOracleComparer(oracle);
var result = comparer.Compare(lowConfidenceGraph);
result.Success.Should().BeTrue("edge-specific threshold 0.3 allows confidence 0.5");
}
#endregion
#region Strict Mode Tests
[Fact]
public void Comparer_StrictModeRejectsUnexpectedNodes()
{
var oracle = new PatchOracleDefinition
{
Id = "test-strict",
CaseRef = "test-case",
Variant = "reachable",
StrictMode = true,
ExpectedFunctions = new[]
{
new ExpectedFunction { SymbolId = "sym://expected", Required = true }
}
};
var graph = new RichGraph(
Nodes: new[]
{
new RichGraphNode("sym://expected", "sym://expected", null, null, "test", "function", null, null, null, null, null),
new RichGraphNode("sym://unexpected", "sym://unexpected", null, null, "test", "function", null, null, null, null, null)
},
Edges: Array.Empty<RichGraphEdge>(),
Roots: Array.Empty<RichGraphRoot>(),
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
);
var comparer = new PatchOracleComparer(oracle);
var result = comparer.Compare(graph);
result.Success.Should().BeFalse();
result.Violations.Should().Contain(v => v.Type == ViolationType.UnexpectedFunction);
result.Summary.UnexpectedFunctions.Should().Be(1);
}
#endregion
#region Report Generation Tests
[Fact]
public void Result_GeneratesReadableReport()
{
var oracle = new PatchOracleDefinition
{
Id = "test-report",
CaseRef = "test-case",
Variant = "reachable",
ExpectedFunctions = new[]
{
new ExpectedFunction { SymbolId = "sym://missing", Required = true, Reason = "Critical sink" }
}
};
var graph = new RichGraph(
Nodes: new[]
{
new RichGraphNode("sym://other", "sym://other", null, null, "test", "function", null, null, null, null, null)
},
Edges: Array.Empty<RichGraphEdge>(),
Roots: Array.Empty<RichGraphRoot>(),
Analyzer: new RichGraphAnalyzer("test", "1.0", null)
);
var comparer = new PatchOracleComparer(oracle);
var result = comparer.Compare(graph);
var report = result.ToReport();
report.Should().Contain("FAIL");
report.Should().Contain("test-report");
report.Should().Contain("MissingFunction");
report.Should().Contain("sym://missing");
}
#endregion
#region Integration with Fixture Data
public static IEnumerable<object[]> AllOracleData()
{
var loader = new PatchOracleLoader(PatchOracleRoot);
if (!loader.IndexExists())
{
yield break;
}
foreach (var entry in loader.EnumerateOracles())
{
yield return new object[] { entry.Id, entry.CaseRef, entry.Variant };
}
}
[Theory]
[MemberData(nameof(AllOracleData))]
public void AllOracles_HaveValidStructure(string oracleId, string caseRef, string variant)
{
var loader = new PatchOracleLoader(PatchOracleRoot);
var oracle = loader.LoadOracle(oracleId);
oracle.Id.Should().Be(oracleId);
oracle.CaseRef.Should().Be(caseRef);
oracle.Variant.Should().Be(variant);
oracle.SchemaVersion.Should().Be("patch-oracle/v1");
// At least one expectation should be defined
var hasExpectations = oracle.ExpectedFunctions.Count > 0
|| oracle.ExpectedEdges.Count > 0
|| oracle.ExpectedRoots.Count > 0
|| oracle.ForbiddenFunctions.Count > 0
|| oracle.ForbiddenEdges.Count > 0;
hasExpectations.Should().BeTrue($"Oracle '{oracleId}' should define at least one expectation");
}
#endregion
}

View File

@@ -0,0 +1,32 @@
{
"version": "1.0",
"schema": "patch-oracle/v1",
"generated_at": "2025-12-13T00:00:00Z",
"description": "Patch-oracle fixtures for CI graph validation. Each oracle defines expected functions/edges that must be present (or absent) in generated reachability graphs.",
"oracles": [
{
"id": "curl-CVE-2023-38545-socks5-heap-reachable",
"case_ref": "curl-CVE-2023-38545-socks5-heap",
"variant": "reachable",
"path": "cases/curl-CVE-2023-38545-socks5-heap/reachable.oracle.json"
},
{
"id": "curl-CVE-2023-38545-socks5-heap-unreachable",
"case_ref": "curl-CVE-2023-38545-socks5-heap",
"variant": "unreachable",
"path": "cases/curl-CVE-2023-38545-socks5-heap/unreachable.oracle.json"
},
{
"id": "java-log4j-CVE-2021-44228-log4shell-reachable",
"case_ref": "java-log4j-CVE-2021-44228-log4shell",
"variant": "reachable",
"path": "cases/java-log4j-CVE-2021-44228-log4shell/reachable.oracle.json"
},
{
"id": "dotnet-kestrel-CVE-2023-44487-http2-rapid-reset-reachable",
"case_ref": "dotnet-kestrel-CVE-2023-44487-http2-rapid-reset",
"variant": "reachable",
"path": "cases/dotnet-kestrel-CVE-2023-44487-http2-rapid-reset/reachable.oracle.json"
}
]
}

View File

@@ -0,0 +1,56 @@
{
"schema_version": "patch-oracle/v1",
"id": "curl-CVE-2023-38545-socks5-heap-reachable",
"case_ref": "curl-CVE-2023-38545-socks5-heap",
"variant": "reachable",
"description": "Validates that the SOCKS5 heap overflow vulnerability path is reachable from network handler to vulnerable sink",
"expected_functions": [
{
"symbol_id": "sym://net:handler#read",
"kind": "entrypoint",
"required": true,
"reason": "Network read handler is the entry point for external data"
},
{
"symbol_id": "sym://curl:curl.c#entry",
"kind": "function",
"required": true,
"reason": "SOCKS5 protocol handling entry point"
},
{
"symbol_id": "sym://curl:curl.c#sink",
"kind": "function",
"required": true,
"reason": "Vulnerable buffer handling function"
}
],
"expected_edges": [
{
"from": "sym://net:handler#read",
"to": "sym://curl:curl.c#entry",
"kind": "call",
"min_confidence": 0.8,
"required": true,
"reason": "Data flows from network handler to SOCKS5 handler"
},
{
"from": "sym://curl:curl.c#entry",
"to": "sym://curl:curl.c#sink",
"kind": "call",
"min_confidence": 0.8,
"required": true,
"reason": "SOCKS5 handler invokes vulnerable buffer function"
}
],
"expected_roots": [
{
"id": "sym://net:handler#read",
"phase": "runtime",
"required": true,
"reason": "Network handler is the runtime entry point"
}
],
"min_confidence": 0.5,
"strict_mode": false,
"created_at": "2025-12-13T00:00:00Z"
}

View File

@@ -0,0 +1,32 @@
{
"schema_version": "patch-oracle/v1",
"id": "curl-CVE-2023-38545-socks5-heap-unreachable",
"case_ref": "curl-CVE-2023-38545-socks5-heap",
"variant": "unreachable",
"description": "Validates that the SOCKS5 heap overflow vulnerability path is NOT reachable when SOCKS5 is disabled",
"expected_functions": [
{
"symbol_id": "sym://net:handler#read",
"kind": "entrypoint",
"required": true,
"reason": "Network read handler still exists but cannot reach vulnerable code"
}
],
"expected_edges": [],
"forbidden_functions": [
{
"symbol_id": "sym://curl:curl.c#sink",
"reason": "Vulnerable sink should not be in call graph when SOCKS5 disabled"
}
],
"forbidden_edges": [
{
"from": "sym://curl:curl.c#entry",
"to": "sym://curl:curl.c#sink",
"reason": "This edge should not exist when SOCKS5 is disabled"
}
],
"min_confidence": 0.5,
"strict_mode": false,
"created_at": "2025-12-13T00:00:00Z"
}

View File

@@ -0,0 +1,44 @@
{
"schema_version": "patch-oracle/v1",
"id": "dotnet-kestrel-CVE-2023-44487-http2-rapid-reset-reachable",
"case_ref": "dotnet-kestrel-CVE-2023-44487-http2-rapid-reset",
"variant": "reachable",
"description": "Validates that the HTTP/2 Rapid Reset DoS vulnerability path is reachable",
"expected_functions": [
{
"symbol_id": "sym://dotnet:Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2.Http2Connection#ProcessRequestsAsync",
"lang": "dotnet",
"kind": "method",
"required": true,
"reason": "HTTP/2 connection handler entry point"
},
{
"symbol_id": "sym://dotnet:Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2.Http2Stream#*",
"lang": "dotnet",
"kind": "method",
"required": true,
"reason": "HTTP/2 stream management affected by rapid reset"
}
],
"expected_edges": [
{
"from": "sym://dotnet:Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2.Http2Connection#ProcessRequestsAsync",
"to": "sym://dotnet:Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2.Http2Stream#*",
"kind": "call",
"min_confidence": 0.7,
"required": true,
"reason": "Connection handler creates/manages streams"
}
],
"expected_roots": [
{
"id": "sym://dotnet:Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2.Http2Connection#ProcessRequestsAsync",
"phase": "runtime",
"required": true,
"reason": "HTTP/2 processing is a runtime entry point"
}
],
"min_confidence": 0.5,
"strict_mode": false,
"created_at": "2025-12-13T00:00:00Z"
}

View File

@@ -0,0 +1,64 @@
{
"schema_version": "patch-oracle/v1",
"id": "java-log4j-CVE-2021-44228-log4shell-reachable",
"case_ref": "java-log4j-CVE-2021-44228-log4shell",
"variant": "reachable",
"description": "Validates that the Log4Shell JNDI injection path is reachable from logger to JNDI lookup",
"expected_functions": [
{
"symbol_id": "sym://java:org.apache.logging.log4j.core.Logger#logMessage",
"lang": "java",
"kind": "method",
"required": true,
"reason": "Logger entry point that processes user-controlled format strings"
},
{
"symbol_id": "sym://java:org.apache.logging.log4j.core.pattern.MessagePatternConverter#format",
"lang": "java",
"kind": "method",
"required": true,
"reason": "Pattern converter that triggers lookup substitution"
},
{
"symbol_id": "sym://java:org.apache.logging.log4j.core.lookup.StrSubstitutor#replace",
"lang": "java",
"kind": "method",
"required": true,
"reason": "String substitution that invokes lookups"
},
{
"symbol_id": "sym://java:org.apache.logging.log4j.core.lookup.JndiLookup#lookup",
"lang": "java",
"kind": "method",
"required": true,
"reason": "Vulnerable JNDI lookup method"
}
],
"expected_edges": [
{
"from": "sym://java:org.apache.logging.log4j.core.Logger#logMessage",
"to": "sym://java:org.apache.logging.log4j.core.pattern.MessagePatternConverter#format",
"kind": "call",
"required": true,
"reason": "Logger delegates to pattern converter"
},
{
"from": "sym://java:org.apache.logging.log4j.core.lookup.StrSubstitutor#replace",
"to": "sym://java:org.apache.logging.log4j.core.lookup.JndiLookup#lookup",
"kind": "call",
"required": true,
"reason": "String substitution invokes JNDI lookup"
}
],
"expected_roots": [
{
"id": "sym://java:org.apache.logging.log4j.core.Logger#*",
"phase": "runtime",
"required": true,
"reason": "Logger methods are runtime entry points"
}
],
"min_confidence": 0.6,
"strict_mode": false,
"created_at": "2025-12-13T00:00:00Z"
}

View File

@@ -0,0 +1,179 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "stellaops:patch-oracle/v1",
"title": "Patch Oracle Schema v1",
"description": "Defines expected functions/edges for reachability graph validation. CI fails when expected elements are missing.",
"type": "object",
"properties": {
"schema_version": {
"type": "string",
"const": "patch-oracle/v1",
"description": "Schema version identifier"
},
"id": {
"type": "string",
"description": "Unique oracle identifier (e.g., 'curl-CVE-2023-38545-socks5-heap-reachable')"
},
"case_ref": {
"type": "string",
"description": "Reference to parent reachbench case (e.g., 'curl-CVE-2023-38545-socks5-heap')"
},
"variant": {
"type": "string",
"enum": ["reachable", "unreachable"],
"description": "Which variant this oracle applies to"
},
"description": {
"type": "string",
"description": "Human-readable description of what this oracle validates"
},
"expected_functions": {
"type": "array",
"description": "Functions that MUST be present in the generated graph",
"items": {
"$ref": "#/definitions/expected_function"
}
},
"expected_edges": {
"type": "array",
"description": "Edges that MUST be present in the generated graph",
"items": {
"$ref": "#/definitions/expected_edge"
}
},
"expected_roots": {
"type": "array",
"description": "Root nodes that MUST be present in the generated graph",
"items": {
"$ref": "#/definitions/expected_root"
}
},
"forbidden_functions": {
"type": "array",
"description": "Functions that MUST NOT be present (for unreachable variants)",
"items": {
"$ref": "#/definitions/expected_function"
}
},
"forbidden_edges": {
"type": "array",
"description": "Edges that MUST NOT be present (for unreachable variants)",
"items": {
"$ref": "#/definitions/expected_edge"
}
},
"min_confidence": {
"type": "number",
"minimum": 0.0,
"maximum": 1.0,
"default": 0.5,
"description": "Minimum confidence threshold for edge matching"
},
"strict_mode": {
"type": "boolean",
"default": false,
"description": "If true, extra functions/edges not in oracle cause failure"
},
"created_at": {
"type": "string",
"format": "date-time",
"description": "When this oracle was created"
},
"updated_at": {
"type": "string",
"format": "date-time",
"description": "When this oracle was last updated"
}
},
"required": ["schema_version", "id", "case_ref", "variant"],
"definitions": {
"expected_function": {
"type": "object",
"properties": {
"symbol_id": {
"type": "string",
"description": "Expected symbol ID (exact match or pattern with '*' wildcards)"
},
"lang": {
"type": "string",
"description": "Expected language (optional, for filtering)"
},
"kind": {
"type": "string",
"description": "Expected node kind (e.g., 'function', 'method', 'entrypoint')"
},
"purl_pattern": {
"type": "string",
"description": "Expected purl pattern (optional, supports wildcards)"
},
"required": {
"type": "boolean",
"default": true,
"description": "If true, missing this function fails CI"
},
"reason": {
"type": "string",
"description": "Why this function is expected (for documentation)"
}
},
"required": ["symbol_id"]
},
"expected_edge": {
"type": "object",
"properties": {
"from": {
"type": "string",
"description": "Source node symbol ID (exact match or pattern)"
},
"to": {
"type": "string",
"description": "Target node symbol ID (exact match or pattern)"
},
"kind": {
"type": "string",
"description": "Expected edge kind (e.g., 'call', 'plt', 'indirect')"
},
"min_confidence": {
"type": "number",
"minimum": 0.0,
"maximum": 1.0,
"description": "Minimum confidence for this specific edge"
},
"required": {
"type": "boolean",
"default": true,
"description": "If true, missing this edge fails CI"
},
"reason": {
"type": "string",
"description": "Why this edge is expected (for documentation)"
}
},
"required": ["from", "to"]
},
"expected_root": {
"type": "object",
"properties": {
"id": {
"type": "string",
"description": "Root node ID (exact match or pattern)"
},
"phase": {
"type": "string",
"enum": ["load", "init", "main", "runtime", "fini"],
"description": "Expected execution phase"
},
"required": {
"type": "boolean",
"default": true,
"description": "If true, missing this root fails CI"
},
"reason": {
"type": "string",
"description": "Why this root is expected"
}
},
"required": ["id"]
}
}
}