Add call graph fixtures for various languages and scenarios
Some checks failed
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Findings Ledger CI / build-test (push) Has been cancelled
Findings Ledger CI / migration-validation (push) Has been cancelled
Findings Ledger CI / generate-manifest (push) Has been cancelled
Lighthouse CI / Lighthouse Audit (push) Has been cancelled
Lighthouse CI / Axe Accessibility Audit (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Reachability Corpus Validation / validate-corpus (push) Has been cancelled
Reachability Corpus Validation / validate-ground-truths (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Reachability Corpus Validation / determinism-check (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled

- Introduced `all-edge-reasons.json` to test edge resolution reasons in .NET.
- Added `all-visibility-levels.json` to validate method visibility levels in .NET.
- Created `dotnet-aspnetcore-minimal.json` for a minimal ASP.NET Core application.
- Included `go-gin-api.json` for a Go Gin API application structure.
- Added `java-spring-boot.json` for the Spring PetClinic application in Java.
- Introduced `legacy-no-schema.json` for legacy application structure without schema.
- Created `node-express-api.json` for an Express.js API application structure.
This commit is contained in:
master
2025-12-16 10:44:24 +02:00
parent 4391f35d8a
commit 5a480a3c2a
223 changed files with 19367 additions and 727 deletions

View File

@@ -0,0 +1,72 @@
namespace StellaOps.Scanner.Worker.Determinism.Calculators;
/// <summary>
/// Calculates Bitwise Fidelity (BF) by comparing SHA-256 hashes of outputs.
/// </summary>
public sealed class BitwiseFidelityCalculator
{
/// <summary>
/// Computes BF by comparing hashes across replay runs.
/// </summary>
/// <param name="baselineHashes">Hashes from baseline run (artifact -> hash)</param>
/// <param name="replayHashes">Hashes from each replay run</param>
/// <returns>BF score and mismatch details</returns>
public (double Score, int IdenticalCount, List<FidelityMismatch> Mismatches) Calculate(
IReadOnlyDictionary<string, string> baselineHashes,
IReadOnlyList<IReadOnlyDictionary<string, string>> replayHashes)
{
ArgumentNullException.ThrowIfNull(baselineHashes);
ArgumentNullException.ThrowIfNull(replayHashes);
if (replayHashes.Count == 0)
return (1.0, 0, []);
var identicalCount = 0;
var mismatches = new List<FidelityMismatch>();
for (var i = 0; i < replayHashes.Count; i++)
{
var replay = replayHashes[i];
var identical = true;
var diffArtifacts = new List<string>();
foreach (var (artifact, baselineHash) in baselineHashes)
{
if (!replay.TryGetValue(artifact, out var replayHash) ||
!string.Equals(baselineHash, replayHash, StringComparison.OrdinalIgnoreCase))
{
identical = false;
diffArtifacts.Add(artifact);
}
}
// Also check for artifacts in replay but not in baseline
foreach (var artifact in replay.Keys)
{
if (!baselineHashes.ContainsKey(artifact) && !diffArtifacts.Contains(artifact))
{
identical = false;
diffArtifacts.Add(artifact);
}
}
if (identical)
{
identicalCount++;
}
else
{
mismatches.Add(new FidelityMismatch
{
RunIndex = i,
Type = FidelityMismatchType.BitwiseOnly,
Description = $"Hash mismatch in {diffArtifacts.Count} artifact(s)",
AffectedArtifacts = diffArtifacts.OrderBy(a => a, StringComparer.Ordinal).ToList()
});
}
}
var score = (double)identicalCount / replayHashes.Count;
return (score, identicalCount, mismatches);
}
}

View File

@@ -0,0 +1,107 @@
namespace StellaOps.Scanner.Worker.Determinism.Calculators;
/// <summary>
/// Calculates Policy Fidelity (PF) by comparing final policy decisions.
/// </summary>
public sealed class PolicyFidelityCalculator
{
/// <summary>
/// Computes PF by comparing policy decisions.
/// </summary>
public (double Score, int MatchCount, List<FidelityMismatch> Mismatches) Calculate(
PolicyDecision baseline,
IReadOnlyList<PolicyDecision> replays)
{
ArgumentNullException.ThrowIfNull(baseline);
ArgumentNullException.ThrowIfNull(replays);
if (replays.Count == 0)
return (1.0, 0, []);
var matchCount = 0;
var mismatches = new List<FidelityMismatch>();
for (var i = 0; i < replays.Count; i++)
{
var replay = replays[i];
var (isMatch, differences) = CompareDecisions(baseline, replay);
if (isMatch)
{
matchCount++;
}
else
{
mismatches.Add(new FidelityMismatch
{
RunIndex = i,
Type = FidelityMismatchType.PolicyDrift,
Description = $"Policy decision differs: {string.Join(", ", differences)}",
AffectedArtifacts = differences
});
}
}
var score = (double)matchCount / replays.Count;
return (score, matchCount, mismatches);
}
private static (bool IsMatch, List<string> Differences) CompareDecisions(
PolicyDecision a,
PolicyDecision b)
{
var differences = new List<string>();
// Compare overall outcome
if (a.Passed != b.Passed)
differences.Add($"outcome:{a.Passed}→{b.Passed}");
// Compare reason codes (order-independent)
var aReasons = a.ReasonCodes.OrderBy(r => r, StringComparer.Ordinal).ToList();
var bReasons = b.ReasonCodes.OrderBy(r => r, StringComparer.Ordinal).ToList();
if (!aReasons.SequenceEqual(bReasons))
differences.Add("reason_codes");
// Compare violation count
if (a.ViolationCount != b.ViolationCount)
differences.Add($"violations:{a.ViolationCount}→{b.ViolationCount}");
// Compare block level
if (!string.Equals(a.BlockLevel, b.BlockLevel, StringComparison.Ordinal))
differences.Add($"block_level:{a.BlockLevel}→{b.BlockLevel}");
return (differences.Count == 0, differences);
}
}
/// <summary>
/// Represents a policy decision for fidelity comparison.
/// </summary>
public sealed record PolicyDecision
{
/// <summary>
/// Whether the policy evaluation passed (true) or failed (false).
/// </summary>
public required bool Passed { get; init; }
/// <summary>
/// List of reason codes explaining the decision.
/// </summary>
public required IReadOnlyList<string> ReasonCodes { get; init; }
/// <summary>
/// Number of policy violations.
/// </summary>
public required int ViolationCount { get; init; }
/// <summary>
/// Block level: "none", "warn", "block"
/// </summary>
public required string BlockLevel { get; init; }
/// <summary>
/// Policy hash used for this decision.
/// </summary>
public string? PolicyHash { get; init; }
}

View File

@@ -0,0 +1,106 @@
namespace StellaOps.Scanner.Worker.Determinism.Calculators;
/// <summary>
/// Calculates Semantic Fidelity (SF) by comparing normalized object structures.
/// Ignores formatting differences; compares packages, versions, CVEs, severities, verdicts.
/// </summary>
public sealed class SemanticFidelityCalculator
{
/// <summary>
/// Computes SF by comparing normalized findings.
/// </summary>
public (double Score, int MatchCount, List<FidelityMismatch> Mismatches) Calculate(
NormalizedFindings baseline,
IReadOnlyList<NormalizedFindings> replays)
{
ArgumentNullException.ThrowIfNull(baseline);
ArgumentNullException.ThrowIfNull(replays);
if (replays.Count == 0)
return (1.0, 0, []);
var matchCount = 0;
var mismatches = new List<FidelityMismatch>();
for (var i = 0; i < replays.Count; i++)
{
var replay = replays[i];
var (isMatch, differences) = CompareNormalized(baseline, replay);
if (isMatch)
{
matchCount++;
}
else
{
mismatches.Add(new FidelityMismatch
{
RunIndex = i,
Type = FidelityMismatchType.SemanticOnly,
Description = $"Semantic differences: {string.Join(", ", differences)}",
AffectedArtifacts = differences
});
}
}
var score = (double)matchCount / replays.Count;
return (score, matchCount, mismatches);
}
private static (bool IsMatch, List<string> Differences) CompareNormalized(
NormalizedFindings a,
NormalizedFindings b)
{
var differences = new List<string>();
// Compare package sets (order-independent)
var aPackages = a.Packages.OrderBy(p => p.Purl, StringComparer.Ordinal)
.ThenBy(p => p.Version, StringComparer.Ordinal)
.ToList();
var bPackages = b.Packages.OrderBy(p => p.Purl, StringComparer.Ordinal)
.ThenBy(p => p.Version, StringComparer.Ordinal)
.ToList();
if (!aPackages.SequenceEqual(bPackages))
differences.Add("packages");
// Compare CVE sets (order-independent)
var aCves = a.Cves.OrderBy(c => c, StringComparer.Ordinal).ToList();
var bCves = b.Cves.OrderBy(c => c, StringComparer.Ordinal).ToList();
if (!aCves.SequenceEqual(bCves))
differences.Add("cves");
// Compare severity counts (order-independent)
var aSeverities = a.SeverityCounts.OrderBy(kvp => kvp.Key, StringComparer.Ordinal).ToList();
var bSeverities = b.SeverityCounts.OrderBy(kvp => kvp.Key, StringComparer.Ordinal).ToList();
if (!aSeverities.SequenceEqual(bSeverities))
differences.Add("severities");
// Compare verdicts (order-independent)
var aVerdicts = a.Verdicts.OrderBy(v => v.Key, StringComparer.Ordinal).ToList();
var bVerdicts = b.Verdicts.OrderBy(v => v.Key, StringComparer.Ordinal).ToList();
if (!aVerdicts.SequenceEqual(bVerdicts))
differences.Add("verdicts");
return (differences.Count == 0, differences);
}
}
/// <summary>
/// Normalized findings for semantic comparison.
/// </summary>
public sealed record NormalizedFindings
{
public required IReadOnlyList<NormalizedPackage> Packages { get; init; }
public required IReadOnlySet<string> Cves { get; init; }
public required IReadOnlyDictionary<string, int> SeverityCounts { get; init; }
public required IReadOnlyDictionary<string, string> Verdicts { get; init; }
}
/// <summary>
/// Normalized package representation for comparison.
/// </summary>
public sealed record NormalizedPackage(string Purl, string Version) : IEquatable<NormalizedPackage>;

View File

@@ -0,0 +1,86 @@
namespace StellaOps.Scanner.Worker.Determinism;
/// <summary>
/// Three-tier fidelity metrics for deterministic reproducibility measurement.
/// All scores are ratios in range [0.0, 1.0].
/// </summary>
public sealed record FidelityMetrics
{
/// <summary>
/// Bitwise Fidelity (BF): identical_outputs / total_replays
/// Target: >= 0.98 (general), >= 0.95 (regulated)
/// </summary>
public required double BitwiseFidelity { get; init; }
/// <summary>
/// Semantic Fidelity (SF): normalized object comparison match ratio
/// Allows formatting differences, compares: packages, versions, CVEs, severities, verdicts
/// </summary>
public required double SemanticFidelity { get; init; }
/// <summary>
/// Policy Fidelity (PF): policy decision match ratio
/// Compares: pass/fail + reason codes
/// Target: ~1.0 unless policy changed intentionally
/// </summary>
public required double PolicyFidelity { get; init; }
/// <summary>
/// Number of replay runs compared.
/// </summary>
public required int TotalReplays { get; init; }
/// <summary>
/// Number of bitwise-identical outputs.
/// </summary>
public required int IdenticalOutputs { get; init; }
/// <summary>
/// Number of semantically-equivalent outputs.
/// </summary>
public required int SemanticMatches { get; init; }
/// <summary>
/// Number of policy-decision matches.
/// </summary>
public required int PolicyMatches { get; init; }
/// <summary>
/// Computed timestamp (UTC).
/// </summary>
public required DateTimeOffset ComputedAt { get; init; }
/// <summary>
/// Diagnostic information for non-identical runs.
/// </summary>
public IReadOnlyList<FidelityMismatch>? Mismatches { get; init; }
}
/// <summary>
/// Diagnostic information about a fidelity mismatch.
/// </summary>
public sealed record FidelityMismatch
{
public required int RunIndex { get; init; }
public required FidelityMismatchType Type { get; init; }
public required string Description { get; init; }
public IReadOnlyList<string>? AffectedArtifacts { get; init; }
}
/// <summary>
/// Type of fidelity mismatch.
/// </summary>
public enum FidelityMismatchType
{
/// <summary>Hash differs but content semantically equivalent</summary>
BitwiseOnly,
/// <summary>Content differs but policy decision matches</summary>
SemanticOnly,
/// <summary>Policy decision differs</summary>
PolicyDrift,
/// <summary>All tiers differ</summary>
Full
}

View File

@@ -0,0 +1,209 @@
using StellaOps.Scanner.Worker.Determinism.Calculators;
namespace StellaOps.Scanner.Worker.Determinism;
/// <summary>
/// Service that orchestrates fidelity metric calculation across all three tiers.
/// </summary>
public sealed class FidelityMetricsService
{
private readonly BitwiseFidelityCalculator _bitwiseCalculator;
private readonly SemanticFidelityCalculator _semanticCalculator;
private readonly PolicyFidelityCalculator _policyCalculator;
public FidelityMetricsService()
{
_bitwiseCalculator = new BitwiseFidelityCalculator();
_semanticCalculator = new SemanticFidelityCalculator();
_policyCalculator = new PolicyFidelityCalculator();
}
/// <summary>
/// Computes all three fidelity metrics for a set of replay runs.
/// </summary>
/// <param name="baselineHashes">Artifact hashes from baseline run</param>
/// <param name="replayHashes">Artifact hashes from each replay run</param>
/// <param name="baselineFindings">Normalized findings from baseline</param>
/// <param name="replayFindings">Normalized findings from each replay</param>
/// <param name="baselineDecision">Policy decision from baseline</param>
/// <param name="replayDecisions">Policy decisions from each replay</param>
/// <returns>Complete fidelity metrics</returns>
public FidelityMetrics Calculate(
IReadOnlyDictionary<string, string> baselineHashes,
IReadOnlyList<IReadOnlyDictionary<string, string>> replayHashes,
NormalizedFindings baselineFindings,
IReadOnlyList<NormalizedFindings> replayFindings,
PolicyDecision baselineDecision,
IReadOnlyList<PolicyDecision> replayDecisions)
{
ArgumentNullException.ThrowIfNull(baselineHashes);
ArgumentNullException.ThrowIfNull(replayHashes);
ArgumentNullException.ThrowIfNull(baselineFindings);
ArgumentNullException.ThrowIfNull(replayFindings);
ArgumentNullException.ThrowIfNull(baselineDecision);
ArgumentNullException.ThrowIfNull(replayDecisions);
// Calculate bitwise fidelity
var (bfScore, bfIdentical, bfMismatches) = _bitwiseCalculator.Calculate(
baselineHashes, replayHashes);
// Calculate semantic fidelity
var (sfScore, sfMatches, sfMismatches) = _semanticCalculator.Calculate(
baselineFindings, replayFindings);
// Calculate policy fidelity
var (pfScore, pfMatches, pfMismatches) = _policyCalculator.Calculate(
baselineDecision, replayDecisions);
// Combine mismatches with proper classification
var allMismatches = CombineMismatches(bfMismatches, sfMismatches, pfMismatches);
return new FidelityMetrics
{
BitwiseFidelity = bfScore,
SemanticFidelity = sfScore,
PolicyFidelity = pfScore,
TotalReplays = replayHashes.Count,
IdenticalOutputs = bfIdentical,
SemanticMatches = sfMatches,
PolicyMatches = pfMatches,
ComputedAt = DateTimeOffset.UtcNow,
Mismatches = allMismatches.Count > 0 ? allMismatches : null
};
}
/// <summary>
/// Evaluates whether the fidelity metrics meet the specified thresholds.
/// </summary>
/// <param name="metrics">Computed fidelity metrics</param>
/// <param name="thresholds">Thresholds to check against</param>
/// <param name="isRegulated">Whether this is a regulated project</param>
/// <returns>Evaluation result with pass/fail and reason</returns>
public FidelityEvaluation Evaluate(
FidelityMetrics metrics,
FidelityThresholds thresholds,
bool isRegulated = false)
{
ArgumentNullException.ThrowIfNull(metrics);
ArgumentNullException.ThrowIfNull(thresholds);
var failures = new List<string>();
var bfThreshold = isRegulated
? thresholds.BitwiseFidelityRegulated
: thresholds.BitwiseFidelityGeneral;
if (metrics.BitwiseFidelity < bfThreshold)
failures.Add($"BF {metrics.BitwiseFidelity:P2} < {bfThreshold:P2}");
if (metrics.SemanticFidelity < thresholds.SemanticFidelity)
failures.Add($"SF {metrics.SemanticFidelity:P2} < {thresholds.SemanticFidelity:P2}");
if (metrics.PolicyFidelity < thresholds.PolicyFidelity)
failures.Add($"PF {metrics.PolicyFidelity:P2} < {thresholds.PolicyFidelity:P2}");
var shouldBlock = metrics.BitwiseFidelity < thresholds.BitwiseFidelityBlockThreshold;
return new FidelityEvaluation
{
Passed = failures.Count == 0,
ShouldBlockRelease = shouldBlock,
FailureReasons = failures,
EvaluatedAt = DateTimeOffset.UtcNow
};
}
private static List<FidelityMismatch> CombineMismatches(
List<FidelityMismatch> bfMismatches,
List<FidelityMismatch> sfMismatches,
List<FidelityMismatch> pfMismatches)
{
var combined = new Dictionary<int, FidelityMismatch>();
// Start with bitwise mismatches
foreach (var m in bfMismatches)
{
combined[m.RunIndex] = m;
}
// Upgrade or add semantic mismatches
foreach (var m in sfMismatches)
{
if (combined.TryGetValue(m.RunIndex, out var existing))
{
// Both bitwise and semantic differ
combined[m.RunIndex] = existing with
{
Type = FidelityMismatchType.Full,
Description = $"{existing.Description}; {m.Description}",
AffectedArtifacts = (existing.AffectedArtifacts ?? [])
.Concat(m.AffectedArtifacts ?? [])
.Distinct()
.OrderBy(a => a, StringComparer.Ordinal)
.ToList()
};
}
else
{
combined[m.RunIndex] = m;
}
}
// Upgrade or add policy mismatches
foreach (var m in pfMismatches)
{
if (combined.TryGetValue(m.RunIndex, out var existing))
{
var newType = existing.Type switch
{
FidelityMismatchType.Full => FidelityMismatchType.Full,
_ => FidelityMismatchType.Full
};
combined[m.RunIndex] = existing with
{
Type = newType,
Description = $"{existing.Description}; {m.Description}",
AffectedArtifacts = (existing.AffectedArtifacts ?? [])
.Concat(m.AffectedArtifacts ?? [])
.Distinct()
.OrderBy(a => a, StringComparer.Ordinal)
.ToList()
};
}
else
{
combined[m.RunIndex] = m;
}
}
return combined.Values
.OrderBy(m => m.RunIndex)
.ToList();
}
}
/// <summary>
/// Result of evaluating fidelity metrics against thresholds.
/// </summary>
public sealed record FidelityEvaluation
{
/// <summary>
/// Whether all thresholds were met.
/// </summary>
public required bool Passed { get; init; }
/// <summary>
/// Whether the release should be blocked (BF below critical threshold).
/// </summary>
public required bool ShouldBlockRelease { get; init; }
/// <summary>
/// List of threshold violations.
/// </summary>
public required IReadOnlyList<string> FailureReasons { get; init; }
/// <summary>
/// Timestamp of evaluation.
/// </summary>
public required DateTimeOffset EvaluatedAt { get; init; }
}

View File

@@ -0,0 +1,42 @@
namespace StellaOps.Scanner.Worker.Determinism;
/// <summary>
/// SLO thresholds for fidelity metrics.
/// </summary>
public sealed record FidelityThresholds
{
/// <summary>
/// Minimum BF for general workloads (default: 0.98)
/// </summary>
public double BitwiseFidelityGeneral { get; init; } = 0.98;
/// <summary>
/// Minimum BF for regulated projects (default: 0.95)
/// </summary>
public double BitwiseFidelityRegulated { get; init; } = 0.95;
/// <summary>
/// Minimum SF (default: 0.99)
/// </summary>
public double SemanticFidelity { get; init; } = 0.99;
/// <summary>
/// Minimum PF (default: 1.0 unless policy changed)
/// </summary>
public double PolicyFidelity { get; init; } = 1.0;
/// <summary>
/// Week-over-week BF drop that triggers warning (default: 0.02 = 2%)
/// </summary>
public double BitwiseFidelityWarnDrop { get; init; } = 0.02;
/// <summary>
/// Overall BF that triggers page/block release (default: 0.90)
/// </summary>
public double BitwiseFidelityBlockThreshold { get; init; } = 0.90;
/// <summary>
/// Default thresholds.
/// </summary>
public static FidelityThresholds Default => new();
}