Add call graph fixtures for various languages and scenarios
Some checks failed
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Findings Ledger CI / build-test (push) Has been cancelled
Findings Ledger CI / migration-validation (push) Has been cancelled
Findings Ledger CI / generate-manifest (push) Has been cancelled
Lighthouse CI / Lighthouse Audit (push) Has been cancelled
Lighthouse CI / Axe Accessibility Audit (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Reachability Corpus Validation / validate-corpus (push) Has been cancelled
Reachability Corpus Validation / validate-ground-truths (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Reachability Corpus Validation / determinism-check (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Some checks failed
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Findings Ledger CI / build-test (push) Has been cancelled
Findings Ledger CI / migration-validation (push) Has been cancelled
Findings Ledger CI / generate-manifest (push) Has been cancelled
Lighthouse CI / Lighthouse Audit (push) Has been cancelled
Lighthouse CI / Axe Accessibility Audit (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Reachability Corpus Validation / validate-corpus (push) Has been cancelled
Reachability Corpus Validation / validate-ground-truths (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Reachability Corpus Validation / determinism-check (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
- Introduced `all-edge-reasons.json` to test edge resolution reasons in .NET. - Added `all-visibility-levels.json` to validate method visibility levels in .NET. - Created `dotnet-aspnetcore-minimal.json` for a minimal ASP.NET Core application. - Included `go-gin-api.json` for a Go Gin API application structure. - Added `java-spring-boot.json` for the Spring PetClinic application in Java. - Introduced `legacy-no-schema.json` for legacy application structure without schema. - Created `node-express-api.json` for an Express.js API application structure.
This commit is contained in:
@@ -0,0 +1,72 @@
|
||||
namespace StellaOps.Scanner.Worker.Determinism.Calculators;
|
||||
|
||||
/// <summary>
|
||||
/// Calculates Bitwise Fidelity (BF) by comparing SHA-256 hashes of outputs.
|
||||
/// </summary>
|
||||
public sealed class BitwiseFidelityCalculator
|
||||
{
|
||||
/// <summary>
|
||||
/// Computes BF by comparing hashes across replay runs.
|
||||
/// </summary>
|
||||
/// <param name="baselineHashes">Hashes from baseline run (artifact -> hash)</param>
|
||||
/// <param name="replayHashes">Hashes from each replay run</param>
|
||||
/// <returns>BF score and mismatch details</returns>
|
||||
public (double Score, int IdenticalCount, List<FidelityMismatch> Mismatches) Calculate(
|
||||
IReadOnlyDictionary<string, string> baselineHashes,
|
||||
IReadOnlyList<IReadOnlyDictionary<string, string>> replayHashes)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(baselineHashes);
|
||||
ArgumentNullException.ThrowIfNull(replayHashes);
|
||||
|
||||
if (replayHashes.Count == 0)
|
||||
return (1.0, 0, []);
|
||||
|
||||
var identicalCount = 0;
|
||||
var mismatches = new List<FidelityMismatch>();
|
||||
|
||||
for (var i = 0; i < replayHashes.Count; i++)
|
||||
{
|
||||
var replay = replayHashes[i];
|
||||
var identical = true;
|
||||
var diffArtifacts = new List<string>();
|
||||
|
||||
foreach (var (artifact, baselineHash) in baselineHashes)
|
||||
{
|
||||
if (!replay.TryGetValue(artifact, out var replayHash) ||
|
||||
!string.Equals(baselineHash, replayHash, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
identical = false;
|
||||
diffArtifacts.Add(artifact);
|
||||
}
|
||||
}
|
||||
|
||||
// Also check for artifacts in replay but not in baseline
|
||||
foreach (var artifact in replay.Keys)
|
||||
{
|
||||
if (!baselineHashes.ContainsKey(artifact) && !diffArtifacts.Contains(artifact))
|
||||
{
|
||||
identical = false;
|
||||
diffArtifacts.Add(artifact);
|
||||
}
|
||||
}
|
||||
|
||||
if (identical)
|
||||
{
|
||||
identicalCount++;
|
||||
}
|
||||
else
|
||||
{
|
||||
mismatches.Add(new FidelityMismatch
|
||||
{
|
||||
RunIndex = i,
|
||||
Type = FidelityMismatchType.BitwiseOnly,
|
||||
Description = $"Hash mismatch in {diffArtifacts.Count} artifact(s)",
|
||||
AffectedArtifacts = diffArtifacts.OrderBy(a => a, StringComparer.Ordinal).ToList()
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
var score = (double)identicalCount / replayHashes.Count;
|
||||
return (score, identicalCount, mismatches);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
namespace StellaOps.Scanner.Worker.Determinism.Calculators;
|
||||
|
||||
/// <summary>
|
||||
/// Calculates Policy Fidelity (PF) by comparing final policy decisions.
|
||||
/// </summary>
|
||||
public sealed class PolicyFidelityCalculator
|
||||
{
|
||||
/// <summary>
|
||||
/// Computes PF by comparing policy decisions.
|
||||
/// </summary>
|
||||
public (double Score, int MatchCount, List<FidelityMismatch> Mismatches) Calculate(
|
||||
PolicyDecision baseline,
|
||||
IReadOnlyList<PolicyDecision> replays)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(baseline);
|
||||
ArgumentNullException.ThrowIfNull(replays);
|
||||
|
||||
if (replays.Count == 0)
|
||||
return (1.0, 0, []);
|
||||
|
||||
var matchCount = 0;
|
||||
var mismatches = new List<FidelityMismatch>();
|
||||
|
||||
for (var i = 0; i < replays.Count; i++)
|
||||
{
|
||||
var replay = replays[i];
|
||||
var (isMatch, differences) = CompareDecisions(baseline, replay);
|
||||
|
||||
if (isMatch)
|
||||
{
|
||||
matchCount++;
|
||||
}
|
||||
else
|
||||
{
|
||||
mismatches.Add(new FidelityMismatch
|
||||
{
|
||||
RunIndex = i,
|
||||
Type = FidelityMismatchType.PolicyDrift,
|
||||
Description = $"Policy decision differs: {string.Join(", ", differences)}",
|
||||
AffectedArtifacts = differences
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
var score = (double)matchCount / replays.Count;
|
||||
return (score, matchCount, mismatches);
|
||||
}
|
||||
|
||||
private static (bool IsMatch, List<string> Differences) CompareDecisions(
|
||||
PolicyDecision a,
|
||||
PolicyDecision b)
|
||||
{
|
||||
var differences = new List<string>();
|
||||
|
||||
// Compare overall outcome
|
||||
if (a.Passed != b.Passed)
|
||||
differences.Add($"outcome:{a.Passed}→{b.Passed}");
|
||||
|
||||
// Compare reason codes (order-independent)
|
||||
var aReasons = a.ReasonCodes.OrderBy(r => r, StringComparer.Ordinal).ToList();
|
||||
var bReasons = b.ReasonCodes.OrderBy(r => r, StringComparer.Ordinal).ToList();
|
||||
|
||||
if (!aReasons.SequenceEqual(bReasons))
|
||||
differences.Add("reason_codes");
|
||||
|
||||
// Compare violation count
|
||||
if (a.ViolationCount != b.ViolationCount)
|
||||
differences.Add($"violations:{a.ViolationCount}→{b.ViolationCount}");
|
||||
|
||||
// Compare block level
|
||||
if (!string.Equals(a.BlockLevel, b.BlockLevel, StringComparison.Ordinal))
|
||||
differences.Add($"block_level:{a.BlockLevel}→{b.BlockLevel}");
|
||||
|
||||
return (differences.Count == 0, differences);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Represents a policy decision for fidelity comparison.
|
||||
/// </summary>
|
||||
public sealed record PolicyDecision
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether the policy evaluation passed (true) or failed (false).
|
||||
/// </summary>
|
||||
public required bool Passed { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// List of reason codes explaining the decision.
|
||||
/// </summary>
|
||||
public required IReadOnlyList<string> ReasonCodes { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Number of policy violations.
|
||||
/// </summary>
|
||||
public required int ViolationCount { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Block level: "none", "warn", "block"
|
||||
/// </summary>
|
||||
public required string BlockLevel { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Policy hash used for this decision.
|
||||
/// </summary>
|
||||
public string? PolicyHash { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
namespace StellaOps.Scanner.Worker.Determinism.Calculators;
|
||||
|
||||
/// <summary>
|
||||
/// Calculates Semantic Fidelity (SF) by comparing normalized object structures.
|
||||
/// Ignores formatting differences; compares packages, versions, CVEs, severities, verdicts.
|
||||
/// </summary>
|
||||
public sealed class SemanticFidelityCalculator
|
||||
{
|
||||
/// <summary>
|
||||
/// Computes SF by comparing normalized findings.
|
||||
/// </summary>
|
||||
public (double Score, int MatchCount, List<FidelityMismatch> Mismatches) Calculate(
|
||||
NormalizedFindings baseline,
|
||||
IReadOnlyList<NormalizedFindings> replays)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(baseline);
|
||||
ArgumentNullException.ThrowIfNull(replays);
|
||||
|
||||
if (replays.Count == 0)
|
||||
return (1.0, 0, []);
|
||||
|
||||
var matchCount = 0;
|
||||
var mismatches = new List<FidelityMismatch>();
|
||||
|
||||
for (var i = 0; i < replays.Count; i++)
|
||||
{
|
||||
var replay = replays[i];
|
||||
var (isMatch, differences) = CompareNormalized(baseline, replay);
|
||||
|
||||
if (isMatch)
|
||||
{
|
||||
matchCount++;
|
||||
}
|
||||
else
|
||||
{
|
||||
mismatches.Add(new FidelityMismatch
|
||||
{
|
||||
RunIndex = i,
|
||||
Type = FidelityMismatchType.SemanticOnly,
|
||||
Description = $"Semantic differences: {string.Join(", ", differences)}",
|
||||
AffectedArtifacts = differences
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
var score = (double)matchCount / replays.Count;
|
||||
return (score, matchCount, mismatches);
|
||||
}
|
||||
|
||||
private static (bool IsMatch, List<string> Differences) CompareNormalized(
|
||||
NormalizedFindings a,
|
||||
NormalizedFindings b)
|
||||
{
|
||||
var differences = new List<string>();
|
||||
|
||||
// Compare package sets (order-independent)
|
||||
var aPackages = a.Packages.OrderBy(p => p.Purl, StringComparer.Ordinal)
|
||||
.ThenBy(p => p.Version, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
var bPackages = b.Packages.OrderBy(p => p.Purl, StringComparer.Ordinal)
|
||||
.ThenBy(p => p.Version, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
|
||||
if (!aPackages.SequenceEqual(bPackages))
|
||||
differences.Add("packages");
|
||||
|
||||
// Compare CVE sets (order-independent)
|
||||
var aCves = a.Cves.OrderBy(c => c, StringComparer.Ordinal).ToList();
|
||||
var bCves = b.Cves.OrderBy(c => c, StringComparer.Ordinal).ToList();
|
||||
|
||||
if (!aCves.SequenceEqual(bCves))
|
||||
differences.Add("cves");
|
||||
|
||||
// Compare severity counts (order-independent)
|
||||
var aSeverities = a.SeverityCounts.OrderBy(kvp => kvp.Key, StringComparer.Ordinal).ToList();
|
||||
var bSeverities = b.SeverityCounts.OrderBy(kvp => kvp.Key, StringComparer.Ordinal).ToList();
|
||||
|
||||
if (!aSeverities.SequenceEqual(bSeverities))
|
||||
differences.Add("severities");
|
||||
|
||||
// Compare verdicts (order-independent)
|
||||
var aVerdicts = a.Verdicts.OrderBy(v => v.Key, StringComparer.Ordinal).ToList();
|
||||
var bVerdicts = b.Verdicts.OrderBy(v => v.Key, StringComparer.Ordinal).ToList();
|
||||
|
||||
if (!aVerdicts.SequenceEqual(bVerdicts))
|
||||
differences.Add("verdicts");
|
||||
|
||||
return (differences.Count == 0, differences);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Normalized findings for semantic comparison.
|
||||
/// </summary>
|
||||
public sealed record NormalizedFindings
|
||||
{
|
||||
public required IReadOnlyList<NormalizedPackage> Packages { get; init; }
|
||||
public required IReadOnlySet<string> Cves { get; init; }
|
||||
public required IReadOnlyDictionary<string, int> SeverityCounts { get; init; }
|
||||
public required IReadOnlyDictionary<string, string> Verdicts { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Normalized package representation for comparison.
|
||||
/// </summary>
|
||||
public sealed record NormalizedPackage(string Purl, string Version) : IEquatable<NormalizedPackage>;
|
||||
@@ -0,0 +1,86 @@
|
||||
namespace StellaOps.Scanner.Worker.Determinism;
|
||||
|
||||
/// <summary>
|
||||
/// Three-tier fidelity metrics for deterministic reproducibility measurement.
|
||||
/// All scores are ratios in range [0.0, 1.0].
|
||||
/// </summary>
|
||||
public sealed record FidelityMetrics
|
||||
{
|
||||
/// <summary>
|
||||
/// Bitwise Fidelity (BF): identical_outputs / total_replays
|
||||
/// Target: >= 0.98 (general), >= 0.95 (regulated)
|
||||
/// </summary>
|
||||
public required double BitwiseFidelity { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Semantic Fidelity (SF): normalized object comparison match ratio
|
||||
/// Allows formatting differences, compares: packages, versions, CVEs, severities, verdicts
|
||||
/// </summary>
|
||||
public required double SemanticFidelity { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Policy Fidelity (PF): policy decision match ratio
|
||||
/// Compares: pass/fail + reason codes
|
||||
/// Target: ~1.0 unless policy changed intentionally
|
||||
/// </summary>
|
||||
public required double PolicyFidelity { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Number of replay runs compared.
|
||||
/// </summary>
|
||||
public required int TotalReplays { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Number of bitwise-identical outputs.
|
||||
/// </summary>
|
||||
public required int IdenticalOutputs { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Number of semantically-equivalent outputs.
|
||||
/// </summary>
|
||||
public required int SemanticMatches { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Number of policy-decision matches.
|
||||
/// </summary>
|
||||
public required int PolicyMatches { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Computed timestamp (UTC).
|
||||
/// </summary>
|
||||
public required DateTimeOffset ComputedAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Diagnostic information for non-identical runs.
|
||||
/// </summary>
|
||||
public IReadOnlyList<FidelityMismatch>? Mismatches { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Diagnostic information about a fidelity mismatch.
|
||||
/// </summary>
|
||||
public sealed record FidelityMismatch
|
||||
{
|
||||
public required int RunIndex { get; init; }
|
||||
public required FidelityMismatchType Type { get; init; }
|
||||
public required string Description { get; init; }
|
||||
public IReadOnlyList<string>? AffectedArtifacts { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Type of fidelity mismatch.
|
||||
/// </summary>
|
||||
public enum FidelityMismatchType
|
||||
{
|
||||
/// <summary>Hash differs but content semantically equivalent</summary>
|
||||
BitwiseOnly,
|
||||
|
||||
/// <summary>Content differs but policy decision matches</summary>
|
||||
SemanticOnly,
|
||||
|
||||
/// <summary>Policy decision differs</summary>
|
||||
PolicyDrift,
|
||||
|
||||
/// <summary>All tiers differ</summary>
|
||||
Full
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
using StellaOps.Scanner.Worker.Determinism.Calculators;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Determinism;
|
||||
|
||||
/// <summary>
|
||||
/// Service that orchestrates fidelity metric calculation across all three tiers.
|
||||
/// </summary>
|
||||
public sealed class FidelityMetricsService
|
||||
{
|
||||
private readonly BitwiseFidelityCalculator _bitwiseCalculator;
|
||||
private readonly SemanticFidelityCalculator _semanticCalculator;
|
||||
private readonly PolicyFidelityCalculator _policyCalculator;
|
||||
|
||||
public FidelityMetricsService()
|
||||
{
|
||||
_bitwiseCalculator = new BitwiseFidelityCalculator();
|
||||
_semanticCalculator = new SemanticFidelityCalculator();
|
||||
_policyCalculator = new PolicyFidelityCalculator();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computes all three fidelity metrics for a set of replay runs.
|
||||
/// </summary>
|
||||
/// <param name="baselineHashes">Artifact hashes from baseline run</param>
|
||||
/// <param name="replayHashes">Artifact hashes from each replay run</param>
|
||||
/// <param name="baselineFindings">Normalized findings from baseline</param>
|
||||
/// <param name="replayFindings">Normalized findings from each replay</param>
|
||||
/// <param name="baselineDecision">Policy decision from baseline</param>
|
||||
/// <param name="replayDecisions">Policy decisions from each replay</param>
|
||||
/// <returns>Complete fidelity metrics</returns>
|
||||
public FidelityMetrics Calculate(
|
||||
IReadOnlyDictionary<string, string> baselineHashes,
|
||||
IReadOnlyList<IReadOnlyDictionary<string, string>> replayHashes,
|
||||
NormalizedFindings baselineFindings,
|
||||
IReadOnlyList<NormalizedFindings> replayFindings,
|
||||
PolicyDecision baselineDecision,
|
||||
IReadOnlyList<PolicyDecision> replayDecisions)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(baselineHashes);
|
||||
ArgumentNullException.ThrowIfNull(replayHashes);
|
||||
ArgumentNullException.ThrowIfNull(baselineFindings);
|
||||
ArgumentNullException.ThrowIfNull(replayFindings);
|
||||
ArgumentNullException.ThrowIfNull(baselineDecision);
|
||||
ArgumentNullException.ThrowIfNull(replayDecisions);
|
||||
|
||||
// Calculate bitwise fidelity
|
||||
var (bfScore, bfIdentical, bfMismatches) = _bitwiseCalculator.Calculate(
|
||||
baselineHashes, replayHashes);
|
||||
|
||||
// Calculate semantic fidelity
|
||||
var (sfScore, sfMatches, sfMismatches) = _semanticCalculator.Calculate(
|
||||
baselineFindings, replayFindings);
|
||||
|
||||
// Calculate policy fidelity
|
||||
var (pfScore, pfMatches, pfMismatches) = _policyCalculator.Calculate(
|
||||
baselineDecision, replayDecisions);
|
||||
|
||||
// Combine mismatches with proper classification
|
||||
var allMismatches = CombineMismatches(bfMismatches, sfMismatches, pfMismatches);
|
||||
|
||||
return new FidelityMetrics
|
||||
{
|
||||
BitwiseFidelity = bfScore,
|
||||
SemanticFidelity = sfScore,
|
||||
PolicyFidelity = pfScore,
|
||||
TotalReplays = replayHashes.Count,
|
||||
IdenticalOutputs = bfIdentical,
|
||||
SemanticMatches = sfMatches,
|
||||
PolicyMatches = pfMatches,
|
||||
ComputedAt = DateTimeOffset.UtcNow,
|
||||
Mismatches = allMismatches.Count > 0 ? allMismatches : null
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates whether the fidelity metrics meet the specified thresholds.
|
||||
/// </summary>
|
||||
/// <param name="metrics">Computed fidelity metrics</param>
|
||||
/// <param name="thresholds">Thresholds to check against</param>
|
||||
/// <param name="isRegulated">Whether this is a regulated project</param>
|
||||
/// <returns>Evaluation result with pass/fail and reason</returns>
|
||||
public FidelityEvaluation Evaluate(
|
||||
FidelityMetrics metrics,
|
||||
FidelityThresholds thresholds,
|
||||
bool isRegulated = false)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(metrics);
|
||||
ArgumentNullException.ThrowIfNull(thresholds);
|
||||
|
||||
var failures = new List<string>();
|
||||
var bfThreshold = isRegulated
|
||||
? thresholds.BitwiseFidelityRegulated
|
||||
: thresholds.BitwiseFidelityGeneral;
|
||||
|
||||
if (metrics.BitwiseFidelity < bfThreshold)
|
||||
failures.Add($"BF {metrics.BitwiseFidelity:P2} < {bfThreshold:P2}");
|
||||
|
||||
if (metrics.SemanticFidelity < thresholds.SemanticFidelity)
|
||||
failures.Add($"SF {metrics.SemanticFidelity:P2} < {thresholds.SemanticFidelity:P2}");
|
||||
|
||||
if (metrics.PolicyFidelity < thresholds.PolicyFidelity)
|
||||
failures.Add($"PF {metrics.PolicyFidelity:P2} < {thresholds.PolicyFidelity:P2}");
|
||||
|
||||
var shouldBlock = metrics.BitwiseFidelity < thresholds.BitwiseFidelityBlockThreshold;
|
||||
|
||||
return new FidelityEvaluation
|
||||
{
|
||||
Passed = failures.Count == 0,
|
||||
ShouldBlockRelease = shouldBlock,
|
||||
FailureReasons = failures,
|
||||
EvaluatedAt = DateTimeOffset.UtcNow
|
||||
};
|
||||
}
|
||||
|
||||
private static List<FidelityMismatch> CombineMismatches(
|
||||
List<FidelityMismatch> bfMismatches,
|
||||
List<FidelityMismatch> sfMismatches,
|
||||
List<FidelityMismatch> pfMismatches)
|
||||
{
|
||||
var combined = new Dictionary<int, FidelityMismatch>();
|
||||
|
||||
// Start with bitwise mismatches
|
||||
foreach (var m in bfMismatches)
|
||||
{
|
||||
combined[m.RunIndex] = m;
|
||||
}
|
||||
|
||||
// Upgrade or add semantic mismatches
|
||||
foreach (var m in sfMismatches)
|
||||
{
|
||||
if (combined.TryGetValue(m.RunIndex, out var existing))
|
||||
{
|
||||
// Both bitwise and semantic differ
|
||||
combined[m.RunIndex] = existing with
|
||||
{
|
||||
Type = FidelityMismatchType.Full,
|
||||
Description = $"{existing.Description}; {m.Description}",
|
||||
AffectedArtifacts = (existing.AffectedArtifacts ?? [])
|
||||
.Concat(m.AffectedArtifacts ?? [])
|
||||
.Distinct()
|
||||
.OrderBy(a => a, StringComparer.Ordinal)
|
||||
.ToList()
|
||||
};
|
||||
}
|
||||
else
|
||||
{
|
||||
combined[m.RunIndex] = m;
|
||||
}
|
||||
}
|
||||
|
||||
// Upgrade or add policy mismatches
|
||||
foreach (var m in pfMismatches)
|
||||
{
|
||||
if (combined.TryGetValue(m.RunIndex, out var existing))
|
||||
{
|
||||
var newType = existing.Type switch
|
||||
{
|
||||
FidelityMismatchType.Full => FidelityMismatchType.Full,
|
||||
_ => FidelityMismatchType.Full
|
||||
};
|
||||
|
||||
combined[m.RunIndex] = existing with
|
||||
{
|
||||
Type = newType,
|
||||
Description = $"{existing.Description}; {m.Description}",
|
||||
AffectedArtifacts = (existing.AffectedArtifacts ?? [])
|
||||
.Concat(m.AffectedArtifacts ?? [])
|
||||
.Distinct()
|
||||
.OrderBy(a => a, StringComparer.Ordinal)
|
||||
.ToList()
|
||||
};
|
||||
}
|
||||
else
|
||||
{
|
||||
combined[m.RunIndex] = m;
|
||||
}
|
||||
}
|
||||
|
||||
return combined.Values
|
||||
.OrderBy(m => m.RunIndex)
|
||||
.ToList();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of evaluating fidelity metrics against thresholds.
|
||||
/// </summary>
|
||||
public sealed record FidelityEvaluation
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether all thresholds were met.
|
||||
/// </summary>
|
||||
public required bool Passed { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether the release should be blocked (BF below critical threshold).
|
||||
/// </summary>
|
||||
public required bool ShouldBlockRelease { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// List of threshold violations.
|
||||
/// </summary>
|
||||
public required IReadOnlyList<string> FailureReasons { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Timestamp of evaluation.
|
||||
/// </summary>
|
||||
public required DateTimeOffset EvaluatedAt { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
namespace StellaOps.Scanner.Worker.Determinism;
|
||||
|
||||
/// <summary>
|
||||
/// SLO thresholds for fidelity metrics.
|
||||
/// </summary>
|
||||
public sealed record FidelityThresholds
|
||||
{
|
||||
/// <summary>
|
||||
/// Minimum BF for general workloads (default: 0.98)
|
||||
/// </summary>
|
||||
public double BitwiseFidelityGeneral { get; init; } = 0.98;
|
||||
|
||||
/// <summary>
|
||||
/// Minimum BF for regulated projects (default: 0.95)
|
||||
/// </summary>
|
||||
public double BitwiseFidelityRegulated { get; init; } = 0.95;
|
||||
|
||||
/// <summary>
|
||||
/// Minimum SF (default: 0.99)
|
||||
/// </summary>
|
||||
public double SemanticFidelity { get; init; } = 0.99;
|
||||
|
||||
/// <summary>
|
||||
/// Minimum PF (default: 1.0 unless policy changed)
|
||||
/// </summary>
|
||||
public double PolicyFidelity { get; init; } = 1.0;
|
||||
|
||||
/// <summary>
|
||||
/// Week-over-week BF drop that triggers warning (default: 0.02 = 2%)
|
||||
/// </summary>
|
||||
public double BitwiseFidelityWarnDrop { get; init; } = 0.02;
|
||||
|
||||
/// <summary>
|
||||
/// Overall BF that triggers page/block release (default: 0.90)
|
||||
/// </summary>
|
||||
public double BitwiseFidelityBlockThreshold { get; init; } = 0.90;
|
||||
|
||||
/// <summary>
|
||||
/// Default thresholds.
|
||||
/// </summary>
|
||||
public static FidelityThresholds Default => new();
|
||||
}
|
||||
Reference in New Issue
Block a user