Some checks failed
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
812 lines
28 KiB
C#
812 lines
28 KiB
C#
using System.Collections.Immutable;
|
|
using System.Security.Cryptography;
|
|
using System.Text;
|
|
using StellaOps.Policy.Engine.Telemetry;
|
|
|
|
namespace StellaOps.Policy.Engine.Simulation;
|
|
|
|
/// <summary>
|
|
/// Service for computing simulation analytics including rule firing counts, heatmaps,
|
|
/// sampled traces, and delta summaries.
|
|
/// </summary>
|
|
public sealed class SimulationAnalyticsService
|
|
{
|
|
private static readonly ImmutableArray<string> OutcomeSeverityOrder = ImmutableArray.Create(
|
|
"allow", "info", "warn", "review", "block", "deny", "critical");
|
|
|
|
private static readonly ImmutableArray<string> SeverityOrder = ImmutableArray.Create(
|
|
"informational", "low", "medium", "high", "critical");
|
|
|
|
/// <summary>
|
|
/// Computes full simulation analytics from rule hit traces.
|
|
/// </summary>
|
|
public SimulationAnalytics ComputeAnalytics(
|
|
string policyRef,
|
|
IReadOnlyList<RuleHitTrace> traces,
|
|
IReadOnlyList<SimulationFinding> findings,
|
|
SimulationAnalyticsOptions? options = null)
|
|
{
|
|
options ??= SimulationAnalyticsOptions.Default;
|
|
|
|
var firingCounts = ComputeRuleFiringCounts(traces, findings.Count);
|
|
var heatmap = ComputeHeatmap(traces, findings, options);
|
|
var sampledTraces = ComputeSampledTraces(traces, findings, options);
|
|
|
|
return new SimulationAnalytics(
|
|
firingCounts,
|
|
heatmap,
|
|
sampledTraces,
|
|
DeltaSummary: null);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Computes delta summary comparing base and candidate simulation results.
|
|
/// </summary>
|
|
public SimulationDeltaSummary ComputeDeltaSummary(
|
|
string basePolicyRef,
|
|
string candidatePolicyRef,
|
|
IReadOnlyList<SimulationFindingResult> baseResults,
|
|
IReadOnlyList<SimulationFindingResult> candidateResults,
|
|
SimulationComparisonType comparisonType = SimulationComparisonType.VersionCompare)
|
|
{
|
|
var baseByFinding = baseResults.ToDictionary(r => r.FindingId);
|
|
var candidateByFinding = candidateResults.ToDictionary(r => r.FindingId);
|
|
|
|
var outcomeChanges = ComputeOutcomeChanges(baseByFinding, candidateByFinding);
|
|
var severityChanges = ComputeSeverityChanges(baseByFinding, candidateByFinding);
|
|
var ruleChanges = ComputeRuleChanges(baseResults, candidateResults);
|
|
var highImpact = ComputeHighImpactFindings(baseByFinding, candidateByFinding);
|
|
|
|
var hashInput = $"{basePolicyRef}:{candidatePolicyRef}:{baseResults.Count}:{candidateResults.Count}";
|
|
var determinismHash = ComputeHash(hashInput);
|
|
|
|
return new SimulationDeltaSummary(
|
|
comparisonType,
|
|
basePolicyRef,
|
|
candidatePolicyRef,
|
|
TotalFindings: baseResults.Count,
|
|
outcomeChanges,
|
|
severityChanges,
|
|
ruleChanges,
|
|
highImpact,
|
|
determinismHash);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Computes rule firing counts from traces.
|
|
/// </summary>
|
|
public RuleFiringCounts ComputeRuleFiringCounts(
|
|
IReadOnlyList<RuleHitTrace> traces,
|
|
int totalEvaluations)
|
|
{
|
|
var ruleStats = new Dictionary<string, RuleStats>();
|
|
var byPriority = new Dictionary<int, int>();
|
|
var byOutcome = new Dictionary<string, int>();
|
|
var byCategory = new Dictionary<string, int>();
|
|
var vexByVendor = new Dictionary<string, int>();
|
|
var vexByStatus = new Dictionary<string, int>();
|
|
var vexByJustification = new Dictionary<string, int>();
|
|
var totalFired = 0;
|
|
var totalVexOverrides = 0;
|
|
|
|
foreach (var trace in traces)
|
|
{
|
|
if (!trace.ExpressionResult)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
totalFired++;
|
|
|
|
// Rule stats
|
|
if (!ruleStats.TryGetValue(trace.RuleName, out var stats))
|
|
{
|
|
stats = new RuleStats(trace.RuleName, trace.RulePriority, trace.RuleCategory);
|
|
ruleStats[trace.RuleName] = stats;
|
|
}
|
|
|
|
stats.FireCount++;
|
|
stats.TotalEvaluationUs += trace.EvaluationMicroseconds;
|
|
stats.IncrementOutcome(trace.Outcome);
|
|
|
|
// Priority aggregation
|
|
byPriority.TryGetValue(trace.RulePriority, out var priorityCount);
|
|
byPriority[trace.RulePriority] = priorityCount + 1;
|
|
|
|
// Outcome aggregation
|
|
byOutcome.TryGetValue(trace.Outcome, out var outcomeCount);
|
|
byOutcome[trace.Outcome] = outcomeCount + 1;
|
|
|
|
// Category aggregation
|
|
if (!string.IsNullOrWhiteSpace(trace.RuleCategory))
|
|
{
|
|
byCategory.TryGetValue(trace.RuleCategory, out var categoryCount);
|
|
byCategory[trace.RuleCategory] = categoryCount + 1;
|
|
}
|
|
|
|
// VEX overrides
|
|
if (trace.IsVexOverride)
|
|
{
|
|
totalVexOverrides++;
|
|
|
|
if (!string.IsNullOrWhiteSpace(trace.VexVendor))
|
|
{
|
|
vexByVendor.TryGetValue(trace.VexVendor, out var vendorCount);
|
|
vexByVendor[trace.VexVendor] = vendorCount + 1;
|
|
}
|
|
|
|
if (!string.IsNullOrWhiteSpace(trace.VexStatus))
|
|
{
|
|
vexByStatus.TryGetValue(trace.VexStatus, out var statusCount);
|
|
vexByStatus[trace.VexStatus] = statusCount + 1;
|
|
}
|
|
|
|
if (!string.IsNullOrWhiteSpace(trace.VexJustification))
|
|
{
|
|
vexByJustification.TryGetValue(trace.VexJustification, out var justCount);
|
|
vexByJustification[trace.VexJustification] = justCount + 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Build rule fire counts
|
|
var ruleFireCounts = ruleStats.Values
|
|
.Select(s => new RuleFireCount(
|
|
s.RuleName,
|
|
s.Priority,
|
|
s.Category,
|
|
s.FireCount,
|
|
totalEvaluations > 0 ? (double)s.FireCount / totalEvaluations * 100 : 0,
|
|
s.OutcomeCounts.ToImmutableDictionary(),
|
|
s.FireCount > 0 ? (double)s.TotalEvaluationUs / s.FireCount : 0))
|
|
.ToImmutableDictionary(r => r.RuleName);
|
|
|
|
var topRules = ruleFireCounts.Values
|
|
.OrderByDescending(r => r.FireCount)
|
|
.Take(10)
|
|
.ToImmutableArray();
|
|
|
|
var vexOverrides = new VexOverrideCounts(
|
|
totalVexOverrides,
|
|
vexByVendor.ToImmutableDictionary(),
|
|
vexByStatus.ToImmutableDictionary(),
|
|
vexByJustification.ToImmutableDictionary());
|
|
|
|
return new RuleFiringCounts(
|
|
totalEvaluations,
|
|
totalFired,
|
|
ruleFireCounts,
|
|
byPriority.ToImmutableDictionary(),
|
|
byOutcome.ToImmutableDictionary(),
|
|
byCategory.ToImmutableDictionary(),
|
|
topRules,
|
|
vexOverrides);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Computes heatmap aggregates for visualization.
|
|
/// </summary>
|
|
public SimulationHeatmap ComputeHeatmap(
|
|
IReadOnlyList<RuleHitTrace> traces,
|
|
IReadOnlyList<SimulationFinding> findings,
|
|
SimulationAnalyticsOptions options)
|
|
{
|
|
var ruleSeverityMatrix = ComputeRuleSeverityMatrix(traces);
|
|
var ruleOutcomeMatrix = ComputeRuleOutcomeMatrix(traces);
|
|
var findingCoverage = ComputeFindingRuleCoverage(traces, findings);
|
|
var temporalDist = ComputeTemporalDistribution(traces, options.TemporalBucketMs);
|
|
|
|
return new SimulationHeatmap(
|
|
ruleSeverityMatrix,
|
|
ruleOutcomeMatrix,
|
|
findingCoverage,
|
|
temporalDist);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Computes sampled explain traces with deterministic ordering.
|
|
/// </summary>
|
|
public SampledExplainTraces ComputeSampledTraces(
|
|
IReadOnlyList<RuleHitTrace> traces,
|
|
IReadOnlyList<SimulationFinding> findings,
|
|
SimulationAnalyticsOptions options)
|
|
{
|
|
// Group traces by finding
|
|
var tracesByFinding = traces
|
|
.GroupBy(t => t.ComponentPurl ?? t.AdvisoryId ?? "unknown")
|
|
.ToDictionary(g => g.Key, g => g.ToList());
|
|
|
|
var findingsById = findings.ToDictionary(f => f.FindingId);
|
|
|
|
// Deterministic ordering by finding_id, then rule_priority
|
|
var ordering = new TraceOrdering("finding_id", "rule_priority", "ascending");
|
|
|
|
// Sample traces deterministically
|
|
var sampledList = new List<SampledTrace>();
|
|
var totalTraceCount = 0;
|
|
|
|
foreach (var finding in findings.OrderBy(f => f.FindingId, StringComparer.Ordinal))
|
|
{
|
|
var key = finding.ComponentPurl ?? finding.AdvisoryId ?? finding.FindingId;
|
|
if (!tracesByFinding.TryGetValue(key, out var findingTraces))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
totalTraceCount += findingTraces.Count;
|
|
|
|
// Deterministic sampling based on finding_id hash
|
|
var sampleHash = ComputeHash(finding.FindingId);
|
|
var sampleValue = Math.Abs(sampleHash.GetHashCode()) % 100;
|
|
var shouldSample = sampleValue < (int)(options.TraceSampleRate * 100);
|
|
|
|
if (!shouldSample && sampledList.Count >= options.MaxSampledTraces)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
// Always sample high-impact findings
|
|
var hasFiredRule = findingTraces.Any(t => t.ExpressionResult);
|
|
var isHighSeverity = findingTraces.Any(t =>
|
|
t.AssignedSeverity?.Equals("critical", StringComparison.OrdinalIgnoreCase) == true ||
|
|
t.AssignedSeverity?.Equals("high", StringComparison.OrdinalIgnoreCase) == true);
|
|
var hasVexOverride = findingTraces.Any(t => t.IsVexOverride);
|
|
|
|
var sampleReason = DetermineSampleReason(shouldSample, isHighSeverity, hasVexOverride);
|
|
|
|
if (!shouldSample && !isHighSeverity && !hasVexOverride)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
var orderedTraces = findingTraces.OrderBy(t => t.RulePriority).ToList();
|
|
var finalTrace = orderedTraces.LastOrDefault(t => t.ExpressionResult) ?? orderedTraces.LastOrDefault();
|
|
|
|
if (finalTrace == null)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
var ruleSequence = orderedTraces
|
|
.Where(t => t.ExpressionResult)
|
|
.Select(t => t.RuleName)
|
|
.ToImmutableArray();
|
|
|
|
sampledList.Add(new SampledTrace(
|
|
TraceId: $"{finding.FindingId}:{finalTrace.SpanId}",
|
|
FindingId: finding.FindingId,
|
|
ComponentPurl: finding.ComponentPurl,
|
|
AdvisoryId: finding.AdvisoryId,
|
|
FinalOutcome: finalTrace.Outcome,
|
|
AssignedSeverity: finalTrace.AssignedSeverity,
|
|
RulesEvaluated: findingTraces.Count,
|
|
RulesFired: findingTraces.Count(t => t.ExpressionResult),
|
|
VexApplied: hasVexOverride,
|
|
EvaluationMs: findingTraces.Sum(t => t.EvaluationMicroseconds) / 1000.0,
|
|
RuleSequence: ruleSequence,
|
|
SampleReason: sampleReason));
|
|
|
|
if (sampledList.Count >= options.MaxSampledTraces)
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Compute determinism hash from ordered sample
|
|
var hashBuilder = new StringBuilder();
|
|
foreach (var sample in sampledList.OrderBy(s => s.FindingId, StringComparer.Ordinal))
|
|
{
|
|
hashBuilder.Append(sample.FindingId);
|
|
hashBuilder.Append(':');
|
|
hashBuilder.Append(sample.FinalOutcome);
|
|
hashBuilder.Append(';');
|
|
}
|
|
|
|
var determinismHash = ComputeHash(hashBuilder.ToString());
|
|
|
|
return new SampledExplainTraces(
|
|
options.TraceSampleRate,
|
|
totalTraceCount,
|
|
sampledList.Count,
|
|
ordering,
|
|
sampledList.ToImmutableArray(),
|
|
determinismHash);
|
|
}
|
|
|
|
private ImmutableArray<HeatmapCell> ComputeRuleSeverityMatrix(IReadOnlyList<RuleHitTrace> traces)
|
|
{
|
|
var matrix = new Dictionary<(string rule, string severity), int>();
|
|
|
|
foreach (var trace in traces.Where(t => t.ExpressionResult && !string.IsNullOrWhiteSpace(t.AssignedSeverity)))
|
|
{
|
|
var key = (trace.RuleName, trace.AssignedSeverity!);
|
|
matrix.TryGetValue(key, out var count);
|
|
matrix[key] = count + 1;
|
|
}
|
|
|
|
var maxValue = matrix.Values.DefaultIfEmpty(1).Max();
|
|
|
|
return matrix
|
|
.Select(kvp => new HeatmapCell(
|
|
kvp.Key.rule,
|
|
kvp.Key.severity,
|
|
kvp.Value,
|
|
maxValue > 0 ? (double)kvp.Value / maxValue : 0))
|
|
.OrderBy(c => c.X, StringComparer.Ordinal)
|
|
.ThenBy(c => SeverityOrder.IndexOf(c.Y.ToLowerInvariant()))
|
|
.ToImmutableArray();
|
|
}
|
|
|
|
private ImmutableArray<HeatmapCell> ComputeRuleOutcomeMatrix(IReadOnlyList<RuleHitTrace> traces)
|
|
{
|
|
var matrix = new Dictionary<(string rule, string outcome), int>();
|
|
|
|
foreach (var trace in traces.Where(t => t.ExpressionResult))
|
|
{
|
|
var key = (trace.RuleName, trace.Outcome);
|
|
matrix.TryGetValue(key, out var count);
|
|
matrix[key] = count + 1;
|
|
}
|
|
|
|
var maxValue = matrix.Values.DefaultIfEmpty(1).Max();
|
|
|
|
return matrix
|
|
.Select(kvp => new HeatmapCell(
|
|
kvp.Key.rule,
|
|
kvp.Key.outcome,
|
|
kvp.Value,
|
|
maxValue > 0 ? (double)kvp.Value / maxValue : 0))
|
|
.OrderBy(c => c.X, StringComparer.Ordinal)
|
|
.ThenBy(c => OutcomeSeverityOrder.IndexOf(c.Y.ToLowerInvariant()))
|
|
.ToImmutableArray();
|
|
}
|
|
|
|
private FindingRuleCoverage ComputeFindingRuleCoverage(
|
|
IReadOnlyList<RuleHitTrace> traces,
|
|
IReadOnlyList<SimulationFinding> findings)
|
|
{
|
|
var rulesThatFired = traces
|
|
.Where(t => t.ExpressionResult)
|
|
.Select(t => t.RuleName)
|
|
.ToHashSet();
|
|
|
|
var allRules = traces
|
|
.Select(t => t.RuleName)
|
|
.Distinct()
|
|
.ToHashSet();
|
|
|
|
var rulesNeverFired = allRules.Except(rulesThatFired).ToImmutableArray();
|
|
|
|
// Group by finding to count matches per finding
|
|
var findingMatchCounts = traces
|
|
.Where(t => t.ExpressionResult)
|
|
.GroupBy(t => t.ComponentPurl ?? t.AdvisoryId ?? "unknown")
|
|
.ToDictionary(g => g.Key, g => g.Select(t => t.RuleName).Distinct().Count());
|
|
|
|
var matchCountDistribution = findingMatchCounts.Values
|
|
.GroupBy(c => c)
|
|
.ToDictionary(g => g.Key, g => g.Count())
|
|
.ToImmutableDictionary();
|
|
|
|
var findingsMatched = findingMatchCounts.Count;
|
|
var findingsUnmatched = findings.Count - findingsMatched;
|
|
|
|
return new FindingRuleCoverage(
|
|
findings.Count,
|
|
findingsMatched,
|
|
findingsUnmatched,
|
|
findings.Count > 0 ? (double)findingsMatched / findings.Count * 100 : 0,
|
|
rulesNeverFired,
|
|
matchCountDistribution);
|
|
}
|
|
|
|
private ImmutableArray<TemporalBucket> ComputeTemporalDistribution(
|
|
IReadOnlyList<RuleHitTrace> traces,
|
|
long bucketMs)
|
|
{
|
|
if (traces.Count == 0)
|
|
{
|
|
return ImmutableArray<TemporalBucket>.Empty;
|
|
}
|
|
|
|
var minTime = traces.Min(t => t.EvaluationTimestamp);
|
|
var maxTime = traces.Max(t => t.EvaluationTimestamp);
|
|
var totalMs = (long)(maxTime - minTime).TotalMilliseconds;
|
|
|
|
if (totalMs <= 0)
|
|
{
|
|
return ImmutableArray.Create(new TemporalBucket(0, bucketMs, traces.Count, traces.Count(t => t.ExpressionResult)));
|
|
}
|
|
|
|
var buckets = new Dictionary<long, (int evalCount, int fireCount)>();
|
|
|
|
foreach (var trace in traces)
|
|
{
|
|
var offsetMs = (long)(trace.EvaluationTimestamp - minTime).TotalMilliseconds;
|
|
var bucketStart = (offsetMs / bucketMs) * bucketMs;
|
|
|
|
buckets.TryGetValue(bucketStart, out var counts);
|
|
buckets[bucketStart] = (counts.evalCount + 1, counts.fireCount + (trace.ExpressionResult ? 1 : 0));
|
|
}
|
|
|
|
return buckets
|
|
.OrderBy(kvp => kvp.Key)
|
|
.Select(kvp => new TemporalBucket(kvp.Key, kvp.Key + bucketMs, kvp.Value.evalCount, kvp.Value.fireCount))
|
|
.ToImmutableArray();
|
|
}
|
|
|
|
private OutcomeChangeSummary ComputeOutcomeChanges(
|
|
Dictionary<string, SimulationFindingResult> baseResults,
|
|
Dictionary<string, SimulationFindingResult> candidateResults)
|
|
{
|
|
var unchanged = 0;
|
|
var improved = 0;
|
|
var regressed = 0;
|
|
var transitionCounts = new Dictionary<(string from, string to), int>();
|
|
|
|
foreach (var (findingId, baseResult) in baseResults)
|
|
{
|
|
if (!candidateResults.TryGetValue(findingId, out var candidateResult))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (baseResult.Outcome == candidateResult.Outcome)
|
|
{
|
|
unchanged++;
|
|
}
|
|
else
|
|
{
|
|
var key = (baseResult.Outcome, candidateResult.Outcome);
|
|
transitionCounts.TryGetValue(key, out var count);
|
|
transitionCounts[key] = count + 1;
|
|
|
|
var isImprovement = IsOutcomeImprovement(baseResult.Outcome, candidateResult.Outcome);
|
|
if (isImprovement)
|
|
{
|
|
improved++;
|
|
}
|
|
else
|
|
{
|
|
regressed++;
|
|
}
|
|
}
|
|
}
|
|
|
|
var total = baseResults.Count;
|
|
var transitions = transitionCounts
|
|
.Select(kvp => new OutcomeTransition(
|
|
kvp.Key.from,
|
|
kvp.Key.to,
|
|
kvp.Value,
|
|
total > 0 ? (double)kvp.Value / total * 100 : 0,
|
|
IsOutcomeImprovement(kvp.Key.from, kvp.Key.to)))
|
|
.OrderByDescending(t => t.Count)
|
|
.ToImmutableArray();
|
|
|
|
return new OutcomeChangeSummary(unchanged, improved, regressed, transitions);
|
|
}
|
|
|
|
private SeverityChangeSummary ComputeSeverityChanges(
|
|
Dictionary<string, SimulationFindingResult> baseResults,
|
|
Dictionary<string, SimulationFindingResult> candidateResults)
|
|
{
|
|
var unchanged = 0;
|
|
var escalated = 0;
|
|
var deescalated = 0;
|
|
var transitionCounts = new Dictionary<(string from, string to), int>();
|
|
|
|
foreach (var (findingId, baseResult) in baseResults)
|
|
{
|
|
if (!candidateResults.TryGetValue(findingId, out var candidateResult))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
var baseSeverity = baseResult.Severity ?? "unknown";
|
|
var candidateSeverity = candidateResult.Severity ?? "unknown";
|
|
|
|
if (baseSeverity == candidateSeverity)
|
|
{
|
|
unchanged++;
|
|
}
|
|
else
|
|
{
|
|
var key = (baseSeverity, candidateSeverity);
|
|
transitionCounts.TryGetValue(key, out var count);
|
|
transitionCounts[key] = count + 1;
|
|
|
|
var baseIdx = SeverityOrder.IndexOf(baseSeverity.ToLowerInvariant());
|
|
var candidateIdx = SeverityOrder.IndexOf(candidateSeverity.ToLowerInvariant());
|
|
|
|
if (candidateIdx > baseIdx)
|
|
{
|
|
escalated++;
|
|
}
|
|
else
|
|
{
|
|
deescalated++;
|
|
}
|
|
}
|
|
}
|
|
|
|
var total = baseResults.Count;
|
|
var transitions = transitionCounts
|
|
.Select(kvp => new SeverityTransition(
|
|
kvp.Key.from,
|
|
kvp.Key.to,
|
|
kvp.Value,
|
|
total > 0 ? (double)kvp.Value / total * 100 : 0))
|
|
.OrderByDescending(t => t.Count)
|
|
.ToImmutableArray();
|
|
|
|
return new SeverityChangeSummary(unchanged, escalated, deescalated, transitions);
|
|
}
|
|
|
|
private RuleChangeSummary ComputeRuleChanges(
|
|
IReadOnlyList<SimulationFindingResult> baseResults,
|
|
IReadOnlyList<SimulationFindingResult> candidateResults)
|
|
{
|
|
var baseRules = baseResults
|
|
.SelectMany(r => r.FiredRules ?? Array.Empty<string>())
|
|
.Distinct()
|
|
.ToHashSet();
|
|
|
|
var candidateRules = candidateResults
|
|
.SelectMany(r => r.FiredRules ?? Array.Empty<string>())
|
|
.Distinct()
|
|
.ToHashSet();
|
|
|
|
var rulesAdded = candidateRules.Except(baseRules).ToImmutableArray();
|
|
var rulesRemoved = baseRules.Except(candidateRules).ToImmutableArray();
|
|
|
|
// Compute fire rate changes for common rules
|
|
var baseFireRates = ComputeFireRates(baseResults);
|
|
var candidateFireRates = ComputeFireRates(candidateResults);
|
|
|
|
var fireRateChanges = baseRules.Intersect(candidateRules)
|
|
.Select(rule =>
|
|
{
|
|
var baseRate = baseFireRates.GetValueOrDefault(rule, 0);
|
|
var candidateRate = candidateFireRates.GetValueOrDefault(rule, 0);
|
|
var change = candidateRate - baseRate;
|
|
return new RuleFireRateChange(
|
|
rule,
|
|
baseRate,
|
|
candidateRate,
|
|
change,
|
|
Math.Abs(change) > 5.0); // >5% change is significant
|
|
})
|
|
.Where(c => Math.Abs(c.ChangePercentage) > 1.0) // Only show changes > 1%
|
|
.OrderByDescending(c => Math.Abs(c.ChangePercentage))
|
|
.Take(20)
|
|
.ToImmutableArray();
|
|
|
|
return new RuleChangeSummary(
|
|
rulesAdded,
|
|
rulesRemoved,
|
|
ImmutableArray<RuleModification>.Empty, // Would require policy diff analysis
|
|
fireRateChanges);
|
|
}
|
|
|
|
private Dictionary<string, double> ComputeFireRates(IReadOnlyList<SimulationFindingResult> results)
|
|
{
|
|
var ruleCounts = new Dictionary<string, int>();
|
|
|
|
foreach (var result in results)
|
|
{
|
|
foreach (var rule in result.FiredRules ?? Array.Empty<string>())
|
|
{
|
|
ruleCounts.TryGetValue(rule, out var count);
|
|
ruleCounts[rule] = count + 1;
|
|
}
|
|
}
|
|
|
|
var total = results.Count;
|
|
return ruleCounts.ToDictionary(
|
|
kvp => kvp.Key,
|
|
kvp => total > 0 ? (double)kvp.Value / total * 100 : 0);
|
|
}
|
|
|
|
private ImmutableArray<HighImpactFinding> ComputeHighImpactFindings(
|
|
Dictionary<string, SimulationFindingResult> baseResults,
|
|
Dictionary<string, SimulationFindingResult> candidateResults)
|
|
{
|
|
var highImpact = new List<HighImpactFinding>();
|
|
|
|
foreach (var (findingId, baseResult) in baseResults)
|
|
{
|
|
if (!candidateResults.TryGetValue(findingId, out var candidateResult))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
var impactScore = ComputeImpactScore(baseResult, candidateResult);
|
|
if (impactScore < 0.3) // Threshold for high impact
|
|
{
|
|
continue;
|
|
}
|
|
|
|
var impactReason = DetermineImpactReason(baseResult, candidateResult);
|
|
|
|
highImpact.Add(new HighImpactFinding(
|
|
findingId,
|
|
baseResult.ComponentPurl,
|
|
baseResult.AdvisoryId,
|
|
baseResult.Outcome,
|
|
candidateResult.Outcome,
|
|
baseResult.Severity,
|
|
candidateResult.Severity,
|
|
impactScore,
|
|
impactReason));
|
|
}
|
|
|
|
return highImpact
|
|
.OrderByDescending(f => f.ImpactScore)
|
|
.Take(50)
|
|
.ToImmutableArray();
|
|
}
|
|
|
|
private double ComputeImpactScore(SimulationFindingResult baseResult, SimulationFindingResult candidateResult)
|
|
{
|
|
var score = 0.0;
|
|
|
|
// Outcome change weight
|
|
if (baseResult.Outcome != candidateResult.Outcome)
|
|
{
|
|
var baseIdx = OutcomeSeverityOrder.IndexOf(baseResult.Outcome.ToLowerInvariant());
|
|
var candidateIdx = OutcomeSeverityOrder.IndexOf(candidateResult.Outcome.ToLowerInvariant());
|
|
score += Math.Abs(candidateIdx - baseIdx) * 0.2;
|
|
}
|
|
|
|
// Severity change weight
|
|
var baseSeverity = baseResult.Severity ?? "unknown";
|
|
var candidateSeverity = candidateResult.Severity ?? "unknown";
|
|
if (baseSeverity != candidateSeverity)
|
|
{
|
|
var baseIdx = SeverityOrder.IndexOf(baseSeverity.ToLowerInvariant());
|
|
var candidateIdx = SeverityOrder.IndexOf(candidateSeverity.ToLowerInvariant());
|
|
score += Math.Abs(candidateIdx - baseIdx) * 0.15;
|
|
}
|
|
|
|
return Math.Min(1.0, score);
|
|
}
|
|
|
|
private string DetermineImpactReason(SimulationFindingResult baseResult, SimulationFindingResult candidateResult)
|
|
{
|
|
var reasons = new List<string>();
|
|
|
|
if (baseResult.Outcome != candidateResult.Outcome)
|
|
{
|
|
reasons.Add($"Outcome changed from '{baseResult.Outcome}' to '{candidateResult.Outcome}'");
|
|
}
|
|
|
|
if (baseResult.Severity != candidateResult.Severity)
|
|
{
|
|
reasons.Add($"Severity changed from '{baseResult.Severity}' to '{candidateResult.Severity}'");
|
|
}
|
|
|
|
return string.Join("; ", reasons);
|
|
}
|
|
|
|
private bool IsOutcomeImprovement(string from, string to)
|
|
{
|
|
var fromIdx = OutcomeSeverityOrder.IndexOf(from.ToLowerInvariant());
|
|
var toIdx = OutcomeSeverityOrder.IndexOf(to.ToLowerInvariant());
|
|
|
|
// Lower index = less severe = improvement
|
|
return toIdx < fromIdx;
|
|
}
|
|
|
|
private static string DetermineSampleReason(bool randomSample, bool highSeverity, bool vexOverride)
|
|
{
|
|
if (vexOverride)
|
|
{
|
|
return "vex_override";
|
|
}
|
|
|
|
if (highSeverity)
|
|
{
|
|
return "high_severity";
|
|
}
|
|
|
|
return randomSample ? "random_sample" : "coverage";
|
|
}
|
|
|
|
private static string ComputeHash(string input)
|
|
{
|
|
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(input));
|
|
return Convert.ToHexString(bytes)[..16].ToLowerInvariant();
|
|
}
|
|
|
|
private sealed class RuleStats
|
|
{
|
|
public string RuleName { get; }
|
|
public int Priority { get; }
|
|
public string? Category { get; }
|
|
public int FireCount { get; set; }
|
|
public long TotalEvaluationUs { get; set; }
|
|
public Dictionary<string, int> OutcomeCounts { get; } = new();
|
|
|
|
public RuleStats(string ruleName, int priority, string? category)
|
|
{
|
|
RuleName = ruleName;
|
|
Priority = priority;
|
|
Category = category;
|
|
}
|
|
|
|
public void IncrementOutcome(string outcome)
|
|
{
|
|
OutcomeCounts.TryGetValue(outcome, out var count);
|
|
OutcomeCounts[outcome] = count + 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Options for simulation analytics computation.
|
|
/// </summary>
|
|
public sealed record SimulationAnalyticsOptions
|
|
{
|
|
/// <summary>
|
|
/// Sample rate for traces (0.0 to 1.0).
|
|
/// </summary>
|
|
public double TraceSampleRate { get; init; } = 0.1;
|
|
|
|
/// <summary>
|
|
/// Maximum number of sampled traces to include.
|
|
/// </summary>
|
|
public int MaxSampledTraces { get; init; } = 100;
|
|
|
|
/// <summary>
|
|
/// Temporal bucket size in milliseconds.
|
|
/// </summary>
|
|
public long TemporalBucketMs { get; init; } = 100;
|
|
|
|
/// <summary>
|
|
/// Maximum number of top rules to include.
|
|
/// </summary>
|
|
public int MaxTopRules { get; init; } = 10;
|
|
|
|
/// <summary>
|
|
/// Significance threshold for fire rate changes (percentage).
|
|
/// </summary>
|
|
public double FireRateSignificanceThreshold { get; init; } = 5.0;
|
|
|
|
/// <summary>
|
|
/// Default options.
|
|
/// </summary>
|
|
public static SimulationAnalyticsOptions Default { get; } = new();
|
|
|
|
/// <summary>
|
|
/// Options for quick simulations (lower sampling, faster).
|
|
/// </summary>
|
|
public static SimulationAnalyticsOptions Quick { get; } = new()
|
|
{
|
|
TraceSampleRate = 0.01,
|
|
MaxSampledTraces = 20,
|
|
TemporalBucketMs = 500
|
|
};
|
|
|
|
/// <summary>
|
|
/// Options for batch simulations (balanced).
|
|
/// </summary>
|
|
public static SimulationAnalyticsOptions Batch { get; } = new()
|
|
{
|
|
TraceSampleRate = 0.05,
|
|
MaxSampledTraces = 50,
|
|
TemporalBucketMs = 200
|
|
};
|
|
}
|
|
|
|
/// <summary>
|
|
/// Result of a single finding simulation (for delta comparison).
|
|
/// </summary>
|
|
public sealed record SimulationFindingResult(
|
|
string FindingId,
|
|
string? ComponentPurl,
|
|
string? AdvisoryId,
|
|
string Outcome,
|
|
string? Severity,
|
|
IReadOnlyList<string>? FiredRules);
|