using System.Collections.Immutable; using System.Security.Cryptography; using System.Text; using StellaOps.Policy.Engine.Telemetry; namespace StellaOps.Policy.Engine.Simulation; /// /// Service for computing simulation analytics including rule firing counts, heatmaps, /// sampled traces, and delta summaries. /// public sealed class SimulationAnalyticsService { private static readonly ImmutableArray OutcomeSeverityOrder = ImmutableArray.Create( "allow", "info", "warn", "review", "block", "deny", "critical"); private static readonly ImmutableArray SeverityOrder = ImmutableArray.Create( "informational", "low", "medium", "high", "critical"); /// /// Computes full simulation analytics from rule hit traces. /// public SimulationAnalytics ComputeAnalytics( string policyRef, IReadOnlyList traces, IReadOnlyList findings, SimulationAnalyticsOptions? options = null) { options ??= SimulationAnalyticsOptions.Default; var firingCounts = ComputeRuleFiringCounts(traces, findings.Count); var heatmap = ComputeHeatmap(traces, findings, options); var sampledTraces = ComputeSampledTraces(traces, findings, options); return new SimulationAnalytics( firingCounts, heatmap, sampledTraces, DeltaSummary: null); } /// /// Computes delta summary comparing base and candidate simulation results. /// public SimulationDeltaSummary ComputeDeltaSummary( string basePolicyRef, string candidatePolicyRef, IReadOnlyList baseResults, IReadOnlyList candidateResults, SimulationComparisonType comparisonType = SimulationComparisonType.VersionCompare) { var baseByFinding = baseResults.ToDictionary(r => r.FindingId); var candidateByFinding = candidateResults.ToDictionary(r => r.FindingId); var outcomeChanges = ComputeOutcomeChanges(baseByFinding, candidateByFinding); var severityChanges = ComputeSeverityChanges(baseByFinding, candidateByFinding); var ruleChanges = ComputeRuleChanges(baseResults, candidateResults); var highImpact = ComputeHighImpactFindings(baseByFinding, candidateByFinding); var hashInput = $"{basePolicyRef}:{candidatePolicyRef}:{baseResults.Count}:{candidateResults.Count}"; var determinismHash = ComputeHash(hashInput); return new SimulationDeltaSummary( comparisonType, basePolicyRef, candidatePolicyRef, TotalFindings: baseResults.Count, outcomeChanges, severityChanges, ruleChanges, highImpact, determinismHash); } /// /// Computes rule firing counts from traces. /// public RuleFiringCounts ComputeRuleFiringCounts( IReadOnlyList traces, int totalEvaluations) { var ruleStats = new Dictionary(); var byPriority = new Dictionary(); var byOutcome = new Dictionary(); var byCategory = new Dictionary(); var vexByVendor = new Dictionary(); var vexByStatus = new Dictionary(); var vexByJustification = new Dictionary(); var totalFired = 0; var totalVexOverrides = 0; foreach (var trace in traces) { if (!trace.ExpressionResult) { continue; } totalFired++; // Rule stats if (!ruleStats.TryGetValue(trace.RuleName, out var stats)) { stats = new RuleStats(trace.RuleName, trace.RulePriority, trace.RuleCategory); ruleStats[trace.RuleName] = stats; } stats.FireCount++; stats.TotalEvaluationUs += trace.EvaluationMicroseconds; stats.IncrementOutcome(trace.Outcome); // Priority aggregation byPriority.TryGetValue(trace.RulePriority, out var priorityCount); byPriority[trace.RulePriority] = priorityCount + 1; // Outcome aggregation byOutcome.TryGetValue(trace.Outcome, out var outcomeCount); byOutcome[trace.Outcome] = outcomeCount + 1; // Category aggregation if (!string.IsNullOrWhiteSpace(trace.RuleCategory)) { byCategory.TryGetValue(trace.RuleCategory, out var categoryCount); byCategory[trace.RuleCategory] = categoryCount + 1; } // VEX overrides if (trace.IsVexOverride) { totalVexOverrides++; if (!string.IsNullOrWhiteSpace(trace.VexVendor)) { vexByVendor.TryGetValue(trace.VexVendor, out var vendorCount); vexByVendor[trace.VexVendor] = vendorCount + 1; } if (!string.IsNullOrWhiteSpace(trace.VexStatus)) { vexByStatus.TryGetValue(trace.VexStatus, out var statusCount); vexByStatus[trace.VexStatus] = statusCount + 1; } if (!string.IsNullOrWhiteSpace(trace.VexJustification)) { vexByJustification.TryGetValue(trace.VexJustification, out var justCount); vexByJustification[trace.VexJustification] = justCount + 1; } } } // Build rule fire counts var ruleFireCounts = ruleStats.Values .Select(s => new RuleFireCount( s.RuleName, s.Priority, s.Category, s.FireCount, totalEvaluations > 0 ? (double)s.FireCount / totalEvaluations * 100 : 0, s.OutcomeCounts.ToImmutableDictionary(), s.FireCount > 0 ? (double)s.TotalEvaluationUs / s.FireCount : 0)) .ToImmutableDictionary(r => r.RuleName); var topRules = ruleFireCounts.Values .OrderByDescending(r => r.FireCount) .Take(10) .ToImmutableArray(); var vexOverrides = new VexOverrideCounts( totalVexOverrides, vexByVendor.ToImmutableDictionary(), vexByStatus.ToImmutableDictionary(), vexByJustification.ToImmutableDictionary()); return new RuleFiringCounts( totalEvaluations, totalFired, ruleFireCounts, byPriority.ToImmutableDictionary(), byOutcome.ToImmutableDictionary(), byCategory.ToImmutableDictionary(), topRules, vexOverrides); } /// /// Computes heatmap aggregates for visualization. /// public SimulationHeatmap ComputeHeatmap( IReadOnlyList traces, IReadOnlyList findings, SimulationAnalyticsOptions options) { var ruleSeverityMatrix = ComputeRuleSeverityMatrix(traces); var ruleOutcomeMatrix = ComputeRuleOutcomeMatrix(traces); var findingCoverage = ComputeFindingRuleCoverage(traces, findings); var temporalDist = ComputeTemporalDistribution(traces, options.TemporalBucketMs); return new SimulationHeatmap( ruleSeverityMatrix, ruleOutcomeMatrix, findingCoverage, temporalDist); } /// /// Computes sampled explain traces with deterministic ordering. /// public SampledExplainTraces ComputeSampledTraces( IReadOnlyList traces, IReadOnlyList findings, SimulationAnalyticsOptions options) { // Group traces by finding var tracesByFinding = traces .GroupBy(t => t.ComponentPurl ?? t.AdvisoryId ?? "unknown") .ToDictionary(g => g.Key, g => g.ToList()); var findingsById = findings.ToDictionary(f => f.FindingId); // Deterministic ordering by finding_id, then rule_priority var ordering = new TraceOrdering("finding_id", "rule_priority", "ascending"); // Sample traces deterministically var sampledList = new List(); var totalTraceCount = 0; foreach (var finding in findings.OrderBy(f => f.FindingId, StringComparer.Ordinal)) { var key = finding.ComponentPurl ?? finding.AdvisoryId ?? finding.FindingId; if (!tracesByFinding.TryGetValue(key, out var findingTraces)) { continue; } totalTraceCount += findingTraces.Count; // Deterministic sampling based on finding_id hash var sampleHash = ComputeHash(finding.FindingId); var sampleValue = Math.Abs(sampleHash.GetHashCode()) % 100; var shouldSample = sampleValue < (int)(options.TraceSampleRate * 100); if (!shouldSample && sampledList.Count >= options.MaxSampledTraces) { continue; } // Always sample high-impact findings var hasFiredRule = findingTraces.Any(t => t.ExpressionResult); var isHighSeverity = findingTraces.Any(t => t.AssignedSeverity?.Equals("critical", StringComparison.OrdinalIgnoreCase) == true || t.AssignedSeverity?.Equals("high", StringComparison.OrdinalIgnoreCase) == true); var hasVexOverride = findingTraces.Any(t => t.IsVexOverride); var sampleReason = DetermineSampleReason(shouldSample, isHighSeverity, hasVexOverride); if (!shouldSample && !isHighSeverity && !hasVexOverride) { continue; } var orderedTraces = findingTraces.OrderBy(t => t.RulePriority).ToList(); var finalTrace = orderedTraces.LastOrDefault(t => t.ExpressionResult) ?? orderedTraces.LastOrDefault(); if (finalTrace == null) { continue; } var ruleSequence = orderedTraces .Where(t => t.ExpressionResult) .Select(t => t.RuleName) .ToImmutableArray(); sampledList.Add(new SampledTrace( TraceId: $"{finding.FindingId}:{finalTrace.SpanId}", FindingId: finding.FindingId, ComponentPurl: finding.ComponentPurl, AdvisoryId: finding.AdvisoryId, FinalOutcome: finalTrace.Outcome, AssignedSeverity: finalTrace.AssignedSeverity, RulesEvaluated: findingTraces.Count, RulesFired: findingTraces.Count(t => t.ExpressionResult), VexApplied: hasVexOverride, EvaluationMs: findingTraces.Sum(t => t.EvaluationMicroseconds) / 1000.0, RuleSequence: ruleSequence, SampleReason: sampleReason)); if (sampledList.Count >= options.MaxSampledTraces) { break; } } // Compute determinism hash from ordered sample var hashBuilder = new StringBuilder(); foreach (var sample in sampledList.OrderBy(s => s.FindingId, StringComparer.Ordinal)) { hashBuilder.Append(sample.FindingId); hashBuilder.Append(':'); hashBuilder.Append(sample.FinalOutcome); hashBuilder.Append(';'); } var determinismHash = ComputeHash(hashBuilder.ToString()); return new SampledExplainTraces( options.TraceSampleRate, totalTraceCount, sampledList.Count, ordering, sampledList.ToImmutableArray(), determinismHash); } private ImmutableArray ComputeRuleSeverityMatrix(IReadOnlyList traces) { var matrix = new Dictionary<(string rule, string severity), int>(); foreach (var trace in traces.Where(t => t.ExpressionResult && !string.IsNullOrWhiteSpace(t.AssignedSeverity))) { var key = (trace.RuleName, trace.AssignedSeverity!); matrix.TryGetValue(key, out var count); matrix[key] = count + 1; } var maxValue = matrix.Values.DefaultIfEmpty(1).Max(); return matrix .Select(kvp => new HeatmapCell( kvp.Key.rule, kvp.Key.severity, kvp.Value, maxValue > 0 ? (double)kvp.Value / maxValue : 0)) .OrderBy(c => c.X, StringComparer.Ordinal) .ThenBy(c => SeverityOrder.IndexOf(c.Y.ToLowerInvariant())) .ToImmutableArray(); } private ImmutableArray ComputeRuleOutcomeMatrix(IReadOnlyList traces) { var matrix = new Dictionary<(string rule, string outcome), int>(); foreach (var trace in traces.Where(t => t.ExpressionResult)) { var key = (trace.RuleName, trace.Outcome); matrix.TryGetValue(key, out var count); matrix[key] = count + 1; } var maxValue = matrix.Values.DefaultIfEmpty(1).Max(); return matrix .Select(kvp => new HeatmapCell( kvp.Key.rule, kvp.Key.outcome, kvp.Value, maxValue > 0 ? (double)kvp.Value / maxValue : 0)) .OrderBy(c => c.X, StringComparer.Ordinal) .ThenBy(c => OutcomeSeverityOrder.IndexOf(c.Y.ToLowerInvariant())) .ToImmutableArray(); } private FindingRuleCoverage ComputeFindingRuleCoverage( IReadOnlyList traces, IReadOnlyList findings) { var rulesThatFired = traces .Where(t => t.ExpressionResult) .Select(t => t.RuleName) .ToHashSet(); var allRules = traces .Select(t => t.RuleName) .Distinct() .ToHashSet(); var rulesNeverFired = allRules.Except(rulesThatFired).ToImmutableArray(); // Group by finding to count matches per finding var findingMatchCounts = traces .Where(t => t.ExpressionResult) .GroupBy(t => t.ComponentPurl ?? t.AdvisoryId ?? "unknown") .ToDictionary(g => g.Key, g => g.Select(t => t.RuleName).Distinct().Count()); var matchCountDistribution = findingMatchCounts.Values .GroupBy(c => c) .ToDictionary(g => g.Key, g => g.Count()) .ToImmutableDictionary(); var findingsMatched = findingMatchCounts.Count; var findingsUnmatched = findings.Count - findingsMatched; return new FindingRuleCoverage( findings.Count, findingsMatched, findingsUnmatched, findings.Count > 0 ? (double)findingsMatched / findings.Count * 100 : 0, rulesNeverFired, matchCountDistribution); } private ImmutableArray ComputeTemporalDistribution( IReadOnlyList traces, long bucketMs) { if (traces.Count == 0) { return ImmutableArray.Empty; } var minTime = traces.Min(t => t.EvaluationTimestamp); var maxTime = traces.Max(t => t.EvaluationTimestamp); var totalMs = (long)(maxTime - minTime).TotalMilliseconds; if (totalMs <= 0) { return ImmutableArray.Create(new TemporalBucket(0, bucketMs, traces.Count, traces.Count(t => t.ExpressionResult))); } var buckets = new Dictionary(); foreach (var trace in traces) { var offsetMs = (long)(trace.EvaluationTimestamp - minTime).TotalMilliseconds; var bucketStart = (offsetMs / bucketMs) * bucketMs; buckets.TryGetValue(bucketStart, out var counts); buckets[bucketStart] = (counts.evalCount + 1, counts.fireCount + (trace.ExpressionResult ? 1 : 0)); } return buckets .OrderBy(kvp => kvp.Key) .Select(kvp => new TemporalBucket(kvp.Key, kvp.Key + bucketMs, kvp.Value.evalCount, kvp.Value.fireCount)) .ToImmutableArray(); } private OutcomeChangeSummary ComputeOutcomeChanges( Dictionary baseResults, Dictionary candidateResults) { var unchanged = 0; var improved = 0; var regressed = 0; var transitionCounts = new Dictionary<(string from, string to), int>(); foreach (var (findingId, baseResult) in baseResults) { if (!candidateResults.TryGetValue(findingId, out var candidateResult)) { continue; } if (baseResult.Outcome == candidateResult.Outcome) { unchanged++; } else { var key = (baseResult.Outcome, candidateResult.Outcome); transitionCounts.TryGetValue(key, out var count); transitionCounts[key] = count + 1; var isImprovement = IsOutcomeImprovement(baseResult.Outcome, candidateResult.Outcome); if (isImprovement) { improved++; } else { regressed++; } } } var total = baseResults.Count; var transitions = transitionCounts .Select(kvp => new OutcomeTransition( kvp.Key.from, kvp.Key.to, kvp.Value, total > 0 ? (double)kvp.Value / total * 100 : 0, IsOutcomeImprovement(kvp.Key.from, kvp.Key.to))) .OrderByDescending(t => t.Count) .ToImmutableArray(); return new OutcomeChangeSummary(unchanged, improved, regressed, transitions); } private SeverityChangeSummary ComputeSeverityChanges( Dictionary baseResults, Dictionary candidateResults) { var unchanged = 0; var escalated = 0; var deescalated = 0; var transitionCounts = new Dictionary<(string from, string to), int>(); foreach (var (findingId, baseResult) in baseResults) { if (!candidateResults.TryGetValue(findingId, out var candidateResult)) { continue; } var baseSeverity = baseResult.Severity ?? "unknown"; var candidateSeverity = candidateResult.Severity ?? "unknown"; if (baseSeverity == candidateSeverity) { unchanged++; } else { var key = (baseSeverity, candidateSeverity); transitionCounts.TryGetValue(key, out var count); transitionCounts[key] = count + 1; var baseIdx = SeverityOrder.IndexOf(baseSeverity.ToLowerInvariant()); var candidateIdx = SeverityOrder.IndexOf(candidateSeverity.ToLowerInvariant()); if (candidateIdx > baseIdx) { escalated++; } else { deescalated++; } } } var total = baseResults.Count; var transitions = transitionCounts .Select(kvp => new SeverityTransition( kvp.Key.from, kvp.Key.to, kvp.Value, total > 0 ? (double)kvp.Value / total * 100 : 0)) .OrderByDescending(t => t.Count) .ToImmutableArray(); return new SeverityChangeSummary(unchanged, escalated, deescalated, transitions); } private RuleChangeSummary ComputeRuleChanges( IReadOnlyList baseResults, IReadOnlyList candidateResults) { var baseRules = baseResults .SelectMany(r => r.FiredRules ?? Array.Empty()) .Distinct() .ToHashSet(); var candidateRules = candidateResults .SelectMany(r => r.FiredRules ?? Array.Empty()) .Distinct() .ToHashSet(); var rulesAdded = candidateRules.Except(baseRules).ToImmutableArray(); var rulesRemoved = baseRules.Except(candidateRules).ToImmutableArray(); // Compute fire rate changes for common rules var baseFireRates = ComputeFireRates(baseResults); var candidateFireRates = ComputeFireRates(candidateResults); var fireRateChanges = baseRules.Intersect(candidateRules) .Select(rule => { var baseRate = baseFireRates.GetValueOrDefault(rule, 0); var candidateRate = candidateFireRates.GetValueOrDefault(rule, 0); var change = candidateRate - baseRate; return new RuleFireRateChange( rule, baseRate, candidateRate, change, Math.Abs(change) > 5.0); // >5% change is significant }) .Where(c => Math.Abs(c.ChangePercentage) > 1.0) // Only show changes > 1% .OrderByDescending(c => Math.Abs(c.ChangePercentage)) .Take(20) .ToImmutableArray(); return new RuleChangeSummary( rulesAdded, rulesRemoved, ImmutableArray.Empty, // Would require policy diff analysis fireRateChanges); } private Dictionary ComputeFireRates(IReadOnlyList results) { var ruleCounts = new Dictionary(); foreach (var result in results) { foreach (var rule in result.FiredRules ?? Array.Empty()) { ruleCounts.TryGetValue(rule, out var count); ruleCounts[rule] = count + 1; } } var total = results.Count; return ruleCounts.ToDictionary( kvp => kvp.Key, kvp => total > 0 ? (double)kvp.Value / total * 100 : 0); } private ImmutableArray ComputeHighImpactFindings( Dictionary baseResults, Dictionary candidateResults) { var highImpact = new List(); foreach (var (findingId, baseResult) in baseResults) { if (!candidateResults.TryGetValue(findingId, out var candidateResult)) { continue; } var impactScore = ComputeImpactScore(baseResult, candidateResult); if (impactScore < 0.3) // Threshold for high impact { continue; } var impactReason = DetermineImpactReason(baseResult, candidateResult); highImpact.Add(new HighImpactFinding( findingId, baseResult.ComponentPurl, baseResult.AdvisoryId, baseResult.Outcome, candidateResult.Outcome, baseResult.Severity, candidateResult.Severity, impactScore, impactReason)); } return highImpact .OrderByDescending(f => f.ImpactScore) .Take(50) .ToImmutableArray(); } private double ComputeImpactScore(SimulationFindingResult baseResult, SimulationFindingResult candidateResult) { var score = 0.0; // Outcome change weight if (baseResult.Outcome != candidateResult.Outcome) { var baseIdx = OutcomeSeverityOrder.IndexOf(baseResult.Outcome.ToLowerInvariant()); var candidateIdx = OutcomeSeverityOrder.IndexOf(candidateResult.Outcome.ToLowerInvariant()); score += Math.Abs(candidateIdx - baseIdx) * 0.2; } // Severity change weight var baseSeverity = baseResult.Severity ?? "unknown"; var candidateSeverity = candidateResult.Severity ?? "unknown"; if (baseSeverity != candidateSeverity) { var baseIdx = SeverityOrder.IndexOf(baseSeverity.ToLowerInvariant()); var candidateIdx = SeverityOrder.IndexOf(candidateSeverity.ToLowerInvariant()); score += Math.Abs(candidateIdx - baseIdx) * 0.15; } return Math.Min(1.0, score); } private string DetermineImpactReason(SimulationFindingResult baseResult, SimulationFindingResult candidateResult) { var reasons = new List(); if (baseResult.Outcome != candidateResult.Outcome) { reasons.Add($"Outcome changed from '{baseResult.Outcome}' to '{candidateResult.Outcome}'"); } if (baseResult.Severity != candidateResult.Severity) { reasons.Add($"Severity changed from '{baseResult.Severity}' to '{candidateResult.Severity}'"); } return string.Join("; ", reasons); } private bool IsOutcomeImprovement(string from, string to) { var fromIdx = OutcomeSeverityOrder.IndexOf(from.ToLowerInvariant()); var toIdx = OutcomeSeverityOrder.IndexOf(to.ToLowerInvariant()); // Lower index = less severe = improvement return toIdx < fromIdx; } private static string DetermineSampleReason(bool randomSample, bool highSeverity, bool vexOverride) { if (vexOverride) { return "vex_override"; } if (highSeverity) { return "high_severity"; } return randomSample ? "random_sample" : "coverage"; } private static string ComputeHash(string input) { var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(input)); return Convert.ToHexString(bytes)[..16].ToLowerInvariant(); } private sealed class RuleStats { public string RuleName { get; } public int Priority { get; } public string? Category { get; } public int FireCount { get; set; } public long TotalEvaluationUs { get; set; } public Dictionary OutcomeCounts { get; } = new(); public RuleStats(string ruleName, int priority, string? category) { RuleName = ruleName; Priority = priority; Category = category; } public void IncrementOutcome(string outcome) { OutcomeCounts.TryGetValue(outcome, out var count); OutcomeCounts[outcome] = count + 1; } } } /// /// Options for simulation analytics computation. /// public sealed record SimulationAnalyticsOptions { /// /// Sample rate for traces (0.0 to 1.0). /// public double TraceSampleRate { get; init; } = 0.1; /// /// Maximum number of sampled traces to include. /// public int MaxSampledTraces { get; init; } = 100; /// /// Temporal bucket size in milliseconds. /// public long TemporalBucketMs { get; init; } = 100; /// /// Maximum number of top rules to include. /// public int MaxTopRules { get; init; } = 10; /// /// Significance threshold for fire rate changes (percentage). /// public double FireRateSignificanceThreshold { get; init; } = 5.0; /// /// Default options. /// public static SimulationAnalyticsOptions Default { get; } = new(); /// /// Options for quick simulations (lower sampling, faster). /// public static SimulationAnalyticsOptions Quick { get; } = new() { TraceSampleRate = 0.01, MaxSampledTraces = 20, TemporalBucketMs = 500 }; /// /// Options for batch simulations (balanced). /// public static SimulationAnalyticsOptions Batch { get; } = new() { TraceSampleRate = 0.05, MaxSampledTraces = 50, TemporalBucketMs = 200 }; } /// /// Result of a single finding simulation (for delta comparison). /// public sealed record SimulationFindingResult( string FindingId, string? ComponentPurl, string? AdvisoryId, string Outcome, string? Severity, IReadOnlyList? FiredRules);