using System.Collections.Immutable; using FluentAssertions; using StellaOps.Policy.Engine.Simulation; using StellaOps.Policy.Engine.Telemetry; using Xunit; namespace StellaOps.Policy.Engine.Tests.Simulation; public sealed class SimulationAnalyticsServiceTests { private readonly SimulationAnalyticsService _service = new(); [Fact] public void ComputeRuleFiringCounts_EmptyTraces_ReturnsEmptyCounts() { // Arrange var traces = Array.Empty(); // Act var result = _service.ComputeRuleFiringCounts(traces, 10); // Assert result.TotalEvaluations.Should().Be(10); result.TotalRulesFired.Should().Be(0); result.RulesByName.Should().BeEmpty(); result.RulesByPriority.Should().BeEmpty(); result.RulesByOutcome.Should().BeEmpty(); result.TopRules.Should().BeEmpty(); } [Fact] public void ComputeRuleFiringCounts_WithFiredRules_CountsCorrectly() { // Arrange var traces = new[] { CreateTrace("rule_a", 1, "block", expressionResult: true), CreateTrace("rule_a", 1, "block", expressionResult: true), CreateTrace("rule_b", 2, "allow", expressionResult: true), CreateTrace("rule_c", 3, "warn", expressionResult: false), // Not fired }; // Act var result = _service.ComputeRuleFiringCounts(traces, 10); // Assert result.TotalRulesFired.Should().Be(3); result.RulesByName.Should().HaveCount(2); result.RulesByName["rule_a"].FireCount.Should().Be(2); result.RulesByName["rule_b"].FireCount.Should().Be(1); result.RulesByPriority[1].Should().Be(2); result.RulesByPriority[2].Should().Be(1); result.RulesByOutcome["block"].Should().Be(2); result.RulesByOutcome["allow"].Should().Be(1); } [Fact] public void ComputeRuleFiringCounts_TopRules_OrderedByFireCount() { // Arrange var traces = new List(); for (var i = 0; i < 15; i++) { traces.Add(CreateTrace("frequently_fired", 1, "block", expressionResult: true)); } for (var i = 0; i < 5; i++) { traces.Add(CreateTrace("sometimes_fired", 2, "warn", expressionResult: true)); } traces.Add(CreateTrace("rarely_fired", 3, "allow", expressionResult: true)); // Act var result = _service.ComputeRuleFiringCounts(traces, 100); // Assert result.TopRules.Should().HaveCount(3); result.TopRules[0].RuleName.Should().Be("frequently_fired"); result.TopRules[0].FireCount.Should().Be(15); result.TopRules[1].RuleName.Should().Be("sometimes_fired"); result.TopRules[1].FireCount.Should().Be(5); result.TopRules[2].RuleName.Should().Be("rarely_fired"); result.TopRules[2].FireCount.Should().Be(1); } [Fact] public void ComputeRuleFiringCounts_VexOverrides_CountedCorrectly() { // Arrange var traces = new[] { CreateTrace("rule_a", 1, "allow", expressionResult: true, isVexOverride: true, vexVendor: "vendor_a", vexStatus: "not_affected"), CreateTrace("rule_a", 1, "allow", expressionResult: true, isVexOverride: true, vexVendor: "vendor_a", vexStatus: "fixed"), CreateTrace("rule_b", 2, "allow", expressionResult: true, isVexOverride: true, vexVendor: "vendor_b", vexStatus: "not_affected"), CreateTrace("rule_c", 3, "block", expressionResult: true), }; // Act var result = _service.ComputeRuleFiringCounts(traces, 10); // Assert result.VexOverrides.TotalOverrides.Should().Be(3); result.VexOverrides.ByVendor["vendor_a"].Should().Be(2); result.VexOverrides.ByVendor["vendor_b"].Should().Be(1); result.VexOverrides.ByStatus["not_affected"].Should().Be(2); result.VexOverrides.ByStatus["fixed"].Should().Be(1); } [Fact] public void ComputeHeatmap_RuleSeverityMatrix_BuildsCorrectly() { // Arrange var traces = new[] { CreateTrace("rule_a", 1, "block", expressionResult: true, severity: "critical"), CreateTrace("rule_a", 1, "block", expressionResult: true, severity: "critical"), CreateTrace("rule_a", 1, "block", expressionResult: true, severity: "high"), CreateTrace("rule_b", 2, "warn", expressionResult: true, severity: "medium"), }; var findings = CreateFindings(4); // Act var result = _service.ComputeHeatmap(traces, findings, SimulationAnalyticsOptions.Default); // Assert result.RuleSeverityMatrix.Should().NotBeEmpty(); var criticalCell = result.RuleSeverityMatrix.FirstOrDefault(c => c.X == "rule_a" && c.Y == "critical"); criticalCell.Should().NotBeNull(); criticalCell!.Value.Should().Be(2); } [Fact] public void ComputeHeatmap_FindingRuleCoverage_CalculatesCorrectly() { // Arrange var traces = new[] { CreateTrace("rule_a", 1, "block", expressionResult: true, componentPurl: "pkg:npm/lodash@4.0.0"), CreateTrace("rule_b", 2, "allow", expressionResult: true, componentPurl: "pkg:npm/lodash@4.0.0"), CreateTrace("rule_a", 1, "block", expressionResult: false, componentPurl: "pkg:npm/express@5.0.0"), }; var findings = new[] { new SimulationFinding("f1", "pkg:npm/lodash@4.0.0", "GHSA-123", new Dictionary()), new SimulationFinding("f2", "pkg:npm/express@5.0.0", "GHSA-456", new Dictionary()), new SimulationFinding("f3", "pkg:npm/axios@1.0.0", "GHSA-789", new Dictionary()), }; // Act var result = _service.ComputeHeatmap(traces, findings, SimulationAnalyticsOptions.Default); // Assert result.FindingRuleCoverage.TotalFindings.Should().Be(3); result.FindingRuleCoverage.FindingsMatched.Should().Be(1); result.FindingRuleCoverage.CoveragePercentage.Should().BeApproximately(33.33, 0.1); } [Fact] public void ComputeSampledTraces_DeterministicOrdering_OrdersByFindingId() { // Arrange var traces = new[] { CreateTrace("rule_a", 1, "block", expressionResult: true, componentPurl: "pkg:npm/z-package@1.0.0"), CreateTrace("rule_a", 1, "allow", expressionResult: true, componentPurl: "pkg:npm/a-package@1.0.0"), CreateTrace("rule_b", 2, "warn", expressionResult: true, componentPurl: "pkg:npm/m-package@1.0.0"), }; var findings = new[] { new SimulationFinding("finding-z", "pkg:npm/z-package@1.0.0", null, new Dictionary()), new SimulationFinding("finding-a", "pkg:npm/a-package@1.0.0", null, new Dictionary()), new SimulationFinding("finding-m", "pkg:npm/m-package@1.0.0", null, new Dictionary()), }; var options = new SimulationAnalyticsOptions { TraceSampleRate = 1.0, MaxSampledTraces = 100 }; // Act var result = _service.ComputeSampledTraces(traces, findings, options); // Assert result.Ordering.PrimaryKey.Should().Be("finding_id"); result.Ordering.Direction.Should().Be("ascending"); } [Fact] public void ComputeSampledTraces_DeterminismHash_ConsistentForSameInput() { // Arrange var traces = new[] { CreateTrace("rule_a", 1, "block", expressionResult: true, componentPurl: "pkg:npm/lodash@4.0.0"), }; var findings = new[] { new SimulationFinding("f1", "pkg:npm/lodash@4.0.0", "GHSA-123", new Dictionary()), }; var options = new SimulationAnalyticsOptions { TraceSampleRate = 1.0 }; // Act var result1 = _service.ComputeSampledTraces(traces, findings, options); var result2 = _service.ComputeSampledTraces(traces, findings, options); // Assert result1.DeterminismHash.Should().Be(result2.DeterminismHash); } [Fact] public void ComputeSampledTraces_HighSeverity_AlwaysSampled() { // Arrange var traces = new[] { CreateTrace("rule_a", 1, "block", expressionResult: true, componentPurl: "pkg:npm/critical@1.0.0", severity: "critical"), }; var findings = new[] { new SimulationFinding("f1", "pkg:npm/critical@1.0.0", null, new Dictionary()), }; var options = new SimulationAnalyticsOptions { TraceSampleRate = 0.0 }; // Zero base rate // Act var result = _service.ComputeSampledTraces(traces, findings, options); // Assert result.SampledCount.Should().BeGreaterThan(0); result.Traces.Should().Contain(t => t.SampleReason == "high_severity"); } [Fact] public void ComputeDeltaSummary_OutcomeChanges_CalculatesCorrectly() { // Arrange var baseResults = new[] { new SimulationFindingResult("f1", "pkg:a", null, "block", "critical", new[] { "rule_a" }), new SimulationFindingResult("f2", "pkg:b", null, "warn", "medium", new[] { "rule_b" }), new SimulationFindingResult("f3", "pkg:c", null, "allow", "low", new[] { "rule_c" }), }; var candidateResults = new[] { new SimulationFindingResult("f1", "pkg:a", null, "warn", "high", new[] { "rule_a" }), // Improved new SimulationFindingResult("f2", "pkg:b", null, "block", "critical", new[] { "rule_b" }), // Regressed new SimulationFindingResult("f3", "pkg:c", null, "allow", "low", new[] { "rule_c" }), // Unchanged }; // Act var result = _service.ComputeDeltaSummary("v1", "v2", baseResults, candidateResults); // Assert result.OutcomeChanges.Unchanged.Should().Be(1); result.OutcomeChanges.Improved.Should().Be(1); result.OutcomeChanges.Regressed.Should().Be(1); result.OutcomeChanges.Transitions.Should().HaveCount(2); } [Fact] public void ComputeDeltaSummary_SeverityChanges_TracksEscalationAndDeescalation() { // Arrange var baseResults = new[] { new SimulationFindingResult("f1", "pkg:a", null, "block", "medium", Array.Empty()), new SimulationFindingResult("f2", "pkg:b", null, "block", "high", Array.Empty()), new SimulationFindingResult("f3", "pkg:c", null, "warn", "low", Array.Empty()), }; var candidateResults = new[] { new SimulationFindingResult("f1", "pkg:a", null, "block", "critical", Array.Empty()), // Escalated new SimulationFindingResult("f2", "pkg:b", null, "block", "medium", Array.Empty()), // Deescalated new SimulationFindingResult("f3", "pkg:c", null, "warn", "low", Array.Empty()), // Unchanged }; // Act var result = _service.ComputeDeltaSummary("v1", "v2", baseResults, candidateResults); // Assert result.SeverityChanges.Unchanged.Should().Be(1); result.SeverityChanges.Escalated.Should().Be(1); result.SeverityChanges.Deescalated.Should().Be(1); } [Fact] public void ComputeDeltaSummary_RuleChanges_DetectsAddedAndRemovedRules() { // Arrange var baseResults = new[] { new SimulationFindingResult("f1", "pkg:a", null, "block", "high", new[] { "rule_old", "rule_common" }), }; var candidateResults = new[] { new SimulationFindingResult("f1", "pkg:a", null, "block", "high", new[] { "rule_new", "rule_common" }), }; // Act var result = _service.ComputeDeltaSummary("v1", "v2", baseResults, candidateResults); // Assert result.RuleChanges.RulesAdded.Should().Contain("rule_new"); result.RuleChanges.RulesRemoved.Should().Contain("rule_old"); } [Fact] public void ComputeDeltaSummary_HighImpactFindings_IdentifiedCorrectly() { // Arrange var baseResults = new[] { new SimulationFindingResult("f1", "pkg:critical", "CVE-2024-001", "allow", "low", Array.Empty()), }; var candidateResults = new[] { new SimulationFindingResult("f1", "pkg:critical", "CVE-2024-001", "block", "critical", Array.Empty()), }; // Act var result = _service.ComputeDeltaSummary("v1", "v2", baseResults, candidateResults); // Assert result.HighImpactFindings.Should().NotBeEmpty(); result.HighImpactFindings[0].FindingId.Should().Be("f1"); result.HighImpactFindings[0].ImpactScore.Should().BeGreaterThan(0.5); } [Fact] public void ComputeDeltaSummary_DeterminismHash_ConsistentForSameInput() { // Arrange var baseResults = new[] { new SimulationFindingResult("f1", "pkg:a", null, "block", "high", Array.Empty()), }; var candidateResults = new[] { new SimulationFindingResult("f1", "pkg:a", null, "warn", "medium", Array.Empty()), }; // Act var result1 = _service.ComputeDeltaSummary("v1", "v2", baseResults, candidateResults); var result2 = _service.ComputeDeltaSummary("v1", "v2", baseResults, candidateResults); // Assert result1.DeterminismHash.Should().Be(result2.DeterminismHash); } [Fact] public void ComputeAnalytics_FullAnalysis_ReturnsAllComponents() { // Arrange var traces = new[] { CreateTrace("rule_a", 1, "block", expressionResult: true, componentPurl: "pkg:npm/lodash@4.0.0", severity: "high"), CreateTrace("rule_b", 2, "allow", expressionResult: true, componentPurl: "pkg:npm/express@5.0.0", severity: "low"), }; var findings = new[] { new SimulationFinding("f1", "pkg:npm/lodash@4.0.0", "GHSA-123", new Dictionary()), new SimulationFinding("f2", "pkg:npm/express@5.0.0", "GHSA-456", new Dictionary()), }; // Act var result = _service.ComputeAnalytics("policy-v1", traces, findings); // Assert result.RuleFiringCounts.Should().NotBeNull(); result.Heatmap.Should().NotBeNull(); result.SampledTraces.Should().NotBeNull(); result.DeltaSummary.Should().BeNull(); // No delta for single policy analysis } private static RuleHitTrace CreateTrace( string ruleName, int priority, string outcome, bool expressionResult, string? severity = null, bool isVexOverride = false, string? vexVendor = null, string? vexStatus = null, string? componentPurl = null) { return new RuleHitTrace { TraceId = Guid.NewGuid().ToString(), SpanId = Guid.NewGuid().ToString("N")[..16], TenantId = "test-tenant", PolicyId = "test-policy", RunId = "test-run", RuleName = ruleName, RulePriority = priority, Outcome = outcome, AssignedSeverity = severity, ComponentPurl = componentPurl, ExpressionResult = expressionResult, EvaluationTimestamp = DateTimeOffset.UtcNow, RecordedAt = DateTimeOffset.UtcNow, EvaluationMicroseconds = 100, IsVexOverride = isVexOverride, VexVendor = vexVendor, VexStatus = vexStatus, IsSampled = true, Attributes = ImmutableDictionary.Empty }; } private static SimulationFinding[] CreateFindings(int count) { return Enumerable.Range(1, count) .Select(i => new SimulationFinding( $"finding-{i}", $"pkg:npm/package-{i}@1.0.0", $"GHSA-{i:D3}", new Dictionary())) .ToArray(); } }