up

src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Simulation/SimulationAnalyticsServiceTests.cs

@@ -0,0 +1,414 @@
+using System.Collections.Immutable;
+using FluentAssertions;
+using StellaOps.Policy.Engine.Simulation;
+using StellaOps.Policy.Engine.Telemetry;
+using Xunit;
+
+namespace StellaOps.Policy.Engine.Tests.Simulation;
+
+public sealed class SimulationAnalyticsServiceTests
+{
+    private readonly SimulationAnalyticsService _service = new();
+
+    [Fact]
+    public void ComputeRuleFiringCounts_EmptyTraces_ReturnsEmptyCounts()
+    {
+        // Arrange
+        var traces = Array.Empty<RuleHitTrace>();
+
+        // Act
+        var result = _service.ComputeRuleFiringCounts(traces, 10);
+
+        // Assert
+        result.TotalEvaluations.Should().Be(10);
+        result.TotalRulesFired.Should().Be(0);
+        result.RulesByName.Should().BeEmpty();
+        result.RulesByPriority.Should().BeEmpty();
+        result.RulesByOutcome.Should().BeEmpty();
+        result.TopRules.Should().BeEmpty();
+    }
+
+    [Fact]
+    public void ComputeRuleFiringCounts_WithFiredRules_CountsCorrectly()
+    {
+        // Arrange
+        var traces = new[]
+        {
+            CreateTrace("rule_a", 1, "block", expressionResult: true),
+            CreateTrace("rule_a", 1, "block", expressionResult: true),
+            CreateTrace("rule_b", 2, "allow", expressionResult: true),
+            CreateTrace("rule_c", 3, "warn", expressionResult: false), // Not fired
+        };
+
+        // Act
+        var result = _service.ComputeRuleFiringCounts(traces, 10);
+
+        // Assert
+        result.TotalRulesFired.Should().Be(3);
+        result.RulesByName.Should().HaveCount(2);
+        result.RulesByName["rule_a"].FireCount.Should().Be(2);
+        result.RulesByName["rule_b"].FireCount.Should().Be(1);
+        result.RulesByPriority[1].Should().Be(2);
+        result.RulesByPriority[2].Should().Be(1);
+        result.RulesByOutcome["block"].Should().Be(2);
+        result.RulesByOutcome["allow"].Should().Be(1);
+    }
+
+    [Fact]
+    public void ComputeRuleFiringCounts_TopRules_OrderedByFireCount()
+    {
+        // Arrange
+        var traces = new List<RuleHitTrace>();
+        for (var i = 0; i < 15; i++)
+        {
+            traces.Add(CreateTrace("frequently_fired", 1, "block", expressionResult: true));
+        }
+        for (var i = 0; i < 5; i++)
+        {
+            traces.Add(CreateTrace("sometimes_fired", 2, "warn", expressionResult: true));
+        }
+        traces.Add(CreateTrace("rarely_fired", 3, "allow", expressionResult: true));
+
+        // Act
+        var result = _service.ComputeRuleFiringCounts(traces, 100);
+
+        // Assert
+        result.TopRules.Should().HaveCount(3);
+        result.TopRules[0].RuleName.Should().Be("frequently_fired");
+        result.TopRules[0].FireCount.Should().Be(15);
+        result.TopRules[1].RuleName.Should().Be("sometimes_fired");
+        result.TopRules[1].FireCount.Should().Be(5);
+        result.TopRules[2].RuleName.Should().Be("rarely_fired");
+        result.TopRules[2].FireCount.Should().Be(1);
+    }
+
+    [Fact]
+    public void ComputeRuleFiringCounts_VexOverrides_CountedCorrectly()
+    {
+        // Arrange
+        var traces = new[]
+        {
+            CreateTrace("rule_a", 1, "allow", expressionResult: true, isVexOverride: true, vexVendor: "vendor_a", vexStatus: "not_affected"),
+            CreateTrace("rule_a", 1, "allow", expressionResult: true, isVexOverride: true, vexVendor: "vendor_a", vexStatus: "fixed"),
+            CreateTrace("rule_b", 2, "allow", expressionResult: true, isVexOverride: true, vexVendor: "vendor_b", vexStatus: "not_affected"),
+            CreateTrace("rule_c", 3, "block", expressionResult: true),
+        };
+
+        // Act
+        var result = _service.ComputeRuleFiringCounts(traces, 10);
+
+        // Assert
+        result.VexOverrides.TotalOverrides.Should().Be(3);
+        result.VexOverrides.ByVendor["vendor_a"].Should().Be(2);
+        result.VexOverrides.ByVendor["vendor_b"].Should().Be(1);
+        result.VexOverrides.ByStatus["not_affected"].Should().Be(2);
+        result.VexOverrides.ByStatus["fixed"].Should().Be(1);
+    }
+
+    [Fact]
+    public void ComputeHeatmap_RuleSeverityMatrix_BuildsCorrectly()
+    {
+        // Arrange
+        var traces = new[]
+        {
+            CreateTrace("rule_a", 1, "block", expressionResult: true, severity: "critical"),
+            CreateTrace("rule_a", 1, "block", expressionResult: true, severity: "critical"),
+            CreateTrace("rule_a", 1, "block", expressionResult: true, severity: "high"),
+            CreateTrace("rule_b", 2, "warn", expressionResult: true, severity: "medium"),
+        };
+        var findings = CreateFindings(4);
+
+        // Act
+        var result = _service.ComputeHeatmap(traces, findings, SimulationAnalyticsOptions.Default);
+
+        // Assert
+        result.RuleSeverityMatrix.Should().NotBeEmpty();
+        var criticalCell = result.RuleSeverityMatrix.FirstOrDefault(c => c.X == "rule_a" && c.Y == "critical");
+        criticalCell.Should().NotBeNull();
+        criticalCell!.Value.Should().Be(2);
+    }
+
+    [Fact]
+    public void ComputeHeatmap_FindingRuleCoverage_CalculatesCorrectly()
+    {
+        // Arrange
+        var traces = new[]
+        {
+            CreateTrace("rule_a", 1, "block", expressionResult: true, componentPurl: "pkg:npm/lodash@4.0.0"),
+            CreateTrace("rule_b", 2, "allow", expressionResult: true, componentPurl: "pkg:npm/lodash@4.0.0"),
+            CreateTrace("rule_a", 1, "block", expressionResult: false, componentPurl: "pkg:npm/express@5.0.0"),
+        };
+        var findings = new[]
+        {
+            new SimulationFinding("f1", "pkg:npm/lodash@4.0.0", "GHSA-123", new Dictionary<string, object?>()),
+            new SimulationFinding("f2", "pkg:npm/express@5.0.0", "GHSA-456", new Dictionary<string, object?>()),
+            new SimulationFinding("f3", "pkg:npm/axios@1.0.0", "GHSA-789", new Dictionary<string, object?>()),
+        };
+
+        // Act
+        var result = _service.ComputeHeatmap(traces, findings, SimulationAnalyticsOptions.Default);
+
+        // Assert
+        result.FindingRuleCoverage.TotalFindings.Should().Be(3);
+        result.FindingRuleCoverage.FindingsMatched.Should().Be(1);
+        result.FindingRuleCoverage.CoveragePercentage.Should().BeApproximately(33.33, 0.1);
+    }
+
+    [Fact]
+    public void ComputeSampledTraces_DeterministicOrdering_OrdersByFindingId()
+    {
+        // Arrange
+        var traces = new[]
+        {
+            CreateTrace("rule_a", 1, "block", expressionResult: true, componentPurl: "pkg:npm/z-package@1.0.0"),
+            CreateTrace("rule_a", 1, "allow", expressionResult: true, componentPurl: "pkg:npm/a-package@1.0.0"),
+            CreateTrace("rule_b", 2, "warn", expressionResult: true, componentPurl: "pkg:npm/m-package@1.0.0"),
+        };
+        var findings = new[]
+        {
+            new SimulationFinding("finding-z", "pkg:npm/z-package@1.0.0", null, new Dictionary<string, object?>()),
+            new SimulationFinding("finding-a", "pkg:npm/a-package@1.0.0", null, new Dictionary<string, object?>()),
+            new SimulationFinding("finding-m", "pkg:npm/m-package@1.0.0", null, new Dictionary<string, object?>()),
+        };
+        var options = new SimulationAnalyticsOptions { TraceSampleRate = 1.0, MaxSampledTraces = 100 };
+
+        // Act
+        var result = _service.ComputeSampledTraces(traces, findings, options);
+
+        // Assert
+        result.Ordering.PrimaryKey.Should().Be("finding_id");
+        result.Ordering.Direction.Should().Be("ascending");
+    }
+
+    [Fact]
+    public void ComputeSampledTraces_DeterminismHash_ConsistentForSameInput()
+    {
+        // Arrange
+        var traces = new[]
+        {
+            CreateTrace("rule_a", 1, "block", expressionResult: true, componentPurl: "pkg:npm/lodash@4.0.0"),
+        };
+        var findings = new[]
+        {
+            new SimulationFinding("f1", "pkg:npm/lodash@4.0.0", "GHSA-123", new Dictionary<string, object?>()),
+        };
+        var options = new SimulationAnalyticsOptions { TraceSampleRate = 1.0 };
+
+        // Act
+        var result1 = _service.ComputeSampledTraces(traces, findings, options);
+        var result2 = _service.ComputeSampledTraces(traces, findings, options);
+
+        // Assert
+        result1.DeterminismHash.Should().Be(result2.DeterminismHash);
+    }
+
+    [Fact]
+    public void ComputeSampledTraces_HighSeverity_AlwaysSampled()
+    {
+        // Arrange
+        var traces = new[]
+        {
+            CreateTrace("rule_a", 1, "block", expressionResult: true, componentPurl: "pkg:npm/critical@1.0.0", severity: "critical"),
+        };
+        var findings = new[]
+        {
+            new SimulationFinding("f1", "pkg:npm/critical@1.0.0", null, new Dictionary<string, object?>()),
+        };
+        var options = new SimulationAnalyticsOptions { TraceSampleRate = 0.0 }; // Zero base rate
+
+        // Act
+        var result = _service.ComputeSampledTraces(traces, findings, options);
+
+        // Assert
+        result.SampledCount.Should().BeGreaterThan(0);
+        result.Traces.Should().Contain(t => t.SampleReason == "high_severity");
+    }
+
+    [Fact]
+    public void ComputeDeltaSummary_OutcomeChanges_CalculatesCorrectly()
+    {
+        // Arrange
+        var baseResults = new[]
+        {
+            new SimulationFindingResult("f1", "pkg:a", null, "block", "critical", new[] { "rule_a" }),
+            new SimulationFindingResult("f2", "pkg:b", null, "warn", "medium", new[] { "rule_b" }),
+            new SimulationFindingResult("f3", "pkg:c", null, "allow", "low", new[] { "rule_c" }),
+        };
+        var candidateResults = new[]
+        {
+            new SimulationFindingResult("f1", "pkg:a", null, "warn", "high", new[] { "rule_a" }), // Improved
+            new SimulationFindingResult("f2", "pkg:b", null, "block", "critical", new[] { "rule_b" }), // Regressed
+            new SimulationFindingResult("f3", "pkg:c", null, "allow", "low", new[] { "rule_c" }), // Unchanged
+        };
+
+        // Act
+        var result = _service.ComputeDeltaSummary("v1", "v2", baseResults, candidateResults);
+
+        // Assert
+        result.OutcomeChanges.Unchanged.Should().Be(1);
+        result.OutcomeChanges.Improved.Should().Be(1);
+        result.OutcomeChanges.Regressed.Should().Be(1);
+        result.OutcomeChanges.Transitions.Should().HaveCount(2);
+    }
+
+    [Fact]
+    public void ComputeDeltaSummary_SeverityChanges_TracksEscalationAndDeescalation()
+    {
+        // Arrange
+        var baseResults = new[]
+        {
+            new SimulationFindingResult("f1", "pkg:a", null, "block", "medium", Array.Empty<string>()),
+            new SimulationFindingResult("f2", "pkg:b", null, "block", "high", Array.Empty<string>()),
+            new SimulationFindingResult("f3", "pkg:c", null, "warn", "low", Array.Empty<string>()),
+        };
+        var candidateResults = new[]
+        {
+            new SimulationFindingResult("f1", "pkg:a", null, "block", "critical", Array.Empty<string>()), // Escalated
+            new SimulationFindingResult("f2", "pkg:b", null, "block", "medium", Array.Empty<string>()), // Deescalated
+            new SimulationFindingResult("f3", "pkg:c", null, "warn", "low", Array.Empty<string>()), // Unchanged
+        };
+
+        // Act
+        var result = _service.ComputeDeltaSummary("v1", "v2", baseResults, candidateResults);
+
+        // Assert
+        result.SeverityChanges.Unchanged.Should().Be(1);
+        result.SeverityChanges.Escalated.Should().Be(1);
+        result.SeverityChanges.Deescalated.Should().Be(1);
+    }
+
+    [Fact]
+    public void ComputeDeltaSummary_RuleChanges_DetectsAddedAndRemovedRules()
+    {
+        // Arrange
+        var baseResults = new[]
+        {
+            new SimulationFindingResult("f1", "pkg:a", null, "block", "high", new[] { "rule_old", "rule_common" }),
+        };
+        var candidateResults = new[]
+        {
+            new SimulationFindingResult("f1", "pkg:a", null, "block", "high", new[] { "rule_new", "rule_common" }),
+        };
+
+        // Act
+        var result = _service.ComputeDeltaSummary("v1", "v2", baseResults, candidateResults);
+
+        // Assert
+        result.RuleChanges.RulesAdded.Should().Contain("rule_new");
+        result.RuleChanges.RulesRemoved.Should().Contain("rule_old");
+    }
+
+    [Fact]
+    public void ComputeDeltaSummary_HighImpactFindings_IdentifiedCorrectly()
+    {
+        // Arrange
+        var baseResults = new[]
+        {
+            new SimulationFindingResult("f1", "pkg:critical", "CVE-2024-001", "allow", "low", Array.Empty<string>()),
+        };
+        var candidateResults = new[]
+        {
+            new SimulationFindingResult("f1", "pkg:critical", "CVE-2024-001", "block", "critical", Array.Empty<string>()),
+        };
+
+        // Act
+        var result = _service.ComputeDeltaSummary("v1", "v2", baseResults, candidateResults);
+
+        // Assert
+        result.HighImpactFindings.Should().NotBeEmpty();
+        result.HighImpactFindings[0].FindingId.Should().Be("f1");
+        result.HighImpactFindings[0].ImpactScore.Should().BeGreaterThan(0.5);
+    }
+
+    [Fact]
+    public void ComputeDeltaSummary_DeterminismHash_ConsistentForSameInput()
+    {
+        // Arrange
+        var baseResults = new[]
+        {
+            new SimulationFindingResult("f1", "pkg:a", null, "block", "high", Array.Empty<string>()),
+        };
+        var candidateResults = new[]
+        {
+            new SimulationFindingResult("f1", "pkg:a", null, "warn", "medium", Array.Empty<string>()),
+        };
+
+        // Act
+        var result1 = _service.ComputeDeltaSummary("v1", "v2", baseResults, candidateResults);
+        var result2 = _service.ComputeDeltaSummary("v1", "v2", baseResults, candidateResults);
+
+        // Assert
+        result1.DeterminismHash.Should().Be(result2.DeterminismHash);
+    }
+
+    [Fact]
+    public void ComputeAnalytics_FullAnalysis_ReturnsAllComponents()
+    {
+        // Arrange
+        var traces = new[]
+        {
+            CreateTrace("rule_a", 1, "block", expressionResult: true, componentPurl: "pkg:npm/lodash@4.0.0", severity: "high"),
+            CreateTrace("rule_b", 2, "allow", expressionResult: true, componentPurl: "pkg:npm/express@5.0.0", severity: "low"),
+        };
+        var findings = new[]
+        {
+            new SimulationFinding("f1", "pkg:npm/lodash@4.0.0", "GHSA-123", new Dictionary<string, object?>()),
+            new SimulationFinding("f2", "pkg:npm/express@5.0.0", "GHSA-456", new Dictionary<string, object?>()),
+        };
+
+        // Act
+        var result = _service.ComputeAnalytics("policy-v1", traces, findings);
+
+        // Assert
+        result.RuleFiringCounts.Should().NotBeNull();
+        result.Heatmap.Should().NotBeNull();
+        result.SampledTraces.Should().NotBeNull();
+        result.DeltaSummary.Should().BeNull(); // No delta for single policy analysis
+    }
+
+    private static RuleHitTrace CreateTrace(
+        string ruleName,
+        int priority,
+        string outcome,
+        bool expressionResult,
+        string? severity = null,
+        bool isVexOverride = false,
+        string? vexVendor = null,
+        string? vexStatus = null,
+        string? componentPurl = null)
+    {
+        return new RuleHitTrace
+        {
+            TraceId = Guid.NewGuid().ToString(),
+            SpanId = Guid.NewGuid().ToString("N")[..16],
+            TenantId = "test-tenant",
+            PolicyId = "test-policy",
+            RunId = "test-run",
+            RuleName = ruleName,
+            RulePriority = priority,
+            Outcome = outcome,
+            AssignedSeverity = severity,
+            ComponentPurl = componentPurl,
+            ExpressionResult = expressionResult,
+            EvaluationTimestamp = DateTimeOffset.UtcNow,
+            RecordedAt = DateTimeOffset.UtcNow,
+            EvaluationMicroseconds = 100,
+            IsVexOverride = isVexOverride,
+            VexVendor = vexVendor,
+            VexStatus = vexStatus,
+            IsSampled = true,
+            Attributes = ImmutableDictionary<string, string>.Empty
+        };
+    }
+
+    private static SimulationFinding[] CreateFindings(int count)
+    {
+        return Enumerable.Range(1, count)
+            .Select(i => new SimulationFinding(
+                $"finding-{i}",
+                $"pkg:npm/package-{i}@1.0.0",
+                $"GHSA-{i:D3}",
+                new Dictionary<string, object?>()))
+            .ToArray();
+    }
+}