// ----------------------------------------------------------------------------- // PolicyDeterminismTests.cs // Sprint: SPRINT_5100_0007_0003 - Epic B (Determinism Gate) // Task: T5 - Policy Verdict Determinism // Description: Tests to validate policy verdict generation determinism // ----------------------------------------------------------------------------- using System.Collections.Immutable; using System.Text; using FluentAssertions; using StellaOps.Canonical.Json; using StellaOps.Testing.Determinism; using Xunit; namespace StellaOps.Integration.Determinism; /// /// Determinism validation tests for policy verdict generation. /// Ensures identical inputs produce identical verdicts across: /// - Single verdict generation /// - Batch verdict generation /// - Verdict serialization /// - Multiple runs with frozen time /// - Parallel execution /// public class PolicyDeterminismTests { #region Single Verdict Determinism Tests [Fact] public void PolicyVerdict_WithIdenticalInput_ProducesDeterministicOutput() { // Arrange var input = CreateSamplePolicyInput(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); // Act - Generate verdict multiple times var verdict1 = EvaluatePolicy(input, frozenTime); var verdict2 = EvaluatePolicy(input, frozenTime); var verdict3 = EvaluatePolicy(input, frozenTime); // Assert - All outputs should be identical verdict1.Should().BeEquivalentTo(verdict2); verdict2.Should().BeEquivalentTo(verdict3); } [Fact] public void PolicyVerdict_CanonicalHash_IsStable() { // Arrange var input = CreateSamplePolicyInput(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); // Act - Generate verdict and compute canonical hash twice var verdict1 = EvaluatePolicy(input, frozenTime); var json1 = SerializeVerdict(verdict1); var hash1 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(json1)); var verdict2 = EvaluatePolicy(input, frozenTime); var json2 = SerializeVerdict(verdict2); var hash2 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(json2)); // Assert hash1.Should().Be(hash2, "Same input should produce same canonical hash"); hash1.Should().MatchRegex("^[0-9a-f]{64}$"); } [Fact] public void PolicyVerdict_DeterminismManifest_CanBeCreated() { // Arrange var input = CreateSamplePolicyInput(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); var verdict = EvaluatePolicy(input, frozenTime); var json = SerializeVerdict(verdict); var verdictBytes = Encoding.UTF8.GetBytes(json); var artifactInfo = new ArtifactInfo { Type = "policy-verdict", Name = "test-finding-verdict", Version = "1.0.0", Format = "PolicyVerdict JSON" }; var toolchain = new ToolchainInfo { Platform = ".NET 10.0", Components = new[] { new ComponentInfo { Name = "StellaOps.Policy.Engine", Version = "1.0.0" } } }; // Act - Create determinism manifest var manifest = DeterminismManifestWriter.CreateManifest( verdictBytes, artifactInfo, toolchain); // Assert manifest.SchemaVersion.Should().Be("1.0"); manifest.Artifact.Format.Should().Be("PolicyVerdict JSON"); manifest.CanonicalHash.Algorithm.Should().Be("SHA-256"); manifest.CanonicalHash.Value.Should().MatchRegex("^[0-9a-f]{64}$"); } [Fact] public async Task PolicyVerdict_ParallelGeneration_ProducesDeterministicOutput() { // Arrange var input = CreateSamplePolicyInput(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); // Act - Generate in parallel 20 times var tasks = Enumerable.Range(0, 20) .Select(_ => Task.Run(() => EvaluatePolicy(input, frozenTime))) .ToArray(); var verdicts = await Task.WhenAll(tasks); // Assert - All outputs should be identical var first = verdicts[0]; verdicts.Should().AllSatisfy(v => v.Should().BeEquivalentTo(first)); } #endregion #region Batch Verdict Determinism Tests [Fact] public void PolicyVerdictBatch_WithIdenticalInput_ProducesDeterministicOutput() { // Arrange var inputs = CreateSampleBatchPolicyInputs(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); // Act - Generate batch verdicts multiple times var batch1 = EvaluatePolicyBatch(inputs, frozenTime); var batch2 = EvaluatePolicyBatch(inputs, frozenTime); var batch3 = EvaluatePolicyBatch(inputs, frozenTime); // Assert - All batches should be identical batch1.Should().BeEquivalentTo(batch2); batch2.Should().BeEquivalentTo(batch3); } [Fact] public void PolicyVerdictBatch_Ordering_IsDeterministic() { // Arrange - Findings in random order var inputs = CreateSampleBatchPolicyInputs(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); // Act - Generate batch verdicts multiple times var batch1 = EvaluatePolicyBatch(inputs, frozenTime); var batch2 = EvaluatePolicyBatch(inputs, frozenTime); // Assert - Order should be deterministic var json1 = SerializeBatch(batch1); var json2 = SerializeBatch(batch2); json1.Should().Be(json2); } [Fact] public void PolicyVerdictBatch_CanonicalHash_IsStable() { // Arrange var inputs = CreateSampleBatchPolicyInputs(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); // Act var batch1 = EvaluatePolicyBatch(inputs, frozenTime); var json1 = SerializeBatch(batch1); var hash1 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(json1)); var batch2 = EvaluatePolicyBatch(inputs, frozenTime); var json2 = SerializeBatch(batch2); var hash2 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(json2)); // Assert hash1.Should().Be(hash2); hash1.Should().MatchRegex("^[0-9a-f]{64}$"); } #endregion #region Verdict Status Determinism Tests [Theory] [InlineData(PolicyVerdictStatus.Pass)] [InlineData(PolicyVerdictStatus.Blocked)] [InlineData(PolicyVerdictStatus.Ignored)] [InlineData(PolicyVerdictStatus.Warned)] [InlineData(PolicyVerdictStatus.Deferred)] [InlineData(PolicyVerdictStatus.Escalated)] [InlineData(PolicyVerdictStatus.RequiresVex)] public void PolicyVerdict_WithStatus_IsDeterministic(PolicyVerdictStatus status) { // Arrange var input = CreatePolicyInputWithExpectedStatus(status); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); // Act var verdict1 = EvaluatePolicy(input, frozenTime); var verdict2 = EvaluatePolicy(input, frozenTime); // Assert verdict1.Status.Should().Be(status); verdict2.Status.Should().Be(status); verdict1.Should().BeEquivalentTo(verdict2); } #endregion #region Score Calculation Determinism Tests [Fact] public void PolicyScore_WithSameInputs_ProducesDeterministicScore() { // Arrange var input = CreateSamplePolicyInput(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); // Act var verdict1 = EvaluatePolicy(input, frozenTime); var verdict2 = EvaluatePolicy(input, frozenTime); // Assert - Scores should be identical (not floating point approximate) verdict1.Score.Should().Be(verdict2.Score); } [Fact] public void PolicyScore_InputOrdering_DoesNotAffectScore() { // Arrange - Same inputs but in different order var inputs1 = new Dictionary { { "cvss", 7.5 }, { "epss", 0.001 }, { "kev", 0.0 }, { "reachability", 0.8 } }; var inputs2 = new Dictionary { { "reachability", 0.8 }, { "kev", 0.0 }, { "epss", 0.001 }, { "cvss", 7.5 } }; var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); // Act var verdict1 = EvaluatePolicyWithInputs("CVE-2024-1234", inputs1, frozenTime); var verdict2 = EvaluatePolicyWithInputs("CVE-2024-1234", inputs2, frozenTime); // Assert verdict1.Score.Should().Be(verdict2.Score); verdict1.Status.Should().Be(verdict2.Status); } [Fact] public void PolicyScore_FloatingPointPrecision_IsConsistent() { // Arrange - Inputs that might cause floating point issues var inputs = new Dictionary { { "cvss", 0.1 + 0.2 }, // Classic floating point precision test { "epss", 1.0 / 3.0 }, { "weight_a", 0.33333333333333333 }, { "weight_b", 0.66666666666666666 } }; var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); // Act var verdict1 = EvaluatePolicyWithInputs("CVE-2024-5678", inputs, frozenTime); var verdict2 = EvaluatePolicyWithInputs("CVE-2024-5678", inputs, frozenTime); // Assert - Score should be rounded to consistent precision verdict1.Score.Should().Be(verdict2.Score); } #endregion #region Rule Matching Determinism Tests [Fact] public void PolicyRuleMatching_WithMultipleMatchingRules_SelectsDeterministically() { // Arrange - Input that matches multiple rules var input = CreateInputMatchingMultipleRules(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); // Act var verdict1 = EvaluatePolicy(input, frozenTime); var verdict2 = EvaluatePolicy(input, frozenTime); // Assert - Same rule should be selected each time verdict1.RuleName.Should().Be(verdict2.RuleName); verdict1.RuleAction.Should().Be(verdict2.RuleAction); } [Fact] public void PolicyQuieting_IsDeterministic() { // Arrange - Input that triggers quieting var input = CreateQuietedInput(); var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z"); // Act var verdict1 = EvaluatePolicy(input, frozenTime); var verdict2 = EvaluatePolicy(input, frozenTime); // Assert verdict1.Quiet.Should().Be(verdict2.Quiet); verdict1.QuietedBy.Should().Be(verdict2.QuietedBy); } #endregion #region Helper Methods private static PolicyInput CreateSamplePolicyInput() { return new PolicyInput { FindingId = "CVE-2024-1234", CvssScore = 7.5, EpssScore = 0.001, IsKev = false, ReachabilityScore = 0.8, SourceTrust = "high", PackageType = "npm", Severity = "high" }; } private static PolicyInput[] CreateSampleBatchPolicyInputs() { return new[] { new PolicyInput { FindingId = "CVE-2024-1111", CvssScore = 9.8, EpssScore = 0.5, IsKev = true, ReachabilityScore = 1.0, SourceTrust = "high", PackageType = "npm", Severity = "critical" }, new PolicyInput { FindingId = "CVE-2024-2222", CvssScore = 5.5, EpssScore = 0.01, IsKev = false, ReachabilityScore = 0.3, SourceTrust = "medium", PackageType = "pypi", Severity = "medium" }, new PolicyInput { FindingId = "CVE-2024-3333", CvssScore = 3.2, EpssScore = 0.001, IsKev = false, ReachabilityScore = 0.1, SourceTrust = "low", PackageType = "maven", Severity = "low" } }; } private static PolicyInput CreatePolicyInputWithExpectedStatus(PolicyVerdictStatus status) { return status switch { PolicyVerdictStatus.Pass => new PolicyInput { FindingId = "CVE-PASS-001", CvssScore = 2.0, EpssScore = 0.0001, IsKev = false, ReachabilityScore = 0.0, SourceTrust = "high", PackageType = "npm", Severity = "low" }, PolicyVerdictStatus.Blocked => new PolicyInput { FindingId = "CVE-BLOCKED-001", CvssScore = 9.8, EpssScore = 0.9, IsKev = true, ReachabilityScore = 1.0, SourceTrust = "high", PackageType = "npm", Severity = "critical" }, PolicyVerdictStatus.Warned => new PolicyInput { FindingId = "CVE-WARNED-001", CvssScore = 7.0, EpssScore = 0.05, IsKev = false, ReachabilityScore = 0.5, SourceTrust = "medium", PackageType = "npm", Severity = "high" }, PolicyVerdictStatus.RequiresVex => new PolicyInput { FindingId = "CVE-VEXREQ-001", CvssScore = 7.5, EpssScore = 0.1, IsKev = false, ReachabilityScore = null, // Unknown reachability SourceTrust = "high", PackageType = "npm", Severity = "high" }, _ => new PolicyInput { FindingId = $"CVE-{status}-001", CvssScore = 5.0, EpssScore = 0.01, IsKev = false, ReachabilityScore = 0.5, SourceTrust = "medium", PackageType = "npm", Severity = "medium" } }; } private static PolicyInput CreateInputMatchingMultipleRules() { return new PolicyInput { FindingId = "CVE-MULTIRULE-001", CvssScore = 7.0, EpssScore = 0.1, IsKev = false, ReachabilityScore = 0.5, SourceTrust = "high", PackageType = "npm", Severity = "high" }; } private static PolicyInput CreateQuietedInput() { return new PolicyInput { FindingId = "CVE-2024-QUIETED", CvssScore = 9.0, EpssScore = 0.5, IsKev = false, ReachabilityScore = 1.0, SourceTrust = "high", PackageType = "npm", Severity = "critical", QuietedBy = "waiver:WAIVER-2024-001" }; } private static PolicyVerdictResult EvaluatePolicy(PolicyInput input, DateTimeOffset timestamp) { // TODO: Integrate with actual PolicyEngine // For now, return deterministic stub var status = DetermineStatus(input); var score = CalculateScore(input); var ruleName = DetermineRuleName(input); return new PolicyVerdictResult { FindingId = input.FindingId, Status = status, Score = score, RuleName = ruleName, RuleAction = status == PolicyVerdictStatus.Pass ? "allow" : "block", Notes = null, ConfigVersion = "1.0", Inputs = new Dictionary { { "cvss", input.CvssScore }, { "epss", input.EpssScore }, { "kev", input.IsKev ? 1.0 : 0.0 }, { "reachability", input.ReachabilityScore ?? 0.5 } }.ToImmutableDictionary(), Quiet = input.QuietedBy != null, QuietedBy = input.QuietedBy, Timestamp = timestamp }; } private static PolicyVerdictResult EvaluatePolicyWithInputs( string findingId, Dictionary inputs, DateTimeOffset timestamp) { // Calculate score from inputs var cvss = inputs.GetValueOrDefault("cvss", 0); var epss = inputs.GetValueOrDefault("epss", 0); var score = Math.Round((cvss * 10 + epss * 100) / 2, 4); var status = score > 70 ? PolicyVerdictStatus.Blocked : score > 40 ? PolicyVerdictStatus.Warned : PolicyVerdictStatus.Pass; return new PolicyVerdictResult { FindingId = findingId, Status = status, Score = score, RuleName = "calculated-score-rule", RuleAction = status == PolicyVerdictStatus.Pass ? "allow" : "block", Notes = null, ConfigVersion = "1.0", Inputs = inputs.ToImmutableDictionary(), Quiet = false, QuietedBy = null, Timestamp = timestamp }; } private static PolicyVerdictResult[] EvaluatePolicyBatch(PolicyInput[] inputs, DateTimeOffset timestamp) { return inputs .Select(input => EvaluatePolicy(input, timestamp)) .OrderBy(v => v.FindingId, StringComparer.Ordinal) .ToArray(); } private static PolicyVerdictStatus DetermineStatus(PolicyInput input) { if (input.QuietedBy != null) return PolicyVerdictStatus.Ignored; if (input.ReachabilityScore == null) return PolicyVerdictStatus.RequiresVex; if (input.IsKev || input.CvssScore >= 9.0 || input.EpssScore >= 0.5) return PolicyVerdictStatus.Blocked; if (input.CvssScore >= 7.0 || input.EpssScore >= 0.05) return PolicyVerdictStatus.Warned; return PolicyVerdictStatus.Pass; } private static double CalculateScore(PolicyInput input) { var baseScore = input.CvssScore * 10; var epssMultiplier = 1 + (input.EpssScore * 10); var kevBonus = input.IsKev ? 20 : 0; var reachabilityFactor = input.ReachabilityScore ?? 0.5; var rawScore = (baseScore * epssMultiplier + kevBonus) * reachabilityFactor; return Math.Round(rawScore, 4); } private static string DetermineRuleName(PolicyInput input) { if (input.IsKev) return "kev-critical-block"; if (input.CvssScore >= 9.0) return "critical-cvss-block"; if (input.EpssScore >= 0.5) return "high-exploit-likelihood-block"; if (input.CvssScore >= 7.0) return "high-cvss-warn"; return "default-pass"; } private static string SerializeVerdict(PolicyVerdictResult verdict) { // Canonical JSON serialization var inputsJson = string.Join(", ", verdict.Inputs .OrderBy(kvp => kvp.Key, StringComparer.Ordinal) .Select(kvp => $"\"{kvp.Key}\": {kvp.Value}")); return $$""" { "configVersion": "{{verdict.ConfigVersion}}", "findingId": "{{verdict.FindingId}}", "inputs": {{{inputsJson}}}, "notes": {{(verdict.Notes == null ? "null" : $"\"{verdict.Notes}\"")}}, "quiet": {{verdict.Quiet.ToString().ToLowerInvariant()}}, "quietedBy": {{(verdict.QuietedBy == null ? "null" : $"\"{verdict.QuietedBy}\"")}}, "ruleAction": "{{verdict.RuleAction}}", "ruleName": "{{verdict.RuleName}}", "score": {{verdict.Score}}, "status": "{{verdict.Status}}", "timestamp": "{{verdict.Timestamp:O}}" } """; } private static string SerializeBatch(PolicyVerdictResult[] verdicts) { var items = verdicts.Select(SerializeVerdict); return $"[\n {string.Join(",\n ", items)}\n]"; } #endregion #region DTOs private sealed record PolicyInput { public required string FindingId { get; init; } public required double CvssScore { get; init; } public required double EpssScore { get; init; } public required bool IsKev { get; init; } public double? ReachabilityScore { get; init; } public required string SourceTrust { get; init; } public required string PackageType { get; init; } public required string Severity { get; init; } public string? QuietedBy { get; init; } } private sealed record PolicyVerdictResult { public required string FindingId { get; init; } public required PolicyVerdictStatus Status { get; init; } public required double Score { get; init; } public required string RuleName { get; init; } public required string RuleAction { get; init; } public string? Notes { get; init; } public required string ConfigVersion { get; init; } public required ImmutableDictionary Inputs { get; init; } public required bool Quiet { get; init; } public string? QuietedBy { get; init; } public required DateTimeOffset Timestamp { get; init; } } private enum PolicyVerdictStatus { Pass, Blocked, Ignored, Warned, Deferred, Escalated, RequiresVex } #endregion }