// SPDX-License-Identifier: AGPL-3.0-or-later // SPDX-FileCopyrightText: 2025 StellaOps Contributors // Sprint: SPRINT_5100_0009_0004 - Policy Module Test Implementation // Tasks: POLICY-5100-014, POLICY-5100-015 using FluentAssertions; using StellaOps.Policy.Engine; using StellaOps.DeltaVerdict; using StellaOps.Excititor.Core; using StellaOps.Policy.Unknowns; using Xunit; namespace StellaOps.Policy.Engine.Tests.Determinism; /// /// Determinism tests for Policy Engine. /// Verifies that same policy + same inputs produces identical outputs (including hashes). /// [Trait("Category", "Determinism")] [Trait("Category", "L0")] public sealed class PolicyEngineDeterminismTests { #region POLICY-5100-014: Same policy + same inputs = same verdict hash [Fact] public void SameInputs_ProduceIdenticalVerdictHash_AcrossMultipleRuns() { // Arrange var policy = CreateTestPolicy(); var input = CreateTestInput(); var evaluator = CreateEvaluator(); // Act - run multiple times var results = Enumerable.Range(0, 10) .Select(_ => evaluator.Evaluate(policy, input)) .ToList(); // Assert - all results should have identical hashes var firstHash = results[0].VerdictHash; results.Should().AllSatisfy(r => r.VerdictHash.Should().Be(firstHash)); } [Fact] public void SameInputs_ProduceIdenticalVerdictJson_AcrossMultipleRuns() { // Arrange var policy = CreateTestPolicy(); var input = CreateTestInput(); var evaluator = CreateEvaluator(); // Act - run multiple times var results = Enumerable.Range(0, 10) .Select(_ => evaluator.Evaluate(policy, input)) .ToList(); // Assert - canonical JSON should be byte-identical var firstJson = results[0].ToCanonicalJson(); results.Should().AllSatisfy(r => r.ToCanonicalJson().Should().Be(firstJson)); } [Fact] public void InputOrder_DoesNotAffect_VerdictHash() { // Arrange var policy = CreateTestPolicy(); var evaluator = CreateEvaluator(); // Create two inputs with same findings in different order var findings1 = new[] { CreateFinding("CVE-2024-0001", "high"), CreateFinding("CVE-2024-0002", "medium"), CreateFinding("CVE-2024-0003", "low") }; var findings2 = new[] { CreateFinding("CVE-2024-0003", "low"), CreateFinding("CVE-2024-0001", "high"), CreateFinding("CVE-2024-0002", "medium") }; var input1 = CreateTestInputWithFindings(findings1); var input2 = CreateTestInputWithFindings(findings2); // Act var result1 = evaluator.Evaluate(policy, input1); var result2 = evaluator.Evaluate(policy, input2); // Assert - same findings (different order) should produce same verdict result1.VerdictHash.Should().Be(result2.VerdictHash, "verdict hash should be order-independent"); } [Fact] public void TimestampVariation_DoesNotAffect_VerdictHash() { // Arrange var policy = CreateTestPolicy(); var evaluator = CreateEvaluator(); // Create inputs at different (simulated) timestamps var input1 = CreateTestInput(); var input2 = CreateTestInput(); // Act var result1 = evaluator.Evaluate(policy, input1); // Simulate time passing (if timestamps are used, they should be from input, not wall clock) var result2 = evaluator.Evaluate(policy, input2); // Assert result1.VerdictHash.Should().Be(result2.VerdictHash, "verdict hash should be deterministic regardless of evaluation time"); } [Fact] public async Task ConcurrentEvaluations_ProduceIdenticalResults() { // Arrange var policy = CreateTestPolicy(); var input = CreateTestInput(); var evaluator = CreateEvaluator(); // Act - evaluate concurrently var tasks = Enumerable.Range(0, 20) .Select(_ => Task.Run(() => evaluator.Evaluate(policy, input))) .ToArray(); var results = await Task.WhenAll(tasks); // Assert var firstHash = results[0].VerdictHash; results.Should().AllSatisfy(r => r.VerdictHash.Should().Be(firstHash)); } [Fact] public void VexMergeOrder_DoesNotAffect_VerdictHash() { // Arrange var policy = CreateTestPolicyWithVex(); var evaluator = CreateEvaluator(); // Same VEX statements in different order var vexStatements1 = new[] { CreateVexStatement("CVE-2024-0001", VexStatus.NotAffected), CreateVexStatement("CVE-2024-0002", VexStatus.Fixed) }; var vexStatements2 = new[] { CreateVexStatement("CVE-2024-0002", VexStatus.Fixed), CreateVexStatement("CVE-2024-0001", VexStatus.NotAffected) }; var input1 = CreateTestInputWithVex(vexStatements1); var input2 = CreateTestInputWithVex(vexStatements2); // Act var result1 = evaluator.Evaluate(policy, input1); var result2 = evaluator.Evaluate(policy, input2); // Assert result1.VerdictHash.Should().Be(result2.VerdictHash, "VEX merge should be order-independent"); } #endregion #region POLICY-5100-015: Unknown budget enforcement [Fact] public void UnknownsBudget_FailsVerdict_WhenExceeded() { // Arrange var policy = CreatePolicyWithUnknownsBudget(maxUnknowns: 3); var evaluator = CreateEvaluator(); // Input with 5 unknown findings (exceeds budget of 3) var findings = Enumerable.Range(1, 5) .Select(i => CreateUnknownFinding($"CVE-2024-{i:D4}")) .ToArray(); var input = CreateTestInputWithFindings(findings); // Act var result = evaluator.Evaluate(policy, input); // Assert result.Status.Should().Be(VerdictStatus.Failed); result.Violations.Should().Contain(v => v.Code == "UNKNOWNS_BUDGET_EXCEEDED" || v.Message.Contains("unknowns", StringComparison.OrdinalIgnoreCase)); } [Fact] public void UnknownsBudget_PassesVerdict_WhenWithinLimit() { // Arrange var policy = CreatePolicyWithUnknownsBudget(maxUnknowns: 10); var evaluator = CreateEvaluator(); // Input with 3 unknown findings (within budget of 10) var findings = Enumerable.Range(1, 3) .Select(i => CreateUnknownFinding($"CVE-2024-{i:D4}")) .ToArray(); var input = CreateTestInputWithFindings(findings); // Act var result = evaluator.Evaluate(policy, input); // Assert - should not fail due to unknowns budget result.Violations.Should().NotContain(v => v.Code == "UNKNOWNS_BUDGET_EXCEEDED"); } [Fact] public void UnknownsBudget_CountsOnlyUnknownSeverity() { // Arrange var policy = CreatePolicyWithUnknownsBudget(maxUnknowns: 2); var evaluator = CreateEvaluator(); // Mix of known and unknown severities var findings = new[] { CreateFinding("CVE-2024-0001", "high"), // known CreateFinding("CVE-2024-0002", "medium"), // known CreateUnknownFinding("CVE-2024-0003"), // unknown CreateFinding("CVE-2024-0004", "low"), // known CreateUnknownFinding("CVE-2024-0005"), // unknown CreateFinding("CVE-2024-0006", "critical") // known }; var input = CreateTestInputWithFindings(findings); // Act var result = evaluator.Evaluate(policy, input); // Assert - should pass (only 2 unknowns, exactly at budget) result.Violations.Should().NotContain(v => v.Code == "UNKNOWNS_BUDGET_EXCEEDED"); } [Fact] public void UnknownsBudget_ZeroBudget_FailsOnAnyUnknown() { // Arrange var policy = CreatePolicyWithUnknownsBudget(maxUnknowns: 0); var evaluator = CreateEvaluator(); // Single unknown finding var findings = new[] { CreateUnknownFinding("CVE-2024-0001") }; var input = CreateTestInputWithFindings(findings); // Act var result = evaluator.Evaluate(policy, input); // Assert result.Status.Should().Be(VerdictStatus.Failed); result.Violations.Should().Contain(v => v.Code == "UNKNOWNS_BUDGET_EXCEEDED" || v.Message.Contains("unknowns", StringComparison.OrdinalIgnoreCase)); } [Fact] public void UnknownsBudget_PerSeverity_EnforcedCorrectly() { // Arrange - budget per severity level var policy = CreatePolicyWithPerSeverityUnknownsBudget( criticalMax: 0, highMax: 1, mediumMax: 5, lowMax: 10); var evaluator = CreateEvaluator(); // 2 high-unknown findings (exceeds high budget of 1) var findings = new[] { CreateFindingWithUnknownImpact("CVE-2024-0001", baseLevel: "high"), CreateFindingWithUnknownImpact("CVE-2024-0002", baseLevel: "high") }; var input = CreateTestInputWithFindings(findings); // Act var result = evaluator.Evaluate(policy, input); // Assert result.Violations.Should().Contain(v => v.Code.Contains("BUDGET", StringComparison.OrdinalIgnoreCase) || v.Message.Contains("budget", StringComparison.OrdinalIgnoreCase)); } [Fact] public void UnknownsBudget_ReportedInVerdictArtifact() { // Arrange var policy = CreatePolicyWithUnknownsBudget(maxUnknowns: 5); var evaluator = CreateEvaluator(); var findings = Enumerable.Range(1, 3) .Select(i => CreateUnknownFinding($"CVE-2024-{i:D4}")) .ToArray(); var input = CreateTestInputWithFindings(findings); // Act var result = evaluator.Evaluate(policy, input); // Assert - verdict should include unknowns count result.Metrics.Should().ContainKey("unknowns_count"); result.Metrics["unknowns_count"].Should().Be(3); } #endregion #region Test Helpers private static PolicyEvaluator CreateEvaluator() { return new PolicyEvaluator(new PolicyEvaluatorOptions { DeterministicMode = true }); } private static PolicyDefinition CreateTestPolicy() { return new PolicyDefinition { Id = "test-policy", Version = "1.0.0", Rules = [ new PolicyRule { Name = "fail-on-critical", Priority = 1, Condition = "severity == 'critical'", Action = PolicyAction.Block, Reason = "Critical vulnerabilities must be addressed" } ] }; } private static PolicyDefinition CreateTestPolicyWithVex() { return new PolicyDefinition { Id = "test-policy-vex", Version = "1.0.0", VexEnabled = true, Rules = [ new PolicyRule { Name = "respect-vex", Priority = 1, Condition = "vex.status != 'not_affected'", Action = PolicyAction.Evaluate, Reason = "Apply VEX status" } ] }; } private static PolicyDefinition CreatePolicyWithUnknownsBudget(int maxUnknowns) { return new PolicyDefinition { Id = "test-policy-unknowns", Version = "1.0.0", UnknownsBudget = new UnknownsBudget { MaxTotal = maxUnknowns, FailOnExceed = true }, Rules = [] }; } private static PolicyDefinition CreatePolicyWithPerSeverityUnknownsBudget( int criticalMax, int highMax, int mediumMax, int lowMax) { return new PolicyDefinition { Id = "test-policy-unknowns-severity", Version = "1.0.0", UnknownsBudget = new UnknownsBudget { MaxTotal = criticalMax + highMax + mediumMax + lowMax, PerSeverity = new Dictionary { ["critical"] = criticalMax, ["high"] = highMax, ["medium"] = mediumMax, ["low"] = lowMax }, FailOnExceed = true }, Rules = [] }; } private static EvaluationInput CreateTestInput() { return new EvaluationInput { ArtifactDigest = "sha256:abc123", Findings = [ CreateFinding("CVE-2024-0001", "high"), CreateFinding("CVE-2024-0002", "medium") ] }; } private static EvaluationInput CreateTestInputWithFindings(Finding[] findings) { return new EvaluationInput { ArtifactDigest = "sha256:abc123", Findings = findings.ToList() }; } private static EvaluationInput CreateTestInputWithVex(VexStatement[] statements) { return new EvaluationInput { ArtifactDigest = "sha256:abc123", Findings = [ CreateFinding("CVE-2024-0001", "high"), CreateFinding("CVE-2024-0002", "medium") ], VexStatements = statements.ToList() }; } private static Finding CreateFinding(string id, string severity) { return new Finding { VulnerabilityId = id, Severity = severity, Package = new PackageRef { Purl = $"pkg:npm/test@1.0.0" } }; } private static Finding CreateUnknownFinding(string id) { return new Finding { VulnerabilityId = id, Severity = "unknown", Package = new PackageRef { Purl = $"pkg:npm/test@1.0.0" } }; } private static Finding CreateFindingWithUnknownImpact(string id, string baseLevel) { return new Finding { VulnerabilityId = id, Severity = baseLevel, ImpactUnknown = true, Package = new PackageRef { Purl = $"pkg:npm/test@1.0.0" } }; } private static VexStatement CreateVexStatement(string vulnerabilityId, VexStatus status) { return new VexStatement { VulnerabilityId = vulnerabilityId, Status = status, Justification = "Test justification" }; } #endregion } #region Supporting Types (stubs for compilation) // These types should be replaced with actual project references // They are placeholders for the test structure public enum VerdictStatus { Passed, Failed, Warning } public record PolicyEvaluatorOptions { public bool DeterministicMode { get; init; } } public class PolicyEvaluator(PolicyEvaluatorOptions options) { private readonly PolicyEvaluatorOptions _options = options; public EvaluationResult Evaluate(PolicyDefinition policy, EvaluationInput input) { // Stub implementation - actual implementation would come from Policy.Engine var violations = new List(); var metrics = new Dictionary(); // Count unknowns var unknownsCount = input.Findings.Count(f => f.Severity == "unknown" || f.ImpactUnknown); metrics["unknowns_count"] = unknownsCount; metrics["deterministic_mode"] = _options.DeterministicMode ? 1 : 0; // Check unknowns budget if (policy.UnknownsBudget is not null && policy.UnknownsBudget.FailOnExceed) { if (unknownsCount > policy.UnknownsBudget.MaxTotal) { violations.Add(new Violation { Code = "UNKNOWNS_BUDGET_EXCEEDED", Message = $"Unknowns count {unknownsCount} exceeds budget {policy.UnknownsBudget.MaxTotal}" }); } if (policy.UnknownsBudget.PerSeverity is not null) { foreach (var (severity, max) in policy.UnknownsBudget.PerSeverity) { var count = input.Findings.Count(f => f.Severity == severity && f.ImpactUnknown); if (count > max) { violations.Add(new Violation { Code = "UNKNOWNS_BUDGET_EXCEEDED", Message = $"{severity} unknowns count {count} exceeds budget {max}" }); } } } } // Deterministic hash calculation var hash = ComputeDeterministicHash(policy, input); return new EvaluationResult { Status = violations.Count > 0 ? VerdictStatus.Failed : VerdictStatus.Passed, VerdictHash = hash, Violations = violations, Metrics = metrics }; } private static string ComputeDeterministicHash(PolicyDefinition policy, EvaluationInput input) { // Sort findings for determinism var sortedFindings = input.Findings .OrderBy(f => f.VulnerabilityId) .ThenBy(f => f.Package.Purl) .ToList(); var hashInput = $"{policy.Id}:{policy.Version}:{string.Join(",", sortedFindings.Select(f => $"{f.VulnerabilityId}:{f.Severity}"))}"; using var sha = System.Security.Cryptography.SHA256.Create(); var bytes = sha.ComputeHash(System.Text.Encoding.UTF8.GetBytes(hashInput)); return Convert.ToHexString(bytes).ToLowerInvariant(); } } public record EvaluationResult { public VerdictStatus Status { get; init; } public string VerdictHash { get; init; } = string.Empty; public List Violations { get; init; } = []; public Dictionary Metrics { get; init; } = []; public string ToCanonicalJson() { return System.Text.Json.JsonSerializer.Serialize(this, new System.Text.Json.JsonSerializerOptions { WriteIndented = false, PropertyNamingPolicy = System.Text.Json.JsonNamingPolicy.SnakeCaseLower }); } } public record PolicyDefinition { public string Id { get; init; } = string.Empty; public string Version { get; init; } = string.Empty; public bool VexEnabled { get; init; } public UnknownsBudget? UnknownsBudget { get; init; } public List Rules { get; init; } = []; } public record PolicyRule { public string Name { get; init; } = string.Empty; public int Priority { get; init; } public string Condition { get; init; } = string.Empty; public PolicyAction Action { get; init; } public string Reason { get; init; } = string.Empty; } public enum PolicyAction { Evaluate, Block, Warn, Allow } public record UnknownsBudget { public int MaxTotal { get; init; } public bool FailOnExceed { get; init; } public Dictionary? PerSeverity { get; init; } } public record EvaluationInput { public string ArtifactDigest { get; init; } = string.Empty; public List Findings { get; init; } = []; public List VexStatements { get; init; } = []; } public record Finding { public string VulnerabilityId { get; init; } = string.Empty; public string Severity { get; init; } = string.Empty; public bool ImpactUnknown { get; init; } public PackageRef Package { get; init; } = new(); } public record PackageRef { public string Purl { get; init; } = string.Empty; } public record VexStatement { public string VulnerabilityId { get; init; } = string.Empty; public VexStatus Status { get; init; } public string Justification { get; init; } = string.Empty; } public enum VexStatus { Unknown, NotAffected, Affected, Fixed, UnderInvestigation } public record Violation { public string Code { get; init; } = string.Empty; public string Message { get; init; } = string.Empty; } #endregion