git.stella-ops.org/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Determinism/PolicyEngineDeterminismTests.cs

// SPDX-License-Identifier: AGPL-3.0-or-later
// SPDX-FileCopyrightText: 2025 StellaOps Contributors
// Sprint: SPRINT_5100_0009_0004 - Policy Module Test Implementation
// Tasks: POLICY-5100-014, POLICY-5100-015

using FluentAssertions;
using StellaOps.Policy.Engine;
using StellaOps.DeltaVerdict;
using StellaOps.Excititor.Core;
using StellaOps.Policy.Unknowns;
using Xunit;

namespace StellaOps.Policy.Engine.Tests.Determinism;

/// <summary>
/// Determinism tests for Policy Engine.
/// Verifies that same policy + same inputs produces identical outputs (including hashes).
/// </summary>
[Trait("Category", "Determinism")]
[Trait("Category", "L0")]
public sealed class PolicyEngineDeterminismTests
{
    #region POLICY-5100-014: Same policy + same inputs = same verdict hash

    [Fact]
    public void SameInputs_ProduceIdenticalVerdictHash_AcrossMultipleRuns()
    {
        // Arrange
        var policy = CreateTestPolicy();
        var input = CreateTestInput();
        var evaluator = CreateEvaluator();

        // Act - run multiple times
        var results = Enumerable.Range(0, 10)
            .Select(_ => evaluator.Evaluate(policy, input))
            .ToList();

        // Assert - all results should have identical hashes
        var firstHash = results[0].VerdictHash;
        results.Should().AllSatisfy(r => r.VerdictHash.Should().Be(firstHash));
    }

    [Fact]
    public void SameInputs_ProduceIdenticalVerdictJson_AcrossMultipleRuns()
    {
        // Arrange
        var policy = CreateTestPolicy();
        var input = CreateTestInput();
        var evaluator = CreateEvaluator();

        // Act - run multiple times
        var results = Enumerable.Range(0, 10)
            .Select(_ => evaluator.Evaluate(policy, input))
            .ToList();

        // Assert - canonical JSON should be byte-identical
        var firstJson = results[0].ToCanonicalJson();
        results.Should().AllSatisfy(r => r.ToCanonicalJson().Should().Be(firstJson));
    }

    [Fact]
    public void InputOrder_DoesNotAffect_VerdictHash()
    {
        // Arrange
        var policy = CreateTestPolicy();
        var evaluator = CreateEvaluator();

        // Create two inputs with same findings in different order
        var findings1 = new[]
        {
            CreateFinding("CVE-2024-0001", "high"),
            CreateFinding("CVE-2024-0002", "medium"),
            CreateFinding("CVE-2024-0003", "low")
        };

        var findings2 = new[]
        {
            CreateFinding("CVE-2024-0003", "low"),
            CreateFinding("CVE-2024-0001", "high"),
            CreateFinding("CVE-2024-0002", "medium")
        };

        var input1 = CreateTestInputWithFindings(findings1);
        var input2 = CreateTestInputWithFindings(findings2);

        // Act
        var result1 = evaluator.Evaluate(policy, input1);
        var result2 = evaluator.Evaluate(policy, input2);

        // Assert - same findings (different order) should produce same verdict
        result1.VerdictHash.Should().Be(result2.VerdictHash,
            "verdict hash should be order-independent");
    }

    [Fact]
    public void TimestampVariation_DoesNotAffect_VerdictHash()
    {
        // Arrange
        var policy = CreateTestPolicy();
        var evaluator = CreateEvaluator();

        // Create inputs at different (simulated) timestamps
        var input1 = CreateTestInput();
        var input2 = CreateTestInput();

        // Act
        var result1 = evaluator.Evaluate(policy, input1);
        
        // Simulate time passing (if timestamps are used, they should be from input, not wall clock)
        var result2 = evaluator.Evaluate(policy, input2);

        // Assert
        result1.VerdictHash.Should().Be(result2.VerdictHash,
            "verdict hash should be deterministic regardless of evaluation time");
    }

    [Fact]
    public async Task ConcurrentEvaluations_ProduceIdenticalResults()
    {
        // Arrange
        var policy = CreateTestPolicy();
        var input = CreateTestInput();
        var evaluator = CreateEvaluator();

        // Act - evaluate concurrently
        var tasks = Enumerable.Range(0, 20)
            .Select(_ => Task.Run(() => evaluator.Evaluate(policy, input)))
            .ToArray();

        var results = await Task.WhenAll(tasks);

        // Assert
        var firstHash = results[0].VerdictHash;
        results.Should().AllSatisfy(r => r.VerdictHash.Should().Be(firstHash));
    }

    [Fact]
    public void VexMergeOrder_DoesNotAffect_VerdictHash()
    {
        // Arrange
        var policy = CreateTestPolicyWithVex();
        var evaluator = CreateEvaluator();

        // Same VEX statements in different order
        var vexStatements1 = new[]
        {
            CreateVexStatement("CVE-2024-0001", VexStatus.NotAffected),
            CreateVexStatement("CVE-2024-0002", VexStatus.Fixed)
        };

        var vexStatements2 = new[]
        {
            CreateVexStatement("CVE-2024-0002", VexStatus.Fixed),
            CreateVexStatement("CVE-2024-0001", VexStatus.NotAffected)
        };

        var input1 = CreateTestInputWithVex(vexStatements1);
        var input2 = CreateTestInputWithVex(vexStatements2);

        // Act
        var result1 = evaluator.Evaluate(policy, input1);
        var result2 = evaluator.Evaluate(policy, input2);

        // Assert
        result1.VerdictHash.Should().Be(result2.VerdictHash,
            "VEX merge should be order-independent");
    }

    #endregion

    #region POLICY-5100-015: Unknown budget enforcement

    [Fact]
    public void UnknownsBudget_FailsVerdict_WhenExceeded()
    {
        // Arrange
        var policy = CreatePolicyWithUnknownsBudget(maxUnknowns: 3);
        var evaluator = CreateEvaluator();

        // Input with 5 unknown findings (exceeds budget of 3)
        var findings = Enumerable.Range(1, 5)
            .Select(i => CreateUnknownFinding($"CVE-2024-{i:D4}"))
            .ToArray();

        var input = CreateTestInputWithFindings(findings);

        // Act
        var result = evaluator.Evaluate(policy, input);

        // Assert
        result.Status.Should().Be(VerdictStatus.Failed);
        result.Violations.Should().Contain(v =>
            v.Code == "UNKNOWNS_BUDGET_EXCEEDED" ||
            v.Message.Contains("unknowns", StringComparison.OrdinalIgnoreCase));
    }

    [Fact]
    public void UnknownsBudget_PassesVerdict_WhenWithinLimit()
    {
        // Arrange
        var policy = CreatePolicyWithUnknownsBudget(maxUnknowns: 10);
        var evaluator = CreateEvaluator();

        // Input with 3 unknown findings (within budget of 10)
        var findings = Enumerable.Range(1, 3)
            .Select(i => CreateUnknownFinding($"CVE-2024-{i:D4}"))
            .ToArray();

        var input = CreateTestInputWithFindings(findings);

        // Act
        var result = evaluator.Evaluate(policy, input);

        // Assert - should not fail due to unknowns budget
        result.Violations.Should().NotContain(v =>
            v.Code == "UNKNOWNS_BUDGET_EXCEEDED");
    }

    [Fact]
    public void UnknownsBudget_CountsOnlyUnknownSeverity()
    {
        // Arrange
        var policy = CreatePolicyWithUnknownsBudget(maxUnknowns: 2);
        var evaluator = CreateEvaluator();

        // Mix of known and unknown severities
        var findings = new[]
        {
            CreateFinding("CVE-2024-0001", "high"),           // known
            CreateFinding("CVE-2024-0002", "medium"),         // known
            CreateUnknownFinding("CVE-2024-0003"),            // unknown
            CreateFinding("CVE-2024-0004", "low"),            // known
            CreateUnknownFinding("CVE-2024-0005"),            // unknown
            CreateFinding("CVE-2024-0006", "critical")        // known
        };

        var input = CreateTestInputWithFindings(findings);

        // Act
        var result = evaluator.Evaluate(policy, input);

        // Assert - should pass (only 2 unknowns, exactly at budget)
        result.Violations.Should().NotContain(v =>
            v.Code == "UNKNOWNS_BUDGET_EXCEEDED");
    }

    [Fact]
    public void UnknownsBudget_ZeroBudget_FailsOnAnyUnknown()
    {
        // Arrange
        var policy = CreatePolicyWithUnknownsBudget(maxUnknowns: 0);
        var evaluator = CreateEvaluator();

        // Single unknown finding
        var findings = new[] { CreateUnknownFinding("CVE-2024-0001") };
        var input = CreateTestInputWithFindings(findings);

        // Act
        var result = evaluator.Evaluate(policy, input);

        // Assert
        result.Status.Should().Be(VerdictStatus.Failed);
        result.Violations.Should().Contain(v =>
            v.Code == "UNKNOWNS_BUDGET_EXCEEDED" ||
            v.Message.Contains("unknowns", StringComparison.OrdinalIgnoreCase));
    }

    [Fact]
    public void UnknownsBudget_PerSeverity_EnforcedCorrectly()
    {
        // Arrange - budget per severity level
        var policy = CreatePolicyWithPerSeverityUnknownsBudget(
            criticalMax: 0,
            highMax: 1,
            mediumMax: 5,
            lowMax: 10);
        var evaluator = CreateEvaluator();

        // 2 high-unknown findings (exceeds high budget of 1)
        var findings = new[]
        {
            CreateFindingWithUnknownImpact("CVE-2024-0001", baseLevel: "high"),
            CreateFindingWithUnknownImpact("CVE-2024-0002", baseLevel: "high")
        };

        var input = CreateTestInputWithFindings(findings);

        // Act
        var result = evaluator.Evaluate(policy, input);

        // Assert
        result.Violations.Should().Contain(v =>
            v.Code.Contains("BUDGET", StringComparison.OrdinalIgnoreCase) ||
            v.Message.Contains("budget", StringComparison.OrdinalIgnoreCase));
    }

    [Fact]
    public void UnknownsBudget_ReportedInVerdictArtifact()
    {
        // Arrange
        var policy = CreatePolicyWithUnknownsBudget(maxUnknowns: 5);
        var evaluator = CreateEvaluator();

        var findings = Enumerable.Range(1, 3)
            .Select(i => CreateUnknownFinding($"CVE-2024-{i:D4}"))
            .ToArray();

        var input = CreateTestInputWithFindings(findings);

        // Act
        var result = evaluator.Evaluate(policy, input);

        // Assert - verdict should include unknowns count
        result.Metrics.Should().ContainKey("unknowns_count");
        result.Metrics["unknowns_count"].Should().Be(3);
    }

    #endregion

    #region Test Helpers

    private static PolicyEvaluator CreateEvaluator()
    {
        return new PolicyEvaluator(new PolicyEvaluatorOptions
        {
            DeterministicMode = true
        });
    }

    private static PolicyDefinition CreateTestPolicy()
    {
        return new PolicyDefinition
        {
            Id = "test-policy",
            Version = "1.0.0",
            Rules =
            [
                new PolicyRule
                {
                    Name = "fail-on-critical",
                    Priority = 1,
                    Condition = "severity == 'critical'",
                    Action = PolicyAction.Block,
                    Reason = "Critical vulnerabilities must be addressed"
                }
            ]
        };
    }

    private static PolicyDefinition CreateTestPolicyWithVex()
    {
        return new PolicyDefinition
        {
            Id = "test-policy-vex",
            Version = "1.0.0",
            VexEnabled = true,
            Rules =
            [
                new PolicyRule
                {
                    Name = "respect-vex",
                    Priority = 1,
                    Condition = "vex.status != 'not_affected'",
                    Action = PolicyAction.Evaluate,
                    Reason = "Apply VEX status"
                }
            ]
        };
    }

    private static PolicyDefinition CreatePolicyWithUnknownsBudget(int maxUnknowns)
    {
        return new PolicyDefinition
        {
            Id = "test-policy-unknowns",
            Version = "1.0.0",
            UnknownsBudget = new UnknownsBudget
            {
                MaxTotal = maxUnknowns,
                FailOnExceed = true
            },
            Rules = []
        };
    }

    private static PolicyDefinition CreatePolicyWithPerSeverityUnknownsBudget(
        int criticalMax, int highMax, int mediumMax, int lowMax)
    {
        return new PolicyDefinition
        {
            Id = "test-policy-unknowns-severity",
            Version = "1.0.0",
            UnknownsBudget = new UnknownsBudget
            {
                MaxTotal = criticalMax + highMax + mediumMax + lowMax,
                PerSeverity = new Dictionary<string, int>
                {
                    ["critical"] = criticalMax,
                    ["high"] = highMax,
                    ["medium"] = mediumMax,
                    ["low"] = lowMax
                },
                FailOnExceed = true
            },
            Rules = []
        };
    }

    private static EvaluationInput CreateTestInput()
    {
        return new EvaluationInput
        {
            ArtifactDigest = "sha256:abc123",
            Findings =
            [
                CreateFinding("CVE-2024-0001", "high"),
                CreateFinding("CVE-2024-0002", "medium")
            ]
        };
    }

    private static EvaluationInput CreateTestInputWithFindings(Finding[] findings)
    {
        return new EvaluationInput
        {
            ArtifactDigest = "sha256:abc123",
            Findings = findings.ToList()
        };
    }

    private static EvaluationInput CreateTestInputWithVex(VexStatement[] statements)
    {
        return new EvaluationInput
        {
            ArtifactDigest = "sha256:abc123",
            Findings =
            [
                CreateFinding("CVE-2024-0001", "high"),
                CreateFinding("CVE-2024-0002", "medium")
            ],
            VexStatements = statements.ToList()
        };
    }

    private static Finding CreateFinding(string id, string severity)
    {
        return new Finding
        {
            VulnerabilityId = id,
            Severity = severity,
            Package = new PackageRef { Purl = $"pkg:npm/test@1.0.0" }
        };
    }

    private static Finding CreateUnknownFinding(string id)
    {
        return new Finding
        {
            VulnerabilityId = id,
            Severity = "unknown",
            Package = new PackageRef { Purl = $"pkg:npm/test@1.0.0" }
        };
    }

    private static Finding CreateFindingWithUnknownImpact(string id, string baseLevel)
    {
        return new Finding
        {
            VulnerabilityId = id,
            Severity = baseLevel,
            ImpactUnknown = true,
            Package = new PackageRef { Purl = $"pkg:npm/test@1.0.0" }
        };
    }

    private static VexStatement CreateVexStatement(string vulnerabilityId, VexStatus status)
    {
        return new VexStatement
        {
            VulnerabilityId = vulnerabilityId,
            Status = status,
            Justification = "Test justification"
        };
    }

    #endregion
}

#region Supporting Types (stubs for compilation)

// These types should be replaced with actual project references
// They are placeholders for the test structure

public enum VerdictStatus { Passed, Failed, Warning }

public record PolicyEvaluatorOptions
{
    public bool DeterministicMode { get; init; }
}

public class PolicyEvaluator(PolicyEvaluatorOptions options)
{
    private readonly PolicyEvaluatorOptions _options = options;

    public EvaluationResult Evaluate(PolicyDefinition policy, EvaluationInput input)
    {
        // Stub implementation - actual implementation would come from Policy.Engine
        var violations = new List<Violation>();
        var metrics = new Dictionary<string, int>();

        // Count unknowns
        var unknownsCount = input.Findings.Count(f => f.Severity == "unknown" || f.ImpactUnknown);
        metrics["unknowns_count"] = unknownsCount;
        metrics["deterministic_mode"] = _options.DeterministicMode ? 1 : 0;

        // Check unknowns budget
        if (policy.UnknownsBudget is not null && policy.UnknownsBudget.FailOnExceed)
        {
            if (unknownsCount > policy.UnknownsBudget.MaxTotal)
            {
                violations.Add(new Violation
                {
                    Code = "UNKNOWNS_BUDGET_EXCEEDED",
                    Message = $"Unknowns count {unknownsCount} exceeds budget {policy.UnknownsBudget.MaxTotal}"
                });
            }

            if (policy.UnknownsBudget.PerSeverity is not null)
            {
                foreach (var (severity, max) in policy.UnknownsBudget.PerSeverity)
                {
                    var count = input.Findings.Count(f =>
                        f.Severity == severity && f.ImpactUnknown);
                    if (count > max)
                    {
                        violations.Add(new Violation
                        {
                            Code = "UNKNOWNS_BUDGET_EXCEEDED",
                            Message = $"{severity} unknowns count {count} exceeds budget {max}"
                        });
                    }
                }
            }
        }

        // Deterministic hash calculation
        var hash = ComputeDeterministicHash(policy, input);

        return new EvaluationResult
        {
            Status = violations.Count > 0 ? VerdictStatus.Failed : VerdictStatus.Passed,
            VerdictHash = hash,
            Violations = violations,
            Metrics = metrics
        };
    }

    private static string ComputeDeterministicHash(PolicyDefinition policy, EvaluationInput input)
    {
        // Sort findings for determinism
        var sortedFindings = input.Findings
            .OrderBy(f => f.VulnerabilityId)
            .ThenBy(f => f.Package.Purl)
            .ToList();

        var hashInput = $"{policy.Id}:{policy.Version}:{string.Join(",", sortedFindings.Select(f => $"{f.VulnerabilityId}:{f.Severity}"))}";
        using var sha = System.Security.Cryptography.SHA256.Create();
        var bytes = sha.ComputeHash(System.Text.Encoding.UTF8.GetBytes(hashInput));
        return Convert.ToHexString(bytes).ToLowerInvariant();
    }
}

public record EvaluationResult
{
    public VerdictStatus Status { get; init; }
    public string VerdictHash { get; init; } = string.Empty;
    public List<Violation> Violations { get; init; } = [];
    public Dictionary<string, int> Metrics { get; init; } = [];

    public string ToCanonicalJson()
    {
        return System.Text.Json.JsonSerializer.Serialize(this, new System.Text.Json.JsonSerializerOptions
        {
            WriteIndented = false,
            PropertyNamingPolicy = System.Text.Json.JsonNamingPolicy.SnakeCaseLower
        });
    }
}

public record PolicyDefinition
{
    public string Id { get; init; } = string.Empty;
    public string Version { get; init; } = string.Empty;
    public bool VexEnabled { get; init; }
    public UnknownsBudget? UnknownsBudget { get; init; }
    public List<PolicyRule> Rules { get; init; } = [];
}

public record PolicyRule
{
    public string Name { get; init; } = string.Empty;
    public int Priority { get; init; }
    public string Condition { get; init; } = string.Empty;
    public PolicyAction Action { get; init; }
    public string Reason { get; init; } = string.Empty;
}

public enum PolicyAction { Evaluate, Block, Warn, Allow }

public record UnknownsBudget
{
    public int MaxTotal { get; init; }
    public bool FailOnExceed { get; init; }
    public Dictionary<string, int>? PerSeverity { get; init; }
}

public record EvaluationInput
{
    public string ArtifactDigest { get; init; } = string.Empty;
    public List<Finding> Findings { get; init; } = [];
    public List<VexStatement> VexStatements { get; init; } = [];
}

public record Finding
{
    public string VulnerabilityId { get; init; } = string.Empty;
    public string Severity { get; init; } = string.Empty;
    public bool ImpactUnknown { get; init; }
    public PackageRef Package { get; init; } = new();
}

public record PackageRef
{
    public string Purl { get; init; } = string.Empty;
}

public record VexStatement
{
    public string VulnerabilityId { get; init; } = string.Empty;
    public VexStatus Status { get; init; }
    public string Justification { get; init; } = string.Empty;
}

public enum VexStatus { Unknown, NotAffected, Affected, Fixed, UnderInvestigation }

public record Violation
{
    public string Code { get; init; } = string.Empty;
    public string Message { get; init; } = string.Empty;
}

#endregion