Files
git.stella-ops.org/tests/integration/StellaOps.Integration.Determinism/PolicyDeterminismTests.cs
master 5590a99a1a Add tests for SBOM generation determinism across multiple formats
- Created `StellaOps.TestKit.Tests` project for unit tests related to determinism.
- Implemented `DeterminismManifestTests` to validate deterministic output for canonical bytes and strings, file read/write operations, and error handling for invalid schema versions.
- Added `SbomDeterminismTests` to ensure identical inputs produce consistent SBOMs across SPDX 3.0.1 and CycloneDX 1.6/1.7 formats, including parallel execution tests.
- Updated project references in `StellaOps.Integration.Determinism` to include the new determinism testing library.
2025-12-23 23:51:58 +02:00

659 lines
21 KiB
C#

// -----------------------------------------------------------------------------
// PolicyDeterminismTests.cs
// Sprint: SPRINT_5100_0007_0003 - Epic B (Determinism Gate)
// Task: T5 - Policy Verdict Determinism
// Description: Tests to validate policy verdict generation determinism
// -----------------------------------------------------------------------------
using System.Collections.Immutable;
using System.Text;
using FluentAssertions;
using StellaOps.Canonical.Json;
using StellaOps.Testing.Determinism;
using Xunit;
namespace StellaOps.Integration.Determinism;
/// <summary>
/// Determinism validation tests for policy verdict generation.
/// Ensures identical inputs produce identical verdicts across:
/// - Single verdict generation
/// - Batch verdict generation
/// - Verdict serialization
/// - Multiple runs with frozen time
/// - Parallel execution
/// </summary>
public class PolicyDeterminismTests
{
#region Single Verdict Determinism Tests
[Fact]
public void PolicyVerdict_WithIdenticalInput_ProducesDeterministicOutput()
{
// Arrange
var input = CreateSamplePolicyInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act - Generate verdict multiple times
var verdict1 = EvaluatePolicy(input, frozenTime);
var verdict2 = EvaluatePolicy(input, frozenTime);
var verdict3 = EvaluatePolicy(input, frozenTime);
// Assert - All outputs should be identical
verdict1.Should().BeEquivalentTo(verdict2);
verdict2.Should().BeEquivalentTo(verdict3);
}
[Fact]
public void PolicyVerdict_CanonicalHash_IsStable()
{
// Arrange
var input = CreateSamplePolicyInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act - Generate verdict and compute canonical hash twice
var verdict1 = EvaluatePolicy(input, frozenTime);
var json1 = SerializeVerdict(verdict1);
var hash1 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(json1));
var verdict2 = EvaluatePolicy(input, frozenTime);
var json2 = SerializeVerdict(verdict2);
var hash2 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(json2));
// Assert
hash1.Should().Be(hash2, "Same input should produce same canonical hash");
hash1.Should().MatchRegex("^[0-9a-f]{64}$");
}
[Fact]
public void PolicyVerdict_DeterminismManifest_CanBeCreated()
{
// Arrange
var input = CreateSamplePolicyInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
var verdict = EvaluatePolicy(input, frozenTime);
var json = SerializeVerdict(verdict);
var verdictBytes = Encoding.UTF8.GetBytes(json);
var artifactInfo = new ArtifactInfo
{
Type = "policy-verdict",
Name = "test-finding-verdict",
Version = "1.0.0",
Format = "PolicyVerdict JSON"
};
var toolchain = new ToolchainInfo
{
Platform = ".NET 10.0",
Components = new[]
{
new ComponentInfo { Name = "StellaOps.Policy.Engine", Version = "1.0.0" }
}
};
// Act - Create determinism manifest
var manifest = DeterminismManifestWriter.CreateManifest(
verdictBytes,
artifactInfo,
toolchain);
// Assert
manifest.SchemaVersion.Should().Be("1.0");
manifest.Artifact.Format.Should().Be("PolicyVerdict JSON");
manifest.CanonicalHash.Algorithm.Should().Be("SHA-256");
manifest.CanonicalHash.Value.Should().MatchRegex("^[0-9a-f]{64}$");
}
[Fact]
public async Task PolicyVerdict_ParallelGeneration_ProducesDeterministicOutput()
{
// Arrange
var input = CreateSamplePolicyInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act - Generate in parallel 20 times
var tasks = Enumerable.Range(0, 20)
.Select(_ => Task.Run(() => EvaluatePolicy(input, frozenTime)))
.ToArray();
var verdicts = await Task.WhenAll(tasks);
// Assert - All outputs should be identical
var first = verdicts[0];
verdicts.Should().AllSatisfy(v => v.Should().BeEquivalentTo(first));
}
#endregion
#region Batch Verdict Determinism Tests
[Fact]
public void PolicyVerdictBatch_WithIdenticalInput_ProducesDeterministicOutput()
{
// Arrange
var inputs = CreateSampleBatchPolicyInputs();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act - Generate batch verdicts multiple times
var batch1 = EvaluatePolicyBatch(inputs, frozenTime);
var batch2 = EvaluatePolicyBatch(inputs, frozenTime);
var batch3 = EvaluatePolicyBatch(inputs, frozenTime);
// Assert - All batches should be identical
batch1.Should().BeEquivalentTo(batch2);
batch2.Should().BeEquivalentTo(batch3);
}
[Fact]
public void PolicyVerdictBatch_Ordering_IsDeterministic()
{
// Arrange - Findings in random order
var inputs = CreateSampleBatchPolicyInputs();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act - Generate batch verdicts multiple times
var batch1 = EvaluatePolicyBatch(inputs, frozenTime);
var batch2 = EvaluatePolicyBatch(inputs, frozenTime);
// Assert - Order should be deterministic
var json1 = SerializeBatch(batch1);
var json2 = SerializeBatch(batch2);
json1.Should().Be(json2);
}
[Fact]
public void PolicyVerdictBatch_CanonicalHash_IsStable()
{
// Arrange
var inputs = CreateSampleBatchPolicyInputs();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var batch1 = EvaluatePolicyBatch(inputs, frozenTime);
var json1 = SerializeBatch(batch1);
var hash1 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(json1));
var batch2 = EvaluatePolicyBatch(inputs, frozenTime);
var json2 = SerializeBatch(batch2);
var hash2 = CanonJson.Sha256Hex(Encoding.UTF8.GetBytes(json2));
// Assert
hash1.Should().Be(hash2);
hash1.Should().MatchRegex("^[0-9a-f]{64}$");
}
#endregion
#region Verdict Status Determinism Tests
[Theory]
[InlineData(PolicyVerdictStatus.Pass)]
[InlineData(PolicyVerdictStatus.Blocked)]
[InlineData(PolicyVerdictStatus.Ignored)]
[InlineData(PolicyVerdictStatus.Warned)]
[InlineData(PolicyVerdictStatus.Deferred)]
[InlineData(PolicyVerdictStatus.Escalated)]
[InlineData(PolicyVerdictStatus.RequiresVex)]
public void PolicyVerdict_WithStatus_IsDeterministic(PolicyVerdictStatus status)
{
// Arrange
var input = CreatePolicyInputWithExpectedStatus(status);
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var verdict1 = EvaluatePolicy(input, frozenTime);
var verdict2 = EvaluatePolicy(input, frozenTime);
// Assert
verdict1.Status.Should().Be(status);
verdict2.Status.Should().Be(status);
verdict1.Should().BeEquivalentTo(verdict2);
}
#endregion
#region Score Calculation Determinism Tests
[Fact]
public void PolicyScore_WithSameInputs_ProducesDeterministicScore()
{
// Arrange
var input = CreateSamplePolicyInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var verdict1 = EvaluatePolicy(input, frozenTime);
var verdict2 = EvaluatePolicy(input, frozenTime);
// Assert - Scores should be identical (not floating point approximate)
verdict1.Score.Should().Be(verdict2.Score);
}
[Fact]
public void PolicyScore_InputOrdering_DoesNotAffectScore()
{
// Arrange - Same inputs but in different order
var inputs1 = new Dictionary<string, double>
{
{ "cvss", 7.5 },
{ "epss", 0.001 },
{ "kev", 0.0 },
{ "reachability", 0.8 }
};
var inputs2 = new Dictionary<string, double>
{
{ "reachability", 0.8 },
{ "kev", 0.0 },
{ "epss", 0.001 },
{ "cvss", 7.5 }
};
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var verdict1 = EvaluatePolicyWithInputs("CVE-2024-1234", inputs1, frozenTime);
var verdict2 = EvaluatePolicyWithInputs("CVE-2024-1234", inputs2, frozenTime);
// Assert
verdict1.Score.Should().Be(verdict2.Score);
verdict1.Status.Should().Be(verdict2.Status);
}
[Fact]
public void PolicyScore_FloatingPointPrecision_IsConsistent()
{
// Arrange - Inputs that might cause floating point issues
var inputs = new Dictionary<string, double>
{
{ "cvss", 0.1 + 0.2 }, // Classic floating point precision test
{ "epss", 1.0 / 3.0 },
{ "weight_a", 0.33333333333333333 },
{ "weight_b", 0.66666666666666666 }
};
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var verdict1 = EvaluatePolicyWithInputs("CVE-2024-5678", inputs, frozenTime);
var verdict2 = EvaluatePolicyWithInputs("CVE-2024-5678", inputs, frozenTime);
// Assert - Score should be rounded to consistent precision
verdict1.Score.Should().Be(verdict2.Score);
}
#endregion
#region Rule Matching Determinism Tests
[Fact]
public void PolicyRuleMatching_WithMultipleMatchingRules_SelectsDeterministically()
{
// Arrange - Input that matches multiple rules
var input = CreateInputMatchingMultipleRules();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var verdict1 = EvaluatePolicy(input, frozenTime);
var verdict2 = EvaluatePolicy(input, frozenTime);
// Assert - Same rule should be selected each time
verdict1.RuleName.Should().Be(verdict2.RuleName);
verdict1.RuleAction.Should().Be(verdict2.RuleAction);
}
[Fact]
public void PolicyQuieting_IsDeterministic()
{
// Arrange - Input that triggers quieting
var input = CreateQuietedInput();
var frozenTime = DateTimeOffset.Parse("2025-12-23T18:00:00Z");
// Act
var verdict1 = EvaluatePolicy(input, frozenTime);
var verdict2 = EvaluatePolicy(input, frozenTime);
// Assert
verdict1.Quiet.Should().Be(verdict2.Quiet);
verdict1.QuietedBy.Should().Be(verdict2.QuietedBy);
}
#endregion
#region Helper Methods
private static PolicyInput CreateSamplePolicyInput()
{
return new PolicyInput
{
FindingId = "CVE-2024-1234",
CvssScore = 7.5,
EpssScore = 0.001,
IsKev = false,
ReachabilityScore = 0.8,
SourceTrust = "high",
PackageType = "npm",
Severity = "high"
};
}
private static PolicyInput[] CreateSampleBatchPolicyInputs()
{
return new[]
{
new PolicyInput
{
FindingId = "CVE-2024-1111",
CvssScore = 9.8,
EpssScore = 0.5,
IsKev = true,
ReachabilityScore = 1.0,
SourceTrust = "high",
PackageType = "npm",
Severity = "critical"
},
new PolicyInput
{
FindingId = "CVE-2024-2222",
CvssScore = 5.5,
EpssScore = 0.01,
IsKev = false,
ReachabilityScore = 0.3,
SourceTrust = "medium",
PackageType = "pypi",
Severity = "medium"
},
new PolicyInput
{
FindingId = "CVE-2024-3333",
CvssScore = 3.2,
EpssScore = 0.001,
IsKev = false,
ReachabilityScore = 0.1,
SourceTrust = "low",
PackageType = "maven",
Severity = "low"
}
};
}
private static PolicyInput CreatePolicyInputWithExpectedStatus(PolicyVerdictStatus status)
{
return status switch
{
PolicyVerdictStatus.Pass => new PolicyInput
{
FindingId = "CVE-PASS-001",
CvssScore = 2.0,
EpssScore = 0.0001,
IsKev = false,
ReachabilityScore = 0.0,
SourceTrust = "high",
PackageType = "npm",
Severity = "low"
},
PolicyVerdictStatus.Blocked => new PolicyInput
{
FindingId = "CVE-BLOCKED-001",
CvssScore = 9.8,
EpssScore = 0.9,
IsKev = true,
ReachabilityScore = 1.0,
SourceTrust = "high",
PackageType = "npm",
Severity = "critical"
},
PolicyVerdictStatus.Warned => new PolicyInput
{
FindingId = "CVE-WARNED-001",
CvssScore = 7.0,
EpssScore = 0.05,
IsKev = false,
ReachabilityScore = 0.5,
SourceTrust = "medium",
PackageType = "npm",
Severity = "high"
},
PolicyVerdictStatus.RequiresVex => new PolicyInput
{
FindingId = "CVE-VEXREQ-001",
CvssScore = 7.5,
EpssScore = 0.1,
IsKev = false,
ReachabilityScore = null, // Unknown reachability
SourceTrust = "high",
PackageType = "npm",
Severity = "high"
},
_ => new PolicyInput
{
FindingId = $"CVE-{status}-001",
CvssScore = 5.0,
EpssScore = 0.01,
IsKev = false,
ReachabilityScore = 0.5,
SourceTrust = "medium",
PackageType = "npm",
Severity = "medium"
}
};
}
private static PolicyInput CreateInputMatchingMultipleRules()
{
return new PolicyInput
{
FindingId = "CVE-MULTIRULE-001",
CvssScore = 7.0,
EpssScore = 0.1,
IsKev = false,
ReachabilityScore = 0.5,
SourceTrust = "high",
PackageType = "npm",
Severity = "high"
};
}
private static PolicyInput CreateQuietedInput()
{
return new PolicyInput
{
FindingId = "CVE-2024-QUIETED",
CvssScore = 9.0,
EpssScore = 0.5,
IsKev = false,
ReachabilityScore = 1.0,
SourceTrust = "high",
PackageType = "npm",
Severity = "critical",
QuietedBy = "waiver:WAIVER-2024-001"
};
}
private static PolicyVerdictResult EvaluatePolicy(PolicyInput input, DateTimeOffset timestamp)
{
// TODO: Integrate with actual PolicyEngine
// For now, return deterministic stub
var status = DetermineStatus(input);
var score = CalculateScore(input);
var ruleName = DetermineRuleName(input);
return new PolicyVerdictResult
{
FindingId = input.FindingId,
Status = status,
Score = score,
RuleName = ruleName,
RuleAction = status == PolicyVerdictStatus.Pass ? "allow" : "block",
Notes = null,
ConfigVersion = "1.0",
Inputs = new Dictionary<string, double>
{
{ "cvss", input.CvssScore },
{ "epss", input.EpssScore },
{ "kev", input.IsKev ? 1.0 : 0.0 },
{ "reachability", input.ReachabilityScore ?? 0.5 }
}.ToImmutableDictionary(),
Quiet = input.QuietedBy != null,
QuietedBy = input.QuietedBy,
Timestamp = timestamp
};
}
private static PolicyVerdictResult EvaluatePolicyWithInputs(
string findingId,
Dictionary<string, double> inputs,
DateTimeOffset timestamp)
{
// Calculate score from inputs
var cvss = inputs.GetValueOrDefault("cvss", 0);
var epss = inputs.GetValueOrDefault("epss", 0);
var score = Math.Round((cvss * 10 + epss * 100) / 2, 4);
var status = score > 70 ? PolicyVerdictStatus.Blocked :
score > 40 ? PolicyVerdictStatus.Warned :
PolicyVerdictStatus.Pass;
return new PolicyVerdictResult
{
FindingId = findingId,
Status = status,
Score = score,
RuleName = "calculated-score-rule",
RuleAction = status == PolicyVerdictStatus.Pass ? "allow" : "block",
Notes = null,
ConfigVersion = "1.0",
Inputs = inputs.ToImmutableDictionary(),
Quiet = false,
QuietedBy = null,
Timestamp = timestamp
};
}
private static PolicyVerdictResult[] EvaluatePolicyBatch(PolicyInput[] inputs, DateTimeOffset timestamp)
{
return inputs
.Select(input => EvaluatePolicy(input, timestamp))
.OrderBy(v => v.FindingId, StringComparer.Ordinal)
.ToArray();
}
private static PolicyVerdictStatus DetermineStatus(PolicyInput input)
{
if (input.QuietedBy != null)
return PolicyVerdictStatus.Ignored;
if (input.ReachabilityScore == null)
return PolicyVerdictStatus.RequiresVex;
if (input.IsKev || input.CvssScore >= 9.0 || input.EpssScore >= 0.5)
return PolicyVerdictStatus.Blocked;
if (input.CvssScore >= 7.0 || input.EpssScore >= 0.05)
return PolicyVerdictStatus.Warned;
return PolicyVerdictStatus.Pass;
}
private static double CalculateScore(PolicyInput input)
{
var baseScore = input.CvssScore * 10;
var epssMultiplier = 1 + (input.EpssScore * 10);
var kevBonus = input.IsKev ? 20 : 0;
var reachabilityFactor = input.ReachabilityScore ?? 0.5;
var rawScore = (baseScore * epssMultiplier + kevBonus) * reachabilityFactor;
return Math.Round(rawScore, 4);
}
private static string DetermineRuleName(PolicyInput input)
{
if (input.IsKev)
return "kev-critical-block";
if (input.CvssScore >= 9.0)
return "critical-cvss-block";
if (input.EpssScore >= 0.5)
return "high-exploit-likelihood-block";
if (input.CvssScore >= 7.0)
return "high-cvss-warn";
return "default-pass";
}
private static string SerializeVerdict(PolicyVerdictResult verdict)
{
// Canonical JSON serialization
var inputsJson = string.Join(", ", verdict.Inputs
.OrderBy(kvp => kvp.Key, StringComparer.Ordinal)
.Select(kvp => $"\"{kvp.Key}\": {kvp.Value}"));
return $$"""
{
"configVersion": "{{verdict.ConfigVersion}}",
"findingId": "{{verdict.FindingId}}",
"inputs": {{{inputsJson}}},
"notes": {{(verdict.Notes == null ? "null" : $"\"{verdict.Notes}\"")}},
"quiet": {{verdict.Quiet.ToString().ToLowerInvariant()}},
"quietedBy": {{(verdict.QuietedBy == null ? "null" : $"\"{verdict.QuietedBy}\"")}},
"ruleAction": "{{verdict.RuleAction}}",
"ruleName": "{{verdict.RuleName}}",
"score": {{verdict.Score}},
"status": "{{verdict.Status}}",
"timestamp": "{{verdict.Timestamp:O}}"
}
""";
}
private static string SerializeBatch(PolicyVerdictResult[] verdicts)
{
var items = verdicts.Select(SerializeVerdict);
return $"[\n {string.Join(",\n ", items)}\n]";
}
#endregion
#region DTOs
private sealed record PolicyInput
{
public required string FindingId { get; init; }
public required double CvssScore { get; init; }
public required double EpssScore { get; init; }
public required bool IsKev { get; init; }
public double? ReachabilityScore { get; init; }
public required string SourceTrust { get; init; }
public required string PackageType { get; init; }
public required string Severity { get; init; }
public string? QuietedBy { get; init; }
}
private sealed record PolicyVerdictResult
{
public required string FindingId { get; init; }
public required PolicyVerdictStatus Status { get; init; }
public required double Score { get; init; }
public required string RuleName { get; init; }
public required string RuleAction { get; init; }
public string? Notes { get; init; }
public required string ConfigVersion { get; init; }
public required ImmutableDictionary<string, double> Inputs { get; init; }
public required bool Quiet { get; init; }
public string? QuietedBy { get; init; }
public required DateTimeOffset Timestamp { get; init; }
}
private enum PolicyVerdictStatus
{
Pass,
Blocked,
Ignored,
Warned,
Deferred,
Escalated,
RequiresVex
}
#endregion
}