product advisories, stella router improval, tests streghthening
This commit is contained in:
@@ -0,0 +1,649 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
// SPDX-FileCopyrightText: 2025 StellaOps Contributors
|
||||
// Sprint: SPRINT_5100_0009_0004 - Policy Module Test Implementation
|
||||
// Tasks: POLICY-5100-014, POLICY-5100-015
|
||||
|
||||
using FluentAssertions;
|
||||
using StellaOps.Policy.Engine;
|
||||
using StellaOps.DeltaVerdict;
|
||||
using StellaOps.Excititor.Core.Vex;
|
||||
using StellaOps.Policy.Unknowns;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Policy.Engine.Tests.Determinism;
|
||||
|
||||
/// <summary>
|
||||
/// Determinism tests for Policy Engine.
|
||||
/// Verifies that same policy + same inputs produces identical outputs (including hashes).
|
||||
/// </summary>
|
||||
[Trait("Category", "Determinism")]
|
||||
[Trait("Category", "L0")]
|
||||
public sealed class PolicyEngineDeterminismTests
|
||||
{
|
||||
#region POLICY-5100-014: Same policy + same inputs = same verdict hash
|
||||
|
||||
[Fact]
|
||||
public void SameInputs_ProduceIdenticalVerdictHash_AcrossMultipleRuns()
|
||||
{
|
||||
// Arrange
|
||||
var policy = CreateTestPolicy();
|
||||
var input = CreateTestInput();
|
||||
var evaluator = CreateEvaluator();
|
||||
|
||||
// Act - run multiple times
|
||||
var results = Enumerable.Range(0, 10)
|
||||
.Select(_ => evaluator.Evaluate(policy, input))
|
||||
.ToList();
|
||||
|
||||
// Assert - all results should have identical hashes
|
||||
var firstHash = results[0].VerdictHash;
|
||||
results.Should().AllSatisfy(r => r.VerdictHash.Should().Be(firstHash));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SameInputs_ProduceIdenticalVerdictJson_AcrossMultipleRuns()
|
||||
{
|
||||
// Arrange
|
||||
var policy = CreateTestPolicy();
|
||||
var input = CreateTestInput();
|
||||
var evaluator = CreateEvaluator();
|
||||
|
||||
// Act - run multiple times
|
||||
var results = Enumerable.Range(0, 10)
|
||||
.Select(_ => evaluator.Evaluate(policy, input))
|
||||
.ToList();
|
||||
|
||||
// Assert - canonical JSON should be byte-identical
|
||||
var firstJson = results[0].ToCanonicalJson();
|
||||
results.Should().AllSatisfy(r => r.ToCanonicalJson().Should().Be(firstJson));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void InputOrder_DoesNotAffect_VerdictHash()
|
||||
{
|
||||
// Arrange
|
||||
var policy = CreateTestPolicy();
|
||||
var evaluator = CreateEvaluator();
|
||||
|
||||
// Create two inputs with same findings in different order
|
||||
var findings1 = new[]
|
||||
{
|
||||
CreateFinding("CVE-2024-0001", "high"),
|
||||
CreateFinding("CVE-2024-0002", "medium"),
|
||||
CreateFinding("CVE-2024-0003", "low")
|
||||
};
|
||||
|
||||
var findings2 = new[]
|
||||
{
|
||||
CreateFinding("CVE-2024-0003", "low"),
|
||||
CreateFinding("CVE-2024-0001", "high"),
|
||||
CreateFinding("CVE-2024-0002", "medium")
|
||||
};
|
||||
|
||||
var input1 = CreateTestInputWithFindings(findings1);
|
||||
var input2 = CreateTestInputWithFindings(findings2);
|
||||
|
||||
// Act
|
||||
var result1 = evaluator.Evaluate(policy, input1);
|
||||
var result2 = evaluator.Evaluate(policy, input2);
|
||||
|
||||
// Assert - same findings (different order) should produce same verdict
|
||||
result1.VerdictHash.Should().Be(result2.VerdictHash,
|
||||
"verdict hash should be order-independent");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TimestampVariation_DoesNotAffect_VerdictHash()
|
||||
{
|
||||
// Arrange
|
||||
var policy = CreateTestPolicy();
|
||||
var evaluator = CreateEvaluator();
|
||||
|
||||
// Create inputs at different (simulated) timestamps
|
||||
var input1 = CreateTestInput();
|
||||
var input2 = CreateTestInput();
|
||||
|
||||
// Act
|
||||
var result1 = evaluator.Evaluate(policy, input1);
|
||||
|
||||
// Simulate time passing (if timestamps are used, they should be from input, not wall clock)
|
||||
var result2 = evaluator.Evaluate(policy, input2);
|
||||
|
||||
// Assert
|
||||
result1.VerdictHash.Should().Be(result2.VerdictHash,
|
||||
"verdict hash should be deterministic regardless of evaluation time");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConcurrentEvaluations_ProduceIdenticalResults()
|
||||
{
|
||||
// Arrange
|
||||
var policy = CreateTestPolicy();
|
||||
var input = CreateTestInput();
|
||||
var evaluator = CreateEvaluator();
|
||||
|
||||
// Act - evaluate concurrently
|
||||
var tasks = Enumerable.Range(0, 20)
|
||||
.Select(_ => Task.Run(() => evaluator.Evaluate(policy, input)))
|
||||
.ToArray();
|
||||
|
||||
var results = Task.WhenAll(tasks).GetAwaiter().GetResult();
|
||||
|
||||
// Assert
|
||||
var firstHash = results[0].VerdictHash;
|
||||
results.Should().AllSatisfy(r => r.VerdictHash.Should().Be(firstHash));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void VexMergeOrder_DoesNotAffect_VerdictHash()
|
||||
{
|
||||
// Arrange
|
||||
var policy = CreateTestPolicyWithVex();
|
||||
var evaluator = CreateEvaluator();
|
||||
|
||||
// Same VEX statements in different order
|
||||
var vexStatements1 = new[]
|
||||
{
|
||||
CreateVexStatement("CVE-2024-0001", VexStatus.NotAffected),
|
||||
CreateVexStatement("CVE-2024-0002", VexStatus.Fixed)
|
||||
};
|
||||
|
||||
var vexStatements2 = new[]
|
||||
{
|
||||
CreateVexStatement("CVE-2024-0002", VexStatus.Fixed),
|
||||
CreateVexStatement("CVE-2024-0001", VexStatus.NotAffected)
|
||||
};
|
||||
|
||||
var input1 = CreateTestInputWithVex(vexStatements1);
|
||||
var input2 = CreateTestInputWithVex(vexStatements2);
|
||||
|
||||
// Act
|
||||
var result1 = evaluator.Evaluate(policy, input1);
|
||||
var result2 = evaluator.Evaluate(policy, input2);
|
||||
|
||||
// Assert
|
||||
result1.VerdictHash.Should().Be(result2.VerdictHash,
|
||||
"VEX merge should be order-independent");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region POLICY-5100-015: Unknown budget enforcement
|
||||
|
||||
[Fact]
|
||||
public void UnknownsBudget_FailsVerdict_WhenExceeded()
|
||||
{
|
||||
// Arrange
|
||||
var policy = CreatePolicyWithUnknownsBudget(maxUnknowns: 3);
|
||||
var evaluator = CreateEvaluator();
|
||||
|
||||
// Input with 5 unknown findings (exceeds budget of 3)
|
||||
var findings = Enumerable.Range(1, 5)
|
||||
.Select(i => CreateUnknownFinding($"CVE-2024-{i:D4}"))
|
||||
.ToArray();
|
||||
|
||||
var input = CreateTestInputWithFindings(findings);
|
||||
|
||||
// Act
|
||||
var result = evaluator.Evaluate(policy, input);
|
||||
|
||||
// Assert
|
||||
result.Status.Should().Be(VerdictStatus.Failed);
|
||||
result.Violations.Should().Contain(v =>
|
||||
v.Code == "UNKNOWNS_BUDGET_EXCEEDED" ||
|
||||
v.Message.Contains("unknowns", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UnknownsBudget_PassesVerdict_WhenWithinLimit()
|
||||
{
|
||||
// Arrange
|
||||
var policy = CreatePolicyWithUnknownsBudget(maxUnknowns: 10);
|
||||
var evaluator = CreateEvaluator();
|
||||
|
||||
// Input with 3 unknown findings (within budget of 10)
|
||||
var findings = Enumerable.Range(1, 3)
|
||||
.Select(i => CreateUnknownFinding($"CVE-2024-{i:D4}"))
|
||||
.ToArray();
|
||||
|
||||
var input = CreateTestInputWithFindings(findings);
|
||||
|
||||
// Act
|
||||
var result = evaluator.Evaluate(policy, input);
|
||||
|
||||
// Assert - should not fail due to unknowns budget
|
||||
result.Violations.Should().NotContain(v =>
|
||||
v.Code == "UNKNOWNS_BUDGET_EXCEEDED");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UnknownsBudget_CountsOnlyUnknownSeverity()
|
||||
{
|
||||
// Arrange
|
||||
var policy = CreatePolicyWithUnknownsBudget(maxUnknowns: 2);
|
||||
var evaluator = CreateEvaluator();
|
||||
|
||||
// Mix of known and unknown severities
|
||||
var findings = new[]
|
||||
{
|
||||
CreateFinding("CVE-2024-0001", "high"), // known
|
||||
CreateFinding("CVE-2024-0002", "medium"), // known
|
||||
CreateUnknownFinding("CVE-2024-0003"), // unknown
|
||||
CreateFinding("CVE-2024-0004", "low"), // known
|
||||
CreateUnknownFinding("CVE-2024-0005"), // unknown
|
||||
CreateFinding("CVE-2024-0006", "critical") // known
|
||||
};
|
||||
|
||||
var input = CreateTestInputWithFindings(findings);
|
||||
|
||||
// Act
|
||||
var result = evaluator.Evaluate(policy, input);
|
||||
|
||||
// Assert - should pass (only 2 unknowns, exactly at budget)
|
||||
result.Violations.Should().NotContain(v =>
|
||||
v.Code == "UNKNOWNS_BUDGET_EXCEEDED");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UnknownsBudget_ZeroBudget_FailsOnAnyUnknown()
|
||||
{
|
||||
// Arrange
|
||||
var policy = CreatePolicyWithUnknownsBudget(maxUnknowns: 0);
|
||||
var evaluator = CreateEvaluator();
|
||||
|
||||
// Single unknown finding
|
||||
var findings = new[] { CreateUnknownFinding("CVE-2024-0001") };
|
||||
var input = CreateTestInputWithFindings(findings);
|
||||
|
||||
// Act
|
||||
var result = evaluator.Evaluate(policy, input);
|
||||
|
||||
// Assert
|
||||
result.Status.Should().Be(VerdictStatus.Failed);
|
||||
result.Violations.Should().Contain(v =>
|
||||
v.Code == "UNKNOWNS_BUDGET_EXCEEDED" ||
|
||||
v.Message.Contains("unknowns", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UnknownsBudget_PerSeverity_EnforcedCorrectly()
|
||||
{
|
||||
// Arrange - budget per severity level
|
||||
var policy = CreatePolicyWithPerSeverityUnknownsBudget(
|
||||
criticalMax: 0,
|
||||
highMax: 1,
|
||||
mediumMax: 5,
|
||||
lowMax: 10);
|
||||
var evaluator = CreateEvaluator();
|
||||
|
||||
// 2 high-unknown findings (exceeds high budget of 1)
|
||||
var findings = new[]
|
||||
{
|
||||
CreateFindingWithUnknownImpact("CVE-2024-0001", baseLevel: "high"),
|
||||
CreateFindingWithUnknownImpact("CVE-2024-0002", baseLevel: "high")
|
||||
};
|
||||
|
||||
var input = CreateTestInputWithFindings(findings);
|
||||
|
||||
// Act
|
||||
var result = evaluator.Evaluate(policy, input);
|
||||
|
||||
// Assert
|
||||
result.Violations.Should().Contain(v =>
|
||||
v.Code.Contains("BUDGET", StringComparison.OrdinalIgnoreCase) ||
|
||||
v.Message.Contains("budget", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UnknownsBudget_ReportedInVerdictArtifact()
|
||||
{
|
||||
// Arrange
|
||||
var policy = CreatePolicyWithUnknownsBudget(maxUnknowns: 5);
|
||||
var evaluator = CreateEvaluator();
|
||||
|
||||
var findings = Enumerable.Range(1, 3)
|
||||
.Select(i => CreateUnknownFinding($"CVE-2024-{i:D4}"))
|
||||
.ToArray();
|
||||
|
||||
var input = CreateTestInputWithFindings(findings);
|
||||
|
||||
// Act
|
||||
var result = evaluator.Evaluate(policy, input);
|
||||
|
||||
// Assert - verdict should include unknowns count
|
||||
result.Metrics.Should().ContainKey("unknowns_count");
|
||||
result.Metrics["unknowns_count"].Should().Be(3);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Test Helpers
|
||||
|
||||
private static PolicyEvaluator CreateEvaluator()
|
||||
{
|
||||
return new PolicyEvaluator(new PolicyEvaluatorOptions
|
||||
{
|
||||
DeterministicMode = true
|
||||
});
|
||||
}
|
||||
|
||||
private static PolicyDefinition CreateTestPolicy()
|
||||
{
|
||||
return new PolicyDefinition
|
||||
{
|
||||
Id = "test-policy",
|
||||
Version = "1.0.0",
|
||||
Rules =
|
||||
[
|
||||
new PolicyRule
|
||||
{
|
||||
Name = "fail-on-critical",
|
||||
Priority = 1,
|
||||
Condition = "severity == 'critical'",
|
||||
Action = PolicyAction.Block,
|
||||
Reason = "Critical vulnerabilities must be addressed"
|
||||
}
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
private static PolicyDefinition CreateTestPolicyWithVex()
|
||||
{
|
||||
return new PolicyDefinition
|
||||
{
|
||||
Id = "test-policy-vex",
|
||||
Version = "1.0.0",
|
||||
VexEnabled = true,
|
||||
Rules =
|
||||
[
|
||||
new PolicyRule
|
||||
{
|
||||
Name = "respect-vex",
|
||||
Priority = 1,
|
||||
Condition = "vex.status != 'not_affected'",
|
||||
Action = PolicyAction.Evaluate,
|
||||
Reason = "Apply VEX status"
|
||||
}
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
private static PolicyDefinition CreatePolicyWithUnknownsBudget(int maxUnknowns)
|
||||
{
|
||||
return new PolicyDefinition
|
||||
{
|
||||
Id = "test-policy-unknowns",
|
||||
Version = "1.0.0",
|
||||
UnknownsBudget = new UnknownsBudget
|
||||
{
|
||||
MaxTotal = maxUnknowns,
|
||||
FailOnExceed = true
|
||||
},
|
||||
Rules = []
|
||||
};
|
||||
}
|
||||
|
||||
private static PolicyDefinition CreatePolicyWithPerSeverityUnknownsBudget(
|
||||
int criticalMax, int highMax, int mediumMax, int lowMax)
|
||||
{
|
||||
return new PolicyDefinition
|
||||
{
|
||||
Id = "test-policy-unknowns-severity",
|
||||
Version = "1.0.0",
|
||||
UnknownsBudget = new UnknownsBudget
|
||||
{
|
||||
MaxTotal = criticalMax + highMax + mediumMax + lowMax,
|
||||
PerSeverity = new Dictionary<string, int>
|
||||
{
|
||||
["critical"] = criticalMax,
|
||||
["high"] = highMax,
|
||||
["medium"] = mediumMax,
|
||||
["low"] = lowMax
|
||||
},
|
||||
FailOnExceed = true
|
||||
},
|
||||
Rules = []
|
||||
};
|
||||
}
|
||||
|
||||
private static EvaluationInput CreateTestInput()
|
||||
{
|
||||
return new EvaluationInput
|
||||
{
|
||||
ArtifactDigest = "sha256:abc123",
|
||||
Findings =
|
||||
[
|
||||
CreateFinding("CVE-2024-0001", "high"),
|
||||
CreateFinding("CVE-2024-0002", "medium")
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
private static EvaluationInput CreateTestInputWithFindings(Finding[] findings)
|
||||
{
|
||||
return new EvaluationInput
|
||||
{
|
||||
ArtifactDigest = "sha256:abc123",
|
||||
Findings = findings.ToList()
|
||||
};
|
||||
}
|
||||
|
||||
private static EvaluationInput CreateTestInputWithVex(VexStatement[] statements)
|
||||
{
|
||||
return new EvaluationInput
|
||||
{
|
||||
ArtifactDigest = "sha256:abc123",
|
||||
Findings =
|
||||
[
|
||||
CreateFinding("CVE-2024-0001", "high"),
|
||||
CreateFinding("CVE-2024-0002", "medium")
|
||||
],
|
||||
VexStatements = statements.ToList()
|
||||
};
|
||||
}
|
||||
|
||||
private static Finding CreateFinding(string id, string severity)
|
||||
{
|
||||
return new Finding
|
||||
{
|
||||
VulnerabilityId = id,
|
||||
Severity = severity,
|
||||
Package = new PackageRef { Purl = $"pkg:npm/test@1.0.0" }
|
||||
};
|
||||
}
|
||||
|
||||
private static Finding CreateUnknownFinding(string id)
|
||||
{
|
||||
return new Finding
|
||||
{
|
||||
VulnerabilityId = id,
|
||||
Severity = "unknown",
|
||||
Package = new PackageRef { Purl = $"pkg:npm/test@1.0.0" }
|
||||
};
|
||||
}
|
||||
|
||||
private static Finding CreateFindingWithUnknownImpact(string id, string baseLevel)
|
||||
{
|
||||
return new Finding
|
||||
{
|
||||
VulnerabilityId = id,
|
||||
Severity = baseLevel,
|
||||
ImpactUnknown = true,
|
||||
Package = new PackageRef { Purl = $"pkg:npm/test@1.0.0" }
|
||||
};
|
||||
}
|
||||
|
||||
private static VexStatement CreateVexStatement(string vulnerabilityId, VexStatus status)
|
||||
{
|
||||
return new VexStatement
|
||||
{
|
||||
VulnerabilityId = vulnerabilityId,
|
||||
Status = status,
|
||||
Justification = "Test justification"
|
||||
};
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
#region Supporting Types (stubs for compilation)
|
||||
|
||||
// These types should be replaced with actual project references
|
||||
// They are placeholders for the test structure
|
||||
|
||||
public enum VerdictStatus { Passed, Failed, Warning }
|
||||
|
||||
public record PolicyEvaluatorOptions
|
||||
{
|
||||
public bool DeterministicMode { get; init; }
|
||||
}
|
||||
|
||||
public class PolicyEvaluator(PolicyEvaluatorOptions options)
|
||||
{
|
||||
public EvaluationResult Evaluate(PolicyDefinition policy, EvaluationInput input)
|
||||
{
|
||||
// Stub implementation - actual implementation would come from Policy.Engine
|
||||
var violations = new List<Violation>();
|
||||
var metrics = new Dictionary<string, int>();
|
||||
|
||||
// Count unknowns
|
||||
var unknownsCount = input.Findings.Count(f => f.Severity == "unknown" || f.ImpactUnknown);
|
||||
metrics["unknowns_count"] = unknownsCount;
|
||||
|
||||
// Check unknowns budget
|
||||
if (policy.UnknownsBudget is not null && policy.UnknownsBudget.FailOnExceed)
|
||||
{
|
||||
if (unknownsCount > policy.UnknownsBudget.MaxTotal)
|
||||
{
|
||||
violations.Add(new Violation
|
||||
{
|
||||
Code = "UNKNOWNS_BUDGET_EXCEEDED",
|
||||
Message = $"Unknowns count {unknownsCount} exceeds budget {policy.UnknownsBudget.MaxTotal}"
|
||||
});
|
||||
}
|
||||
|
||||
if (policy.UnknownsBudget.PerSeverity is not null)
|
||||
{
|
||||
foreach (var (severity, max) in policy.UnknownsBudget.PerSeverity)
|
||||
{
|
||||
var count = input.Findings.Count(f =>
|
||||
f.Severity == severity && f.ImpactUnknown);
|
||||
if (count > max)
|
||||
{
|
||||
violations.Add(new Violation
|
||||
{
|
||||
Code = "UNKNOWNS_BUDGET_EXCEEDED",
|
||||
Message = $"{severity} unknowns count {count} exceeds budget {max}"
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Deterministic hash calculation
|
||||
var hash = ComputeDeterministicHash(policy, input);
|
||||
|
||||
return new EvaluationResult
|
||||
{
|
||||
Status = violations.Count > 0 ? VerdictStatus.Failed : VerdictStatus.Passed,
|
||||
VerdictHash = hash,
|
||||
Violations = violations,
|
||||
Metrics = metrics
|
||||
};
|
||||
}
|
||||
|
||||
private static string ComputeDeterministicHash(PolicyDefinition policy, EvaluationInput input)
|
||||
{
|
||||
// Sort findings for determinism
|
||||
var sortedFindings = input.Findings
|
||||
.OrderBy(f => f.VulnerabilityId)
|
||||
.ThenBy(f => f.Package.Purl)
|
||||
.ToList();
|
||||
|
||||
var hashInput = $"{policy.Id}:{policy.Version}:{string.Join(",", sortedFindings.Select(f => $"{f.VulnerabilityId}:{f.Severity}"))}";
|
||||
using var sha = System.Security.Cryptography.SHA256.Create();
|
||||
var bytes = sha.ComputeHash(System.Text.Encoding.UTF8.GetBytes(hashInput));
|
||||
return Convert.ToHexString(bytes).ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
|
||||
public record EvaluationResult
|
||||
{
|
||||
public VerdictStatus Status { get; init; }
|
||||
public string VerdictHash { get; init; } = string.Empty;
|
||||
public List<Violation> Violations { get; init; } = [];
|
||||
public Dictionary<string, int> Metrics { get; init; } = [];
|
||||
|
||||
public string ToCanonicalJson()
|
||||
{
|
||||
return System.Text.Json.JsonSerializer.Serialize(this, new System.Text.Json.JsonSerializerOptions
|
||||
{
|
||||
WriteIndented = false,
|
||||
PropertyNamingPolicy = System.Text.Json.JsonNamingPolicy.SnakeCaseLower
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public record PolicyDefinition
|
||||
{
|
||||
public string Id { get; init; } = string.Empty;
|
||||
public string Version { get; init; } = string.Empty;
|
||||
public bool VexEnabled { get; init; }
|
||||
public UnknownsBudget? UnknownsBudget { get; init; }
|
||||
public List<PolicyRule> Rules { get; init; } = [];
|
||||
}
|
||||
|
||||
public record PolicyRule
|
||||
{
|
||||
public string Name { get; init; } = string.Empty;
|
||||
public int Priority { get; init; }
|
||||
public string Condition { get; init; } = string.Empty;
|
||||
public PolicyAction Action { get; init; }
|
||||
public string Reason { get; init; } = string.Empty;
|
||||
}
|
||||
|
||||
public enum PolicyAction { Evaluate, Block, Warn, Allow }
|
||||
|
||||
public record UnknownsBudget
|
||||
{
|
||||
public int MaxTotal { get; init; }
|
||||
public bool FailOnExceed { get; init; }
|
||||
public Dictionary<string, int>? PerSeverity { get; init; }
|
||||
}
|
||||
|
||||
public record EvaluationInput
|
||||
{
|
||||
public string ArtifactDigest { get; init; } = string.Empty;
|
||||
public List<Finding> Findings { get; init; } = [];
|
||||
public List<VexStatement> VexStatements { get; init; } = [];
|
||||
}
|
||||
|
||||
public record Finding
|
||||
{
|
||||
public string VulnerabilityId { get; init; } = string.Empty;
|
||||
public string Severity { get; init; } = string.Empty;
|
||||
public bool ImpactUnknown { get; init; }
|
||||
public PackageRef Package { get; init; } = new();
|
||||
}
|
||||
|
||||
public record PackageRef
|
||||
{
|
||||
public string Purl { get; init; } = string.Empty;
|
||||
}
|
||||
|
||||
public record VexStatement
|
||||
{
|
||||
public string VulnerabilityId { get; init; } = string.Empty;
|
||||
public VexStatus Status { get; init; }
|
||||
public string Justification { get; init; } = string.Empty;
|
||||
}
|
||||
|
||||
public enum VexStatus { Unknown, NotAffected, Affected, Fixed, UnderInvestigation }
|
||||
|
||||
public record Violation
|
||||
{
|
||||
public string Code { get; init; } = string.Empty;
|
||||
public string Message { get; init; } = string.Empty;
|
||||
}
|
||||
|
||||
#endregion
|
||||
@@ -0,0 +1,551 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
// SPDX-FileCopyrightText: 2025 StellaOps Contributors
|
||||
|
||||
using FluentAssertions;
|
||||
using FsCheck;
|
||||
using FsCheck.Xunit;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Lattice;
|
||||
|
||||
namespace StellaOps.Policy.Engine.Tests.Properties;
|
||||
|
||||
/// <summary>
|
||||
/// Property-based tests for VEX lattice merge semantics.
|
||||
/// Verifies that join/meet operations satisfy lattice algebraic properties.
|
||||
/// </summary>
|
||||
public sealed class VexLatticeMergePropertyTests
|
||||
{
|
||||
private readonly IVexLatticeProvider _lattice;
|
||||
|
||||
public VexLatticeMergePropertyTests()
|
||||
{
|
||||
// Use the default K4 lattice provider
|
||||
_lattice = new K4VexLatticeProvider(NullLogger<K4VexLatticeProvider>.Instance);
|
||||
}
|
||||
|
||||
#region Join Properties (Least Upper Bound)
|
||||
|
||||
/// <summary>
|
||||
/// Property: Join is commutative - Join(a, b) = Join(b, a).
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property Join_IsCommutative()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
VexLatticeArbs.AnyVexClaim(),
|
||||
VexLatticeArbs.AnyVexClaim(),
|
||||
(a, b) =>
|
||||
{
|
||||
var joinAB = _lattice.Join(a, b);
|
||||
var joinBA = _lattice.Join(b, a);
|
||||
|
||||
return (joinAB.ResultStatus == joinBA.ResultStatus)
|
||||
.Label($"Join({a.Status}, {b.Status}) = {joinAB.ResultStatus}, Join({b.Status}, {a.Status}) = {joinBA.ResultStatus}");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Join is idempotent - Join(a, a) = a.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property Join_IsIdempotent()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
VexLatticeArbs.AnyVexClaim(),
|
||||
a =>
|
||||
{
|
||||
var result = _lattice.Join(a, a);
|
||||
|
||||
return (result.ResultStatus == a.Status)
|
||||
.Label($"Join({a.Status}, {a.Status}) = {result.ResultStatus}, expected {a.Status}");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Join with bottom (unknown) yields the other element - Join(a, unknown) = a.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property Join_WithBottom_YieldsOther()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
VexLatticeArbs.AnyVexClaim(),
|
||||
a =>
|
||||
{
|
||||
var bottom = VexLatticeArbs.CreateClaim(VexClaimStatus.Unknown);
|
||||
var result = _lattice.Join(a, bottom);
|
||||
|
||||
// Join with bottom should yield the non-bottom element (or bottom if both are bottom)
|
||||
var expected = a.Status == VexClaimStatus.Unknown ? VexClaimStatus.Unknown : a.Status;
|
||||
|
||||
return (result.ResultStatus == expected)
|
||||
.Label($"Join({a.Status}, Unknown) = {result.ResultStatus}, expected {expected}");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Join with top (affected) yields top - Join(a, affected) = affected.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property Join_WithTop_YieldsTop()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
VexLatticeArbs.AnyVexClaim(),
|
||||
a =>
|
||||
{
|
||||
var top = VexLatticeArbs.CreateClaim(VexClaimStatus.Affected);
|
||||
var result = _lattice.Join(a, top);
|
||||
|
||||
return (result.ResultStatus == VexClaimStatus.Affected)
|
||||
.Label($"Join({a.Status}, Affected) = {result.ResultStatus}, expected Affected");
|
||||
});
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Meet Properties (Greatest Lower Bound)
|
||||
|
||||
/// <summary>
|
||||
/// Property: Meet is commutative - Meet(a, b) = Meet(b, a).
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property Meet_IsCommutative()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
VexLatticeArbs.AnyVexClaim(),
|
||||
VexLatticeArbs.AnyVexClaim(),
|
||||
(a, b) =>
|
||||
{
|
||||
var meetAB = _lattice.Meet(a, b);
|
||||
var meetBA = _lattice.Meet(b, a);
|
||||
|
||||
return (meetAB.ResultStatus == meetBA.ResultStatus)
|
||||
.Label($"Meet({a.Status}, {b.Status}) = {meetAB.ResultStatus}, Meet({b.Status}, {a.Status}) = {meetBA.ResultStatus}");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Meet is idempotent - Meet(a, a) = a.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property Meet_IsIdempotent()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
VexLatticeArbs.AnyVexClaim(),
|
||||
a =>
|
||||
{
|
||||
var result = _lattice.Meet(a, a);
|
||||
|
||||
return (result.ResultStatus == a.Status)
|
||||
.Label($"Meet({a.Status}, {a.Status}) = {result.ResultStatus}, expected {a.Status}");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Meet with bottom (unknown) yields bottom - Meet(a, unknown) = unknown.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property Meet_WithBottom_YieldsBottom()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
VexLatticeArbs.AnyVexClaim(),
|
||||
a =>
|
||||
{
|
||||
var bottom = VexLatticeArbs.CreateClaim(VexClaimStatus.Unknown);
|
||||
var result = _lattice.Meet(a, bottom);
|
||||
|
||||
return (result.ResultStatus == VexClaimStatus.Unknown)
|
||||
.Label($"Meet({a.Status}, Unknown) = {result.ResultStatus}, expected Unknown");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Meet with top (affected) yields the other element - Meet(a, affected) = a.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property Meet_WithTop_YieldsOther()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
VexLatticeArbs.AnyVexClaim(),
|
||||
a =>
|
||||
{
|
||||
var top = VexLatticeArbs.CreateClaim(VexClaimStatus.Affected);
|
||||
var result = _lattice.Meet(a, top);
|
||||
|
||||
var expected = a.Status == VexClaimStatus.Affected ? VexClaimStatus.Affected : a.Status;
|
||||
|
||||
return (result.ResultStatus == expected)
|
||||
.Label($"Meet({a.Status}, Affected) = {result.ResultStatus}, expected {expected}");
|
||||
});
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Absorption Laws
|
||||
|
||||
/// <summary>
|
||||
/// Property: Absorption law 1 - Join(a, Meet(a, b)) = a.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property Absorption_JoinMeet()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
VexLatticeArbs.AnyVexClaim(),
|
||||
VexLatticeArbs.AnyVexClaim(),
|
||||
(a, b) =>
|
||||
{
|
||||
var meet = _lattice.Meet(a, b);
|
||||
var meetClaim = VexLatticeArbs.CreateClaim(meet.ResultStatus);
|
||||
var result = _lattice.Join(a, meetClaim);
|
||||
|
||||
return (result.ResultStatus == a.Status)
|
||||
.Label($"Join({a.Status}, Meet({a.Status}, {b.Status})) = {result.ResultStatus}, expected {a.Status}");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Absorption law 2 - Meet(a, Join(a, b)) = a.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property Absorption_MeetJoin()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
VexLatticeArbs.AnyVexClaim(),
|
||||
VexLatticeArbs.AnyVexClaim(),
|
||||
(a, b) =>
|
||||
{
|
||||
var join = _lattice.Join(a, b);
|
||||
var joinClaim = VexLatticeArbs.CreateClaim(join.ResultStatus);
|
||||
var result = _lattice.Meet(a, joinClaim);
|
||||
|
||||
return (result.ResultStatus == a.Status)
|
||||
.Label($"Meet({a.Status}, Join({a.Status}, {b.Status})) = {result.ResultStatus}, expected {a.Status}");
|
||||
});
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region IsHigher / Ordering Properties
|
||||
|
||||
/// <summary>
|
||||
/// Property: IsHigher is antisymmetric - if a > b and b > a then a = b.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property IsHigher_IsAntisymmetric()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
VexLatticeArbs.AnyVexClaimStatus(),
|
||||
VexLatticeArbs.AnyVexClaimStatus(),
|
||||
(a, b) =>
|
||||
{
|
||||
var aHigherB = _lattice.IsHigher(a, b);
|
||||
var bHigherA = _lattice.IsHigher(b, a);
|
||||
|
||||
// If both are true, they must be equal
|
||||
return (!(aHigherB && bHigherA) || a == b)
|
||||
.Label($"IsHigher({a}, {b}) = {aHigherB}, IsHigher({b}, {a}) = {bHigherA}");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: IsHigher is reflexive for equality - IsHigher(a, a) is well-defined.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property IsHigher_IsReflexive()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
VexLatticeArbs.AnyVexClaimStatus(),
|
||||
a =>
|
||||
{
|
||||
var result = _lattice.IsHigher(a, a);
|
||||
|
||||
// Same status should not be "higher" than itself
|
||||
return (!result)
|
||||
.Label($"IsHigher({a}, {a}) = {result}, expected false");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Top element (Affected) is higher than all non-top elements.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property Top_IsHigherThanAllNonTop()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
VexLatticeArbs.AnyVexClaimStatus(),
|
||||
a =>
|
||||
{
|
||||
if (a == VexClaimStatus.Affected)
|
||||
return true.Label("Skip: comparing top with itself");
|
||||
|
||||
var result = _lattice.IsHigher(VexClaimStatus.Affected, a);
|
||||
|
||||
return result
|
||||
.Label($"IsHigher(Affected, {a}) = {result}, expected true");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Bottom element (Unknown) is not higher than any element.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property Bottom_IsNotHigherThanAnything()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
VexLatticeArbs.AnyVexClaimStatus(),
|
||||
a =>
|
||||
{
|
||||
if (a == VexClaimStatus.Unknown)
|
||||
return true.Label("Skip: comparing bottom with itself");
|
||||
|
||||
var result = _lattice.IsHigher(VexClaimStatus.Unknown, a);
|
||||
|
||||
return (!result)
|
||||
.Label($"IsHigher(Unknown, {a}) = {result}, expected false");
|
||||
});
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Conflict Resolution Properties
|
||||
|
||||
/// <summary>
|
||||
/// Property: Conflict resolution always produces a valid winner.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property ConflictResolution_ProducesValidWinner()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
VexLatticeArbs.AnyVexClaim(),
|
||||
VexLatticeArbs.AnyVexClaim(),
|
||||
(a, b) =>
|
||||
{
|
||||
var resolution = _lattice.ResolveConflict(a, b);
|
||||
|
||||
// Winner must be one of the inputs
|
||||
var winnerIsValid = resolution.Winner.Status == a.Status || resolution.Winner.Status == b.Status;
|
||||
|
||||
return winnerIsValid
|
||||
.Label($"Winner status {resolution.Winner.Status} must be {a.Status} or {b.Status}");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Conflict resolution is deterministic.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property ConflictResolution_IsDeterministic()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
VexLatticeArbs.AnyVexClaim(),
|
||||
VexLatticeArbs.AnyVexClaim(),
|
||||
(a, b) =>
|
||||
{
|
||||
var resolution1 = _lattice.ResolveConflict(a, b);
|
||||
var resolution2 = _lattice.ResolveConflict(a, b);
|
||||
|
||||
return (resolution1.Winner.Status == resolution2.Winner.Status &&
|
||||
resolution1.Reason == resolution2.Reason)
|
||||
.Label($"Determinism: {resolution1.Winner.Status}/{resolution1.Reason} vs {resolution2.Winner.Status}/{resolution2.Reason}");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Higher trust weight wins in conflict.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property ConflictResolution_HigherTrustWins()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
VexLatticeArbs.AnyVexClaim(),
|
||||
VexLatticeArbs.AnyVexClaim(),
|
||||
(a, b) =>
|
||||
{
|
||||
var trustA = _lattice.GetTrustWeight(a);
|
||||
var trustB = _lattice.GetTrustWeight(b);
|
||||
var resolution = _lattice.ResolveConflict(a, b);
|
||||
|
||||
// If trust weights differ significantly, higher should win
|
||||
if (Math.Abs(trustA - trustB) > 0.01m)
|
||||
{
|
||||
var expectedWinner = trustA > trustB ? a : b;
|
||||
return (resolution.Winner.Status == expectedWinner.Status ||
|
||||
resolution.Reason != ConflictResolutionReason.TrustWeight)
|
||||
.Label($"Trust: A={trustA}, B={trustB}, Winner={resolution.Winner.Status}");
|
||||
}
|
||||
|
||||
// Otherwise, any result is acceptable
|
||||
return true.Label("Trust weights equal, any result acceptable");
|
||||
});
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Custom FsCheck arbitraries for VEX lattice types.
|
||||
/// </summary>
|
||||
internal static class VexLatticeArbs
|
||||
{
|
||||
private static readonly VexClaimStatus[] AllStatuses =
|
||||
[
|
||||
VexClaimStatus.Unknown,
|
||||
VexClaimStatus.NotAffected,
|
||||
VexClaimStatus.Fixed,
|
||||
VexClaimStatus.UnderInvestigation,
|
||||
VexClaimStatus.Affected
|
||||
];
|
||||
|
||||
public static Arbitrary<VexClaimStatus> AnyVexClaimStatus() =>
|
||||
Arb.From(Gen.Elements(AllStatuses));
|
||||
|
||||
public static Arbitrary<VexClaim> AnyVexClaim() =>
|
||||
Arb.From(
|
||||
from status in Gen.Elements(AllStatuses)
|
||||
from providerId in Gen.Elements("vendor", "maintainer", "third-party", "scanner")
|
||||
from dayOffset in Gen.Choose(0, 365)
|
||||
select CreateClaim(status, providerId, DateTime.UtcNow.AddDays(-dayOffset)));
|
||||
|
||||
public static VexClaim CreateClaim(
|
||||
VexClaimStatus status,
|
||||
string providerId = "test-provider",
|
||||
DateTime? lastSeen = null)
|
||||
{
|
||||
var now = lastSeen ?? DateTime.UtcNow;
|
||||
return new VexClaim
|
||||
{
|
||||
VulnerabilityId = "CVE-2024-0001",
|
||||
Status = status,
|
||||
ProviderId = providerId,
|
||||
Product = new VexProduct
|
||||
{
|
||||
Key = "test-product",
|
||||
Name = "Test Product",
|
||||
Version = "1.0.0"
|
||||
},
|
||||
Document = new VexDocumentSource
|
||||
{
|
||||
SourceUri = new Uri($"https://example.com/vex/{Guid.NewGuid()}"),
|
||||
Digest = $"sha256:{Guid.NewGuid():N}",
|
||||
Format = VexFormat.OpenVex
|
||||
},
|
||||
FirstSeen = now.AddDays(-30),
|
||||
LastSeen = now
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Default K4 lattice provider for testing.
|
||||
/// The K4 lattice: Unknown < {NotAffected, Fixed, UnderInvestigation} < Affected
|
||||
/// </summary>
|
||||
internal sealed class K4VexLatticeProvider : IVexLatticeProvider
|
||||
{
|
||||
private readonly ILogger<K4VexLatticeProvider> _logger;
|
||||
|
||||
// K4 lattice ordering (higher value = higher in lattice)
|
||||
private static readonly Dictionary<VexClaimStatus, int> LatticeOrder = new()
|
||||
{
|
||||
[VexClaimStatus.Unknown] = 0,
|
||||
[VexClaimStatus.NotAffected] = 1,
|
||||
[VexClaimStatus.Fixed] = 1,
|
||||
[VexClaimStatus.UnderInvestigation] = 1,
|
||||
[VexClaimStatus.Affected] = 2
|
||||
};
|
||||
|
||||
// Trust weights by provider type
|
||||
private static readonly Dictionary<string, decimal> TrustWeights = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["vendor"] = 1.0m,
|
||||
["maintainer"] = 0.9m,
|
||||
["third-party"] = 0.7m,
|
||||
["scanner"] = 0.5m
|
||||
};
|
||||
|
||||
public K4VexLatticeProvider(ILogger<K4VexLatticeProvider> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public VexLatticeResult Join(VexClaim left, VexClaim right)
|
||||
{
|
||||
var leftOrder = LatticeOrder.GetValueOrDefault(left.Status, 0);
|
||||
var rightOrder = LatticeOrder.GetValueOrDefault(right.Status, 0);
|
||||
|
||||
if (leftOrder >= rightOrder)
|
||||
{
|
||||
return new VexLatticeResult(left.Status, left, "Left is higher or equal in lattice", null);
|
||||
}
|
||||
|
||||
return new VexLatticeResult(right.Status, right, "Right is higher in lattice", null);
|
||||
}
|
||||
|
||||
public VexLatticeResult Meet(VexClaim left, VexClaim right)
|
||||
{
|
||||
var leftOrder = LatticeOrder.GetValueOrDefault(left.Status, 0);
|
||||
var rightOrder = LatticeOrder.GetValueOrDefault(right.Status, 0);
|
||||
|
||||
if (leftOrder <= rightOrder)
|
||||
{
|
||||
return new VexLatticeResult(left.Status, left, "Left is lower or equal in lattice", null);
|
||||
}
|
||||
|
||||
return new VexLatticeResult(right.Status, right, "Right is lower in lattice", null);
|
||||
}
|
||||
|
||||
public bool IsHigher(VexClaimStatus a, VexClaimStatus b)
|
||||
{
|
||||
var aOrder = LatticeOrder.GetValueOrDefault(a, 0);
|
||||
var bOrder = LatticeOrder.GetValueOrDefault(b, 0);
|
||||
return aOrder > bOrder;
|
||||
}
|
||||
|
||||
public decimal GetTrustWeight(VexClaim statement)
|
||||
{
|
||||
return TrustWeights.GetValueOrDefault(statement.ProviderId, 0.5m);
|
||||
}
|
||||
|
||||
public VexConflictResolution ResolveConflict(VexClaim left, VexClaim right)
|
||||
{
|
||||
var leftTrust = GetTrustWeight(left);
|
||||
var rightTrust = GetTrustWeight(right);
|
||||
|
||||
ConflictResolutionReason reason;
|
||||
VexClaim winner;
|
||||
VexClaim loser;
|
||||
|
||||
if (Math.Abs(leftTrust - rightTrust) > 0.01m)
|
||||
{
|
||||
winner = leftTrust > rightTrust ? left : right;
|
||||
loser = leftTrust > rightTrust ? right : left;
|
||||
reason = ConflictResolutionReason.TrustWeight;
|
||||
}
|
||||
else if (left.LastSeen != right.LastSeen)
|
||||
{
|
||||
winner = left.LastSeen > right.LastSeen ? left : right;
|
||||
loser = left.LastSeen > right.LastSeen ? right : left;
|
||||
reason = ConflictResolutionReason.Freshness;
|
||||
}
|
||||
else
|
||||
{
|
||||
var leftOrder = LatticeOrder.GetValueOrDefault(left.Status, 0);
|
||||
var rightOrder = LatticeOrder.GetValueOrDefault(right.Status, 0);
|
||||
winner = leftOrder >= rightOrder ? left : right;
|
||||
loser = leftOrder >= rightOrder ? right : left;
|
||||
reason = leftOrder != rightOrder ? ConflictResolutionReason.LatticePosition : ConflictResolutionReason.Tie;
|
||||
}
|
||||
|
||||
return new VexConflictResolution(winner, loser, reason, new MergeTrace
|
||||
{
|
||||
LeftSource = left.ProviderId,
|
||||
RightSource = right.ProviderId,
|
||||
LeftStatus = left.Status,
|
||||
RightStatus = right.Status,
|
||||
LeftTrust = leftTrust,
|
||||
RightTrust = rightTrust,
|
||||
ResultStatus = winner.Status,
|
||||
Explanation = $"Resolved by {reason}: {winner.Status} from {winner.ProviderId}",
|
||||
EvaluatedAt = DateTimeOffset.UtcNow
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,535 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
// SPDX-FileCopyrightText: 2025 StellaOps Contributors
|
||||
|
||||
using FluentAssertions;
|
||||
using StellaOps.TestKit.Assertions;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Policy.Engine.Tests.Snapshots;
|
||||
|
||||
/// <summary>
|
||||
/// Snapshot tests for policy evaluation trace summaries.
|
||||
/// Ensures evaluation traces have stable structure for debugging and auditing.
|
||||
/// </summary>
|
||||
public sealed class PolicyEvaluationTraceSnapshotTests
|
||||
{
|
||||
private static readonly DateTimeOffset FrozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z");
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that a simple evaluation trace produces stable structure.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void SimpleEvaluationTrace_ProducesStableStructure()
|
||||
{
|
||||
// Arrange
|
||||
var trace = CreateSimpleEvaluationTrace();
|
||||
|
||||
// Act
|
||||
SnapshotAssert.MatchesSnapshot(trace, "SimpleEvaluationTrace");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that an evaluation trace with multiple rule evaluations produces stable structure.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void MultiRuleEvaluationTrace_ProducesStableStructure()
|
||||
{
|
||||
// Arrange
|
||||
var trace = CreateMultiRuleEvaluationTrace();
|
||||
|
||||
// Act
|
||||
SnapshotAssert.MatchesSnapshot(trace, "MultiRuleEvaluationTrace");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that an evaluation trace with VEX resolution produces stable structure.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void VexResolutionTrace_ProducesStableStructure()
|
||||
{
|
||||
// Arrange
|
||||
var trace = CreateVexResolutionTrace();
|
||||
|
||||
// Act
|
||||
SnapshotAssert.MatchesSnapshot(trace, "VexResolutionTrace");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that an evaluation trace with profile application produces stable structure.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void ProfileApplicationTrace_ProducesStableStructure()
|
||||
{
|
||||
// Arrange
|
||||
var trace = CreateProfileApplicationTrace();
|
||||
|
||||
// Act
|
||||
SnapshotAssert.MatchesSnapshot(trace, "ProfileApplicationTrace");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that an evaluation trace with severity escalation produces stable structure.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void SeverityEscalationTrace_ProducesStableStructure()
|
||||
{
|
||||
// Arrange
|
||||
var trace = CreateSeverityEscalationTrace();
|
||||
|
||||
// Act
|
||||
SnapshotAssert.MatchesSnapshot(trace, "SeverityEscalationTrace");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that evaluation trace steps are ordered by priority.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void EvaluationTrace_StepsOrderedByPriority()
|
||||
{
|
||||
// Arrange
|
||||
var trace = CreateMultiRuleEvaluationTrace();
|
||||
|
||||
// Assert
|
||||
var priorities = trace.Steps.Select(s => s.Priority).ToList();
|
||||
priorities.Should().BeInDescendingOrder("Steps should be ordered by priority (highest first)");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that evaluation trace includes timing information.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void EvaluationTrace_IncludesTimingInformation()
|
||||
{
|
||||
// Arrange
|
||||
var trace = CreateSimpleEvaluationTrace();
|
||||
|
||||
// Assert
|
||||
trace.StartedAt.Should().Be(FrozenTime);
|
||||
trace.CompletedAt.Should().BeAfter(trace.StartedAt);
|
||||
trace.DurationMs.Should().BeGreaterOrEqualTo(0);
|
||||
trace.Steps.Should().AllSatisfy(s => s.DurationMs.Should().BeGreaterOrEqualTo(0));
|
||||
}
|
||||
|
||||
#region Trace Factories
|
||||
|
||||
private static PolicyEvaluationTrace CreateSimpleEvaluationTrace()
|
||||
{
|
||||
return new PolicyEvaluationTrace
|
||||
{
|
||||
TraceId = "TRACE-2025-001",
|
||||
PolicyId = "POL-PROD-001",
|
||||
PolicyName = "Production Baseline Policy",
|
||||
EvaluationContext = new EvaluationContext
|
||||
{
|
||||
DigestEvaluated = "sha256:abc123def456",
|
||||
TenantId = "TENANT-001",
|
||||
Environment = "production",
|
||||
Exposure = "internet"
|
||||
},
|
||||
StartedAt = FrozenTime,
|
||||
CompletedAt = FrozenTime.AddMilliseconds(42),
|
||||
DurationMs = 42,
|
||||
Outcome = "Pass",
|
||||
Steps =
|
||||
[
|
||||
new EvaluationStep
|
||||
{
|
||||
StepNumber = 1,
|
||||
RuleName = "block_critical",
|
||||
Priority = 5,
|
||||
Phase = EvaluationPhase.RuleMatch,
|
||||
Condition = "severity.normalized >= \"Critical\"",
|
||||
ConditionResult = false,
|
||||
Action = null,
|
||||
Explanation = "No critical vulnerabilities found in scan",
|
||||
DurationMs = 15
|
||||
},
|
||||
new EvaluationStep
|
||||
{
|
||||
StepNumber = 2,
|
||||
RuleName = "allow_low_severity",
|
||||
Priority = 1,
|
||||
Phase = EvaluationPhase.RuleMatch,
|
||||
Condition = "severity.normalized <= \"Low\"",
|
||||
ConditionResult = true,
|
||||
Action = "status := \"allowed\"",
|
||||
Explanation = "All findings are Low severity or below",
|
||||
DurationMs = 12
|
||||
}
|
||||
],
|
||||
FinalStatus = "allowed",
|
||||
MatchedRuleCount = 1,
|
||||
TotalRuleCount = 2
|
||||
};
|
||||
}
|
||||
|
||||
private static PolicyEvaluationTrace CreateMultiRuleEvaluationTrace()
|
||||
{
|
||||
return new PolicyEvaluationTrace
|
||||
{
|
||||
TraceId = "TRACE-2025-002",
|
||||
PolicyId = "POL-COMPLEX-001",
|
||||
PolicyName = "Complex Multi-Rule Policy",
|
||||
EvaluationContext = new EvaluationContext
|
||||
{
|
||||
DigestEvaluated = "sha256:xyz789",
|
||||
TenantId = "TENANT-002",
|
||||
Environment = "staging",
|
||||
Exposure = "internal"
|
||||
},
|
||||
StartedAt = FrozenTime,
|
||||
CompletedAt = FrozenTime.AddMilliseconds(89),
|
||||
DurationMs = 89,
|
||||
Outcome = "Warn",
|
||||
Steps =
|
||||
[
|
||||
new EvaluationStep
|
||||
{
|
||||
StepNumber = 1,
|
||||
RuleName = "block_critical",
|
||||
Priority = 5,
|
||||
Phase = EvaluationPhase.RuleMatch,
|
||||
Condition = "severity.normalized >= \"Critical\"",
|
||||
ConditionResult = false,
|
||||
Action = null,
|
||||
Explanation = "No critical vulnerabilities",
|
||||
DurationMs = 10
|
||||
},
|
||||
new EvaluationStep
|
||||
{
|
||||
StepNumber = 2,
|
||||
RuleName = "escalate_high_internet",
|
||||
Priority = 4,
|
||||
Phase = EvaluationPhase.RuleMatch,
|
||||
Condition = "severity.normalized == \"High\" and env.exposure == \"internet\"",
|
||||
ConditionResult = false,
|
||||
Action = null,
|
||||
Explanation = "Not internet-exposed, skipping escalation",
|
||||
DurationMs = 8
|
||||
},
|
||||
new EvaluationStep
|
||||
{
|
||||
StepNumber = 3,
|
||||
RuleName = "require_vex_justification",
|
||||
Priority = 3,
|
||||
Phase = EvaluationPhase.RuleMatch,
|
||||
Condition = "vex.any(status in [\"not_affected\",\"fixed\"])",
|
||||
ConditionResult = true,
|
||||
Action = "status := vex.status",
|
||||
Explanation = "VEX statement found: not_affected (component_not_present)",
|
||||
DurationMs = 25
|
||||
},
|
||||
new EvaluationStep
|
||||
{
|
||||
StepNumber = 4,
|
||||
RuleName = "warn_eol_runtime",
|
||||
Priority = 1,
|
||||
Phase = EvaluationPhase.RuleMatch,
|
||||
Condition = "severity.normalized <= \"Medium\" and sbom.has_tag(\"runtime:eol\")",
|
||||
ConditionResult = true,
|
||||
Action = "warn message \"Runtime marked as EOL; upgrade recommended.\"",
|
||||
Explanation = "EOL runtime detected: python3.9",
|
||||
DurationMs = 15
|
||||
},
|
||||
new EvaluationStep
|
||||
{
|
||||
StepNumber = 5,
|
||||
RuleName = "block_ruby_dev",
|
||||
Priority = 4,
|
||||
Phase = EvaluationPhase.RuleMatch,
|
||||
Condition = "sbom.any_component(ruby.group(\"development\"))",
|
||||
ConditionResult = false,
|
||||
Action = null,
|
||||
Explanation = "No development-only Ruby gems",
|
||||
DurationMs = 12
|
||||
}
|
||||
],
|
||||
FinalStatus = "warning",
|
||||
MatchedRuleCount = 2,
|
||||
TotalRuleCount = 5
|
||||
};
|
||||
}
|
||||
|
||||
private static PolicyEvaluationTrace CreateVexResolutionTrace()
|
||||
{
|
||||
return new PolicyEvaluationTrace
|
||||
{
|
||||
TraceId = "TRACE-2025-003",
|
||||
PolicyId = "POL-VEX-001",
|
||||
PolicyName = "VEX-Aware Policy",
|
||||
EvaluationContext = new EvaluationContext
|
||||
{
|
||||
DigestEvaluated = "sha256:vex123",
|
||||
TenantId = "TENANT-001",
|
||||
Environment = "production",
|
||||
Exposure = "internet"
|
||||
},
|
||||
StartedAt = FrozenTime,
|
||||
CompletedAt = FrozenTime.AddMilliseconds(67),
|
||||
DurationMs = 67,
|
||||
Outcome = "Pass",
|
||||
Steps =
|
||||
[
|
||||
new EvaluationStep
|
||||
{
|
||||
StepNumber = 1,
|
||||
RuleName = "vex_merge_resolution",
|
||||
Priority = 10,
|
||||
Phase = EvaluationPhase.VexMerge,
|
||||
Condition = "vex.statements.count > 1",
|
||||
ConditionResult = true,
|
||||
Action = null,
|
||||
Explanation = "Multiple VEX statements found; merging via K4 lattice",
|
||||
DurationMs = 20,
|
||||
VexMergeDetail = new VexMergeDetail
|
||||
{
|
||||
VulnerabilityId = "CVE-2024-0001",
|
||||
StatementCount = 3,
|
||||
Sources =
|
||||
[
|
||||
new VexSourceInfo { Source = "vendor", Status = "not_affected", Trust = 1.0m },
|
||||
new VexSourceInfo { Source = "maintainer", Status = "affected", Trust = 0.9m },
|
||||
new VexSourceInfo { Source = "scanner", Status = "unknown", Trust = 0.5m }
|
||||
],
|
||||
WinningSource = "vendor",
|
||||
WinningStatus = "not_affected",
|
||||
ResolutionReason = "TrustWeight",
|
||||
ConflictsResolved = 2
|
||||
}
|
||||
},
|
||||
new EvaluationStep
|
||||
{
|
||||
StepNumber = 2,
|
||||
RuleName = "require_vex_justification",
|
||||
Priority = 3,
|
||||
Phase = EvaluationPhase.RuleMatch,
|
||||
Condition = "vex.justification in [\"component_not_present\",\"vulnerable_code_not_present\"]",
|
||||
ConditionResult = true,
|
||||
Action = "status := vex.status",
|
||||
Explanation = "VEX justification accepted: component_not_present",
|
||||
DurationMs = 12
|
||||
}
|
||||
],
|
||||
FinalStatus = "allowed",
|
||||
MatchedRuleCount = 2,
|
||||
TotalRuleCount = 2
|
||||
};
|
||||
}
|
||||
|
||||
private static PolicyEvaluationTrace CreateProfileApplicationTrace()
|
||||
{
|
||||
return new PolicyEvaluationTrace
|
||||
{
|
||||
TraceId = "TRACE-2025-004",
|
||||
PolicyId = "POL-PROFILE-001",
|
||||
PolicyName = "Profile-Based Policy",
|
||||
EvaluationContext = new EvaluationContext
|
||||
{
|
||||
DigestEvaluated = "sha256:profile123",
|
||||
TenantId = "TENANT-003",
|
||||
Environment = "production",
|
||||
Exposure = "internet"
|
||||
},
|
||||
StartedAt = FrozenTime,
|
||||
CompletedAt = FrozenTime.AddMilliseconds(55),
|
||||
DurationMs = 55,
|
||||
Outcome = "Pass",
|
||||
Steps =
|
||||
[
|
||||
new EvaluationStep
|
||||
{
|
||||
StepNumber = 1,
|
||||
RuleName = "profile_severity",
|
||||
Priority = 100,
|
||||
Phase = EvaluationPhase.ProfileApplication,
|
||||
Condition = "profile.severity.enabled",
|
||||
ConditionResult = true,
|
||||
Action = null,
|
||||
Explanation = "Applying severity profile adjustments",
|
||||
DurationMs = 18,
|
||||
ProfileApplicationDetail = new ProfileApplicationDetail
|
||||
{
|
||||
ProfileName = "severity",
|
||||
Adjustments =
|
||||
[
|
||||
new ProfileAdjustment
|
||||
{
|
||||
Finding = "CVE-2024-0001",
|
||||
OriginalSeverity = "High",
|
||||
AdjustedSeverity = "Critical",
|
||||
Reason = "GHSA source weight +0.5, internet exposure +0.5"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
new EvaluationStep
|
||||
{
|
||||
StepNumber = 2,
|
||||
RuleName = "block_critical",
|
||||
Priority = 5,
|
||||
Phase = EvaluationPhase.RuleMatch,
|
||||
Condition = "severity.normalized >= \"Critical\"",
|
||||
ConditionResult = false,
|
||||
Action = null,
|
||||
Explanation = "Post-profile: no critical vulnerabilities (VEX override applied)",
|
||||
DurationMs = 10
|
||||
}
|
||||
],
|
||||
FinalStatus = "allowed",
|
||||
MatchedRuleCount = 1,
|
||||
TotalRuleCount = 2
|
||||
};
|
||||
}
|
||||
|
||||
private static PolicyEvaluationTrace CreateSeverityEscalationTrace()
|
||||
{
|
||||
return new PolicyEvaluationTrace
|
||||
{
|
||||
TraceId = "TRACE-2025-005",
|
||||
PolicyId = "POL-ESCALATE-001",
|
||||
PolicyName = "Escalation Policy",
|
||||
EvaluationContext = new EvaluationContext
|
||||
{
|
||||
DigestEvaluated = "sha256:escalate123",
|
||||
TenantId = "TENANT-001",
|
||||
Environment = "production",
|
||||
Exposure = "internet"
|
||||
},
|
||||
StartedAt = FrozenTime,
|
||||
CompletedAt = FrozenTime.AddMilliseconds(45),
|
||||
DurationMs = 45,
|
||||
Outcome = "Fail",
|
||||
Steps =
|
||||
[
|
||||
new EvaluationStep
|
||||
{
|
||||
StepNumber = 1,
|
||||
RuleName = "escalate_high_internet",
|
||||
Priority = 4,
|
||||
Phase = EvaluationPhase.SeverityEscalation,
|
||||
Condition = "severity.normalized == \"High\" and env.exposure == \"internet\"",
|
||||
ConditionResult = true,
|
||||
Action = "escalate to severity_band(\"Critical\")",
|
||||
Explanation = "High severity on internet-exposed asset escalated to Critical",
|
||||
DurationMs = 15,
|
||||
EscalationDetail = new EscalationDetail
|
||||
{
|
||||
Finding = "CVE-2024-0001",
|
||||
OriginalSeverity = "High",
|
||||
EscalatedSeverity = "Critical",
|
||||
Reason = "Internet exposure triggers escalation per policy rule"
|
||||
}
|
||||
},
|
||||
new EvaluationStep
|
||||
{
|
||||
StepNumber = 2,
|
||||
RuleName = "block_critical",
|
||||
Priority = 5,
|
||||
Phase = EvaluationPhase.RuleMatch,
|
||||
Condition = "severity.normalized >= \"Critical\"",
|
||||
ConditionResult = true,
|
||||
Action = "status := \"blocked\"",
|
||||
Explanation = "Critical severity (post-escalation) triggers block",
|
||||
DurationMs = 10
|
||||
}
|
||||
],
|
||||
FinalStatus = "blocked",
|
||||
MatchedRuleCount = 2,
|
||||
TotalRuleCount = 2
|
||||
};
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
#region Trace Models
|
||||
|
||||
public sealed record PolicyEvaluationTrace
|
||||
{
|
||||
public required string TraceId { get; init; }
|
||||
public required string PolicyId { get; init; }
|
||||
public required string PolicyName { get; init; }
|
||||
public required EvaluationContext EvaluationContext { get; init; }
|
||||
public required DateTimeOffset StartedAt { get; init; }
|
||||
public required DateTimeOffset CompletedAt { get; init; }
|
||||
public required int DurationMs { get; init; }
|
||||
public required string Outcome { get; init; }
|
||||
public required IReadOnlyList<EvaluationStep> Steps { get; init; }
|
||||
public required string FinalStatus { get; init; }
|
||||
public required int MatchedRuleCount { get; init; }
|
||||
public required int TotalRuleCount { get; init; }
|
||||
}
|
||||
|
||||
public sealed record EvaluationContext
|
||||
{
|
||||
public required string DigestEvaluated { get; init; }
|
||||
public required string TenantId { get; init; }
|
||||
public required string Environment { get; init; }
|
||||
public required string Exposure { get; init; }
|
||||
}
|
||||
|
||||
public sealed record EvaluationStep
|
||||
{
|
||||
public required int StepNumber { get; init; }
|
||||
public required string RuleName { get; init; }
|
||||
public required int Priority { get; init; }
|
||||
public required EvaluationPhase Phase { get; init; }
|
||||
public required string Condition { get; init; }
|
||||
public required bool ConditionResult { get; init; }
|
||||
public string? Action { get; init; }
|
||||
public required string Explanation { get; init; }
|
||||
public required int DurationMs { get; init; }
|
||||
public VexMergeDetail? VexMergeDetail { get; init; }
|
||||
public ProfileApplicationDetail? ProfileApplicationDetail { get; init; }
|
||||
public EscalationDetail? EscalationDetail { get; init; }
|
||||
}
|
||||
|
||||
public enum EvaluationPhase
|
||||
{
|
||||
ProfileApplication,
|
||||
VexMerge,
|
||||
SeverityEscalation,
|
||||
RuleMatch
|
||||
}
|
||||
|
||||
public sealed record VexMergeDetail
|
||||
{
|
||||
public required string VulnerabilityId { get; init; }
|
||||
public required int StatementCount { get; init; }
|
||||
public required IReadOnlyList<VexSourceInfo> Sources { get; init; }
|
||||
public required string WinningSource { get; init; }
|
||||
public required string WinningStatus { get; init; }
|
||||
public required string ResolutionReason { get; init; }
|
||||
public required int ConflictsResolved { get; init; }
|
||||
}
|
||||
|
||||
public sealed record VexSourceInfo
|
||||
{
|
||||
public required string Source { get; init; }
|
||||
public required string Status { get; init; }
|
||||
public required decimal Trust { get; init; }
|
||||
}
|
||||
|
||||
public sealed record ProfileApplicationDetail
|
||||
{
|
||||
public required string ProfileName { get; init; }
|
||||
public required IReadOnlyList<ProfileAdjustment> Adjustments { get; init; }
|
||||
}
|
||||
|
||||
public sealed record ProfileAdjustment
|
||||
{
|
||||
public required string Finding { get; init; }
|
||||
public required string OriginalSeverity { get; init; }
|
||||
public required string AdjustedSeverity { get; init; }
|
||||
public required string Reason { get; init; }
|
||||
}
|
||||
|
||||
public sealed record EscalationDetail
|
||||
{
|
||||
public required string Finding { get; init; }
|
||||
public required string OriginalSeverity { get; init; }
|
||||
public required string EscalatedSeverity { get; init; }
|
||||
public required string Reason { get; init; }
|
||||
}
|
||||
|
||||
#endregion
|
||||
@@ -0,0 +1,566 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
// SPDX-FileCopyrightText: 2025 StellaOps Contributors
|
||||
|
||||
using System.Text.Json;
|
||||
using FluentAssertions;
|
||||
using StellaOps.TestKit.Assertions;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Policy.Engine.Tests.Snapshots;
|
||||
|
||||
/// <summary>
|
||||
/// Snapshot tests for verdict artifact canonical JSON output.
|
||||
/// Ensures verdict artifacts have stable, auditor-facing JSON structure.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// These tests validate:
|
||||
/// - Canonical JSON structure is stable across versions
|
||||
/// - Field ordering is deterministic
|
||||
/// - Timestamps and identifiers use ISO-8601 format
|
||||
/// - Sensitive fields are properly masked in audit output
|
||||
/// </remarks>
|
||||
public sealed class VerdictArtifactSnapshotTests
|
||||
{
|
||||
private static readonly DateTimeOffset FrozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z");
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that a passing verdict produces stable canonical JSON.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void PassingVerdict_ProducesStableCanonicalJson()
|
||||
{
|
||||
// Arrange
|
||||
var verdict = CreatePassingVerdict();
|
||||
|
||||
// Act - Use TestKit SnapshotAssert
|
||||
SnapshotAssert.MatchesSnapshot(verdict, "PassingVerdict_Canonical");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that a failing verdict with violations produces stable canonical JSON.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void FailingVerdict_WithViolations_ProducesStableCanonicalJson()
|
||||
{
|
||||
// Arrange
|
||||
var verdict = CreateFailingVerdict();
|
||||
|
||||
// Act
|
||||
SnapshotAssert.MatchesSnapshot(verdict, "FailingVerdict_WithViolations_Canonical");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that a verdict with unknowns budget violations produces stable canonical JSON.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void VerdictWithUnknowns_ProducesStableCanonicalJson()
|
||||
{
|
||||
// Arrange
|
||||
var verdict = CreateVerdictWithUnknowns();
|
||||
|
||||
// Act
|
||||
SnapshotAssert.MatchesSnapshot(verdict, "VerdictWithUnknowns_Canonical");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that a verdict with VEX merge trace produces stable canonical JSON.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void VerdictWithVexMerge_ProducesStableCanonicalJson()
|
||||
{
|
||||
// Arrange
|
||||
var verdict = CreateVerdictWithVexMerge();
|
||||
|
||||
// Act
|
||||
SnapshotAssert.MatchesSnapshot(verdict, "VerdictWithVexMerge_Canonical");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that a complex verdict with multiple rule matches produces stable canonical JSON.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void ComplexVerdict_WithMultipleRules_ProducesStableCanonicalJson()
|
||||
{
|
||||
// Arrange
|
||||
var verdict = CreateComplexVerdict();
|
||||
|
||||
// Act
|
||||
SnapshotAssert.MatchesSnapshot(verdict, "ComplexVerdict_MultipleRules_Canonical");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that an empty verdict (no findings) produces stable canonical JSON.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void EmptyVerdict_ProducesStableCanonicalJson()
|
||||
{
|
||||
// Arrange
|
||||
var verdict = CreateEmptyVerdict();
|
||||
|
||||
// Act
|
||||
SnapshotAssert.MatchesSnapshot(verdict, "EmptyVerdict_Canonical");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that verdict JSON uses ISO-8601 timestamps consistently.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void VerdictTimestamps_UseIso8601Format()
|
||||
{
|
||||
// Arrange
|
||||
var verdict = CreatePassingVerdict();
|
||||
|
||||
// Act
|
||||
var json = JsonSerializer.Serialize(verdict, new JsonSerializerOptions
|
||||
{
|
||||
WriteIndented = true,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
||||
});
|
||||
|
||||
// Assert - All timestamps should be ISO-8601
|
||||
json.Should().Contain("2025-12-24T12:00:00");
|
||||
json.Should().NotContain("12/24/2025"); // Reject locale-specific formats
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that verdict identifiers follow expected format.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void VerdictIdentifiers_FollowExpectedFormat()
|
||||
{
|
||||
// Arrange
|
||||
var verdict = CreatePassingVerdict();
|
||||
|
||||
// Assert
|
||||
verdict.VerdictId.Should().NotBeNullOrEmpty();
|
||||
verdict.PolicyId.Should().NotBeNullOrEmpty();
|
||||
verdict.TenantId.Should().NotBeNullOrEmpty();
|
||||
}
|
||||
|
||||
#region Verdict Factories
|
||||
|
||||
private static VerdictArtifact CreatePassingVerdict()
|
||||
{
|
||||
return new VerdictArtifact
|
||||
{
|
||||
VerdictId = "VERDICT-2025-001",
|
||||
PolicyId = "POL-PROD-001",
|
||||
PolicyName = "Production Baseline Policy",
|
||||
PolicyVersion = "1.2.3",
|
||||
TenantId = "TENANT-001",
|
||||
EvaluatedAt = FrozenTime,
|
||||
DigestEvaluated = "sha256:abc123def456",
|
||||
Outcome = VerdictOutcome.Pass,
|
||||
RulesMatched = 2,
|
||||
RulesTotal = 5,
|
||||
Violations = [],
|
||||
Warnings = [],
|
||||
MatchedRules =
|
||||
[
|
||||
new RuleMatch
|
||||
{
|
||||
RuleName = "allow_low_severity",
|
||||
Priority = 1,
|
||||
Status = RuleMatchStatus.Matched,
|
||||
Reason = "Severity <= Low, allowing"
|
||||
},
|
||||
new RuleMatch
|
||||
{
|
||||
RuleName = "vex_not_affected",
|
||||
Priority = 2,
|
||||
Status = RuleMatchStatus.Matched,
|
||||
Reason = "VEX status is not_affected"
|
||||
}
|
||||
],
|
||||
Metadata = new VerdictMetadata
|
||||
{
|
||||
EvaluationDurationMs = 42,
|
||||
FeedVersions = new Dictionary<string, string>
|
||||
{
|
||||
["nvd"] = "2025-12-24",
|
||||
["ghsa"] = "2025-12-24"
|
||||
},
|
||||
PolicyChecksum = "sha256:policy123"
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static VerdictArtifact CreateFailingVerdict()
|
||||
{
|
||||
return new VerdictArtifact
|
||||
{
|
||||
VerdictId = "VERDICT-2025-002",
|
||||
PolicyId = "POL-PROD-001",
|
||||
PolicyName = "Production Baseline Policy",
|
||||
PolicyVersion = "1.2.3",
|
||||
TenantId = "TENANT-001",
|
||||
EvaluatedAt = FrozenTime,
|
||||
DigestEvaluated = "sha256:abc123def456",
|
||||
Outcome = VerdictOutcome.Fail,
|
||||
RulesMatched = 3,
|
||||
RulesTotal = 5,
|
||||
Violations =
|
||||
[
|
||||
new Violation
|
||||
{
|
||||
RuleName = "block_critical",
|
||||
Severity = "critical",
|
||||
Message = "Critical vulnerability CVE-2024-0001 found",
|
||||
VulnerabilityId = "CVE-2024-0001",
|
||||
PackagePurl = "pkg:npm/lodash@4.17.20",
|
||||
Remediation = "Upgrade to lodash@4.17.21"
|
||||
},
|
||||
new Violation
|
||||
{
|
||||
RuleName = "block_critical",
|
||||
Severity = "critical",
|
||||
Message = "Critical vulnerability CVE-2024-0002 found",
|
||||
VulnerabilityId = "CVE-2024-0002",
|
||||
PackagePurl = "pkg:npm/express@4.18.0",
|
||||
Remediation = "Upgrade to express@4.19.0"
|
||||
}
|
||||
],
|
||||
Warnings = [],
|
||||
MatchedRules =
|
||||
[
|
||||
new RuleMatch
|
||||
{
|
||||
RuleName = "block_critical",
|
||||
Priority = 5,
|
||||
Status = RuleMatchStatus.Violated,
|
||||
Reason = "2 critical vulnerabilities found"
|
||||
}
|
||||
],
|
||||
Metadata = new VerdictMetadata
|
||||
{
|
||||
EvaluationDurationMs = 67,
|
||||
FeedVersions = new Dictionary<string, string>
|
||||
{
|
||||
["nvd"] = "2025-12-24",
|
||||
["ghsa"] = "2025-12-24"
|
||||
},
|
||||
PolicyChecksum = "sha256:policy123"
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static VerdictArtifact CreateVerdictWithUnknowns()
|
||||
{
|
||||
return new VerdictArtifact
|
||||
{
|
||||
VerdictId = "VERDICT-2025-003",
|
||||
PolicyId = "POL-STRICT-001",
|
||||
PolicyName = "Strict Unknown Budget Policy",
|
||||
PolicyVersion = "2.0.0",
|
||||
TenantId = "TENANT-002",
|
||||
EvaluatedAt = FrozenTime,
|
||||
DigestEvaluated = "sha256:xyz789",
|
||||
Outcome = VerdictOutcome.Fail,
|
||||
RulesMatched = 1,
|
||||
RulesTotal = 3,
|
||||
Violations =
|
||||
[
|
||||
new Violation
|
||||
{
|
||||
RuleName = "unknowns_budget",
|
||||
Severity = "high",
|
||||
Message = "Unknowns budget exceeded: 5 critical unknowns (max: 0)",
|
||||
VulnerabilityId = null,
|
||||
PackagePurl = null,
|
||||
Remediation = "Resolve unknown packages or request VEX statements"
|
||||
}
|
||||
],
|
||||
Warnings = [],
|
||||
MatchedRules =
|
||||
[
|
||||
new RuleMatch
|
||||
{
|
||||
RuleName = "unknowns_budget",
|
||||
Priority = 10,
|
||||
Status = RuleMatchStatus.Violated,
|
||||
Reason = "Critical unknowns (5) exceeds budget (0)"
|
||||
}
|
||||
],
|
||||
UnknownsBudgetResult = new UnknownsBudgetSummary
|
||||
{
|
||||
WithinBudget = false,
|
||||
CriticalCount = 5,
|
||||
HighCount = 12,
|
||||
MediumCount = 45,
|
||||
LowCount = 120,
|
||||
TotalCount = 182,
|
||||
Action = "Block"
|
||||
},
|
||||
Metadata = new VerdictMetadata
|
||||
{
|
||||
EvaluationDurationMs = 89,
|
||||
FeedVersions = new Dictionary<string, string>
|
||||
{
|
||||
["nvd"] = "2025-12-24"
|
||||
},
|
||||
PolicyChecksum = "sha256:policy456"
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static VerdictArtifact CreateVerdictWithVexMerge()
|
||||
{
|
||||
return new VerdictArtifact
|
||||
{
|
||||
VerdictId = "VERDICT-2025-004",
|
||||
PolicyId = "POL-PROD-001",
|
||||
PolicyName = "Production Baseline Policy",
|
||||
PolicyVersion = "1.2.3",
|
||||
TenantId = "TENANT-001",
|
||||
EvaluatedAt = FrozenTime,
|
||||
DigestEvaluated = "sha256:abc123def456",
|
||||
Outcome = VerdictOutcome.Pass,
|
||||
RulesMatched = 2,
|
||||
RulesTotal = 5,
|
||||
Violations = [],
|
||||
Warnings = [],
|
||||
MatchedRules =
|
||||
[
|
||||
new RuleMatch
|
||||
{
|
||||
RuleName = "require_vex_justification",
|
||||
Priority = 3,
|
||||
Status = RuleMatchStatus.Matched,
|
||||
Reason = "VEX statement accepted with strong justification"
|
||||
}
|
||||
],
|
||||
VexMergeTrace = new VexMergeSummary
|
||||
{
|
||||
VulnerabilityId = "CVE-2024-0001",
|
||||
WinningSource = "vendor",
|
||||
WinningStatus = "not_affected",
|
||||
WinningJustification = "component_not_present",
|
||||
ConflictsResolved = 1,
|
||||
SourcesConsidered = 3,
|
||||
ResolutionReason = "TrustWeight"
|
||||
},
|
||||
Metadata = new VerdictMetadata
|
||||
{
|
||||
EvaluationDurationMs = 55,
|
||||
FeedVersions = new Dictionary<string, string>
|
||||
{
|
||||
["nvd"] = "2025-12-24",
|
||||
["ghsa"] = "2025-12-24"
|
||||
},
|
||||
PolicyChecksum = "sha256:policy123"
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static VerdictArtifact CreateComplexVerdict()
|
||||
{
|
||||
return new VerdictArtifact
|
||||
{
|
||||
VerdictId = "VERDICT-2025-005",
|
||||
PolicyId = "POL-COMPLEX-001",
|
||||
PolicyName = "Complex Multi-Rule Policy",
|
||||
PolicyVersion = "3.0.0",
|
||||
TenantId = "TENANT-003",
|
||||
EvaluatedAt = FrozenTime,
|
||||
DigestEvaluated = "sha256:complex123",
|
||||
Outcome = VerdictOutcome.Warn,
|
||||
RulesMatched = 5,
|
||||
RulesTotal = 8,
|
||||
Violations = [],
|
||||
Warnings =
|
||||
[
|
||||
new Warning
|
||||
{
|
||||
RuleName = "warn_eol_runtime",
|
||||
Severity = "medium",
|
||||
Message = "Runtime marked as EOL; upgrade recommended",
|
||||
PackagePurl = "pkg:deb/debian/python3.9@3.9.2"
|
||||
},
|
||||
new Warning
|
||||
{
|
||||
RuleName = "warn_ruby_git_sources",
|
||||
Severity = "low",
|
||||
Message = "Git-sourced Ruby gem present; review required",
|
||||
PackagePurl = "pkg:gem/custom-gem@1.0.0"
|
||||
}
|
||||
],
|
||||
MatchedRules =
|
||||
[
|
||||
new RuleMatch
|
||||
{
|
||||
RuleName = "block_critical",
|
||||
Priority = 5,
|
||||
Status = RuleMatchStatus.NotMatched,
|
||||
Reason = "No critical vulnerabilities"
|
||||
},
|
||||
new RuleMatch
|
||||
{
|
||||
RuleName = "escalate_high_internet",
|
||||
Priority = 4,
|
||||
Status = RuleMatchStatus.NotMatched,
|
||||
Reason = "No high severity on internet-exposed assets"
|
||||
},
|
||||
new RuleMatch
|
||||
{
|
||||
RuleName = "require_vex_justification",
|
||||
Priority = 3,
|
||||
Status = RuleMatchStatus.Matched,
|
||||
Reason = "VEX statement accepted"
|
||||
},
|
||||
new RuleMatch
|
||||
{
|
||||
RuleName = "warn_eol_runtime",
|
||||
Priority = 1,
|
||||
Status = RuleMatchStatus.Warning,
|
||||
Reason = "EOL runtime detected"
|
||||
},
|
||||
new RuleMatch
|
||||
{
|
||||
RuleName = "warn_ruby_git_sources",
|
||||
Priority = 1,
|
||||
Status = RuleMatchStatus.Warning,
|
||||
Reason = "Git-sourced gem detected"
|
||||
}
|
||||
],
|
||||
Metadata = new VerdictMetadata
|
||||
{
|
||||
EvaluationDurationMs = 123,
|
||||
FeedVersions = new Dictionary<string, string>
|
||||
{
|
||||
["nvd"] = "2025-12-24",
|
||||
["ghsa"] = "2025-12-24",
|
||||
["osv"] = "2025-12-24"
|
||||
},
|
||||
PolicyChecksum = "sha256:policy789"
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static VerdictArtifact CreateEmptyVerdict()
|
||||
{
|
||||
return new VerdictArtifact
|
||||
{
|
||||
VerdictId = "VERDICT-2025-006",
|
||||
PolicyId = "POL-PROD-001",
|
||||
PolicyName = "Production Baseline Policy",
|
||||
PolicyVersion = "1.2.3",
|
||||
TenantId = "TENANT-001",
|
||||
EvaluatedAt = FrozenTime,
|
||||
DigestEvaluated = "sha256:empty123",
|
||||
Outcome = VerdictOutcome.Pass,
|
||||
RulesMatched = 0,
|
||||
RulesTotal = 5,
|
||||
Violations = [],
|
||||
Warnings = [],
|
||||
MatchedRules = [],
|
||||
Metadata = new VerdictMetadata
|
||||
{
|
||||
EvaluationDurationMs = 12,
|
||||
FeedVersions = new Dictionary<string, string>
|
||||
{
|
||||
["nvd"] = "2025-12-24"
|
||||
},
|
||||
PolicyChecksum = "sha256:policy123"
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
#region Verdict Models for Snapshots
|
||||
|
||||
/// <summary>
|
||||
/// Verdict artifact for auditor-facing output.
|
||||
/// </summary>
|
||||
public sealed record VerdictArtifact
|
||||
{
|
||||
public required string VerdictId { get; init; }
|
||||
public required string PolicyId { get; init; }
|
||||
public required string PolicyName { get; init; }
|
||||
public required string PolicyVersion { get; init; }
|
||||
public required string TenantId { get; init; }
|
||||
public required DateTimeOffset EvaluatedAt { get; init; }
|
||||
public required string DigestEvaluated { get; init; }
|
||||
public required VerdictOutcome Outcome { get; init; }
|
||||
public required int RulesMatched { get; init; }
|
||||
public required int RulesTotal { get; init; }
|
||||
public required IReadOnlyList<Violation> Violations { get; init; }
|
||||
public required IReadOnlyList<Warning> Warnings { get; init; }
|
||||
public required IReadOnlyList<RuleMatch> MatchedRules { get; init; }
|
||||
public UnknownsBudgetSummary? UnknownsBudgetResult { get; init; }
|
||||
public VexMergeSummary? VexMergeTrace { get; init; }
|
||||
public required VerdictMetadata Metadata { get; init; }
|
||||
}
|
||||
|
||||
public enum VerdictOutcome
|
||||
{
|
||||
Pass,
|
||||
Warn,
|
||||
Fail
|
||||
}
|
||||
|
||||
public sealed record Violation
|
||||
{
|
||||
public required string RuleName { get; init; }
|
||||
public required string Severity { get; init; }
|
||||
public required string Message { get; init; }
|
||||
public string? VulnerabilityId { get; init; }
|
||||
public string? PackagePurl { get; init; }
|
||||
public string? Remediation { get; init; }
|
||||
}
|
||||
|
||||
public sealed record Warning
|
||||
{
|
||||
public required string RuleName { get; init; }
|
||||
public required string Severity { get; init; }
|
||||
public required string Message { get; init; }
|
||||
public string? PackagePurl { get; init; }
|
||||
}
|
||||
|
||||
public sealed record RuleMatch
|
||||
{
|
||||
public required string RuleName { get; init; }
|
||||
public required int Priority { get; init; }
|
||||
public required RuleMatchStatus Status { get; init; }
|
||||
public required string Reason { get; init; }
|
||||
}
|
||||
|
||||
public enum RuleMatchStatus
|
||||
{
|
||||
NotMatched,
|
||||
Matched,
|
||||
Violated,
|
||||
Warning
|
||||
}
|
||||
|
||||
public sealed record UnknownsBudgetSummary
|
||||
{
|
||||
public required bool WithinBudget { get; init; }
|
||||
public required int CriticalCount { get; init; }
|
||||
public required int HighCount { get; init; }
|
||||
public required int MediumCount { get; init; }
|
||||
public required int LowCount { get; init; }
|
||||
public required int TotalCount { get; init; }
|
||||
public required string Action { get; init; }
|
||||
}
|
||||
|
||||
public sealed record VexMergeSummary
|
||||
{
|
||||
public required string VulnerabilityId { get; init; }
|
||||
public required string WinningSource { get; init; }
|
||||
public required string WinningStatus { get; init; }
|
||||
public required string WinningJustification { get; init; }
|
||||
public required int ConflictsResolved { get; init; }
|
||||
public required int SourcesConsidered { get; init; }
|
||||
public required string ResolutionReason { get; init; }
|
||||
}
|
||||
|
||||
public sealed record VerdictMetadata
|
||||
{
|
||||
public required int EvaluationDurationMs { get; init; }
|
||||
public required Dictionary<string, string> FeedVersions { get; init; }
|
||||
public required string PolicyChecksum { get; init; }
|
||||
}
|
||||
|
||||
#endregion
|
||||
@@ -31,5 +31,9 @@
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../../StellaOps.Policy.Engine/StellaOps.Policy.Engine.csproj" />
|
||||
<ProjectReference Include="../../../__Libraries/StellaOps.DeltaVerdict/StellaOps.DeltaVerdict.csproj" />
|
||||
<ProjectReference Include="../../../Excititor/__Libraries/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj" />
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Policy.Unknowns/StellaOps.Policy.Unknowns.csproj" />
|
||||
<ProjectReference Include="../../../__Libraries/StellaOps.TestKit/StellaOps.TestKit.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
Reference in New Issue
Block a user