Add determinism tests for verdict artifact generation and update SHA256 sums script

- Implemented comprehensive tests for verdict artifact generation to ensure deterministic outputs across various scenarios, including identical inputs, parallel execution, and change ordering. - Created helper methods for generating sample verdict inputs and computing canonical hashes. - Added tests to validate the stability of canonical hashes, proof spine ordering, and summary statistics. - Introduced a new PowerShell script to update SHA256 sums for files, ensuring accurate hash generation and file integrity checks.
2025-12-24 02:17:34 +02:00
parent e59921374e
commit 7503c19b8f
390 changed files with 37389 additions and 5380 deletions
--- a/src/Policy/__Tests/StellaOps.Policy.Tests/StellaOps.Policy.Tests.csproj
+++ b/src/Policy/__Tests/StellaOps.Policy.Tests/StellaOps.Policy.Tests.csproj
@@ -11,6 +11,8 @@
  </PropertyGroup>
  <ItemGroup>
    <PackageReference Include="FluentAssertions" Version="8.2.0" />
+    <PackageReference Include="FsCheck" Version="2.16.6" />
+    <PackageReference Include="FsCheck.Xunit" Version="2.16.6" />
    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.0" />
    <PackageReference Include="Moq" Version="4.20.72" />
    <PackageReference Include="xunit" Version="2.9.3" />
--- a/src/Policy/__Tests/StellaOps.Policy.Tests/TrustLattice/ClaimScoreMergerPropertyTests.cs
+++ b/src/Policy/__Tests/StellaOps.Policy.Tests/TrustLattice/ClaimScoreMergerPropertyTests.cs
@@ -0,0 +1,438 @@
+// -----------------------------------------------------------------------------
+// ClaimScoreMergerPropertyTests.cs
+// Sprint: SPRINT_5100_0007_0001 (Testing Strategy)
+// Task: TEST-STRAT-5100-004 - Add property-based tests to critical routing/decision logic
+// Description: Property-based tests for ClaimScoreMerger verifying order independence,
+//              determinism, score clamping, and conflict detection consistency.
+// -----------------------------------------------------------------------------
+
+using FsCheck;
+using FsCheck.Xunit;
+using FluentAssertions;
+using StellaOps.Policy.TrustLattice;
+using VexStatus = StellaOps.Policy.Confidence.Models.VexStatus;
+
+namespace StellaOps.Policy.Tests.TrustLattice;
+
+/// <summary>
+/// Property-based tests for ClaimScoreMerger.
+/// Verifies critical decision logic properties:
+/// - Order independence: shuffling input order doesn't change winner
+/// - Determinism: same inputs always produce same output
+/// - Score clamping: confidence is always in [0, 1]
+/// - Conflict detection: differing statuses always trigger conflict
+/// </summary>
+[Trait("Category", "Property")]
+public sealed class ClaimScoreMergerPropertyTests
+{
+    private static readonly VexStatus[] AllStatuses =
+    [
+        VexStatus.NotAffected,
+        VexStatus.Affected,
+        VexStatus.Fixed,
+        VexStatus.UnderInvestigation
+    ];
+
+    #region Order Independence
+
+    /// <summary>
+    /// Property: Shuffling input order should not change the winning claim.
+    /// This is critical for deterministic VEX decisioning.
+    /// </summary>
+    [Property(MaxTest = 100)]
+    public Property Merge_IsOrderIndependent()
+    {
+        return Prop.ForAll(
+            ClaimListArb(2, 5),
+            claims =>
+            {
+                var merger = new ClaimScoreMerger();
+                var policy = new MergePolicy();
+
+                // Original order
+                var result1 = merger.Merge(claims, policy);
+
+                // Reversed order
+                var reversed = claims.AsEnumerable().Reverse().ToList();
+                var result2 = merger.Merge(reversed, policy);
+
+                // Winner should be the same regardless of input order
+                return result1.WinningClaim.SourceId == result2.WinningClaim.SourceId &&
+                       result1.WinningClaim.Status == result2.WinningClaim.Status &&
+                       Math.Abs(result1.Confidence - result2.Confidence) < 0.0001;
+            });
+    }
+
+    /// <summary>
+    /// Property: Shuffling any permutation produces same winner.
+    /// </summary>
+    [Property(MaxTest = 50)]
+    public Property Merge_AllPermutationsProduceSameWinner()
+    {
+        return Prop.ForAll(
+            ClaimListArb(2, 4),
+            Gen.Choose(0, 100).ToArbitrary(),
+            (claims, seed) =>
+            {
+                if (claims.Count < 2) return true;
+
+                var merger = new ClaimScoreMerger();
+                var policy = new MergePolicy();
+                var random = new System.Random(seed);
+
+                var result1 = merger.Merge(claims, policy);
+
+                // Shuffle using seed
+                var shuffled = claims.OrderBy(_ => random.Next()).ToList();
+                var result2 = merger.Merge(shuffled, policy);
+
+                return result1.WinningClaim.SourceId == result2.WinningClaim.SourceId;
+            });
+    }
+
+    #endregion
+
+    #region Determinism
+
+    /// <summary>
+    /// Property: Same input always produces identical output.
+    /// </summary>
+    [Property(MaxTest = 100)]
+    public Property Merge_IsDeterministic()
+    {
+        return Prop.ForAll(
+            ClaimListArb(1, 5),
+            claims =>
+            {
+                var merger = new ClaimScoreMerger();
+                var policy = new MergePolicy();
+
+                var result1 = merger.Merge(claims, policy);
+                var result2 = merger.Merge(claims, policy);
+
+                return result1.WinningClaim.SourceId == result2.WinningClaim.SourceId &&
+                       result1.Status == result2.Status &&
+                       result1.Confidence == result2.Confidence &&
+                       result1.HasConflicts == result2.HasConflicts;
+            });
+    }
+
+    /// <summary>
+    /// Property: Repeated merges (100x) produce consistent winner.
+    /// </summary>
+    [Property(MaxTest = 50)]
+    public Property Merge_ConsistentAcrossRepeatedCalls()
+    {
+        return Prop.ForAll(
+            ClaimListArb(2, 5),
+            claims =>
+            {
+                var merger = new ClaimScoreMerger();
+                var policy = new MergePolicy();
+                var expectedWinner = merger.Merge(claims, policy).WinningClaim.SourceId;
+
+                for (var i = 0; i < 100; i++)
+                {
+                    var result = merger.Merge(claims, policy);
+                    if (result.WinningClaim.SourceId != expectedWinner)
+                        return false;
+                }
+                return true;
+            });
+    }
+
+    #endregion
+
+    #region Score Clamping
+
+    /// <summary>
+    /// Property: Confidence is always in [0, 1] range.
+    /// </summary>
+    [Property(MaxTest = 100)]
+    public Property Merge_ConfidenceIsClampedToUnitInterval()
+    {
+        return Prop.ForAll(
+            ClaimListArb(1, 5),
+            claims =>
+            {
+                var merger = new ClaimScoreMerger();
+                var policy = new MergePolicy();
+                var result = merger.Merge(claims, policy);
+
+                return result.Confidence >= 0.0 && result.Confidence <= 1.0;
+            });
+    }
+
+    /// <summary>
+    /// Property: Even with extreme penalty values, confidence stays in [0, 1].
+    /// </summary>
+    [Property(MaxTest = 50)]
+    public Property Merge_ExtremeConflictPenalty_StillClamps()
+    {
+        return Prop.ForAll(
+            ClaimListArb(2, 4),
+            Gen.Choose(0, 200).Select(x => x / 100.0).ToArbitrary(),
+            (claims, penalty) =>
+            {
+                var merger = new ClaimScoreMerger();
+                var policy = new MergePolicy { ConflictPenalty = penalty };
+                var result = merger.Merge(claims, policy);
+
+                return result.Confidence >= 0.0 && result.Confidence <= 1.0;
+            });
+    }
+
+    #endregion
+
+    #region Conflict Detection
+
+    /// <summary>
+    /// Property: When all claims have same status, no conflicts.
+    /// </summary>
+    [Property(MaxTest = 100)]
+    public Property Merge_SameStatus_NoConflicts()
+    {
+        return Prop.ForAll(
+            Gen.Elements(AllStatuses).ToArbitrary(),
+            ClaimCountArb(),
+            (status, count) =>
+            {
+                var claims = GenerateClaimsWithStatus(status, count);
+                var merger = new ClaimScoreMerger();
+                var result = merger.Merge(claims, new MergePolicy());
+
+                return !result.HasConflicts && result.Conflicts.Length == 0;
+            });
+    }
+
+    /// <summary>
+    /// Property: Different statuses always trigger conflict detection.
+    /// </summary>
+    [Property(MaxTest = 100)]
+    public Property Merge_DifferentStatuses_HasConflicts()
+    {
+        return Prop.ForAll(
+            Gen.Elements(AllStatuses).ToArbitrary(),
+            Gen.Elements(AllStatuses).ToArbitrary(),
+            (status1, status2) =>
+            {
+                if (status1 == status2) return true; // Skip same status case
+
+                var claims = new List<(VexClaim, ClaimScoreResult)>
+                {
+                    CreateClaim("source-a", status1, 0.8),
+                    CreateClaim("source-b", status2, 0.7)
+                };
+
+                var merger = new ClaimScoreMerger();
+                var result = merger.Merge(claims, new MergePolicy());
+
+                return result.HasConflicts && result.Conflicts.Length > 0;
+            });
+    }
+
+    /// <summary>
+    /// Property: RequiresReplayProof is true when HasConflicts and policy enables it.
+    /// </summary>
+    [Property(MaxTest = 100)]
+    public Property Merge_ConflictWithReplayPolicy_RequiresReplayProof()
+    {
+        return Prop.ForAll(
+            ClaimListArb(2, 4),
+            Gen.Elements(true, false).ToArbitrary(),
+            (claims, requireReplay) =>
+            {
+                var merger = new ClaimScoreMerger();
+                var policy = new MergePolicy { RequireReplayProofOnConflict = requireReplay };
+                var result = merger.Merge(claims, policy);
+
+                if (result.HasConflicts)
+                    return result.RequiresReplayProof == requireReplay;
+                return !result.RequiresReplayProof;
+            });
+    }
+
+    #endregion
+
+    #region Winner Selection
+
+    /// <summary>
+    /// Property: Winner always has highest adjusted score among claims.
+    /// </summary>
+    [Property(MaxTest = 100)]
+    public Property Merge_WinnerHasHighestAdjustedScore()
+    {
+        return Prop.ForAll(
+            ClaimListArb(2, 5),
+            claims =>
+            {
+                var merger = new ClaimScoreMerger();
+                var result = merger.Merge(claims, new MergePolicy());
+
+                var maxAdjusted = result.AllClaims.Max(c => c.AdjustedScore);
+                return result.WinningClaim.AdjustedScore == maxAdjusted;
+            });
+    }
+
+    /// <summary>
+    /// Property: Single claim is always the winner.
+    /// </summary>
+    [Property(MaxTest = 100)]
+    public Property Merge_SingleClaim_IsAlwaysWinner()
+    {
+        return Prop.ForAll(
+            SingleClaimArb(),
+            claim =>
+            {
+                var merger = new ClaimScoreMerger();
+                var result = merger.Merge([claim], new MergePolicy());
+
+                return result.WinningClaim.SourceId == claim.Claim.SourceId &&
+                       result.Status == claim.Claim.Status &&
+                       !result.HasConflicts;
+            });
+    }
+
+    /// <summary>
+    /// Property: Specificity is tie-breaker when scores are equal and policy enables it.
+    /// </summary>
+    [Property(MaxTest = 50)]
+    public Property Merge_EqualScores_SpecificityBreaksTie()
+    {
+        return Prop.ForAll(
+            Gen.Choose(1, 100).Select(x => x / 100.0).ToArbitrary(),
+            Gen.Choose(1, 5).ToArbitrary(),
+            Gen.Choose(6, 10).ToArbitrary(),
+            (score, lowSpec, highSpec) =>
+            {
+                var claims = new List<(VexClaim, ClaimScoreResult)>
+                {
+                    (new VexClaim
+                    {
+                        SourceId = "low-spec",
+                        Status = VexStatus.NotAffected,
+                        ScopeSpecificity = lowSpec,
+                        IssuedAt = DateTimeOffset.UtcNow
+                    }, new ClaimScoreResult { Score = score, BaseTrust = score, StrengthMultiplier = 1, FreshnessMultiplier = 1 }),
+                    (new VexClaim
+                    {
+                        SourceId = "high-spec",
+                        Status = VexStatus.NotAffected,
+                        ScopeSpecificity = highSpec,
+                        IssuedAt = DateTimeOffset.UtcNow
+                    }, new ClaimScoreResult { Score = score, BaseTrust = score, StrengthMultiplier = 1, FreshnessMultiplier = 1 })
+                };
+
+                var merger = new ClaimScoreMerger();
+                var result = merger.Merge(claims, new MergePolicy { PreferSpecificity = true });
+
+                return result.WinningClaim.SourceId == "high-spec";
+            });
+    }
+
+    #endregion
+
+    #region Empty and Edge Cases
+
+    /// <summary>
+    /// Property: Empty claims list produces UnderInvestigation status.
+    /// </summary>
+    [Fact]
+    public void Merge_EmptyClaims_ReturnsUnderInvestigation()
+    {
+        var merger = new ClaimScoreMerger();
+        var result = merger.Merge([], new MergePolicy());
+
+        result.Status.Should().Be(VexStatus.UnderInvestigation);
+        result.Confidence.Should().Be(0);
+        result.HasConflicts.Should().BeFalse();
+    }
+
+    /// <summary>
+    /// Property: All claims have an entry in AllClaims.
+    /// </summary>
+    [Property(MaxTest = 100)]
+    public Property Merge_AllInputClaimsAppearInOutput()
+    {
+        return Prop.ForAll(
+            ClaimListArb(1, 5),
+            claims =>
+            {
+                var merger = new ClaimScoreMerger();
+                var result = merger.Merge(claims, new MergePolicy());
+
+                var inputIds = claims.Select(c => c.Claim.SourceId).ToHashSet();
+                var outputIds = result.AllClaims.Select(c => c.SourceId).ToHashSet();
+
+                return inputIds.SetEquals(outputIds);
+            });
+    }
+
+    #endregion
+
+    #region Generators
+
+    private static Arbitrary<int> ClaimCountArb()
+    {
+        return Gen.Choose(1, 5).ToArbitrary();
+    }
+
+    private static Arbitrary<List<(VexClaim Claim, ClaimScoreResult Score)>> ClaimListArb(int min, int max)
+    {
+        var claimGen = from sourceId in Gen.Elements("src-a", "src-b", "src-c", "src-d", "src-e")
+                       from status in Gen.Elements(AllStatuses)
+                       from score in Gen.Choose(1, 100).Select(x => x / 100.0)
+                       from specificity in Gen.Choose(1, 10)
+                       select CreateClaim(sourceId, status, score, specificity);
+
+        return Gen.ListOf(claimGen)
+            .Select(flist => flist.ToList())
+            .Where(list => list.Count >= min && list.Count <= max)
+            .Select(list => list.DistinctBy(c => c.Claim.SourceId).ToList())
+            .Where(list => list.Count >= min)
+            .ToArbitrary();
+    }
+
+    private static Arbitrary<(VexClaim Claim, ClaimScoreResult Score)> SingleClaimArb()
+    {
+        return (from sourceId in Gen.Elements("source-single")
+                from status in Gen.Elements(AllStatuses)
+                from score in Gen.Choose(1, 100).Select(x => x / 100.0)
+                select CreateClaim(sourceId, status, score)).ToArbitrary();
+    }
+
+    private static (VexClaim Claim, ClaimScoreResult Score) CreateClaim(
+        string sourceId,
+        VexStatus status,
+        double score,
+        int specificity = 1)
+    {
+        return (
+            new VexClaim
+            {
+                SourceId = sourceId,
+                Status = status,
+                ScopeSpecificity = specificity,
+                IssuedAt = DateTimeOffset.Parse("2025-01-01T00:00:00Z")
+            },
+            new ClaimScoreResult
+            {
+                Score = score,
+                BaseTrust = score,
+                StrengthMultiplier = 1,
+                FreshnessMultiplier = 1
+            }
+        );
+    }
+
+    private static List<(VexClaim Claim, ClaimScoreResult Score)> GenerateClaimsWithStatus(
+        VexStatus status,
+        int count)
+    {
+        return Enumerable.Range(0, count)
+            .Select(i => CreateClaim($"source-{i}", status, 0.5 + (i * 0.1)))
+            .ToList();
+    }
+
+    #endregion
+}