using System.Collections.Immutable; using System.Text.Json; using StellaOps.Scanner.SmartDiff.Detection; using Xunit; namespace StellaOps.Scanner.SmartDiffTests; /// /// Golden fixture tests for Smart-Diff state comparison determinism. /// Per Sprint 3500.3 - ensures stable, reproducible change detection. /// public class StateComparisonGoldenTests { private static readonly string FixturePath = Path.Combine( AppContext.BaseDirectory, "Fixtures", "state-comparison.v1.json"); private static readonly JsonSerializerOptions JsonOptions = new() { PropertyNameCaseInsensitive = true }; private readonly MaterialRiskChangeDetector _detector; public StateComparisonGoldenTests() { _detector = new MaterialRiskChangeDetector(); } [Fact] public void GoldenFixture_Exists() { Assert.True(File.Exists(FixturePath), $"Fixture file not found: {FixturePath}"); } [Theory] [MemberData(nameof(GetTestCases))] public void DetectChanges_MatchesGoldenFixture(GoldenTestCase testCase) { // Arrange var previous = ParseSnapshot(testCase.Previous); var current = ParseSnapshot(testCase.Current); // Act var result = _detector.DetectChanges(previous, current); // Assert Assert.Equal(testCase.Expected.HasMaterialChange, result.HasMaterialChange); if (testCase.Expected.ChangeCount.HasValue) { Assert.Equal(testCase.Expected.ChangeCount.Value, result.Changes.Length); } if (testCase.Expected.TotalPriorityScore.HasValue) { Assert.Equal(testCase.Expected.TotalPriorityScore.Value, result.PriorityScore); } if (testCase.Expected.ChangeType is not null && result.Changes.Length > 0) { var expectedType = ParseChangeType(testCase.Expected.ChangeType); Assert.Contains(result.Changes, c => c.ChangeType == expectedType); } if (testCase.Expected.Direction is not null && result.Changes.Length > 0) { var expectedDirection = ParseDirection(testCase.Expected.Direction); Assert.Contains(result.Changes, c => c.Direction == expectedDirection); } } [Fact] public void StateHash_IsDeterministic() { // Arrange var snapshot = new RiskStateSnapshot( FindingKey: new FindingKey("CVE-2024-1111", "pkg:npm/test@1.0.0"), ScanId: "scan-hash-001", CapturedAt: DateTimeOffset.Parse("2024-12-15T10:00:00Z"), Reachable: true, LatticeState: "CR", VexStatus: VexStatusType.Affected, InAffectedRange: true, Kev: false, EpssScore: 0.05, PolicyFlags: ["FLAG_A", "FLAG_B"], PolicyDecision: PolicyDecisionType.Warn); // Act - compute hash multiple times var hash1 = snapshot.ComputeStateHash(); var hash2 = snapshot.ComputeStateHash(); var hash3 = snapshot.ComputeStateHash(); // Assert - all hashes must be identical Assert.Equal(hash1, hash2); Assert.Equal(hash2, hash3); Assert.StartsWith("sha256:", hash1); } [Fact] public void StateHash_DiffersWithReachabilityChange() { // Arrange var baseSnapshot = new RiskStateSnapshot( FindingKey: new FindingKey("CVE-2024-1111", "pkg:npm/test@1.0.0"), ScanId: "scan-hash-001", CapturedAt: DateTimeOffset.Parse("2024-12-15T10:00:00Z"), Reachable: true, LatticeState: "CR", VexStatus: VexStatusType.Affected, InAffectedRange: true, Kev: false, EpssScore: 0.05, PolicyFlags: [], PolicyDecision: null); var modifiedSnapshot = baseSnapshot with { Reachable = false }; // Act var hash1 = baseSnapshot.ComputeStateHash(); var hash2 = modifiedSnapshot.ComputeStateHash(); // Assert - hashes must differ Assert.NotEqual(hash1, hash2); } [Fact] public void StateHash_DiffersWithVexStatusChange() { // Arrange var baseSnapshot = new RiskStateSnapshot( FindingKey: new FindingKey("CVE-2024-1111", "pkg:npm/test@1.0.0"), ScanId: "scan-hash-001", CapturedAt: DateTimeOffset.Parse("2024-12-15T10:00:00Z"), Reachable: true, LatticeState: "CR", VexStatus: VexStatusType.Affected, InAffectedRange: true, Kev: false, EpssScore: 0.05, PolicyFlags: [], PolicyDecision: null); var modifiedSnapshot = baseSnapshot with { VexStatus = VexStatusType.NotAffected }; // Act var hash1 = baseSnapshot.ComputeStateHash(); var hash2 = modifiedSnapshot.ComputeStateHash(); // Assert - hashes must differ Assert.NotEqual(hash1, hash2); } [Fact] public void StateHash_SameForEquivalentStates() { // Arrange - two snapshots with same risk-relevant fields but different scan IDs var snapshot1 = new RiskStateSnapshot( FindingKey: new FindingKey("CVE-2024-1111", "pkg:npm/test@1.0.0"), ScanId: "scan-001", CapturedAt: DateTimeOffset.Parse("2024-12-15T10:00:00Z"), Reachable: true, LatticeState: "CR", VexStatus: VexStatusType.Affected, InAffectedRange: true, Kev: false, EpssScore: 0.05, PolicyFlags: [], PolicyDecision: null); var snapshot2 = new RiskStateSnapshot( FindingKey: new FindingKey("CVE-2024-1111", "pkg:npm/test@1.0.0"), ScanId: "scan-002", // Different scan ID CapturedAt: DateTimeOffset.Parse("2024-12-16T10:00:00Z"), // Different timestamp Reachable: true, LatticeState: "CR", VexStatus: VexStatusType.Affected, InAffectedRange: true, Kev: false, EpssScore: 0.05, PolicyFlags: [], PolicyDecision: null); // Act var hash1 = snapshot1.ComputeStateHash(); var hash2 = snapshot2.ComputeStateHash(); // Assert - hashes should be the same (scan ID and timestamp are not part of state hash) Assert.Equal(hash1, hash2); } [Fact] public void PriorityScore_IsConsistent() { // Arrange - KEV flip should always produce same priority var previous = new RiskStateSnapshot( FindingKey: new FindingKey("CVE-2024-6666", "pkg:npm/axios@0.21.0"), ScanId: "scan-prev", CapturedAt: DateTimeOffset.Parse("2024-12-01T10:00:00Z"), Reachable: true, LatticeState: "SR", VexStatus: VexStatusType.Affected, InAffectedRange: true, Kev: false, EpssScore: 0.08, PolicyFlags: [], PolicyDecision: null); var current = previous with { ScanId = "scan-curr", CapturedAt = DateTimeOffset.Parse("2024-12-15T10:00:00Z"), Kev = true }; // Act - detect multiple times var result1 = _detector.DetectChanges(previous, current); var result2 = _detector.DetectChanges(previous, current); var result3 = _detector.DetectChanges(previous, current); // Assert - priority score should be deterministic Assert.Equal(result1.PriorityScore, result2.PriorityScore); Assert.Equal(result2.PriorityScore, result3.PriorityScore); } #region Data Loading public static IEnumerable GetTestCases() { if (!File.Exists(FixturePath)) { yield break; } var json = File.ReadAllText(FixturePath); var fixture = JsonSerializer.Deserialize(json, JsonOptions); if (fixture?.TestCases is null) { yield break; } foreach (var testCase in fixture.TestCases) { yield return new object[] { testCase }; } } private static RiskStateSnapshot ParseSnapshot(SnapshotData data) { return new RiskStateSnapshot( FindingKey: new FindingKey(data.FindingKey.VulnId, data.FindingKey.Purl), ScanId: data.ScanId, CapturedAt: DateTimeOffset.Parse(data.CapturedAt), Reachable: data.Reachable, LatticeState: data.LatticeState, VexStatus: ParseVexStatus(data.VexStatus), InAffectedRange: data.InAffectedRange, Kev: data.Kev, EpssScore: data.EpssScore, PolicyFlags: data.PolicyFlags?.ToImmutableArray() ?? [], PolicyDecision: ParsePolicyDecision(data.PolicyDecision)); } private static VexStatusType ParseVexStatus(string value) { return value.ToLowerInvariant() switch { "affected" => VexStatusType.Affected, "not_affected" => VexStatusType.NotAffected, "fixed" => VexStatusType.Fixed, "under_investigation" => VexStatusType.UnderInvestigation, _ => VexStatusType.Unknown }; } private static PolicyDecisionType? ParsePolicyDecision(string? value) { if (string.IsNullOrEmpty(value)) return null; return value.ToLowerInvariant() switch { "allow" => PolicyDecisionType.Allow, "warn" => PolicyDecisionType.Warn, "block" => PolicyDecisionType.Block, _ => null }; } private static MaterialChangeType ParseChangeType(string value) { return value.ToLowerInvariant() switch { "reachability_flip" => MaterialChangeType.ReachabilityFlip, "vex_flip" => MaterialChangeType.VexFlip, "range_boundary" => MaterialChangeType.RangeBoundary, "kev_added" => MaterialChangeType.KevAdded, "kev_removed" => MaterialChangeType.KevRemoved, "epss_threshold" => MaterialChangeType.EpssThreshold, "policy_flip" => MaterialChangeType.PolicyFlip, _ => throw new ArgumentException($"Unknown change type: {value}") }; } private static RiskDirection ParseDirection(string value) { return value.ToLowerInvariant() switch { "increased" => RiskDirection.Increased, "decreased" => RiskDirection.Decreased, "neutral" => RiskDirection.Neutral, _ => throw new ArgumentException($"Unknown direction: {value}") }; } #endregion } #region Fixture DTOs public class GoldenFixture { public string? Version { get; set; } public string? Description { get; set; } public List? TestCases { get; set; } } public class GoldenTestCase { public string Id { get; set; } = ""; public string Name { get; set; } = ""; public string? Rule { get; set; } public SnapshotData Previous { get; set; } = new(); public SnapshotData Current { get; set; } = new(); public ExpectedResult Expected { get; set; } = new(); public override string ToString() => $"{Id}: {Name}"; } public class SnapshotData { public FindingKeyData FindingKey { get; set; } = new(); public string ScanId { get; set; } = ""; public string CapturedAt { get; set; } = ""; public bool? Reachable { get; set; } public string? LatticeState { get; set; } public string VexStatus { get; set; } = "unknown"; public bool? InAffectedRange { get; set; } public bool Kev { get; set; } public double? EpssScore { get; set; } public List? PolicyFlags { get; set; } public string? PolicyDecision { get; set; } } public class FindingKeyData { public string VulnId { get; set; } = ""; public string Purl { get; set; } = ""; } public class ExpectedResult { public bool HasMaterialChange { get; set; } public string? Direction { get; set; } public string? ChangeType { get; set; } public int? ChangeCount { get; set; } public int? TotalPriorityScore { get; set; } public int? PriorityScoreContribution { get; set; } } #endregion