Add unit and integration tests for VexCandidateEmitter and SmartDiff repositories
- Implemented comprehensive unit tests for VexCandidateEmitter to validate candidate emission logic based on various scenarios including absent and present APIs, confidence thresholds, and rate limiting. - Added integration tests for SmartDiff PostgreSQL repositories, covering snapshot storage and retrieval, candidate storage, and material risk change handling. - Ensured tests validate correct behavior for storing, retrieving, and querying snapshots and candidates, including edge cases and expected outcomes.
This commit is contained in:
master
@@ -0,0 +1,374 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json;
|
||||
using StellaOps.Scanner.SmartDiff.Detection;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.SmartDiff.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Golden fixture tests for Smart-Diff state comparison determinism.
|
||||
/// Per Sprint 3500.3 - ensures stable, reproducible change detection.
|
||||
/// </summary>
|
||||
public class StateComparisonGoldenTests
|
||||
{
|
||||
private static readonly string FixturePath = Path.Combine(
|
||||
AppContext.BaseDirectory,
|
||||
"Fixtures",
|
||||
"state-comparison.v1.json");
|
||||
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
PropertyNameCaseInsensitive = true
|
||||
};
|
||||
|
||||
private readonly MaterialRiskChangeDetector _detector;
|
||||
|
||||
public StateComparisonGoldenTests()
|
||||
{
|
||||
_detector = new MaterialRiskChangeDetector();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GoldenFixture_Exists()
|
||||
{
|
||||
Assert.True(File.Exists(FixturePath), $"Fixture file not found: {FixturePath}");
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(GetTestCases))]
|
||||
public void DetectChanges_MatchesGoldenFixture(GoldenTestCase testCase)
|
||||
{
|
||||
// Arrange
|
||||
var previous = ParseSnapshot(testCase.Previous);
|
||||
var current = ParseSnapshot(testCase.Current);
|
||||
|
||||
// Act
|
||||
var result = _detector.DetectChanges(previous, current);
|
||||
|
||||
// Assert
|
||||
Assert.Equal(testCase.Expected.HasMaterialChange, result.HasMaterialChange);
|
||||
|
||||
if (testCase.Expected.ChangeCount.HasValue)
|
||||
{
|
||||
Assert.Equal(testCase.Expected.ChangeCount.Value, result.Changes.Length);
|
||||
}
|
||||
|
||||
if (testCase.Expected.TotalPriorityScore.HasValue)
|
||||
{
|
||||
Assert.Equal(testCase.Expected.TotalPriorityScore.Value, result.PriorityScore);
|
||||
}
|
||||
|
||||
if (testCase.Expected.ChangeType is not null && result.Changes.Length > 0)
|
||||
{
|
||||
var expectedType = ParseChangeType(testCase.Expected.ChangeType);
|
||||
Assert.Contains(result.Changes, c => c.ChangeType == expectedType);
|
||||
}
|
||||
|
||||
if (testCase.Expected.Direction is not null && result.Changes.Length > 0)
|
||||
{
|
||||
var expectedDirection = ParseDirection(testCase.Expected.Direction);
|
||||
Assert.Contains(result.Changes, c => c.Direction == expectedDirection);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StateHash_IsDeterministic()
|
||||
{
|
||||
// Arrange
|
||||
var snapshot = new RiskStateSnapshot(
|
||||
FindingKey: new FindingKey("CVE-2024-1111", "pkg:npm/test@1.0.0"),
|
||||
ScanId: "scan-hash-001",
|
||||
CapturedAt: DateTimeOffset.Parse("2024-12-15T10:00:00Z"),
|
||||
Reachable: true,
|
||||
LatticeState: "CR",
|
||||
VexStatus: VexStatusType.Affected,
|
||||
InAffectedRange: true,
|
||||
Kev: false,
|
||||
EpssScore: 0.05,
|
||||
PolicyFlags: ["FLAG_A", "FLAG_B"],
|
||||
PolicyDecision: PolicyDecisionType.Warn);
|
||||
|
||||
// Act - compute hash multiple times
|
||||
var hash1 = snapshot.ComputeStateHash();
|
||||
var hash2 = snapshot.ComputeStateHash();
|
||||
var hash3 = snapshot.ComputeStateHash();
|
||||
|
||||
// Assert - all hashes must be identical
|
||||
Assert.Equal(hash1, hash2);
|
||||
Assert.Equal(hash2, hash3);
|
||||
Assert.StartsWith("sha256:", hash1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StateHash_DiffersWithReachabilityChange()
|
||||
{
|
||||
// Arrange
|
||||
var baseSnapshot = new RiskStateSnapshot(
|
||||
FindingKey: new FindingKey("CVE-2024-1111", "pkg:npm/test@1.0.0"),
|
||||
ScanId: "scan-hash-001",
|
||||
CapturedAt: DateTimeOffset.Parse("2024-12-15T10:00:00Z"),
|
||||
Reachable: true,
|
||||
LatticeState: "CR",
|
||||
VexStatus: VexStatusType.Affected,
|
||||
InAffectedRange: true,
|
||||
Kev: false,
|
||||
EpssScore: 0.05,
|
||||
PolicyFlags: [],
|
||||
PolicyDecision: null);
|
||||
|
||||
var modifiedSnapshot = baseSnapshot with { Reachable = false };
|
||||
|
||||
// Act
|
||||
var hash1 = baseSnapshot.ComputeStateHash();
|
||||
var hash2 = modifiedSnapshot.ComputeStateHash();
|
||||
|
||||
// Assert - hashes must differ
|
||||
Assert.NotEqual(hash1, hash2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StateHash_DiffersWithVexStatusChange()
|
||||
{
|
||||
// Arrange
|
||||
var baseSnapshot = new RiskStateSnapshot(
|
||||
FindingKey: new FindingKey("CVE-2024-1111", "pkg:npm/test@1.0.0"),
|
||||
ScanId: "scan-hash-001",
|
||||
CapturedAt: DateTimeOffset.Parse("2024-12-15T10:00:00Z"),
|
||||
Reachable: true,
|
||||
LatticeState: "CR",
|
||||
VexStatus: VexStatusType.Affected,
|
||||
InAffectedRange: true,
|
||||
Kev: false,
|
||||
EpssScore: 0.05,
|
||||
PolicyFlags: [],
|
||||
PolicyDecision: null);
|
||||
|
||||
var modifiedSnapshot = baseSnapshot with { VexStatus = VexStatusType.NotAffected };
|
||||
|
||||
// Act
|
||||
var hash1 = baseSnapshot.ComputeStateHash();
|
||||
var hash2 = modifiedSnapshot.ComputeStateHash();
|
||||
|
||||
// Assert - hashes must differ
|
||||
Assert.NotEqual(hash1, hash2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StateHash_SameForEquivalentStates()
|
||||
{
|
||||
// Arrange - two snapshots with same risk-relevant fields but different scan IDs
|
||||
var snapshot1 = new RiskStateSnapshot(
|
||||
FindingKey: new FindingKey("CVE-2024-1111", "pkg:npm/test@1.0.0"),
|
||||
ScanId: "scan-001",
|
||||
CapturedAt: DateTimeOffset.Parse("2024-12-15T10:00:00Z"),
|
||||
Reachable: true,
|
||||
LatticeState: "CR",
|
||||
VexStatus: VexStatusType.Affected,
|
||||
InAffectedRange: true,
|
||||
Kev: false,
|
||||
EpssScore: 0.05,
|
||||
PolicyFlags: [],
|
||||
PolicyDecision: null);
|
||||
|
||||
var snapshot2 = new RiskStateSnapshot(
|
||||
FindingKey: new FindingKey("CVE-2024-1111", "pkg:npm/test@1.0.0"),
|
||||
ScanId: "scan-002", // Different scan ID
|
||||
CapturedAt: DateTimeOffset.Parse("2024-12-16T10:00:00Z"), // Different timestamp
|
||||
Reachable: true,
|
||||
LatticeState: "CR",
|
||||
VexStatus: VexStatusType.Affected,
|
||||
InAffectedRange: true,
|
||||
Kev: false,
|
||||
EpssScore: 0.05,
|
||||
PolicyFlags: [],
|
||||
PolicyDecision: null);
|
||||
|
||||
// Act
|
||||
var hash1 = snapshot1.ComputeStateHash();
|
||||
var hash2 = snapshot2.ComputeStateHash();
|
||||
|
||||
// Assert - hashes should be the same (scan ID and timestamp are not part of state hash)
|
||||
Assert.Equal(hash1, hash2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PriorityScore_IsConsistent()
|
||||
{
|
||||
// Arrange - KEV flip should always produce same priority
|
||||
var previous = new RiskStateSnapshot(
|
||||
FindingKey: new FindingKey("CVE-2024-6666", "pkg:npm/axios@0.21.0"),
|
||||
ScanId: "scan-prev",
|
||||
CapturedAt: DateTimeOffset.Parse("2024-12-01T10:00:00Z"),
|
||||
Reachable: true,
|
||||
LatticeState: "SR",
|
||||
VexStatus: VexStatusType.Affected,
|
||||
InAffectedRange: true,
|
||||
Kev: false,
|
||||
EpssScore: 0.08,
|
||||
PolicyFlags: [],
|
||||
PolicyDecision: null);
|
||||
|
||||
var current = previous with
|
||||
{
|
||||
ScanId = "scan-curr",
|
||||
CapturedAt = DateTimeOffset.Parse("2024-12-15T10:00:00Z"),
|
||||
Kev = true
|
||||
};
|
||||
|
||||
// Act - detect multiple times
|
||||
var result1 = _detector.DetectChanges(previous, current);
|
||||
var result2 = _detector.DetectChanges(previous, current);
|
||||
var result3 = _detector.DetectChanges(previous, current);
|
||||
|
||||
// Assert - priority score should be deterministic
|
||||
Assert.Equal(result1.PriorityScore, result2.PriorityScore);
|
||||
Assert.Equal(result2.PriorityScore, result3.PriorityScore);
|
||||
}
|
||||
|
||||
#region Data Loading
|
||||
|
||||
public static IEnumerable<object[]> GetTestCases()
|
||||
{
|
||||
if (!File.Exists(FixturePath))
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
var json = File.ReadAllText(FixturePath);
|
||||
var fixture = JsonSerializer.Deserialize<GoldenFixture>(json, JsonOptions);
|
||||
|
||||
if (fixture?.TestCases is null)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
foreach (var testCase in fixture.TestCases)
|
||||
{
|
||||
yield return new object[] { testCase };
|
||||
}
|
||||
}
|
||||
|
||||
private static RiskStateSnapshot ParseSnapshot(SnapshotData data)
|
||||
{
|
||||
return new RiskStateSnapshot(
|
||||
FindingKey: new FindingKey(data.FindingKey.VulnId, data.FindingKey.Purl),
|
||||
ScanId: data.ScanId,
|
||||
CapturedAt: DateTimeOffset.Parse(data.CapturedAt),
|
||||
Reachable: data.Reachable,
|
||||
LatticeState: data.LatticeState,
|
||||
VexStatus: ParseVexStatus(data.VexStatus),
|
||||
InAffectedRange: data.InAffectedRange,
|
||||
Kev: data.Kev,
|
||||
EpssScore: data.EpssScore,
|
||||
PolicyFlags: data.PolicyFlags?.ToImmutableArray() ?? [],
|
||||
PolicyDecision: ParsePolicyDecision(data.PolicyDecision));
|
||||
}
|
||||
|
||||
private static VexStatusType ParseVexStatus(string value)
|
||||
{
|
||||
return value.ToLowerInvariant() switch
|
||||
{
|
||||
"affected" => VexStatusType.Affected,
|
||||
"not_affected" => VexStatusType.NotAffected,
|
||||
"fixed" => VexStatusType.Fixed,
|
||||
"under_investigation" => VexStatusType.UnderInvestigation,
|
||||
_ => VexStatusType.Unknown
|
||||
};
|
||||
}
|
||||
|
||||
private static PolicyDecisionType? ParsePolicyDecision(string? value)
|
||||
{
|
||||
if (string.IsNullOrEmpty(value))
|
||||
return null;
|
||||
|
||||
return value.ToLowerInvariant() switch
|
||||
{
|
||||
"allow" => PolicyDecisionType.Allow,
|
||||
"warn" => PolicyDecisionType.Warn,
|
||||
"block" => PolicyDecisionType.Block,
|
||||
_ => null
|
||||
};
|
||||
}
|
||||
|
||||
private static MaterialChangeType ParseChangeType(string value)
|
||||
{
|
||||
return value.ToLowerInvariant() switch
|
||||
{
|
||||
"reachability_flip" => MaterialChangeType.ReachabilityFlip,
|
||||
"vex_flip" => MaterialChangeType.VexFlip,
|
||||
"range_boundary" => MaterialChangeType.RangeBoundary,
|
||||
"kev_added" => MaterialChangeType.KevAdded,
|
||||
"kev_removed" => MaterialChangeType.KevRemoved,
|
||||
"epss_threshold" => MaterialChangeType.EpssThreshold,
|
||||
"policy_flip" => MaterialChangeType.PolicyFlip,
|
||||
_ => throw new ArgumentException($"Unknown change type: {value}")
|
||||
};
|
||||
}
|
||||
|
||||
private static RiskDirection ParseDirection(string value)
|
||||
{
|
||||
return value.ToLowerInvariant() switch
|
||||
{
|
||||
"increased" => RiskDirection.Increased,
|
||||
"decreased" => RiskDirection.Decreased,
|
||||
"neutral" => RiskDirection.Neutral,
|
||||
_ => throw new ArgumentException($"Unknown direction: {value}")
|
||||
};
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
#region Fixture DTOs
|
||||
|
||||
public class GoldenFixture
|
||||
{
|
||||
public string? Version { get; set; }
|
||||
public string? Description { get; set; }
|
||||
public List<GoldenTestCase>? TestCases { get; set; }
|
||||
}
|
||||
|
||||
public class GoldenTestCase
|
||||
{
|
||||
public string Id { get; set; } = "";
|
||||
public string Name { get; set; } = "";
|
||||
public string? Rule { get; set; }
|
||||
public SnapshotData Previous { get; set; } = new();
|
||||
public SnapshotData Current { get; set; } = new();
|
||||
public ExpectedResult Expected { get; set; } = new();
|
||||
|
||||
public override string ToString() => $"{Id}: {Name}";
|
||||
}
|
||||
|
||||
public class SnapshotData
|
||||
{
|
||||
public FindingKeyData FindingKey { get; set; } = new();
|
||||
public string ScanId { get; set; } = "";
|
||||
public string CapturedAt { get; set; } = "";
|
||||
public bool? Reachable { get; set; }
|
||||
public string? LatticeState { get; set; }
|
||||
public string VexStatus { get; set; } = "unknown";
|
||||
public bool? InAffectedRange { get; set; }
|
||||
public bool Kev { get; set; }
|
||||
public double? EpssScore { get; set; }
|
||||
public List<string>? PolicyFlags { get; set; }
|
||||
public string? PolicyDecision { get; set; }
|
||||
}
|
||||
|
||||
public class FindingKeyData
|
||||
{
|
||||
public string VulnId { get; set; } = "";
|
||||
public string Purl { get; set; } = "";
|
||||
}
|
||||
|
||||
public class ExpectedResult
|
||||
{
|
||||
public bool HasMaterialChange { get; set; }
|
||||
public string? Direction { get; set; }
|
||||
public string? ChangeType { get; set; }
|
||||
public int? ChangeCount { get; set; }
|
||||
public int? TotalPriorityScore { get; set; }
|
||||
public int? PriorityScoreContribution { get; set; }
|
||||
}
|
||||
|
||||
#endregion
|
||||
Reference in New Issue
Block a user