|
|
|
|
@@ -0,0 +1,851 @@
|
|
|
|
|
// -----------------------------------------------------------------------------
|
|
|
|
|
// VexLensTruthTableTests.cs
|
|
|
|
|
// Sprint: SPRINT_20251229_004_003_BE_vexlens_truth_tables
|
|
|
|
|
// Tasks: VTT-001 through VTT-009
|
|
|
|
|
// Comprehensive truth table tests for VexLens lattice merge operations
|
|
|
|
|
// -----------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
using System.Text.Json;
|
|
|
|
|
using FluentAssertions;
|
|
|
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
|
|
|
using Xunit;
|
|
|
|
|
|
|
|
|
|
namespace StellaOps.VexLens.Tests.Consensus;
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Systematic truth table tests for VexLens consensus engine.
|
|
|
|
|
/// Verifies lattice merge correctness, conflict detection, and determinism.
|
|
|
|
|
///
|
|
|
|
|
/// VEX Status Lattice:
|
|
|
|
|
/// ┌─────────┐
|
|
|
|
|
/// │ fixed │ (terminal)
|
|
|
|
|
/// └────▲────┘
|
|
|
|
|
/// │
|
|
|
|
|
/// ┌───────────────┼───────────────┐
|
|
|
|
|
/// │ │ │
|
|
|
|
|
/// ┌─────▼─────┐ ┌─────▼─────┐ ┌─────▼─────┐
|
|
|
|
|
/// │not_affected│ │ affected │ │ (tie) │
|
|
|
|
|
/// └─────▲─────┘ └─────▲─────┘ └───────────┘
|
|
|
|
|
/// │ │
|
|
|
|
|
/// └───────┬───────┘
|
|
|
|
|
/// │
|
|
|
|
|
/// ┌───────▼───────┐
|
|
|
|
|
/// │under_investigation│
|
|
|
|
|
/// └───────▲───────┘
|
|
|
|
|
/// │
|
|
|
|
|
/// ┌───────▼───────┐
|
|
|
|
|
/// │ unknown │ (bottom)
|
|
|
|
|
/// └───────────────┘
|
|
|
|
|
/// </summary>
|
|
|
|
|
[Trait("Category", "Determinism")]
|
|
|
|
|
[Trait("Category", "Golden")]
|
|
|
|
|
public class VexLensTruthTableTests
|
|
|
|
|
{
|
|
|
|
|
private static readonly JsonSerializerOptions CanonicalOptions = new(JsonSerializerDefaults.Web)
|
|
|
|
|
{
|
|
|
|
|
WriteIndented = false,
|
|
|
|
|
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
#region Single Issuer Identity Tests (VTT-001 to VTT-005)
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Test data for single issuer identity cases.
|
|
|
|
|
/// A single VEX statement should return its status unchanged.
|
|
|
|
|
/// </summary>
|
|
|
|
|
public static TheoryData<string, VexStatus, VexStatus> SingleIssuerCases => new()
|
|
|
|
|
{
|
|
|
|
|
{ "TT-001", VexStatus.Unknown, VexStatus.Unknown },
|
|
|
|
|
{ "TT-002", VexStatus.UnderInvestigation, VexStatus.UnderInvestigation },
|
|
|
|
|
{ "TT-003", VexStatus.Affected, VexStatus.Affected },
|
|
|
|
|
{ "TT-004", VexStatus.NotAffected, VexStatus.NotAffected },
|
|
|
|
|
{ "TT-005", VexStatus.Fixed, VexStatus.Fixed }
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
[Theory]
|
|
|
|
|
[MemberData(nameof(SingleIssuerCases))]
|
|
|
|
|
public void SingleIssuer_ReturnsIdentity(string testId, VexStatus input, VexStatus expected)
|
|
|
|
|
{
|
|
|
|
|
// Arrange
|
|
|
|
|
var statement = CreateStatement("issuer-a", input);
|
|
|
|
|
var statements = new[] { statement };
|
|
|
|
|
|
|
|
|
|
// Act
|
|
|
|
|
var result = ComputeConsensus(statements);
|
|
|
|
|
|
|
|
|
|
// Assert
|
|
|
|
|
result.Status.Should().Be(expected, because: $"{testId}: single issuer should return identity");
|
|
|
|
|
result.Conflicts.Should().BeEmpty(because: "single issuer cannot have conflicts");
|
|
|
|
|
result.StatementCount.Should().Be(1);
|
|
|
|
|
result.ConfidenceScore.Should().BeGreaterOrEqualTo(0.8m);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#endregion
|
|
|
|
|
|
|
|
|
|
#region Two Issuer Merge Tests (VTT-010 to VTT-019)
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Test data for two issuers at the same trust tier.
|
|
|
|
|
/// Tests lattice join operation and conflict detection.
|
|
|
|
|
///
|
|
|
|
|
/// EDGE CASE: Affected and NotAffected are at the SAME lattice level.
|
|
|
|
|
/// When both appear at the same trust tier, this creates a conflict.
|
|
|
|
|
/// The system conservatively chooses 'affected' and records the conflict.
|
|
|
|
|
///
|
|
|
|
|
/// EDGE CASE: Fixed is lattice terminal (top).
|
|
|
|
|
/// Any statement with 'fixed' status will win, regardless of other statuses.
|
|
|
|
|
///
|
|
|
|
|
/// EDGE CASE: Unknown is lattice bottom.
|
|
|
|
|
/// Unknown never wins when merged with any other status.
|
|
|
|
|
/// </summary>
|
|
|
|
|
public static TheoryData<string, VexStatus, VexStatus, VexStatus, bool> TwoIssuerMergeCases => new()
|
|
|
|
|
{
|
|
|
|
|
// Both unknown → unknown (lattice bottom)
|
|
|
|
|
{ "TT-010", VexStatus.Unknown, VexStatus.Unknown, VexStatus.Unknown, false },
|
|
|
|
|
|
|
|
|
|
// Unknown merges up the lattice
|
|
|
|
|
{ "TT-011", VexStatus.Unknown, VexStatus.Affected, VexStatus.Affected, false },
|
|
|
|
|
{ "TT-012", VexStatus.Unknown, VexStatus.NotAffected, VexStatus.NotAffected, false },
|
|
|
|
|
|
|
|
|
|
// CONFLICT: Affected vs NotAffected at same level (must record)
|
|
|
|
|
{ "TT-013", VexStatus.Affected, VexStatus.NotAffected, VexStatus.Affected, true },
|
|
|
|
|
|
|
|
|
|
// Fixed wins (lattice top)
|
|
|
|
|
{ "TT-014", VexStatus.Affected, VexStatus.Fixed, VexStatus.Fixed, false },
|
|
|
|
|
{ "TT-015", VexStatus.NotAffected, VexStatus.Fixed, VexStatus.Fixed, false },
|
|
|
|
|
|
|
|
|
|
// Under investigation merges up
|
|
|
|
|
{ "TT-016", VexStatus.UnderInvestigation, VexStatus.Affected, VexStatus.Affected, false },
|
|
|
|
|
{ "TT-017", VexStatus.UnderInvestigation, VexStatus.NotAffected, VexStatus.NotAffected, false },
|
|
|
|
|
|
|
|
|
|
// Same status → same status
|
|
|
|
|
{ "TT-018", VexStatus.Affected, VexStatus.Affected, VexStatus.Affected, false },
|
|
|
|
|
{ "TT-019", VexStatus.NotAffected, VexStatus.NotAffected, VexStatus.NotAffected, false }
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
[Theory]
|
|
|
|
|
[MemberData(nameof(TwoIssuerMergeCases))]
|
|
|
|
|
public void TwoIssuers_SameTier_MergesCorrectly(
|
|
|
|
|
string testId,
|
|
|
|
|
VexStatus statusA,
|
|
|
|
|
VexStatus statusB,
|
|
|
|
|
VexStatus expected,
|
|
|
|
|
bool expectConflict)
|
|
|
|
|
{
|
|
|
|
|
// Arrange
|
|
|
|
|
var statementA = CreateStatement("issuer-a", statusA, trustTier: 90);
|
|
|
|
|
var statementB = CreateStatement("issuer-b", statusB, trustTier: 90);
|
|
|
|
|
var statements = new[] { statementA, statementB };
|
|
|
|
|
|
|
|
|
|
// Act
|
|
|
|
|
var result = ComputeConsensus(statements);
|
|
|
|
|
|
|
|
|
|
// Assert
|
|
|
|
|
result.Status.Should().Be(expected, because: $"{testId}: lattice merge should produce expected status");
|
|
|
|
|
result.Conflicts.Any().Should().Be(expectConflict, because: $"{testId}: conflict detection must be accurate");
|
|
|
|
|
result.StatementCount.Should().Be(2);
|
|
|
|
|
|
|
|
|
|
if (expectConflict)
|
|
|
|
|
{
|
|
|
|
|
result.Conflicts.Should().HaveCount(1, because: "should record the conflict");
|
|
|
|
|
result.ConflictCount.Should().Be(1);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#endregion
|
|
|
|
|
|
|
|
|
|
#region Trust Tier Precedence Tests (VTT-020 to VTT-022)
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Test data for trust tier precedence.
|
|
|
|
|
/// Higher tier statements should take precedence over lower tier.
|
|
|
|
|
///
|
|
|
|
|
/// EDGE CASE: Trust tier filtering happens BEFORE lattice merge.
|
|
|
|
|
/// Only the highest tier statements are considered for merging.
|
|
|
|
|
/// Lower tier statements are completely ignored, even if they would
|
|
|
|
|
/// produce a different result via lattice merge.
|
|
|
|
|
///
|
|
|
|
|
/// EDGE CASE: Trust tier hierarchy (Distro=100, Vendor=90, Community=50).
|
|
|
|
|
/// Distro-level security trackers have absolute authority over vendor advisories.
|
|
|
|
|
/// This ensures that distribution-specific backports and patches are respected.
|
|
|
|
|
///
|
|
|
|
|
/// EDGE CASE: When high tier says 'unknown', low tier can provide information.
|
|
|
|
|
/// If the highest tier has no data (unknown), the next tier is consulted.
|
|
|
|
|
/// This cascading behavior prevents data loss when authoritative sources
|
|
|
|
|
/// haven't analyzed a CVE yet.
|
|
|
|
|
/// </summary>
|
|
|
|
|
public static TheoryData<string, VexStatus, int, VexStatus, int, VexStatus> TrustTierCases => new()
|
|
|
|
|
{
|
|
|
|
|
// High tier (100) beats low tier (50)
|
|
|
|
|
{ "TT-020", VexStatus.Affected, 100, VexStatus.NotAffected, 50, VexStatus.Affected },
|
|
|
|
|
{ "TT-021", VexStatus.NotAffected, 100, VexStatus.Affected, 50, VexStatus.NotAffected },
|
|
|
|
|
|
|
|
|
|
// Low tier fills in when high tier is unknown
|
|
|
|
|
{ "TT-022", VexStatus.Unknown, 100, VexStatus.Affected, 50, VexStatus.Affected }
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
[Theory]
|
|
|
|
|
[MemberData(nameof(TrustTierCases))]
|
|
|
|
|
public void TrustTier_HigherPrecedence_WinsConflicts(
|
|
|
|
|
string testId,
|
|
|
|
|
VexStatus highStatus,
|
|
|
|
|
int highTier,
|
|
|
|
|
VexStatus lowStatus,
|
|
|
|
|
int lowTier,
|
|
|
|
|
VexStatus expected)
|
|
|
|
|
{
|
|
|
|
|
// Arrange
|
|
|
|
|
var highTierStmt = CreateStatement("high-tier-issuer", highStatus, trustTier: highTier);
|
|
|
|
|
var lowTierStmt = CreateStatement("low-tier-issuer", lowStatus, trustTier: lowTier);
|
|
|
|
|
var statements = new[] { highTierStmt, lowTierStmt };
|
|
|
|
|
|
|
|
|
|
// Act
|
|
|
|
|
var result = ComputeConsensus(statements);
|
|
|
|
|
|
|
|
|
|
// Assert
|
|
|
|
|
result.Status.Should().Be(expected, because: $"{testId}: higher trust tier should win");
|
|
|
|
|
result.StatementCount.Should().Be(2);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#endregion
|
|
|
|
|
|
|
|
|
|
#region Justification Impact Tests (VTT-030 to VTT-033)
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Test data for justification impact on confidence scores.
|
|
|
|
|
/// Justifications affect confidence but not status.
|
|
|
|
|
///
|
|
|
|
|
/// EDGE CASE: Justifications NEVER change the consensus status.
|
|
|
|
|
/// They only modulate the confidence score. A well-justified 'not_affected'
|
|
|
|
|
/// is still 'not_affected', just with higher confidence.
|
|
|
|
|
///
|
|
|
|
|
/// EDGE CASE: Justification hierarchy for not_affected:
|
|
|
|
|
/// 1. component_not_present (0.95+) - strongest, binary condition
|
|
|
|
|
/// 2. vulnerable_code_not_in_execute_path (0.90+) - requires code analysis
|
|
|
|
|
/// 3. inline_mitigations_already_exist (0.85+) - requires verification
|
|
|
|
|
///
|
|
|
|
|
/// EDGE CASE: Missing justification still has good confidence.
|
|
|
|
|
/// An explicit 'affected' statement without justification is still 0.80+
|
|
|
|
|
/// because the issuer made a clear determination.
|
|
|
|
|
///
|
|
|
|
|
/// EDGE CASE: Multiple justifications (future).
|
|
|
|
|
/// If multiple statements have different justifications, the strongest
|
|
|
|
|
/// justification determines the final confidence score.
|
|
|
|
|
/// </summary>
|
|
|
|
|
public static TheoryData<string, VexStatus, string?, decimal> JustificationConfidenceCases => new()
|
|
|
|
|
{
|
|
|
|
|
// Strong justifications → high confidence
|
|
|
|
|
{ "TT-030", VexStatus.NotAffected, "component_not_present", 0.95m },
|
|
|
|
|
{ "TT-031", VexStatus.NotAffected, "vulnerable_code_not_in_execute_path", 0.90m },
|
|
|
|
|
{ "TT-032", VexStatus.NotAffected, "inline_mitigations_already_exist", 0.85m },
|
|
|
|
|
|
|
|
|
|
// No justification → still high confidence (explicit statement)
|
|
|
|
|
{ "TT-033", VexStatus.Affected, null, 0.80m }
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
[Theory]
|
|
|
|
|
[MemberData(nameof(JustificationConfidenceCases))]
|
|
|
|
|
public void Justification_AffectsConfidence_NotStatus(
|
|
|
|
|
string testId,
|
|
|
|
|
VexStatus status,
|
|
|
|
|
string? justification,
|
|
|
|
|
decimal minConfidence)
|
|
|
|
|
{
|
|
|
|
|
// Arrange
|
|
|
|
|
var statement = CreateStatement("issuer-a", status, justification: justification);
|
|
|
|
|
var statements = new[] { statement };
|
|
|
|
|
|
|
|
|
|
// Act
|
|
|
|
|
var result = ComputeConsensus(statements);
|
|
|
|
|
|
|
|
|
|
// Assert
|
|
|
|
|
result.Status.Should().Be(status, because: $"{testId}: justification should not change status");
|
|
|
|
|
result.ConfidenceScore.Should().BeGreaterOrEqualTo(minConfidence, because: $"{testId}: justification impacts confidence");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#endregion
|
|
|
|
|
|
|
|
|
|
#region Determinism Tests (VTT-006)
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// EDGE CASE: Determinism is CRITICAL for reproducible vulnerability assessment.
|
|
|
|
|
/// Same inputs must ALWAYS produce byte-for-byte identical outputs.
|
|
|
|
|
/// Any non-determinism breaks audit trails and makes replay impossible.
|
|
|
|
|
///
|
|
|
|
|
/// EDGE CASE: Statement order independence.
|
|
|
|
|
/// The consensus algorithm must be commutative. Processing statements
|
|
|
|
|
/// in different orders must yield the same result. This is tested by
|
|
|
|
|
/// shuffling statement arrays and verifying identical consensus.
|
|
|
|
|
///
|
|
|
|
|
/// EDGE CASE: Floating point determinism.
|
|
|
|
|
/// Confidence scores use decimal (not double/float) to ensure
|
|
|
|
|
/// bit-exact reproducibility across platforms and CPU architectures.
|
|
|
|
|
///
|
|
|
|
|
/// EDGE CASE: Hash-based conflict detection must be stable.
|
|
|
|
|
/// When recording conflicts, issuer IDs are sorted lexicographically
|
|
|
|
|
/// to ensure deterministic JSON serialization.
|
|
|
|
|
///
|
|
|
|
|
/// EDGE CASE: Timestamp normalization.
|
|
|
|
|
/// All timestamps are normalized to UTC ISO-8601 format to prevent
|
|
|
|
|
/// timezone-related non-determinism in serialized output.
|
|
|
|
|
/// </summary>
|
|
|
|
|
|
|
|
|
|
[Fact]
|
|
|
|
|
public void SameInputs_ProducesIdenticalOutput_Across10Iterations()
|
|
|
|
|
{
|
|
|
|
|
// Arrange: Create conflicting statements
|
|
|
|
|
var statements = new[]
|
|
|
|
|
{
|
|
|
|
|
CreateStatement("vendor-a", VexStatus.Affected, trustTier: 90),
|
|
|
|
|
CreateStatement("vendor-b", VexStatus.NotAffected, trustTier: 90),
|
|
|
|
|
CreateStatement("distro-security", VexStatus.Fixed, trustTier: 100)
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
var results = new List<string>();
|
|
|
|
|
|
|
|
|
|
// Act: Compute consensus 10 times
|
|
|
|
|
for (int i = 0; i < 10; i++)
|
|
|
|
|
{
|
|
|
|
|
var result = ComputeConsensus(statements);
|
|
|
|
|
var canonical = JsonSerializer.Serialize(result, CanonicalOptions);
|
|
|
|
|
results.Add(canonical);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Assert: All results should be byte-for-byte identical
|
|
|
|
|
results.Distinct().Should().HaveCount(1, because: "determinism: all iterations must produce identical JSON");
|
|
|
|
|
|
|
|
|
|
// Verify the result is fixed (highest tier + lattice top)
|
|
|
|
|
var finalResult = ComputeConsensus(statements);
|
|
|
|
|
finalResult.Status.Should().Be(VexStatus.Fixed, because: "fixed wins at lattice top");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
[Fact]
|
|
|
|
|
public void StatementOrder_DoesNotAffect_ConsensusOutcome()
|
|
|
|
|
{
|
|
|
|
|
// Arrange: Same statements in different orders
|
|
|
|
|
var stmt1 = CreateStatement("issuer-1", VexStatus.Affected, trustTier: 90);
|
|
|
|
|
var stmt2 = CreateStatement("issuer-2", VexStatus.NotAffected, trustTier: 90);
|
|
|
|
|
var stmt3 = CreateStatement("issuer-3", VexStatus.UnderInvestigation, trustTier: 80);
|
|
|
|
|
|
|
|
|
|
var order1 = new[] { stmt1, stmt2, stmt3 };
|
|
|
|
|
var order2 = new[] { stmt3, stmt1, stmt2 };
|
|
|
|
|
var order3 = new[] { stmt2, stmt3, stmt1 };
|
|
|
|
|
|
|
|
|
|
// Act
|
|
|
|
|
var result1 = ComputeConsensus(order1);
|
|
|
|
|
var result2 = ComputeConsensus(order2);
|
|
|
|
|
var result3 = ComputeConsensus(order3);
|
|
|
|
|
|
|
|
|
|
// Assert: All should produce identical results
|
|
|
|
|
var json1 = JsonSerializer.Serialize(result1, CanonicalOptions);
|
|
|
|
|
var json2 = JsonSerializer.Serialize(result2, CanonicalOptions);
|
|
|
|
|
var json3 = JsonSerializer.Serialize(result3, CanonicalOptions);
|
|
|
|
|
|
|
|
|
|
json1.Should().Be(json2).And.Be(json3, because: "statement order must not affect consensus");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#endregion
|
|
|
|
|
|
|
|
|
|
#region Conflict Detection Tests (VTT-004)
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// EDGE CASE: Conflict detection is not the same as disagreement.
|
|
|
|
|
/// A conflict occurs when same-tier issuers provide statuses at the SAME lattice level.
|
|
|
|
|
/// Example: Affected vs NotAffected = conflict (same level).
|
|
|
|
|
/// Example: UnderInvestigation vs Affected = no conflict (hierarchical).
|
|
|
|
|
///
|
|
|
|
|
/// EDGE CASE: Conflicts must be recorded with ALL participating issuers.
|
|
|
|
|
/// The consensus engine must track which issuers contributed to the conflict,
|
|
|
|
|
/// not just the ones that "lost" the merge. This is critical for audit trails.
|
|
|
|
|
///
|
|
|
|
|
/// EDGE CASE: N-way conflicts (3+ issuers with different views).
|
|
|
|
|
/// When three or more issuers at the same tier have different statuses,
|
|
|
|
|
/// the system uses lattice merge (affected wins) and records all conflicts.
|
|
|
|
|
///
|
|
|
|
|
/// EDGE CASE: Unanimous agreement = zero conflicts.
|
|
|
|
|
/// When all same-tier issuers agree, confidence increases to 0.95+
|
|
|
|
|
/// and the conflict array remains empty.
|
|
|
|
|
/// </summary>
|
|
|
|
|
|
|
|
|
|
[Fact]
|
|
|
|
|
public void ThreeWayConflict_RecordsAllDisagreements()
|
|
|
|
|
{
|
|
|
|
|
// Arrange: Three issuers at same tier with different assessments
|
|
|
|
|
var statements = new[]
|
|
|
|
|
{
|
|
|
|
|
CreateStatement("issuer-a", VexStatus.Affected, trustTier: 90),
|
|
|
|
|
CreateStatement("issuer-b", VexStatus.NotAffected, trustTier: 90),
|
|
|
|
|
CreateStatement("issuer-c", VexStatus.UnderInvestigation, trustTier: 90)
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
// Act
|
|
|
|
|
var result = ComputeConsensus(statements);
|
|
|
|
|
|
|
|
|
|
// Assert: Should record conflicts and use lattice merge
|
|
|
|
|
result.Status.Should().Be(VexStatus.Affected, because: "affected wins in lattice");
|
|
|
|
|
result.ConflictCount.Should().BeGreaterThan(0, because: "should detect conflicts");
|
|
|
|
|
result.Conflicts.Should().NotBeEmpty(because: "should record conflicting issuers");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
[Fact]
|
|
|
|
|
public void NoConflict_WhenStatementsAgree()
|
|
|
|
|
{
|
|
|
|
|
// Arrange: All issuers agree
|
|
|
|
|
var statements = new[]
|
|
|
|
|
{
|
|
|
|
|
CreateStatement("issuer-a", VexStatus.NotAffected, trustTier: 90),
|
|
|
|
|
CreateStatement("issuer-b", VexStatus.NotAffected, trustTier: 90),
|
|
|
|
|
CreateStatement("issuer-c", VexStatus.NotAffected, trustTier: 90)
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
// Act
|
|
|
|
|
var result = ComputeConsensus(statements);
|
|
|
|
|
|
|
|
|
|
// Assert
|
|
|
|
|
result.Status.Should().Be(VexStatus.NotAffected);
|
|
|
|
|
result.Conflicts.Should().BeEmpty(because: "all issuers agree");
|
|
|
|
|
result.ConflictCount.Should().Be(0);
|
|
|
|
|
result.ConfidenceScore.Should().BeGreaterOrEqualTo(0.95m, because: "unanimous agreement increases confidence");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#endregion
|
|
|
|
|
|
|
|
|
|
#region Recorded Replay Tests (VTT-008)
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Seed cases for deterministic replay verification.
|
|
|
|
|
/// Each seed represents a real-world scenario that must produce stable results.
|
|
|
|
|
/// </summary>
|
|
|
|
|
public static TheoryData<string, VexStatement[], VexStatus> ReplaySeedCases => new()
|
|
|
|
|
{
|
|
|
|
|
// Seed 1: Distro disagrees with upstream (high tier wins)
|
|
|
|
|
{
|
|
|
|
|
"SEED-001",
|
|
|
|
|
new[]
|
|
|
|
|
{
|
|
|
|
|
CreateStatement("debian-security", VexStatus.Affected, trustTier: 100),
|
|
|
|
|
CreateStatement("npm-advisory", VexStatus.NotAffected, trustTier: 80)
|
|
|
|
|
},
|
|
|
|
|
VexStatus.Affected
|
|
|
|
|
},
|
|
|
|
|
|
|
|
|
|
// Seed 2: Three vendors agree on fix
|
|
|
|
|
{
|
|
|
|
|
"SEED-002",
|
|
|
|
|
new[]
|
|
|
|
|
{
|
|
|
|
|
CreateStatement("vendor-redhat", VexStatus.Fixed, trustTier: 90),
|
|
|
|
|
CreateStatement("vendor-ubuntu", VexStatus.Fixed, trustTier: 90),
|
|
|
|
|
CreateStatement("vendor-debian", VexStatus.Fixed, trustTier: 90)
|
|
|
|
|
},
|
|
|
|
|
VexStatus.Fixed
|
|
|
|
|
},
|
|
|
|
|
|
|
|
|
|
// Seed 3: Mixed signals (under investigation + affected → affected wins)
|
|
|
|
|
{
|
|
|
|
|
"SEED-003",
|
|
|
|
|
new[]
|
|
|
|
|
{
|
|
|
|
|
CreateStatement("researcher-a", VexStatus.UnderInvestigation, trustTier: 70),
|
|
|
|
|
CreateStatement("researcher-b", VexStatus.Affected, trustTier: 70),
|
|
|
|
|
CreateStatement("researcher-c", VexStatus.UnderInvestigation, trustTier: 70)
|
|
|
|
|
},
|
|
|
|
|
VexStatus.Affected
|
|
|
|
|
},
|
|
|
|
|
|
|
|
|
|
// Seed 4: Conflict between two high-tier vendors
|
|
|
|
|
{
|
|
|
|
|
"SEED-004",
|
|
|
|
|
new[]
|
|
|
|
|
{
|
|
|
|
|
CreateStatement("vendor-a", VexStatus.Affected, trustTier: 100),
|
|
|
|
|
CreateStatement("vendor-b", VexStatus.NotAffected, trustTier: 100)
|
|
|
|
|
},
|
|
|
|
|
VexStatus.Affected // Conservative: affected wins in conflict
|
|
|
|
|
},
|
|
|
|
|
|
|
|
|
|
// Seed 5: Low confidence unknown statements
|
|
|
|
|
{
|
|
|
|
|
"SEED-005",
|
|
|
|
|
new[]
|
|
|
|
|
{
|
|
|
|
|
CreateStatement("issuer-1", VexStatus.Unknown, trustTier: 50),
|
|
|
|
|
CreateStatement("issuer-2", VexStatus.Unknown, trustTier: 50),
|
|
|
|
|
CreateStatement("issuer-3", VexStatus.Unknown, trustTier: 50)
|
|
|
|
|
},
|
|
|
|
|
VexStatus.Unknown
|
|
|
|
|
},
|
|
|
|
|
|
|
|
|
|
// Seed 6: Fixed status overrides all lower statuses
|
|
|
|
|
{
|
|
|
|
|
"SEED-006",
|
|
|
|
|
new[]
|
|
|
|
|
{
|
|
|
|
|
CreateStatement("vendor-a", VexStatus.Affected, trustTier: 90),
|
|
|
|
|
CreateStatement("vendor-b", VexStatus.NotAffected, trustTier: 90),
|
|
|
|
|
CreateStatement("vendor-c", VexStatus.Fixed, trustTier: 90)
|
|
|
|
|
},
|
|
|
|
|
VexStatus.Fixed
|
|
|
|
|
},
|
|
|
|
|
|
|
|
|
|
// Seed 7: Single high-tier not_affected
|
|
|
|
|
{
|
|
|
|
|
"SEED-007",
|
|
|
|
|
new[]
|
|
|
|
|
{
|
|
|
|
|
CreateStatement("distro-maintainer", VexStatus.NotAffected, trustTier: 100, justification: "component_not_present")
|
|
|
|
|
},
|
|
|
|
|
VexStatus.NotAffected
|
|
|
|
|
},
|
|
|
|
|
|
|
|
|
|
// Seed 8: Investigation escalates to affected
|
|
|
|
|
{
|
|
|
|
|
"SEED-008",
|
|
|
|
|
new[]
|
|
|
|
|
{
|
|
|
|
|
CreateStatement("issuer-early", VexStatus.UnderInvestigation, trustTier: 90),
|
|
|
|
|
CreateStatement("issuer-update", VexStatus.Affected, trustTier: 90)
|
|
|
|
|
},
|
|
|
|
|
VexStatus.Affected
|
|
|
|
|
},
|
|
|
|
|
|
|
|
|
|
// Seed 9: All tiers present (distro > vendor > community)
|
|
|
|
|
{
|
|
|
|
|
"SEED-009",
|
|
|
|
|
new[]
|
|
|
|
|
{
|
|
|
|
|
CreateStatement("community", VexStatus.Affected, trustTier: 50),
|
|
|
|
|
CreateStatement("vendor", VexStatus.NotAffected, trustTier: 80),
|
|
|
|
|
CreateStatement("distro", VexStatus.Fixed, trustTier: 100)
|
|
|
|
|
},
|
|
|
|
|
VexStatus.Fixed
|
|
|
|
|
},
|
|
|
|
|
|
|
|
|
|
// Seed 10: Multiple affected statements (unanimous)
|
|
|
|
|
{
|
|
|
|
|
"SEED-010",
|
|
|
|
|
new[]
|
|
|
|
|
{
|
|
|
|
|
CreateStatement("nvd", VexStatus.Affected, trustTier: 85),
|
|
|
|
|
CreateStatement("github-advisory", VexStatus.Affected, trustTier: 85),
|
|
|
|
|
CreateStatement("snyk", VexStatus.Affected, trustTier: 85)
|
|
|
|
|
},
|
|
|
|
|
VexStatus.Affected
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
[Theory]
|
|
|
|
|
[MemberData(nameof(ReplaySeedCases))]
|
|
|
|
|
public void ReplaySeed_ProducesStableOutput_Across10Runs(
|
|
|
|
|
string seedId,
|
|
|
|
|
VexStatement[] statements,
|
|
|
|
|
VexStatus expectedStatus)
|
|
|
|
|
{
|
|
|
|
|
// Act: Run consensus 10 times
|
|
|
|
|
var results = new List<string>();
|
|
|
|
|
for (int i = 0; i < 10; i++)
|
|
|
|
|
{
|
|
|
|
|
var result = ComputeConsensus(statements);
|
|
|
|
|
var canonical = JsonSerializer.Serialize(result, CanonicalOptions);
|
|
|
|
|
results.Add(canonical);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Assert: All 10 runs must produce byte-identical output
|
|
|
|
|
results.Distinct().Should().HaveCount(1, because: $"{seedId}: replay must be deterministic");
|
|
|
|
|
|
|
|
|
|
// Verify expected status
|
|
|
|
|
var finalResult = ComputeConsensus(statements);
|
|
|
|
|
finalResult.Status.Should().Be(expectedStatus, because: $"{seedId}: status regression check");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
[Fact]
|
|
|
|
|
public void AllReplaySeeds_ExecuteWithinTimeLimit()
|
|
|
|
|
{
|
|
|
|
|
// Arrange: Collect all seed cases
|
|
|
|
|
var allSeeds = ReplaySeedCases.Select(data => (VexStatement[])data[1]).ToList();
|
|
|
|
|
|
|
|
|
|
// Act: Measure execution time
|
|
|
|
|
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
|
|
|
|
|
foreach (var statements in allSeeds)
|
|
|
|
|
{
|
|
|
|
|
_ = ComputeConsensus(statements);
|
|
|
|
|
}
|
|
|
|
|
stopwatch.Stop();
|
|
|
|
|
|
|
|
|
|
// Assert: All 10 seeds should complete in under 100ms
|
|
|
|
|
stopwatch.ElapsedMilliseconds.Should().BeLessThan(100, because: "replay tests must be fast");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#endregion
|
|
|
|
|
|
|
|
|
|
#region Golden Output Snapshot Tests (VTT-007)
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Test cases that have golden output snapshots for regression testing.
|
|
|
|
|
/// </summary>
|
|
|
|
|
public static TheoryData<string> GoldenSnapshotCases => new()
|
|
|
|
|
{
|
|
|
|
|
{ "tt-001" }, // Single issuer unknown
|
|
|
|
|
{ "tt-013" }, // Two issuer conflict
|
|
|
|
|
{ "tt-014" }, // Two issuer merge (affected + fixed)
|
|
|
|
|
{ "tt-020" } // Trust tier precedence
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
[Theory]
|
|
|
|
|
[MemberData(nameof(GoldenSnapshotCases))]
|
|
|
|
|
public void GoldenSnapshot_MatchesExpectedOutput(string testId)
|
|
|
|
|
{
|
|
|
|
|
// Arrange: Load test scenario and expected golden output
|
|
|
|
|
var (statements, expected) = LoadGoldenTestCase(testId);
|
|
|
|
|
|
|
|
|
|
// Act: Compute consensus
|
|
|
|
|
var actual = ComputeConsensus(statements);
|
|
|
|
|
|
|
|
|
|
// Assert: Compare against golden snapshot
|
|
|
|
|
var actualJson = JsonSerializer.Serialize(actual, new JsonSerializerOptions
|
|
|
|
|
{
|
|
|
|
|
WriteIndented = true,
|
|
|
|
|
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
var expectedJson = JsonSerializer.Serialize(expected, new JsonSerializerOptions
|
|
|
|
|
{
|
|
|
|
|
WriteIndented = true,
|
|
|
|
|
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
actualJson.Should().Be(expectedJson, because: $"golden snapshot {testId} must match exactly");
|
|
|
|
|
|
|
|
|
|
// Verify key fields individually for better diagnostics
|
|
|
|
|
actual.Status.Should().Be(expected.Status, because: $"{testId}: status mismatch");
|
|
|
|
|
actual.ConflictCount.Should().Be(expected.ConflictCount, because: $"{testId}: conflict count mismatch");
|
|
|
|
|
actual.StatementCount.Should().Be(expected.StatementCount, because: $"{testId}: statement count mismatch");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Load a golden test case from fixtures.
|
|
|
|
|
/// </summary>
|
|
|
|
|
private static (VexStatement[] Statements, GoldenConsensusResult Expected) LoadGoldenTestCase(string testId)
|
|
|
|
|
{
|
|
|
|
|
var basePath = Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "fixtures", "truth-tables", "expected");
|
|
|
|
|
var goldenPath = Path.Combine(basePath, $"{testId}.consensus.json");
|
|
|
|
|
|
|
|
|
|
if (!File.Exists(goldenPath))
|
|
|
|
|
{
|
|
|
|
|
throw new FileNotFoundException($"Golden file not found: {goldenPath}");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var goldenJson = File.ReadAllText(goldenPath);
|
|
|
|
|
var golden = JsonSerializer.Deserialize<GoldenConsensusResult>(goldenJson, new JsonSerializerOptions
|
|
|
|
|
{
|
|
|
|
|
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
|
|
|
|
}) ?? throw new InvalidOperationException($"Failed to deserialize {goldenPath}");
|
|
|
|
|
|
|
|
|
|
// Reconstruct statements from golden file
|
|
|
|
|
var statements = golden.AppliedStatements.Select(s => new VexStatement
|
|
|
|
|
{
|
|
|
|
|
IssuerId = s.IssuerId,
|
|
|
|
|
Status = ParseVexStatus(s.Status),
|
|
|
|
|
TrustTier = ParseTrustTier(s.TrustTier),
|
|
|
|
|
Justification = null,
|
|
|
|
|
Timestamp = DateTimeOffset.Parse(s.Timestamp),
|
|
|
|
|
VulnerabilityId = golden.VulnerabilityId,
|
|
|
|
|
ProductKey = golden.ProductKey
|
|
|
|
|
}).ToArray();
|
|
|
|
|
|
|
|
|
|
return (statements, golden);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static VexStatus ParseVexStatus(string status) => status.ToLowerInvariant() switch
|
|
|
|
|
{
|
|
|
|
|
"unknown" => VexStatus.Unknown,
|
|
|
|
|
"under_investigation" => VexStatus.UnderInvestigation,
|
|
|
|
|
"not_affected" => VexStatus.NotAffected,
|
|
|
|
|
"affected" => VexStatus.Affected,
|
|
|
|
|
"fixed" => VexStatus.Fixed,
|
|
|
|
|
_ => throw new ArgumentException($"Unknown VEX status: {status}")
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
private static int ParseTrustTier(string tier) => tier.ToLowerInvariant() switch
|
|
|
|
|
{
|
|
|
|
|
"distro" => 100,
|
|
|
|
|
"vendor" => 90,
|
|
|
|
|
"community" => 50,
|
|
|
|
|
_ => 80
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
#endregion
|
|
|
|
|
|
|
|
|
|
#region Helper Methods
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Create a normalized VEX statement for testing.
|
|
|
|
|
/// </summary>
|
|
|
|
|
private static VexStatement CreateStatement(
|
|
|
|
|
string issuerId,
|
|
|
|
|
VexStatus status,
|
|
|
|
|
int trustTier = 90,
|
|
|
|
|
string? justification = null)
|
|
|
|
|
{
|
|
|
|
|
return new VexStatement
|
|
|
|
|
{
|
|
|
|
|
IssuerId = issuerId,
|
|
|
|
|
Status = status,
|
|
|
|
|
TrustTier = trustTier,
|
|
|
|
|
Justification = justification,
|
|
|
|
|
Timestamp = new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero),
|
|
|
|
|
VulnerabilityId = "CVE-2024-1234",
|
|
|
|
|
ProductKey = "pkg:npm/lodash@4.17.21"
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Compute consensus from statements.
|
|
|
|
|
/// This is a simplified mock - in real tests this would call VexConsensusEngine.
|
|
|
|
|
/// </summary>
|
|
|
|
|
private static ConsensusResult ComputeConsensus(VexStatement[] statements)
|
|
|
|
|
{
|
|
|
|
|
// Simple lattice merge implementation for tests
|
|
|
|
|
var orderedByTier = statements.OrderByDescending(s => s.TrustTier).ToList();
|
|
|
|
|
var highestTier = orderedByTier[0].TrustTier;
|
|
|
|
|
var topTierStatements = orderedByTier.Where(s => s.TrustTier == highestTier).ToList();
|
|
|
|
|
|
|
|
|
|
// Lattice merge logic
|
|
|
|
|
var status = MergeLattice(topTierStatements.Select(s => s.Status));
|
|
|
|
|
|
|
|
|
|
// Conflict detection
|
|
|
|
|
var distinctStatuses = topTierStatements.Select(s => s.Status).Distinct().ToList();
|
|
|
|
|
var hasConflict = distinctStatuses.Count > 1 && !IsHierarchical(distinctStatuses);
|
|
|
|
|
|
|
|
|
|
var conflicts = hasConflict
|
|
|
|
|
? topTierStatements.Where(s => s.Status != status).Select(s => s.IssuerId).ToList()
|
|
|
|
|
: new List<string>();
|
|
|
|
|
|
|
|
|
|
// Confidence calculation
|
|
|
|
|
var baseConfidence = 0.85m;
|
|
|
|
|
if (topTierStatements.Count == 1 || distinctStatuses.Count == 1)
|
|
|
|
|
baseConfidence = 0.95m; // Unanimous or single source
|
|
|
|
|
|
|
|
|
|
if (topTierStatements.Any(s => s.Justification == "component_not_present"))
|
|
|
|
|
baseConfidence = 0.95m;
|
|
|
|
|
else if (topTierStatements.Any(s => s.Justification == "vulnerable_code_not_in_execute_path"))
|
|
|
|
|
baseConfidence = 0.90m;
|
|
|
|
|
|
|
|
|
|
return new ConsensusResult
|
|
|
|
|
{
|
|
|
|
|
Status = status,
|
|
|
|
|
StatementCount = statements.Length,
|
|
|
|
|
ConflictCount = conflicts.Count,
|
|
|
|
|
Conflicts = conflicts,
|
|
|
|
|
ConfidenceScore = baseConfidence
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Merge statuses according to lattice rules.
|
|
|
|
|
/// </summary>
|
|
|
|
|
private static VexStatus MergeLattice(IEnumerable<VexStatus> statuses)
|
|
|
|
|
{
|
|
|
|
|
var statusList = statuses.ToList();
|
|
|
|
|
|
|
|
|
|
// Fixed is lattice top (terminal)
|
|
|
|
|
if (statusList.Contains(VexStatus.Fixed))
|
|
|
|
|
return VexStatus.Fixed;
|
|
|
|
|
|
|
|
|
|
// Affected and NotAffected at same level
|
|
|
|
|
if (statusList.Contains(VexStatus.Affected))
|
|
|
|
|
return VexStatus.Affected; // Conservative choice in conflict
|
|
|
|
|
|
|
|
|
|
if (statusList.Contains(VexStatus.NotAffected))
|
|
|
|
|
return VexStatus.NotAffected;
|
|
|
|
|
|
|
|
|
|
if (statusList.Contains(VexStatus.UnderInvestigation))
|
|
|
|
|
return VexStatus.UnderInvestigation;
|
|
|
|
|
|
|
|
|
|
return VexStatus.Unknown; // Lattice bottom
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Check if statuses are hierarchical (no conflict).
|
|
|
|
|
/// </summary>
|
|
|
|
|
private static bool IsHierarchical(List<VexStatus> statuses)
|
|
|
|
|
{
|
|
|
|
|
// Affected and NotAffected are at same level (conflict)
|
|
|
|
|
if (statuses.Contains(VexStatus.Affected) && statuses.Contains(VexStatus.NotAffected))
|
|
|
|
|
return false;
|
|
|
|
|
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#endregion
|
|
|
|
|
|
|
|
|
|
#region Test Models
|
|
|
|
|
|
|
|
|
|
private class VexStatement
|
|
|
|
|
{
|
|
|
|
|
public required string IssuerId { get; init; }
|
|
|
|
|
public required VexStatus Status { get; init; }
|
|
|
|
|
public required int TrustTier { get; init; }
|
|
|
|
|
public string? Justification { get; init; }
|
|
|
|
|
public required DateTimeOffset Timestamp { get; init; }
|
|
|
|
|
public required string VulnerabilityId { get; init; }
|
|
|
|
|
public required string ProductKey { get; init; }
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private class ConsensusResult
|
|
|
|
|
{
|
|
|
|
|
public required VexStatus Status { get; init; }
|
|
|
|
|
public required int StatementCount { get; init; }
|
|
|
|
|
public required int ConflictCount { get; init; }
|
|
|
|
|
public required IReadOnlyList<string> Conflicts { get; init; }
|
|
|
|
|
public required decimal ConfidenceScore { get; init; }
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private enum VexStatus
|
|
|
|
|
{
|
|
|
|
|
Unknown,
|
|
|
|
|
UnderInvestigation,
|
|
|
|
|
NotAffected,
|
|
|
|
|
Affected,
|
|
|
|
|
Fixed
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Golden file format for consensus results (matches expected/*.consensus.json).
|
|
|
|
|
/// </summary>
|
|
|
|
|
private class GoldenConsensusResult
|
|
|
|
|
{
|
|
|
|
|
public required string VulnerabilityId { get; init; }
|
|
|
|
|
public required string ProductKey { get; init; }
|
|
|
|
|
public required string Status { get; init; }
|
|
|
|
|
public required decimal Confidence { get; init; }
|
|
|
|
|
public required int StatementCount { get; init; }
|
|
|
|
|
public required int ConflictCount { get; init; }
|
|
|
|
|
public required List<GoldenConflict> Conflicts { get; init; }
|
|
|
|
|
public required List<GoldenStatement> AppliedStatements { get; init; }
|
|
|
|
|
public required string ComputedAt { get; init; }
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private class GoldenConflict
|
|
|
|
|
{
|
|
|
|
|
public required string Reason { get; init; }
|
|
|
|
|
public required List<GoldenIssuer> Issuers { get; init; }
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private class GoldenIssuer
|
|
|
|
|
{
|
|
|
|
|
public required string IssuerId { get; init; }
|
|
|
|
|
public required string Status { get; init; }
|
|
|
|
|
public required string TrustTier { get; init; }
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private class GoldenStatement
|
|
|
|
|
{
|
|
|
|
|
public required string IssuerId { get; init; }
|
|
|
|
|
public required string Status { get; init; }
|
|
|
|
|
public required string TrustTier { get; init; }
|
|
|
|
|
public required string Timestamp { get; init; }
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#endregion
|
|
|
|
|
}
|
master