sprints and audit work
This commit is contained in:
StellaOps Bot
@@ -0,0 +1,275 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// DeterminismPropertyTests.cs
|
||||
// Sprint: SPRINT_20260106_001_002_LB_determinization_scoring
|
||||
// Task: DCS-023 - Write determinism tests: same snapshot same entropy
|
||||
// Description: Property-based tests ensuring identical inputs produce identical
|
||||
// outputs across multiple invocations and calculator instances.
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using StellaOps.Policy.Determinization.Evidence;
|
||||
using StellaOps.Policy.Determinization.Models;
|
||||
using StellaOps.Policy.Determinization.Scoring;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Policy.Determinization.Tests.PropertyTests;
|
||||
|
||||
/// <summary>
|
||||
/// Property tests verifying determinism.
|
||||
/// DCS-023: same inputs must yield same outputs, always.
|
||||
/// </summary>
|
||||
[Trait("Category", "Unit")]
|
||||
[Trait("Property", "Determinism")]
|
||||
public class DeterminismPropertyTests
|
||||
{
|
||||
private readonly DateTimeOffset _fixedTime = new(2026, 1, 7, 12, 0, 0, TimeSpan.Zero);
|
||||
|
||||
/// <summary>
|
||||
/// Property: Same snapshot produces same entropy on repeated calls.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Entropy_SameSnapshot_ProducesSameResult()
|
||||
{
|
||||
// Arrange
|
||||
var calculator = new UncertaintyScoreCalculator(NullLogger<UncertaintyScoreCalculator>.Instance);
|
||||
var snapshot = CreateDeterministicSnapshot();
|
||||
|
||||
// Act - calculate 10 times
|
||||
var results = new List<double>();
|
||||
for (var i = 0; i < 10; i++)
|
||||
{
|
||||
results.Add(calculator.CalculateEntropy(snapshot));
|
||||
}
|
||||
|
||||
// Assert - all results should be identical
|
||||
results.Distinct().Should().HaveCount(1, "same input should always produce same entropy");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Different calculator instances produce same entropy for same snapshot.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Entropy_DifferentInstances_ProduceSameResult()
|
||||
{
|
||||
// Arrange
|
||||
var snapshot = CreateDeterministicSnapshot();
|
||||
|
||||
// Act - create multiple instances and calculate
|
||||
var results = new List<double>();
|
||||
for (var i = 0; i < 5; i++)
|
||||
{
|
||||
var calculator = new UncertaintyScoreCalculator(NullLogger<UncertaintyScoreCalculator>.Instance);
|
||||
results.Add(calculator.CalculateEntropy(snapshot));
|
||||
}
|
||||
|
||||
// Assert - all results should be identical
|
||||
results.Distinct().Should().HaveCount(1, "different instances should produce same entropy for same input");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Parallel execution produces consistent results.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Entropy_ParallelExecution_ProducesConsistentResults()
|
||||
{
|
||||
// Arrange
|
||||
var calculator = new UncertaintyScoreCalculator(NullLogger<UncertaintyScoreCalculator>.Instance);
|
||||
var snapshot = CreateDeterministicSnapshot();
|
||||
|
||||
// Act - calculate in parallel
|
||||
var tasks = Enumerable.Range(0, 100)
|
||||
.Select(_ => Task.Run(() => calculator.CalculateEntropy(snapshot)))
|
||||
.ToArray();
|
||||
|
||||
Task.WaitAll(tasks);
|
||||
var results = tasks.Select(t => t.Result).ToList();
|
||||
|
||||
// Assert - all results should be identical
|
||||
results.Distinct().Should().HaveCount(1, "parallel execution should produce consistent results");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Same decay calculation produces same result.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Decay_SameInputs_ProducesSameResult()
|
||||
{
|
||||
// Arrange
|
||||
var calculator = new DecayedConfidenceCalculator(NullLogger<DecayedConfidenceCalculator>.Instance);
|
||||
var ageDays = 7.0;
|
||||
var halfLifeDays = 14.0;
|
||||
|
||||
// Act - calculate 10 times
|
||||
var results = new List<double>();
|
||||
for (var i = 0; i < 10; i++)
|
||||
{
|
||||
results.Add(calculator.CalculateDecayFactor(ageDays, halfLifeDays));
|
||||
}
|
||||
|
||||
// Assert - all results should be identical
|
||||
results.Distinct().Should().HaveCount(1, "same input should always produce same decay factor");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Same snapshot with same weights produces same entropy.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(0.25, 0.15, 0.25, 0.15, 0.10, 0.10)]
|
||||
[InlineData(0.30, 0.20, 0.20, 0.10, 0.10, 0.10)]
|
||||
[InlineData(0.16, 0.16, 0.16, 0.16, 0.18, 0.18)]
|
||||
public void Entropy_SameSnapshotSameWeights_ProducesSameResult(
|
||||
double vex, double epss, double reach, double runtime, double backport, double sbom)
|
||||
{
|
||||
// Arrange
|
||||
var calculator = new UncertaintyScoreCalculator(NullLogger<UncertaintyScoreCalculator>.Instance);
|
||||
var snapshot = CreateDeterministicSnapshot();
|
||||
var weights = new SignalWeights
|
||||
{
|
||||
VexWeight = vex,
|
||||
EpssWeight = epss,
|
||||
ReachabilityWeight = reach,
|
||||
RuntimeWeight = runtime,
|
||||
BackportWeight = backport,
|
||||
SbomLineageWeight = sbom
|
||||
};
|
||||
|
||||
// Act - calculate 5 times
|
||||
var results = new List<double>();
|
||||
for (var i = 0; i < 5; i++)
|
||||
{
|
||||
results.Add(calculator.CalculateEntropy(snapshot, weights));
|
||||
}
|
||||
|
||||
// Assert - all results should be identical
|
||||
results.Distinct().Should().HaveCount(1, "same snapshot + weights should always produce same entropy");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Order of snapshot construction doesn't affect entropy.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Entropy_EquivalentSnapshots_ProduceSameResult()
|
||||
{
|
||||
// Arrange
|
||||
var calculator = new UncertaintyScoreCalculator(NullLogger<UncertaintyScoreCalculator>.Instance);
|
||||
|
||||
// Create two snapshots with same values but constructed differently
|
||||
var snapshot1 = CreateSnapshotWithVexFirst();
|
||||
var snapshot2 = CreateSnapshotWithEpssFirst();
|
||||
|
||||
// Act
|
||||
var entropy1 = calculator.CalculateEntropy(snapshot1);
|
||||
var entropy2 = calculator.CalculateEntropy(snapshot2);
|
||||
|
||||
// Assert
|
||||
entropy1.Should().Be(entropy2, "equivalent snapshots should produce identical entropy");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Decay with floor is deterministic.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(1.0, 30, 14.0, 0.1)]
|
||||
[InlineData(0.8, 7, 7.0, 0.05)]
|
||||
[InlineData(0.5, 100, 30.0, 0.2)]
|
||||
public void Decay_WithFloor_IsDeterministic(double baseConfidence, int ageDays, double halfLifeDays, double floor)
|
||||
{
|
||||
// Arrange
|
||||
var calculator = new DecayedConfidenceCalculator(NullLogger<DecayedConfidenceCalculator>.Instance);
|
||||
|
||||
// Act - calculate 10 times
|
||||
var results = new List<double>();
|
||||
for (var i = 0; i < 10; i++)
|
||||
{
|
||||
results.Add(calculator.Calculate(baseConfidence, ageDays, halfLifeDays, floor));
|
||||
}
|
||||
|
||||
// Assert - all results should be identical
|
||||
results.Distinct().Should().HaveCount(1, "decay with floor should be deterministic");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Entropy calculation is independent of external state.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Entropy_IndependentOfGlobalState_ProducesConsistentResults()
|
||||
{
|
||||
// Arrange
|
||||
var snapshot = CreateDeterministicSnapshot();
|
||||
|
||||
// Act - interleave calculations with some "noise"
|
||||
var results = new List<double>();
|
||||
for (var i = 0; i < 10; i++)
|
||||
{
|
||||
// Create new calculator each time to verify no shared state issues
|
||||
var calculator = new UncertaintyScoreCalculator(NullLogger<UncertaintyScoreCalculator>.Instance);
|
||||
|
||||
// Do some unrelated operations
|
||||
_ = Guid.NewGuid();
|
||||
_ = DateTime.UtcNow;
|
||||
|
||||
results.Add(calculator.CalculateEntropy(snapshot));
|
||||
}
|
||||
|
||||
// Assert - all results should be identical
|
||||
results.Distinct().Should().HaveCount(1, "entropy should be independent of external state");
|
||||
}
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private SignalSnapshot CreateDeterministicSnapshot()
|
||||
{
|
||||
return new SignalSnapshot
|
||||
{
|
||||
Cve = "CVE-2024-1234",
|
||||
Purl = "pkg:test@1.0.0",
|
||||
Vex = SignalState<VexClaimSummary>.Queried(
|
||||
new VexClaimSummary { Status = "affected", Confidence = 0.9, StatementCount = 1, ComputedAt = _fixedTime },
|
||||
_fixedTime),
|
||||
Epss = SignalState<EpssEvidence>.Queried(
|
||||
new EpssEvidence { Cve = "CVE-2024-1234", Epss = 0.5, Percentile = 0.8, PublishedAt = _fixedTime },
|
||||
_fixedTime),
|
||||
Reachability = SignalState<ReachabilityEvidence>.Queried(
|
||||
new ReachabilityEvidence { Status = ReachabilityStatus.Reachable, AnalyzedAt = _fixedTime },
|
||||
_fixedTime),
|
||||
Runtime = SignalState<RuntimeEvidence>.NotQueried(),
|
||||
Backport = SignalState<BackportEvidence>.NotQueried(),
|
||||
Sbom = SignalState<SbomLineageEvidence>.NotQueried(),
|
||||
Cvss = SignalState<CvssEvidence>.Queried(
|
||||
new CvssEvidence { Vector = "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", Version = "3.1", BaseScore = 9.8, Severity = "CRITICAL", Source = "NVD", PublishedAt = _fixedTime },
|
||||
_fixedTime),
|
||||
SnapshotAt = _fixedTime
|
||||
};
|
||||
}
|
||||
|
||||
private SignalSnapshot CreateSnapshotWithVexFirst()
|
||||
{
|
||||
var snapshot = SignalSnapshot.Empty("CVE-2024-1234", "pkg:test@1.0", _fixedTime);
|
||||
return snapshot with
|
||||
{
|
||||
Vex = SignalState<VexClaimSummary>.Queried(
|
||||
new VexClaimSummary { Status = "affected", Confidence = 0.9, StatementCount = 1, ComputedAt = _fixedTime },
|
||||
_fixedTime),
|
||||
Epss = SignalState<EpssEvidence>.Queried(
|
||||
new EpssEvidence { Cve = "CVE-2024-1234", Epss = 0.5, Percentile = 0.8, PublishedAt = _fixedTime },
|
||||
_fixedTime)
|
||||
};
|
||||
}
|
||||
|
||||
private SignalSnapshot CreateSnapshotWithEpssFirst()
|
||||
{
|
||||
var snapshot = SignalSnapshot.Empty("CVE-2024-1234", "pkg:test@1.0", _fixedTime);
|
||||
return snapshot with
|
||||
{
|
||||
Epss = SignalState<EpssEvidence>.Queried(
|
||||
new EpssEvidence { Cve = "CVE-2024-1234", Epss = 0.5, Percentile = 0.8, PublishedAt = _fixedTime },
|
||||
_fixedTime),
|
||||
Vex = SignalState<VexClaimSummary>.Queried(
|
||||
new VexClaimSummary { Status = "affected", Confidence = 0.9, StatementCount = 1, ComputedAt = _fixedTime },
|
||||
_fixedTime)
|
||||
};
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
Reference in New Issue
Block a user