5100* tests strengthtenen work
This commit is contained in:
StellaOps Bot
@@ -0,0 +1,326 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
// SPDX-FileCopyrightText: 2025 StellaOps Contributors
|
||||
|
||||
using System.Collections.Immutable;
|
||||
using FluentAssertions;
|
||||
using FsCheck;
|
||||
using FsCheck.Xunit;
|
||||
using StellaOps.DeltaVerdict.Models;
|
||||
using StellaOps.DeltaVerdict.Policy;
|
||||
|
||||
namespace StellaOps.Policy.Engine.Tests.Properties;
|
||||
|
||||
/// <summary>
|
||||
/// Property-based tests for risk budget evaluation monotonicity.
|
||||
/// Verifies that tightening risk budgets cannot decrease severity verdicts.
|
||||
/// </summary>
|
||||
public sealed class RiskBudgetMonotonicityPropertyTests
|
||||
{
|
||||
private readonly RiskBudgetEvaluator _evaluator = new();
|
||||
|
||||
/// <summary>
|
||||
/// Property: Tightening critical vulnerability budget cannot flip a blocking verdict to passing.
|
||||
/// If a delta violates budget B₁, it must also violate any stricter budget B₂ (where B₂ ≤ B₁).
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property TighteningCriticalBudget_CannotReduceViolations()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
DeltaVerdictArbs.AnyDeltaVerdict(),
|
||||
DeltaVerdictArbs.NonNegativeInt(),
|
||||
DeltaVerdictArbs.NonNegativeInt(),
|
||||
(delta, budget1MaxCritical, reductionAmount) =>
|
||||
{
|
||||
// Arrange
|
||||
var budget1 = new RiskBudget
|
||||
{
|
||||
MaxNewCriticalVulnerabilities = budget1MaxCritical,
|
||||
MaxNewHighVulnerabilities = int.MaxValue, // Allow high
|
||||
MaxRiskScoreIncrease = decimal.MaxValue,
|
||||
MaxMagnitude = DeltaMagnitude.Catastrophic
|
||||
};
|
||||
|
||||
var budget2MaxCritical = Math.Max(0, budget1MaxCritical - reductionAmount);
|
||||
var budget2 = budget1 with { MaxNewCriticalVulnerabilities = budget2MaxCritical };
|
||||
|
||||
// Act
|
||||
var result1 = _evaluator.Evaluate(delta, budget1);
|
||||
var result2 = _evaluator.Evaluate(delta, budget2);
|
||||
|
||||
// Assert: If B₁ violates (blocking), B₂ (stricter) must also violate
|
||||
// Contrapositive: If B₂ passes, B₁ must also pass
|
||||
return (result2.IsWithinBudget || !result1.IsWithinBudget)
|
||||
.Label($"Budget1(max={budget1MaxCritical}) within={result1.IsWithinBudget}, " +
|
||||
$"Budget2(max={budget2MaxCritical}) within={result2.IsWithinBudget}");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Tightening high vulnerability budget preserves monotonicity.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property TighteningHighBudget_CannotReduceViolations()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
DeltaVerdictArbs.AnyDeltaVerdict(),
|
||||
DeltaVerdictArbs.NonNegativeInt(),
|
||||
DeltaVerdictArbs.NonNegativeInt(),
|
||||
(delta, budget1MaxHigh, reductionAmount) =>
|
||||
{
|
||||
var budget1 = new RiskBudget
|
||||
{
|
||||
MaxNewCriticalVulnerabilities = int.MaxValue,
|
||||
MaxNewHighVulnerabilities = budget1MaxHigh,
|
||||
MaxRiskScoreIncrease = decimal.MaxValue,
|
||||
MaxMagnitude = DeltaMagnitude.Catastrophic
|
||||
};
|
||||
|
||||
var budget2MaxHigh = Math.Max(0, budget1MaxHigh - reductionAmount);
|
||||
var budget2 = budget1 with { MaxNewHighVulnerabilities = budget2MaxHigh };
|
||||
|
||||
var result1 = _evaluator.Evaluate(delta, budget1);
|
||||
var result2 = _evaluator.Evaluate(delta, budget2);
|
||||
|
||||
return (result2.IsWithinBudget || !result1.IsWithinBudget)
|
||||
.Label($"High budget monotonicity: B1(max={budget1MaxHigh})={result1.IsWithinBudget}, " +
|
||||
$"B2(max={budget2MaxHigh})={result2.IsWithinBudget}");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Tightening risk score budget preserves monotonicity.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property TighteningRiskScoreBudget_CannotReduceViolations()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
DeltaVerdictArbs.AnyDeltaVerdict(),
|
||||
Arb.From(Gen.Choose(0, 1000).Select(x => (decimal)x)),
|
||||
Arb.From(Gen.Choose(0, 500).Select(x => (decimal)x)),
|
||||
(delta, budget1MaxScore, reductionAmount) =>
|
||||
{
|
||||
var budget1 = new RiskBudget
|
||||
{
|
||||
MaxNewCriticalVulnerabilities = int.MaxValue,
|
||||
MaxNewHighVulnerabilities = int.MaxValue,
|
||||
MaxRiskScoreIncrease = budget1MaxScore,
|
||||
MaxMagnitude = DeltaMagnitude.Catastrophic
|
||||
};
|
||||
|
||||
var budget2MaxScore = Math.Max(0, budget1MaxScore - reductionAmount);
|
||||
var budget2 = budget1 with { MaxRiskScoreIncrease = budget2MaxScore };
|
||||
|
||||
var result1 = _evaluator.Evaluate(delta, budget1);
|
||||
var result2 = _evaluator.Evaluate(delta, budget2);
|
||||
|
||||
return (result2.IsWithinBudget || !result1.IsWithinBudget)
|
||||
.Label($"Risk score monotonicity: B1(max={budget1MaxScore})={result1.IsWithinBudget}, " +
|
||||
$"B2(max={budget2MaxScore})={result2.IsWithinBudget}");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Tightening magnitude budget preserves monotonicity.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property TighteningMagnitudeBudget_CannotReduceViolations()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
DeltaVerdictArbs.AnyDeltaVerdict(),
|
||||
DeltaVerdictArbs.AnyMagnitude(),
|
||||
DeltaVerdictArbs.AnyMagnitude(),
|
||||
(delta, magnitude1, magnitude2) =>
|
||||
{
|
||||
// Ensure magnitude2 <= magnitude1 (stricter)
|
||||
var looserMag = (DeltaMagnitude)Math.Max((int)magnitude1, (int)magnitude2);
|
||||
var stricterMag = (DeltaMagnitude)Math.Min((int)magnitude1, (int)magnitude2);
|
||||
|
||||
var budget1 = new RiskBudget
|
||||
{
|
||||
MaxNewCriticalVulnerabilities = int.MaxValue,
|
||||
MaxNewHighVulnerabilities = int.MaxValue,
|
||||
MaxRiskScoreIncrease = decimal.MaxValue,
|
||||
MaxMagnitude = looserMag
|
||||
};
|
||||
|
||||
var budget2 = budget1 with { MaxMagnitude = stricterMag };
|
||||
|
||||
var result1 = _evaluator.Evaluate(delta, budget1);
|
||||
var result2 = _evaluator.Evaluate(delta, budget2);
|
||||
|
||||
return (result2.IsWithinBudget || !result1.IsWithinBudget)
|
||||
.Label($"Magnitude monotonicity: B1(max={looserMag})={result1.IsWithinBudget}, " +
|
||||
$"B2(max={stricterMag})={result2.IsWithinBudget}");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Adding blocked vulnerabilities can only increase violations.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property AddingBlockedVulnerabilities_CanOnlyIncreaseViolations()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
DeltaVerdictArbs.AnyDeltaVerdict(),
|
||||
Arb.From(Gen.ArrayOf(Gen.Elements("CVE-2024-0001", "CVE-2024-0002", "CVE-2024-0003"))),
|
||||
(delta, additionalBlocked) =>
|
||||
{
|
||||
var budget1 = new RiskBudget
|
||||
{
|
||||
MaxNewCriticalVulnerabilities = int.MaxValue,
|
||||
MaxNewHighVulnerabilities = int.MaxValue,
|
||||
MaxRiskScoreIncrease = decimal.MaxValue,
|
||||
MaxMagnitude = DeltaMagnitude.Catastrophic,
|
||||
BlockedVulnerabilities = ImmutableHashSet<string>.Empty
|
||||
};
|
||||
|
||||
var budget2 = budget1 with
|
||||
{
|
||||
BlockedVulnerabilities = additionalBlocked
|
||||
.ToImmutableHashSet(StringComparer.OrdinalIgnoreCase)
|
||||
};
|
||||
|
||||
var result1 = _evaluator.Evaluate(delta, budget1);
|
||||
var result2 = _evaluator.Evaluate(delta, budget2);
|
||||
|
||||
// More blocked CVEs can only add violations, not remove them
|
||||
return (result1.IsWithinBudget || !result2.IsWithinBudget)
|
||||
.Label($"Blocked monotonicity: B1(blocked=0)={result1.IsWithinBudget}, " +
|
||||
$"B2(blocked={additionalBlocked.Length})={result2.IsWithinBudget}");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Violation count is non-decreasing when tightening budgets.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property ViolationCount_NonDecreasing_WhenTighteningBudget()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
DeltaVerdictArbs.AnyDeltaVerdict(),
|
||||
DeltaVerdictArbs.AnyRiskBudget(),
|
||||
DeltaVerdictArbs.AnyRiskBudget(),
|
||||
(delta, budget1, budget2) =>
|
||||
{
|
||||
// Create consistently tighter budget
|
||||
var tighterBudget = new RiskBudget
|
||||
{
|
||||
MaxNewCriticalVulnerabilities = Math.Min(budget1.MaxNewCriticalVulnerabilities, budget2.MaxNewCriticalVulnerabilities),
|
||||
MaxNewHighVulnerabilities = Math.Min(budget1.MaxNewHighVulnerabilities, budget2.MaxNewHighVulnerabilities),
|
||||
MaxRiskScoreIncrease = Math.Min(budget1.MaxRiskScoreIncrease, budget2.MaxRiskScoreIncrease),
|
||||
MaxMagnitude = (DeltaMagnitude)Math.Min((int)budget1.MaxMagnitude, (int)budget2.MaxMagnitude)
|
||||
};
|
||||
|
||||
var looserBudget = new RiskBudget
|
||||
{
|
||||
MaxNewCriticalVulnerabilities = Math.Max(budget1.MaxNewCriticalVulnerabilities, budget2.MaxNewCriticalVulnerabilities),
|
||||
MaxNewHighVulnerabilities = Math.Max(budget1.MaxNewHighVulnerabilities, budget2.MaxNewHighVulnerabilities),
|
||||
MaxRiskScoreIncrease = Math.Max(budget1.MaxRiskScoreIncrease, budget2.MaxRiskScoreIncrease),
|
||||
MaxMagnitude = (DeltaMagnitude)Math.Max((int)budget1.MaxMagnitude, (int)budget2.MaxMagnitude)
|
||||
};
|
||||
|
||||
var looserResult = _evaluator.Evaluate(delta, looserBudget);
|
||||
var tighterResult = _evaluator.Evaluate(delta, tighterBudget);
|
||||
|
||||
return (tighterResult.Violations.Count >= looserResult.Violations.Count)
|
||||
.Label($"Violation count: looser={looserResult.Violations.Count}, tighter={tighterResult.Violations.Count}");
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Custom FsCheck arbitraries for DeltaVerdict types.
|
||||
/// </summary>
|
||||
internal static class DeltaVerdictArbs
|
||||
{
|
||||
public static Arbitrary<int> NonNegativeInt() =>
|
||||
Arb.From(Gen.Choose(0, 50));
|
||||
|
||||
public static Arbitrary<DeltaMagnitude> AnyMagnitude() =>
|
||||
Arb.From(Gen.Elements(
|
||||
DeltaMagnitude.None,
|
||||
DeltaMagnitude.Minimal,
|
||||
DeltaMagnitude.Low,
|
||||
DeltaMagnitude.Medium,
|
||||
DeltaMagnitude.High,
|
||||
DeltaMagnitude.Severe,
|
||||
DeltaMagnitude.Catastrophic));
|
||||
|
||||
public static Arbitrary<DeltaVerdict.Models.DeltaVerdict> AnyDeltaVerdict() =>
|
||||
Arb.From(
|
||||
from criticalCount in Gen.Choose(0, 5)
|
||||
from highCount in Gen.Choose(0, 10)
|
||||
from riskScoreChange in Gen.Choose(-100, 200)
|
||||
from magnitude in Gen.Elements(
|
||||
DeltaMagnitude.None,
|
||||
DeltaMagnitude.Minimal,
|
||||
DeltaMagnitude.Low,
|
||||
DeltaMagnitude.Medium,
|
||||
DeltaMagnitude.High,
|
||||
DeltaMagnitude.Severe,
|
||||
DeltaMagnitude.Catastrophic)
|
||||
select CreateDeltaVerdict(criticalCount, highCount, riskScoreChange, magnitude));
|
||||
|
||||
public static Arbitrary<RiskBudget> AnyRiskBudget() =>
|
||||
Arb.From(
|
||||
from maxCritical in Gen.Choose(0, 10)
|
||||
from maxHigh in Gen.Choose(0, 20)
|
||||
from maxRiskScore in Gen.Choose(0, 200)
|
||||
from maxMagnitude in Gen.Elements(
|
||||
DeltaMagnitude.None,
|
||||
DeltaMagnitude.Minimal,
|
||||
DeltaMagnitude.Low,
|
||||
DeltaMagnitude.Medium,
|
||||
DeltaMagnitude.High,
|
||||
DeltaMagnitude.Severe,
|
||||
DeltaMagnitude.Catastrophic)
|
||||
select new RiskBudget
|
||||
{
|
||||
MaxNewCriticalVulnerabilities = maxCritical,
|
||||
MaxNewHighVulnerabilities = maxHigh,
|
||||
MaxRiskScoreIncrease = maxRiskScore,
|
||||
MaxMagnitude = maxMagnitude
|
||||
});
|
||||
|
||||
private static DeltaVerdict.Models.DeltaVerdict CreateDeltaVerdict(
|
||||
int criticalCount,
|
||||
int highCount,
|
||||
int riskScoreChange,
|
||||
DeltaMagnitude magnitude)
|
||||
{
|
||||
var addedVulns = new List<VulnerabilityDelta>();
|
||||
|
||||
for (var i = 0; i < criticalCount; i++)
|
||||
{
|
||||
addedVulns.Add(new VulnerabilityDelta(
|
||||
$"CVE-2024-{1000 + i}",
|
||||
"Critical",
|
||||
9.8m,
|
||||
VulnerabilityDeltaType.Added,
|
||||
null));
|
||||
}
|
||||
|
||||
for (var i = 0; i < highCount; i++)
|
||||
{
|
||||
addedVulns.Add(new VulnerabilityDelta(
|
||||
$"CVE-2024-{2000 + i}",
|
||||
"High",
|
||||
7.5m,
|
||||
VulnerabilityDeltaType.Added,
|
||||
null));
|
||||
}
|
||||
|
||||
return new DeltaVerdict.Models.DeltaVerdict
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Timestamp = DateTime.UtcNow,
|
||||
BaselineDigest = "sha256:baseline",
|
||||
CurrentDigest = "sha256:current",
|
||||
AddedVulnerabilities = addedVulns,
|
||||
RemovedVulnerabilities = [],
|
||||
ChangedVulnerabilities = [],
|
||||
RiskScoreDelta = new RiskScoreDelta(0, riskScoreChange, riskScoreChange),
|
||||
Summary = new DeltaSummary(magnitude, addedVulns.Count, 0, 0)
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
// SPDX-FileCopyrightText: 2025 StellaOps Contributors
|
||||
|
||||
using FluentAssertions;
|
||||
using FsCheck;
|
||||
using FsCheck.Xunit;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using StellaOps.Policy.Unknowns;
|
||||
|
||||
namespace StellaOps.Policy.Engine.Tests.Properties;
|
||||
|
||||
/// <summary>
|
||||
/// Property-based tests for unknowns budget enforcement.
|
||||
/// Verifies that "fail if unknowns > N" behavior is consistent.
|
||||
/// </summary>
|
||||
public sealed class UnknownsBudgetPropertyTests
|
||||
{
|
||||
private readonly UnknownsBudgetEnforcer _enforcer;
|
||||
|
||||
public UnknownsBudgetPropertyTests()
|
||||
{
|
||||
_enforcer = new UnknownsBudgetEnforcer(NullLogger<UnknownsBudgetEnforcer>.Instance);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: If critical unknowns exceed budget, result is not within budget.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property CriticalUnknownsExceedingBudget_FailsEvaluation()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
UnknownsBudgetArbs.AnyUnknownsCounts(),
|
||||
UnknownsBudgetArbs.AnyUnknownsBudgetConfig(),
|
||||
(counts, budget) =>
|
||||
{
|
||||
var result = _enforcer.Evaluate(counts, budget);
|
||||
|
||||
var criticalExceeded = counts.Critical > budget.MaxCriticalUnknowns;
|
||||
var highExceeded = counts.High > budget.MaxHighUnknowns;
|
||||
var mediumExceeded = counts.Medium > budget.MaxMediumUnknowns;
|
||||
var lowExceeded = counts.Low > budget.MaxLowUnknowns;
|
||||
var totalExceeded = budget.MaxTotalUnknowns.HasValue && counts.Total > budget.MaxTotalUnknowns.Value;
|
||||
|
||||
var anyExceeded = criticalExceeded || highExceeded || mediumExceeded || lowExceeded || totalExceeded;
|
||||
|
||||
return (result.WithinBudget == !anyExceeded)
|
||||
.Label($"Counts={counts}, Budget={budget}, WithinBudget={result.WithinBudget}, AnyExceeded={anyExceeded}");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Zero counts are always within any budget.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property ZeroCounts_AlwaysWithinBudget()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
UnknownsBudgetArbs.AnyUnknownsBudgetConfig(),
|
||||
budget =>
|
||||
{
|
||||
var zeroCounts = new UnknownsCounts
|
||||
{
|
||||
Critical = 0,
|
||||
High = 0,
|
||||
Medium = 0,
|
||||
Low = 0
|
||||
};
|
||||
|
||||
var result = _enforcer.Evaluate(zeroCounts, budget);
|
||||
|
||||
return result.WithinBudget
|
||||
.Label($"Zero counts should always be within budget: {result.WithinBudget}");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Total count equals sum of individual counts.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property TotalCount_EqualsSumOfIndividualCounts()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
UnknownsBudgetArbs.AnyUnknownsCounts(),
|
||||
counts =>
|
||||
{
|
||||
var expectedTotal = counts.Critical + counts.High + counts.Medium + counts.Low;
|
||||
|
||||
return (counts.Total == expectedTotal)
|
||||
.Label($"Total={counts.Total}, Sum={expectedTotal}");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Tightening budget can only add violations, not remove them.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property TighteningBudget_MonotonicallyIncreasesViolations()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
UnknownsBudgetArbs.AnyUnknownsCounts(),
|
||||
UnknownsBudgetArbs.AnyUnknownsBudgetConfig(),
|
||||
UnknownsBudgetArbs.NonNegativeInt(),
|
||||
UnknownsBudgetArbs.NonNegativeInt(),
|
||||
UnknownsBudgetArbs.NonNegativeInt(),
|
||||
UnknownsBudgetArbs.NonNegativeInt(),
|
||||
(counts, baseBudget, criticalReduction, highReduction, mediumReduction, lowReduction) =>
|
||||
{
|
||||
var looserBudget = baseBudget with
|
||||
{
|
||||
MaxCriticalUnknowns = baseBudget.MaxCriticalUnknowns + criticalReduction,
|
||||
MaxHighUnknowns = baseBudget.MaxHighUnknowns + highReduction,
|
||||
MaxMediumUnknowns = baseBudget.MaxMediumUnknowns + mediumReduction,
|
||||
MaxLowUnknowns = baseBudget.MaxLowUnknowns + lowReduction
|
||||
};
|
||||
|
||||
var tighterBudget = baseBudget;
|
||||
|
||||
var looserResult = _enforcer.Evaluate(counts, looserBudget);
|
||||
var tighterResult = _enforcer.Evaluate(counts, tighterBudget);
|
||||
|
||||
// If looser budget fails, tighter must also fail
|
||||
// If tighter budget passes, looser must also pass
|
||||
return (looserResult.WithinBudget || !tighterResult.WithinBudget)
|
||||
.Label($"Monotonicity: Looser={looserResult.WithinBudget}, Tighter={tighterResult.WithinBudget}");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Block action is determined by budget violation status.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property ShouldBlock_CorrectlyReflectsViolationAndAction()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
UnknownsBudgetArbs.AnyUnknownsCounts(),
|
||||
UnknownsBudgetArbs.AnyUnknownsBudgetConfig(),
|
||||
(counts, budget) =>
|
||||
{
|
||||
var result = _enforcer.Evaluate(counts, budget);
|
||||
var shouldBlock = _enforcer.ShouldBlock(result);
|
||||
|
||||
var expectedBlock = !result.WithinBudget && result.Action == UnknownsBudgetAction.Block;
|
||||
|
||||
return (shouldBlock == expectedBlock)
|
||||
.Label($"ShouldBlock={shouldBlock}, Expected={expectedBlock}, " +
|
||||
$"WithinBudget={result.WithinBudget}, Action={result.Action}");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Warn action never blocks, even with violations.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property WarnAction_NeverBlocks()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
UnknownsBudgetArbs.AnyUnknownsCounts(),
|
||||
UnknownsBudgetArbs.AnyUnknownsBudgetConfig(),
|
||||
(counts, baseBudget) =>
|
||||
{
|
||||
var warnBudget = baseBudget with { Action = UnknownsBudgetAction.Warn };
|
||||
var result = _enforcer.Evaluate(counts, warnBudget);
|
||||
var shouldBlock = _enforcer.ShouldBlock(result);
|
||||
|
||||
return (!shouldBlock)
|
||||
.Label($"Warn action should never block: WithinBudget={result.WithinBudget}");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Log action never blocks, even with violations.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property LogAction_NeverBlocks()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
UnknownsBudgetArbs.AnyUnknownsCounts(),
|
||||
UnknownsBudgetArbs.AnyUnknownsBudgetConfig(),
|
||||
(counts, baseBudget) =>
|
||||
{
|
||||
var logBudget = baseBudget with { Action = UnknownsBudgetAction.Log };
|
||||
var result = _enforcer.Evaluate(counts, logBudget);
|
||||
var shouldBlock = _enforcer.ShouldBlock(result);
|
||||
|
||||
return (!shouldBlock)
|
||||
.Label($"Log action should never block: WithinBudget={result.WithinBudget}");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Violation messages accurately describe the exceeded limits.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property ViolationMessages_AccuratelyDescribeExceededLimits()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
UnknownsBudgetArbs.AnyUnknownsCounts(),
|
||||
UnknownsBudgetArbs.AnyUnknownsBudgetConfig(),
|
||||
(counts, budget) =>
|
||||
{
|
||||
var result = _enforcer.Evaluate(counts, budget);
|
||||
|
||||
var criticalViolation = result.Violations?.Any(v => v.Contains("Critical")) ?? false;
|
||||
var highViolation = result.Violations?.Any(v => v.Contains("High")) ?? false;
|
||||
var mediumViolation = result.Violations?.Any(v => v.Contains("Medium")) ?? false;
|
||||
var lowViolation = result.Violations?.Any(v => v.Contains("Low")) ?? false;
|
||||
var totalViolation = result.Violations?.Any(v => v.Contains("Total")) ?? false;
|
||||
|
||||
var criticalExceeded = counts.Critical > budget.MaxCriticalUnknowns;
|
||||
var highExceeded = counts.High > budget.MaxHighUnknowns;
|
||||
var mediumExceeded = counts.Medium > budget.MaxMediumUnknowns;
|
||||
var lowExceeded = counts.Low > budget.MaxLowUnknowns;
|
||||
var totalExceeded = budget.MaxTotalUnknowns.HasValue && counts.Total > budget.MaxTotalUnknowns.Value;
|
||||
|
||||
// Each exceeded limit should have a corresponding violation message
|
||||
return (criticalViolation == criticalExceeded &&
|
||||
highViolation == highExceeded &&
|
||||
mediumViolation == mediumExceeded &&
|
||||
lowViolation == lowExceeded &&
|
||||
totalViolation == totalExceeded)
|
||||
.Label($"Violations match exceeded limits");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Exceeding exactly N unknowns (N+1) should violate budget of N.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property ExceedingByOne_ViolatesBudget()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
UnknownsBudgetArbs.NonNegativeInt(),
|
||||
maxCritical =>
|
||||
{
|
||||
var counts = new UnknownsCounts
|
||||
{
|
||||
Critical = maxCritical + 1,
|
||||
High = 0,
|
||||
Medium = 0,
|
||||
Low = 0
|
||||
};
|
||||
|
||||
var budget = new UnknownsBudgetConfig
|
||||
{
|
||||
MaxCriticalUnknowns = maxCritical,
|
||||
MaxHighUnknowns = int.MaxValue,
|
||||
MaxMediumUnknowns = int.MaxValue,
|
||||
MaxLowUnknowns = int.MaxValue,
|
||||
MaxTotalUnknowns = null,
|
||||
Action = UnknownsBudgetAction.Block
|
||||
};
|
||||
|
||||
var result = _enforcer.Evaluate(counts, budget);
|
||||
|
||||
return (!result.WithinBudget)
|
||||
.Label($"Critical={maxCritical + 1} should violate budget of {maxCritical}");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Property: Meeting exactly N unknowns (N) should NOT violate budget of N.
|
||||
/// </summary>
|
||||
[Property(MaxTest = 100)]
|
||||
public Property MeetingExactly_DoesNotViolateBudget()
|
||||
{
|
||||
return Prop.ForAll(
|
||||
UnknownsBudgetArbs.NonNegativeInt(),
|
||||
maxCritical =>
|
||||
{
|
||||
var counts = new UnknownsCounts
|
||||
{
|
||||
Critical = maxCritical,
|
||||
High = 0,
|
||||
Medium = 0,
|
||||
Low = 0
|
||||
};
|
||||
|
||||
var budget = new UnknownsBudgetConfig
|
||||
{
|
||||
MaxCriticalUnknowns = maxCritical,
|
||||
MaxHighUnknowns = int.MaxValue,
|
||||
MaxMediumUnknowns = int.MaxValue,
|
||||
MaxLowUnknowns = int.MaxValue,
|
||||
MaxTotalUnknowns = null,
|
||||
Action = UnknownsBudgetAction.Block
|
||||
};
|
||||
|
||||
var result = _enforcer.Evaluate(counts, budget);
|
||||
|
||||
return result.WithinBudget
|
||||
.Label($"Critical={maxCritical} should NOT violate budget of {maxCritical}");
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Custom FsCheck arbitraries for UnknownsBudget types.
|
||||
/// </summary>
|
||||
internal static class UnknownsBudgetArbs
|
||||
{
|
||||
public static Arbitrary<int> NonNegativeInt() =>
|
||||
Arb.From(Gen.Choose(0, 100));
|
||||
|
||||
public static Arbitrary<UnknownsCounts> AnyUnknownsCounts() =>
|
||||
Arb.From(
|
||||
from critical in Gen.Choose(0, 20)
|
||||
from high in Gen.Choose(0, 50)
|
||||
from medium in Gen.Choose(0, 100)
|
||||
from low in Gen.Choose(0, 200)
|
||||
select new UnknownsCounts
|
||||
{
|
||||
Critical = critical,
|
||||
High = high,
|
||||
Medium = medium,
|
||||
Low = low
|
||||
});
|
||||
|
||||
public static Arbitrary<UnknownsBudgetConfig> AnyUnknownsBudgetConfig() =>
|
||||
Arb.From(
|
||||
from maxCritical in Gen.Choose(0, 10)
|
||||
from maxHigh in Gen.Choose(0, 30)
|
||||
from maxMedium in Gen.Choose(0, 80)
|
||||
from maxLow in Gen.Choose(0, 150)
|
||||
from maxTotal in Gen.OneOf(
|
||||
Gen.Constant<int?>(null),
|
||||
Gen.Choose(0, 300).Select(x => (int?)x))
|
||||
from action in Gen.Elements(
|
||||
UnknownsBudgetAction.Block,
|
||||
UnknownsBudgetAction.Warn,
|
||||
UnknownsBudgetAction.Log)
|
||||
select new UnknownsBudgetConfig
|
||||
{
|
||||
MaxCriticalUnknowns = maxCritical,
|
||||
MaxHighUnknowns = maxHigh,
|
||||
MaxMediumUnknowns = maxMedium,
|
||||
MaxLowUnknowns = maxLow,
|
||||
MaxTotalUnknowns = maxTotal,
|
||||
Action = action
|
||||
});
|
||||
}
|
||||
@@ -25,6 +25,8 @@
|
||||
</PackageReference>
|
||||
<PackageReference Include="FluentAssertions" Version="6.12.0" />
|
||||
<PackageReference Include="Moq" Version="4.20.70" />
|
||||
<PackageReference Include="FsCheck" Version="2.16.6" />
|
||||
<PackageReference Include="FsCheck.Xunit" Version="2.16.6" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
Reference in New Issue
Block a user