Add property-based tests for SBOM/VEX document ordering and Unicode normalization determinism

- Implement `SbomVexOrderingDeterminismProperties` for testing component list and vulnerability metadata hash consistency.
- Create `UnicodeNormalizationDeterminismProperties` to validate NFC normalization and Unicode string handling.
- Add project file for `StellaOps.Testing.Determinism.Properties` with necessary dependencies.
- Introduce CI/CD template validation tests including YAML syntax checks and documentation content verification.
- Create validation script for CI/CD templates ensuring all required files and structures are present.
This commit is contained in:
StellaOps Bot
2025-12-26 15:17:15 +02:00
parent 7792749bb4
commit 907783f625
354 changed files with 79727 additions and 1346 deletions

View File

@@ -210,6 +210,7 @@ public interface IBudgetStore
Task UpdateAsync(RiskBudget budget, CancellationToken ct);
Task AddEntryAsync(BudgetEntry entry, CancellationToken ct);
Task<IReadOnlyList<BudgetEntry>> GetEntriesAsync(string serviceId, string window, CancellationToken ct);
Task<IReadOnlyList<RiskBudget>> ListAsync(BudgetStatus? status = null, ServiceTier? tier = null, int limit = 50, CancellationToken ct = default);
}
/// <summary>
@@ -275,4 +276,23 @@ public sealed class InMemoryBudgetStore : IBudgetStore
return Task.FromResult<IReadOnlyList<BudgetEntry>>(result);
}
}
public Task<IReadOnlyList<RiskBudget>> ListAsync(BudgetStatus? status, ServiceTier? tier, int limit, CancellationToken ct)
{
ct.ThrowIfCancellationRequested();
lock (_lock)
{
var query = _budgets.Values.AsEnumerable();
if (status.HasValue)
{
query = query.Where(b => b.Status == status.Value);
}
if (tier.HasValue)
{
query = query.Where(b => b.Tier == tier.Value);
}
var result = query.Take(limit).ToList();
return Task.FromResult<IReadOnlyList<RiskBudget>>(result);
}
}
}

View File

@@ -0,0 +1,180 @@
// -----------------------------------------------------------------------------
// BudgetThresholdNotifier.cs
// Sprint: SPRINT_20251226_002_BE_budget_enforcement
// Task: BUDGET-06-07 - Budget threshold notifications
// Description: Publishes notification events when budget thresholds are crossed
// -----------------------------------------------------------------------------
using System.Collections.Immutable;
using System.Text.Json.Nodes;
using Microsoft.Extensions.Logging;
namespace StellaOps.Policy.Gates;
/// <summary>
/// Publishes notification events when budget thresholds are crossed.
/// </summary>
public sealed class BudgetThresholdNotifier
{
private readonly INotifyEventPublisher _publisher;
private readonly ILogger<BudgetThresholdNotifier> _logger;
/// <summary>
/// Thresholds for different budget status levels.
/// </summary>
public static class Thresholds
{
/// <summary>Yellow threshold: 40%</summary>
public const decimal Yellow = 0.40m;
/// <summary>Red threshold: 70%</summary>
public const decimal Red = 0.70m;
/// <summary>Exhausted threshold: 100%</summary>
public const decimal Exhausted = 1.00m;
}
/// <summary>
/// Create a new budget threshold notifier.
/// </summary>
public BudgetThresholdNotifier(
INotifyEventPublisher publisher,
ILogger<BudgetThresholdNotifier> logger)
{
_publisher = publisher ?? throw new ArgumentNullException(nameof(publisher));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
/// <summary>
/// Check if status has crossed a threshold and publish notification if needed.
/// </summary>
/// <param name="before">Budget status before the change.</param>
/// <param name="after">Budget status after the change.</param>
/// <param name="tenantId">Tenant identifier.</param>
/// <param name="ct">Cancellation token.</param>
public async Task NotifyIfThresholdCrossedAsync(
RiskBudget before,
RiskBudget after,
string tenantId,
CancellationToken ct = default)
{
// Check if status has worsened
if (after.Status > before.Status)
{
await PublishThresholdCrossedAsync(before, after, tenantId, ct);
}
}
/// <summary>
/// Publish a warning notification when approaching threshold.
/// </summary>
public async Task NotifyWarningAsync(
RiskBudget budget,
string tenantId,
CancellationToken ct = default)
{
if (budget.Status >= BudgetStatus.Yellow)
{
var payload = CreatePayload(budget, "warning");
await _publisher.PublishAsync(
BudgetEventKinds.PolicyBudgetWarning,
tenantId,
payload,
ct);
_logger.LogInformation(
"Published budget warning for {ServiceId}: {PercentageUsed}% consumed",
budget.ServiceId,
budget.PercentageUsed);
}
}
/// <summary>
/// Publish an exceeded notification when budget is exhausted.
/// </summary>
public async Task NotifyExceededAsync(
RiskBudget budget,
string tenantId,
CancellationToken ct = default)
{
var payload = CreatePayload(budget, "exceeded");
await _publisher.PublishAsync(
BudgetEventKinds.PolicyBudgetExceeded,
tenantId,
payload,
ct);
_logger.LogWarning(
"Published budget exceeded for {ServiceId}: {PercentageUsed}% consumed",
budget.ServiceId,
budget.PercentageUsed);
}
private async Task PublishThresholdCrossedAsync(
RiskBudget before,
RiskBudget after,
string tenantId,
CancellationToken ct)
{
var eventKind = after.Status == BudgetStatus.Exhausted
? BudgetEventKinds.PolicyBudgetExceeded
: BudgetEventKinds.PolicyBudgetWarning;
var payload = CreatePayload(after, after.Status.ToString().ToLowerInvariant());
payload["previousStatus"] = before.Status.ToString().ToLowerInvariant();
await _publisher.PublishAsync(eventKind, tenantId, payload, ct);
_logger.LogInformation(
"Published budget threshold crossed for {ServiceId}: {PreviousStatus} -> {NewStatus}",
after.ServiceId,
before.Status,
after.Status);
}
private static JsonObject CreatePayload(RiskBudget budget, string severity)
{
return new JsonObject
{
["budgetId"] = budget.BudgetId,
["serviceId"] = budget.ServiceId,
["tier"] = budget.Tier.ToString().ToLowerInvariant(),
["window"] = budget.Window,
["allocated"] = budget.Allocated,
["consumed"] = budget.Consumed,
["remaining"] = budget.Remaining,
["percentageUsed"] = budget.PercentageUsed,
["status"] = budget.Status.ToString().ToLowerInvariant(),
["severity"] = severity,
["timestamp"] = DateTimeOffset.UtcNow.ToString("O")
};
}
}
/// <summary>
/// Known budget event kinds.
/// </summary>
public static class BudgetEventKinds
{
/// <summary>Budget warning threshold crossed.</summary>
public const string PolicyBudgetWarning = "policy.budget.warning";
/// <summary>Budget exhausted.</summary>
public const string PolicyBudgetExceeded = "policy.budget.exceeded";
}
/// <summary>
/// Interface for publishing notification events.
/// </summary>
public interface INotifyEventPublisher
{
/// <summary>
/// Publish a notification event.
/// </summary>
/// <param name="eventKind">Event kind identifier.</param>
/// <param name="tenantId">Tenant identifier.</param>
/// <param name="payload">Event payload.</param>
/// <param name="ct">Cancellation token.</param>
Task PublishAsync(
string eventKind,
string tenantId,
JsonNode payload,
CancellationToken ct = default);
}

View File

@@ -0,0 +1,446 @@
// -----------------------------------------------------------------------------
// EarnedCapacityReplenishment.cs
// Sprint: SPRINT_20251226_002_BE_budget_enforcement
// Task: BUDGET-10 - Earned capacity replenishment
// Description: Grants budget increases based on performance improvement over time
// -----------------------------------------------------------------------------
namespace StellaOps.Policy.Gates;
/// <summary>
/// Evaluates service performance metrics to determine earned budget increases.
/// If MTTR and CFR improve for 2 consecutive windows, grants 10-20% budget increase.
/// </summary>
public sealed class EarnedCapacityEvaluator
{
private readonly IPerformanceMetricsStore _metricsStore;
private readonly IBudgetStore _budgetStore;
private readonly EarnedCapacityOptions _options;
/// <summary>
/// Create a new earned capacity evaluator.
/// </summary>
public EarnedCapacityEvaluator(
IPerformanceMetricsStore metricsStore,
IBudgetStore budgetStore,
EarnedCapacityOptions? options = null)
{
_metricsStore = metricsStore ?? throw new ArgumentNullException(nameof(metricsStore));
_budgetStore = budgetStore ?? throw new ArgumentNullException(nameof(budgetStore));
_options = options ?? new EarnedCapacityOptions();
}
/// <summary>
/// Evaluate if a service qualifies for earned capacity increase.
/// </summary>
/// <param name="serviceId">Service identifier.</param>
/// <param name="currentWindow">Current budget window.</param>
/// <param name="ct">Cancellation token.</param>
/// <returns>Evaluation result with eligibility and recommended increase.</returns>
public async Task<EarnedCapacityResult> EvaluateAsync(
string serviceId,
string currentWindow,
CancellationToken ct = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(serviceId);
ArgumentException.ThrowIfNullOrWhiteSpace(currentWindow);
// Get historical windows to evaluate (current + 2 previous)
var windows = GetWindowSequence(currentWindow, _options.RequiredImprovementWindows + 1);
// Fetch metrics for each window
var metricsHistory = new List<WindowMetrics>();
foreach (var window in windows)
{
var metrics = await _metricsStore.GetMetricsAsync(serviceId, window, ct);
if (metrics != null)
{
metricsHistory.Add(metrics);
}
}
// Need at least 3 windows of data (current + 2 prior)
if (metricsHistory.Count < _options.RequiredImprovementWindows + 1)
{
return EarnedCapacityResult.NotEligible(
serviceId,
EarnedCapacityIneligibilityReason.InsufficientHistory,
$"Requires {_options.RequiredImprovementWindows + 1} windows of data, found {metricsHistory.Count}");
}
// Order by window (oldest first)
metricsHistory = metricsHistory.OrderBy(m => m.Window).ToList();
// Check for consistent improvement
var improvementCheck = CheckConsecutiveImprovement(metricsHistory);
if (!improvementCheck.IsImproving)
{
return EarnedCapacityResult.NotEligible(
serviceId,
EarnedCapacityIneligibilityReason.NoImprovement,
improvementCheck.Reason);
}
// Calculate recommended increase based on improvement magnitude
var increasePercentage = CalculateIncreasePercentage(
improvementCheck.MttrImprovementPercent,
improvementCheck.CfrImprovementPercent);
// Get current budget to calculate actual points
var currentBudget = await _budgetStore.GetAsync(serviceId, currentWindow, ct);
var currentAllocation = currentBudget?.Allocated
?? DefaultBudgetAllocations.GetMonthlyAllocation(ServiceTier.CustomerFacingNonCritical);
var additionalPoints = (int)Math.Ceiling(currentAllocation * increasePercentage / 100m);
return EarnedCapacityResult.Eligible(
serviceId,
increasePercentage,
additionalPoints,
improvementCheck.MttrImprovementPercent,
improvementCheck.CfrImprovementPercent);
}
/// <summary>
/// Apply an earned capacity increase to a service's budget.
/// </summary>
public async Task<RiskBudget> ApplyIncreaseAsync(
string serviceId,
string window,
int additionalPoints,
CancellationToken ct = default)
{
var budget = await _budgetStore.GetAsync(serviceId, window, ct)
?? throw new InvalidOperationException($"Budget not found for service {serviceId} window {window}");
var updatedBudget = budget with
{
Allocated = budget.Allocated + additionalPoints,
UpdatedAt = DateTimeOffset.UtcNow
};
await _budgetStore.UpdateAsync(updatedBudget, ct);
return updatedBudget;
}
private ImprovementCheckResult CheckConsecutiveImprovement(List<WindowMetrics> orderedMetrics)
{
// Compare each window to its predecessor
decimal totalMttrImprovement = 0;
decimal totalCfrImprovement = 0;
int improvingWindows = 0;
for (int i = 1; i < orderedMetrics.Count; i++)
{
var prev = orderedMetrics[i - 1];
var curr = orderedMetrics[i];
// Calculate MTTR improvement (lower is better)
var mttrImproved = prev.MttrHours > 0 && curr.MttrHours < prev.MttrHours;
var mttrImprovementPct = prev.MttrHours > 0
? (prev.MttrHours - curr.MttrHours) / prev.MttrHours * 100
: 0;
// Calculate CFR improvement (lower is better)
var cfrImproved = prev.ChangeFailureRate > 0 && curr.ChangeFailureRate < prev.ChangeFailureRate;
var cfrImprovementPct = prev.ChangeFailureRate > 0
? (prev.ChangeFailureRate - curr.ChangeFailureRate) / prev.ChangeFailureRate * 100
: 0;
// Both metrics must improve (or at least not regress significantly)
if (mttrImproved || (mttrImprovementPct >= -_options.RegressionTolerancePercent))
{
totalMttrImprovement += mttrImprovementPct;
}
else
{
return new ImprovementCheckResult(
false,
$"MTTR regressed in window {curr.Window}: {prev.MttrHours:F1}h -> {curr.MttrHours:F1}h",
0, 0);
}
if (cfrImproved || (cfrImprovementPct >= -_options.RegressionTolerancePercent))
{
totalCfrImprovement += cfrImprovementPct;
}
else
{
return new ImprovementCheckResult(
false,
$"CFR regressed in window {curr.Window}: {prev.ChangeFailureRate:F1}% -> {curr.ChangeFailureRate:F1}%",
0, 0);
}
// At least one metric must actually improve
if (mttrImproved || cfrImproved)
{
improvingWindows++;
}
}
// Need improvement for required consecutive windows
if (improvingWindows < _options.RequiredImprovementWindows)
{
return new ImprovementCheckResult(
false,
$"Required {_options.RequiredImprovementWindows} improving windows, found {improvingWindows}",
0, 0);
}
// Average improvement across windows
var avgMttrImprovement = totalMttrImprovement / (orderedMetrics.Count - 1);
var avgCfrImprovement = totalCfrImprovement / (orderedMetrics.Count - 1);
return new ImprovementCheckResult(true, null, avgMttrImprovement, avgCfrImprovement);
}
private decimal CalculateIncreasePercentage(decimal mttrImprovement, decimal cfrImprovement)
{
// Average of both improvements, clamped to min/max
var avgImprovement = (mttrImprovement + cfrImprovement) / 2;
// Scale: 10% improvement in metrics -> 10% budget increase
// 20%+ improvement -> 20% budget increase (capped)
var increase = Math.Min(avgImprovement, _options.MaxIncreasePercent);
return Math.Max(increase, _options.MinIncreasePercent);
}
private static IReadOnlyList<string> GetWindowSequence(string currentWindow, int count)
{
// Parse window format "YYYY-MM"
var windows = new List<string> { currentWindow };
if (currentWindow.Length >= 7 && currentWindow[4] == '-')
{
if (int.TryParse(currentWindow[..4], out var year) &&
int.TryParse(currentWindow[5..7], out var month))
{
for (int i = 1; i < count; i++)
{
month--;
if (month < 1)
{
month = 12;
year--;
}
windows.Add($"{year:D4}-{month:D2}");
}
}
}
return windows;
}
private sealed record ImprovementCheckResult(
bool IsImproving,
string? Reason,
decimal MttrImprovementPercent,
decimal CfrImprovementPercent);
}
/// <summary>
/// Result of earned capacity evaluation.
/// </summary>
public sealed record EarnedCapacityResult
{
/// <summary>
/// Service identifier.
/// </summary>
public required string ServiceId { get; init; }
/// <summary>
/// Whether the service is eligible for an increase.
/// </summary>
public required bool IsEligible { get; init; }
/// <summary>
/// Reason if not eligible.
/// </summary>
public EarnedCapacityIneligibilityReason? IneligibilityReason { get; init; }
/// <summary>
/// Description of ineligibility.
/// </summary>
public string? IneligibilityDescription { get; init; }
/// <summary>
/// Recommended increase percentage (10-20%).
/// </summary>
public decimal IncreasePercentage { get; init; }
/// <summary>
/// Recommended additional points to allocate.
/// </summary>
public int AdditionalPoints { get; init; }
/// <summary>
/// MTTR improvement over evaluation period.
/// </summary>
public decimal MttrImprovementPercent { get; init; }
/// <summary>
/// CFR improvement over evaluation period.
/// </summary>
public decimal CfrImprovementPercent { get; init; }
/// <summary>
/// Create a not-eligible result.
/// </summary>
public static EarnedCapacityResult NotEligible(
string serviceId,
EarnedCapacityIneligibilityReason reason,
string description) => new()
{
ServiceId = serviceId,
IsEligible = false,
IneligibilityReason = reason,
IneligibilityDescription = description
};
/// <summary>
/// Create an eligible result.
/// </summary>
public static EarnedCapacityResult Eligible(
string serviceId,
decimal increasePercentage,
int additionalPoints,
decimal mttrImprovement,
decimal cfrImprovement) => new()
{
ServiceId = serviceId,
IsEligible = true,
IncreasePercentage = increasePercentage,
AdditionalPoints = additionalPoints,
MttrImprovementPercent = mttrImprovement,
CfrImprovementPercent = cfrImprovement
};
}
/// <summary>
/// Reasons why a service is not eligible for earned capacity.
/// </summary>
public enum EarnedCapacityIneligibilityReason
{
/// <summary>
/// Not enough historical data.
/// </summary>
InsufficientHistory,
/// <summary>
/// Metrics did not improve.
/// </summary>
NoImprovement,
/// <summary>
/// Service is in probation period.
/// </summary>
InProbation,
/// <summary>
/// Manual override preventing increase.
/// </summary>
ManualOverride
}
/// <summary>
/// Performance metrics for a service in a budget window.
/// </summary>
public sealed record WindowMetrics
{
/// <summary>
/// Service identifier.
/// </summary>
public required string ServiceId { get; init; }
/// <summary>
/// Budget window.
/// </summary>
public required string Window { get; init; }
/// <summary>
/// Mean Time to Remediate in hours.
/// </summary>
public required decimal MttrHours { get; init; }
/// <summary>
/// Change Failure Rate as percentage (0-100).
/// </summary>
public required decimal ChangeFailureRate { get; init; }
/// <summary>
/// Number of deployments in the window.
/// </summary>
public int DeploymentCount { get; init; }
/// <summary>
/// Number of vulnerabilities remediated.
/// </summary>
public int VulnerabilitiesRemediated { get; init; }
/// <summary>
/// When metrics were calculated.
/// </summary>
public DateTimeOffset CalculatedAt { get; init; }
}
/// <summary>
/// Options for earned capacity evaluation.
/// </summary>
public sealed class EarnedCapacityOptions
{
/// <summary>
/// Number of consecutive improving windows required.
/// Default: 2.
/// </summary>
public int RequiredImprovementWindows { get; set; } = 2;
/// <summary>
/// Minimum budget increase percentage.
/// Default: 10%.
/// </summary>
public decimal MinIncreasePercent { get; set; } = 10m;
/// <summary>
/// Maximum budget increase percentage.
/// Default: 20%.
/// </summary>
public decimal MaxIncreasePercent { get; set; } = 20m;
/// <summary>
/// Tolerance for minor regression before disqualifying.
/// Default: 5% (allows 5% regression without disqualifying).
/// </summary>
public decimal RegressionTolerancePercent { get; set; } = 5m;
}
/// <summary>
/// Interface for performance metrics storage.
/// </summary>
public interface IPerformanceMetricsStore
{
/// <summary>
/// Get metrics for a service in a specific window.
/// </summary>
Task<WindowMetrics?> GetMetricsAsync(
string serviceId,
string window,
CancellationToken ct = default);
/// <summary>
/// Save or update metrics for a service.
/// </summary>
Task SaveMetricsAsync(
WindowMetrics metrics,
CancellationToken ct = default);
/// <summary>
/// List metrics for a service across windows.
/// </summary>
Task<IReadOnlyList<WindowMetrics>> ListMetricsAsync(
string serviceId,
int windowCount,
CancellationToken ct = default);
}
// IBudgetStore is defined in BudgetLedger.cs