using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using System.Collections.Concurrent; namespace StellaOps.Notifier.Worker.Observability; /// /// Chaos testing service for simulating channel outages and failures. /// Enables controlled fault injection to test resilience of notification delivery. /// public interface IChaosTestRunner { /// /// Starts a chaos experiment. /// Task StartExperimentAsync(ChaosExperimentConfig config, CancellationToken ct = default); /// /// Stops a running chaos experiment. /// Task StopExperimentAsync(string experimentId, CancellationToken ct = default); /// /// Gets the current status of an experiment. /// Task GetExperimentAsync(string experimentId, CancellationToken ct = default); /// /// Lists all experiments (optionally filtered by status). /// Task> ListExperimentsAsync( ChaosExperimentStatus? status = null, int limit = 100, CancellationToken ct = default); /// /// Checks if a channel should fail based on active chaos experiments. /// Task ShouldFailAsync(string tenantId, string channelType, string? channelId = null, CancellationToken ct = default); /// /// Records the outcome of a chaos-affected operation. /// Task RecordOutcomeAsync(string experimentId, ChaosOutcome outcome, CancellationToken ct = default); /// /// Gets experiment results/statistics. /// Task GetResultsAsync(string experimentId, CancellationToken ct = default); /// /// Cleans up completed experiments older than the specified age. /// Task CleanupAsync(TimeSpan olderThan, CancellationToken ct = default); } /// /// Configuration for a chaos experiment. /// public sealed record ChaosExperimentConfig { /// /// Human-readable name for the experiment. /// public required string Name { get; init; } /// /// Description of what the experiment tests. /// public string? Description { get; init; } /// /// Target tenant ID (null for all tenants). /// public string? TenantId { get; init; } /// /// Target channel types to affect. /// public IReadOnlyList TargetChannelTypes { get; init; } = []; /// /// Target channel IDs to affect (empty means all channels of specified types). /// public IReadOnlyList TargetChannelIds { get; init; } = []; /// /// Type of fault to inject. /// public required ChaosFaultType FaultType { get; init; } /// /// Fault configuration parameters. /// public ChaosFaultConfig FaultConfig { get; init; } = new(); /// /// Duration of the experiment. /// public TimeSpan Duration { get; init; } = TimeSpan.FromMinutes(5); /// /// Maximum number of operations to affect (0 = unlimited). /// public int MaxAffectedOperations { get; init; } /// /// Tags for categorizing experiments. /// public IReadOnlyDictionary Tags { get; init; } = new Dictionary(); /// /// Who initiated this experiment. /// public required string InitiatedBy { get; init; } } /// /// Configuration for fault behavior. /// public sealed record ChaosFaultConfig { /// /// Failure rate (0.0 to 1.0) for partial/intermittent failures. /// public double FailureRate { get; init; } = 1.0; /// /// Minimum latency to inject. /// public TimeSpan MinLatency { get; init; } = TimeSpan.FromSeconds(1); /// /// Maximum latency to inject. /// public TimeSpan MaxLatency { get; init; } = TimeSpan.FromSeconds(5); /// /// HTTP status code to return for error responses. /// public int ErrorStatusCode { get; init; } = 500; /// /// Error message to include. /// public string? ErrorMessage { get; init; } /// /// Rate limit (requests per minute) for RateLimit fault type. /// public int RateLimitPerMinute { get; init; } = 10; /// /// Timeout duration for Timeout fault type. /// public TimeSpan TimeoutDuration { get; init; } = TimeSpan.FromSeconds(30); /// /// Random seed for reproducible experiments. /// public int? Seed { get; init; } } /// /// Status of a chaos experiment. /// public enum ChaosExperimentStatus { /// /// Experiment is scheduled but not yet started. /// Scheduled, /// /// Experiment is currently running. /// Running, /// /// Experiment completed normally. /// Completed, /// /// Experiment was stopped early. /// Stopped, /// /// Experiment failed to run. /// Failed } /// /// Represents an active or completed chaos experiment. /// public sealed record ChaosExperiment { /// /// Unique experiment identifier. /// public required string Id { get; init; } /// /// Configuration for this experiment. /// public required ChaosExperimentConfig Config { get; init; } /// /// Current status. /// public required ChaosExperimentStatus Status { get; init; } /// /// When the experiment was created. /// public required DateTimeOffset CreatedAt { get; init; } /// /// When the experiment started running. /// public DateTimeOffset? StartedAt { get; init; } /// /// When the experiment ended. /// public DateTimeOffset? EndedAt { get; init; } /// /// Scheduled end time. /// public DateTimeOffset? ScheduledEndAt { get; init; } /// /// Number of operations affected so far. /// public int AffectedOperations { get; init; } /// /// Error message if status is Failed. /// public string? ErrorMessage { get; init; } } /// /// Decision from chaos system about whether to inject a fault. /// public sealed record ChaosDecision { /// /// Whether to inject a fault. /// public required bool ShouldFail { get; init; } /// /// The experiment causing the fault (if any). /// public string? ExperimentId { get; init; } /// /// Type of fault to inject. /// public ChaosFaultType? FaultType { get; init; } /// /// Fault configuration. /// public ChaosFaultConfig? FaultConfig { get; init; } /// /// Latency to inject (if applicable). /// public TimeSpan? InjectedLatency { get; init; } /// /// Error to return (if applicable). /// public string? InjectedError { get; init; } /// /// HTTP status code to return (if applicable). /// public int? InjectedStatusCode { get; init; } /// /// Reason for the decision. /// public string? Reason { get; init; } /// /// Creates a "no fault" decision. /// public static ChaosDecision NoFault() => new() { ShouldFail = false, Reason = "No active chaos experiment" }; } /// /// Records the outcome of a chaos-affected operation. /// public sealed record ChaosOutcome { /// /// Type of outcome. /// public required ChaosOutcomeType Type { get; init; } /// /// Channel type affected. /// public required string ChannelType { get; init; } /// /// Channel ID affected. /// public string? ChannelId { get; init; } /// /// Tenant ID affected. /// public string? TenantId { get; init; } /// /// Duration of the operation. /// public TimeSpan? Duration { get; init; } /// /// Whether fallback was triggered. /// public bool FallbackTriggered { get; init; } /// /// Whether retry was triggered. /// public bool RetryTriggered { get; init; } /// /// Error message if operation failed. /// public string? ErrorMessage { get; init; } /// /// When this outcome was recorded. /// public DateTimeOffset Timestamp { get; init; } = DateTimeOffset.UtcNow; } /// /// Types of chaos outcomes. /// public enum ChaosOutcomeType { /// /// Fault was injected and operation failed. /// FaultInjected, /// /// Fault was injected but operation recovered. /// RecoveredFromFault, /// /// Operation was delayed by latency injection. /// LatencyInjected, /// /// Operation was rate limited. /// RateLimited, /// /// Operation bypassed due to experiment limits. /// Bypassed } /// /// Results and statistics from a chaos experiment. /// public sealed record ChaosExperimentResults { /// /// Experiment identifier. /// public required string ExperimentId { get; init; } /// /// Total operations affected. /// public required int TotalAffected { get; init; } /// /// Operations that failed due to fault injection. /// public required int FailedOperations { get; init; } /// /// Operations that recovered from fault. /// public required int RecoveredOperations { get; init; } /// /// Operations that triggered fallback. /// public required int FallbackTriggered { get; init; } /// /// Operations that triggered retry. /// public required int RetryTriggered { get; init; } /// /// Average injected latency. /// public TimeSpan? AverageInjectedLatency { get; init; } /// /// Breakdown by channel type. /// public IReadOnlyDictionary ByChannelType { get; init; } = new Dictionary(); /// /// Timeline of outcomes. /// public IReadOnlyList Outcomes { get; init; } = []; } /// /// Statistics for a specific channel type. /// public sealed record ChaosChannelStats { /// /// Channel type. /// public required string ChannelType { get; init; } /// /// Total affected operations. /// public required int TotalAffected { get; init; } /// /// Failed operations. /// public required int Failed { get; init; } /// /// Recovered operations. /// public required int Recovered { get; init; } /// /// Fallback triggered count. /// public required int Fallbacks { get; init; } } /// /// Options for chaos testing. /// public sealed class ChaosTestOptions { public const string SectionName = "Notifier:Observability:Chaos"; /// /// Whether chaos testing is enabled. /// public bool Enabled { get; set; } /// /// Maximum concurrent experiments. /// public int MaxConcurrentExperiments { get; set; } = 5; /// /// Maximum duration for any experiment. /// public TimeSpan MaxExperimentDuration { get; set; } = TimeSpan.FromHours(1); /// /// Retention period for completed experiments. /// public TimeSpan ExperimentRetention { get; set; } = TimeSpan.FromDays(7); /// /// Whether to require explicit tenant targeting. /// public bool RequireTenantTarget { get; set; } = true; /// /// Allowed initiators (empty = all allowed). /// public IReadOnlyList AllowedInitiators { get; set; } = []; } /// /// In-memory implementation of chaos test runner. /// public sealed class InMemoryChaosTestRunner : IChaosTestRunner { private readonly ConcurrentDictionary _experiments = new(); private readonly ChaosTestOptions _options; private readonly TimeProvider _timeProvider; private readonly ILogger _logger; public InMemoryChaosTestRunner( IOptions options, TimeProvider timeProvider, ILogger logger) { _options = options?.Value ?? new ChaosTestOptions(); _timeProvider = timeProvider ?? TimeProvider.System; _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } public Task StartExperimentAsync(ChaosExperimentConfig config, CancellationToken ct = default) { ArgumentNullException.ThrowIfNull(config); if (!_options.Enabled) { throw new InvalidOperationException("Chaos testing is not enabled"); } // Validate initiator if (_options.AllowedInitiators.Count > 0 && !_options.AllowedInitiators.Contains(config.InitiatedBy)) { throw new UnauthorizedAccessException($"Initiator '{config.InitiatedBy}' is not allowed to run chaos experiments"); } // Validate tenant targeting if (_options.RequireTenantTarget && string.IsNullOrEmpty(config.TenantId)) { throw new InvalidOperationException("Tenant targeting is required for chaos experiments"); } // Validate duration if (config.Duration > _options.MaxExperimentDuration) { throw new InvalidOperationException($"Experiment duration exceeds maximum of {_options.MaxExperimentDuration}"); } // Check concurrent limit var runningCount = _experiments.Values.Count(e => e.Experiment.Status == ChaosExperimentStatus.Running); if (runningCount >= _options.MaxConcurrentExperiments) { throw new InvalidOperationException($"Maximum concurrent experiments ({_options.MaxConcurrentExperiments}) reached"); } var now = _timeProvider.GetUtcNow(); var experimentId = $"chaos-{Guid.NewGuid():N}"; var experiment = new ChaosExperiment { Id = experimentId, Config = config, Status = ChaosExperimentStatus.Running, CreatedAt = now, StartedAt = now, ScheduledEndAt = now.Add(config.Duration), AffectedOperations = 0 }; var state = new ChaosExperimentState { Experiment = experiment, Random = config.FaultConfig.Seed.HasValue ? new Random(config.FaultConfig.Seed.Value) : new Random(), Outcomes = [], RateLimitBucket = new RateLimitBucket(_timeProvider) }; _experiments[experimentId] = state; _logger.LogInformation( "Started chaos experiment {ExperimentId}: {Name} ({FaultType}) targeting {ChannelTypes}", experimentId, config.Name, config.FaultType, string.Join(", ", config.TargetChannelTypes)); return Task.FromResult(experiment); } public Task StopExperimentAsync(string experimentId, CancellationToken ct = default) { if (_experiments.TryGetValue(experimentId, out var state)) { if (state.Experiment.Status == ChaosExperimentStatus.Running) { var stopped = state.Experiment with { Status = ChaosExperimentStatus.Stopped, EndedAt = _timeProvider.GetUtcNow() }; state.Experiment = stopped; _logger.LogInformation( "Stopped chaos experiment {ExperimentId} after {AffectedOps} affected operations", experimentId, stopped.AffectedOperations); } } return Task.CompletedTask; } public Task GetExperimentAsync(string experimentId, CancellationToken ct = default) { if (_experiments.TryGetValue(experimentId, out var state)) { CheckAndUpdateExperimentStatus(state); return Task.FromResult(state.Experiment); } return Task.FromResult(null); } public Task> ListExperimentsAsync( ChaosExperimentStatus? status = null, int limit = 100, CancellationToken ct = default) { foreach (var state in _experiments.Values) { CheckAndUpdateExperimentStatus(state); } var query = _experiments.Values.Select(s => s.Experiment).AsEnumerable(); if (status.HasValue) { query = query.Where(e => e.Status == status.Value); } var result = query .OrderByDescending(e => e.CreatedAt) .Take(limit) .ToList(); return Task.FromResult>(result); } public Task ShouldFailAsync( string tenantId, string channelType, string? channelId = null, CancellationToken ct = default) { if (!_options.Enabled) { return Task.FromResult(ChaosDecision.NoFault()); } // Find matching active experiments foreach (var state in _experiments.Values) { CheckAndUpdateExperimentStatus(state); if (state.Experiment.Status != ChaosExperimentStatus.Running) { continue; } var config = state.Experiment.Config; // Check tenant match if (!string.IsNullOrEmpty(config.TenantId) && config.TenantId != tenantId) { continue; } // Check channel type match if (config.TargetChannelTypes.Count > 0 && !config.TargetChannelTypes.Contains(channelType)) { continue; } // Check channel ID match if (config.TargetChannelIds.Count > 0 && !string.IsNullOrEmpty(channelId) && !config.TargetChannelIds.Contains(channelId)) { continue; } // Check max affected operations if (config.MaxAffectedOperations > 0 && state.Experiment.AffectedOperations >= config.MaxAffectedOperations) { continue; } // Determine if fault should be injected based on fault type var decision = EvaluateFault(state, config); if (decision.ShouldFail || decision.InjectedLatency.HasValue) { // Increment affected count state.Experiment = state.Experiment with { AffectedOperations = state.Experiment.AffectedOperations + 1 }; return Task.FromResult(decision); } } return Task.FromResult(ChaosDecision.NoFault()); } private ChaosDecision EvaluateFault(ChaosExperimentState state, ChaosExperimentConfig config) { var faultConfig = config.FaultConfig; return config.FaultType switch { ChaosFaultType.Outage => new ChaosDecision { ShouldFail = true, ExperimentId = state.Experiment.Id, FaultType = ChaosFaultType.Outage, FaultConfig = faultConfig, InjectedError = faultConfig.ErrorMessage ?? "Chaos: Simulated outage", InjectedStatusCode = faultConfig.ErrorStatusCode, Reason = "Complete outage simulation" }, ChaosFaultType.PartialFailure => EvaluatePartialFailure(state, faultConfig), ChaosFaultType.Latency => EvaluateLatency(state, faultConfig), ChaosFaultType.Intermittent => EvaluateIntermittent(state, faultConfig), ChaosFaultType.RateLimit => EvaluateRateLimit(state, faultConfig), ChaosFaultType.Timeout => new ChaosDecision { ShouldFail = true, ExperimentId = state.Experiment.Id, FaultType = ChaosFaultType.Timeout, FaultConfig = faultConfig, InjectedLatency = faultConfig.TimeoutDuration, InjectedError = "Chaos: Request timeout", Reason = "Timeout simulation" }, ChaosFaultType.ErrorResponse => new ChaosDecision { ShouldFail = true, ExperimentId = state.Experiment.Id, FaultType = ChaosFaultType.ErrorResponse, FaultConfig = faultConfig, InjectedStatusCode = faultConfig.ErrorStatusCode, InjectedError = faultConfig.ErrorMessage ?? $"Chaos: HTTP {faultConfig.ErrorStatusCode}", Reason = "Error response simulation" }, ChaosFaultType.CorruptResponse => new ChaosDecision { ShouldFail = true, ExperimentId = state.Experiment.Id, FaultType = ChaosFaultType.CorruptResponse, FaultConfig = faultConfig, InjectedError = "Chaos: Corrupted response data", Reason = "Corrupt response simulation" }, _ => ChaosDecision.NoFault() }; } private ChaosDecision EvaluatePartialFailure(ChaosExperimentState state, ChaosFaultConfig faultConfig) { if (state.Random.NextDouble() < faultConfig.FailureRate) { return new ChaosDecision { ShouldFail = true, ExperimentId = state.Experiment.Id, FaultType = ChaosFaultType.PartialFailure, FaultConfig = faultConfig, InjectedError = faultConfig.ErrorMessage ?? "Chaos: Partial failure", InjectedStatusCode = faultConfig.ErrorStatusCode, Reason = $"Partial failure ({faultConfig.FailureRate:P0} rate)" }; } return ChaosDecision.NoFault(); } private ChaosDecision EvaluateLatency(ChaosExperimentState state, ChaosFaultConfig faultConfig) { var latencyRange = faultConfig.MaxLatency - faultConfig.MinLatency; var randomLatency = faultConfig.MinLatency + TimeSpan.FromMilliseconds( state.Random.NextDouble() * latencyRange.TotalMilliseconds); return new ChaosDecision { ShouldFail = false, // Latency doesn't cause failure ExperimentId = state.Experiment.Id, FaultType = ChaosFaultType.Latency, FaultConfig = faultConfig, InjectedLatency = randomLatency, Reason = $"Latency injection ({randomLatency.TotalMilliseconds:F0}ms)" }; } private ChaosDecision EvaluateIntermittent(ChaosExperimentState state, ChaosFaultConfig faultConfig) { // More random pattern than partial failure var shouldFail = state.Random.NextDouble() < faultConfig.FailureRate && state.Random.Next(3) == 0; // Additional randomness if (shouldFail) { return new ChaosDecision { ShouldFail = true, ExperimentId = state.Experiment.Id, FaultType = ChaosFaultType.Intermittent, FaultConfig = faultConfig, InjectedError = faultConfig.ErrorMessage ?? "Chaos: Intermittent failure", InjectedStatusCode = faultConfig.ErrorStatusCode, Reason = "Intermittent failure simulation" }; } return ChaosDecision.NoFault(); } private ChaosDecision EvaluateRateLimit(ChaosExperimentState state, ChaosFaultConfig faultConfig) { if (state.RateLimitBucket.TryConsume(faultConfig.RateLimitPerMinute)) { return ChaosDecision.NoFault(); } return new ChaosDecision { ShouldFail = true, ExperimentId = state.Experiment.Id, FaultType = ChaosFaultType.RateLimit, FaultConfig = faultConfig, InjectedStatusCode = 429, InjectedError = "Chaos: Rate limit exceeded", Reason = $"Rate limited ({faultConfig.RateLimitPerMinute}/min)" }; } public Task RecordOutcomeAsync(string experimentId, ChaosOutcome outcome, CancellationToken ct = default) { if (_experiments.TryGetValue(experimentId, out var state)) { state.Outcomes.Add(outcome); } return Task.CompletedTask; } public Task GetResultsAsync(string experimentId, CancellationToken ct = default) { if (!_experiments.TryGetValue(experimentId, out var state)) { return Task.FromResult(new ChaosExperimentResults { ExperimentId = experimentId, TotalAffected = 0, FailedOperations = 0, RecoveredOperations = 0, FallbackTriggered = 0, RetryTriggered = 0 }); } var outcomes = state.Outcomes.ToList(); var byChannel = outcomes .GroupBy(o => o.ChannelType) .ToDictionary( g => g.Key, g => new ChaosChannelStats { ChannelType = g.Key, TotalAffected = g.Count(), Failed = g.Count(o => o.Type == ChaosOutcomeType.FaultInjected), Recovered = g.Count(o => o.Type == ChaosOutcomeType.RecoveredFromFault), Fallbacks = g.Count(o => o.FallbackTriggered) }); var latencies = outcomes .Where(o => o.Duration.HasValue && o.Type == ChaosOutcomeType.LatencyInjected) .Select(o => o.Duration!.Value) .ToList(); return Task.FromResult(new ChaosExperimentResults { ExperimentId = experimentId, TotalAffected = outcomes.Count, FailedOperations = outcomes.Count(o => o.Type == ChaosOutcomeType.FaultInjected), RecoveredOperations = outcomes.Count(o => o.Type == ChaosOutcomeType.RecoveredFromFault), FallbackTriggered = outcomes.Count(o => o.FallbackTriggered), RetryTriggered = outcomes.Count(o => o.RetryTriggered), AverageInjectedLatency = latencies.Count > 0 ? TimeSpan.FromMilliseconds(latencies.Average(l => l.TotalMilliseconds)) : null, ByChannelType = byChannel, Outcomes = outcomes }); } public Task CleanupAsync(TimeSpan olderThan, CancellationToken ct = default) { var cutoff = _timeProvider.GetUtcNow() - olderThan; var removed = 0; var toRemove = _experiments .Where(kvp => kvp.Value.Experiment.Status is ChaosExperimentStatus.Completed or ChaosExperimentStatus.Stopped or ChaosExperimentStatus.Failed && kvp.Value.Experiment.EndedAt.HasValue && kvp.Value.Experiment.EndedAt.Value < cutoff) .Select(kvp => kvp.Key) .ToList(); foreach (var id in toRemove) { if (_experiments.TryRemove(id, out _)) { removed++; } } if (removed > 0) { _logger.LogInformation("Cleaned up {Count} completed chaos experiments", removed); } return Task.FromResult(removed); } private void CheckAndUpdateExperimentStatus(ChaosExperimentState state) { if (state.Experiment.Status != ChaosExperimentStatus.Running) { return; } var now = _timeProvider.GetUtcNow(); // Check if expired if (state.Experiment.ScheduledEndAt.HasValue && now >= state.Experiment.ScheduledEndAt.Value) { state.Experiment = state.Experiment with { Status = ChaosExperimentStatus.Completed, EndedAt = state.Experiment.ScheduledEndAt }; _logger.LogInformation( "Chaos experiment {ExperimentId} completed after {AffectedOps} affected operations", state.Experiment.Id, state.Experiment.AffectedOperations); } // Check if max operations reached var config = state.Experiment.Config; if (config.MaxAffectedOperations > 0 && state.Experiment.AffectedOperations >= config.MaxAffectedOperations) { state.Experiment = state.Experiment with { Status = ChaosExperimentStatus.Completed, EndedAt = now }; _logger.LogInformation( "Chaos experiment {ExperimentId} completed after reaching max operations ({Max})", state.Experiment.Id, config.MaxAffectedOperations); } } private sealed class ChaosExperimentState { public required ChaosExperiment Experiment { get; set; } public required Random Random { get; init; } public required List Outcomes { get; init; } public required RateLimitBucket RateLimitBucket { get; init; } } private sealed class RateLimitBucket { private readonly TimeProvider _timeProvider; private DateTimeOffset _windowStart; private int _count; private readonly object _lock = new(); public RateLimitBucket(TimeProvider timeProvider) { _timeProvider = timeProvider; _windowStart = timeProvider.GetUtcNow(); _count = 0; } public bool TryConsume(int limit) { lock (_lock) { var now = _timeProvider.GetUtcNow(); if ((now - _windowStart).TotalMinutes >= 1) { _windowStart = now; _count = 0; } if (_count < limit) { _count++; return true; } return false; } } } } /// /// Extension methods for chaos testing. /// public static class ChaosTestExtensions { /// /// Applies chaos decision to an operation, injecting faults as configured. /// public static async Task ApplyChaosAsync(this ChaosDecision decision, CancellationToken ct = default) { if (!decision.ShouldFail && decision.InjectedLatency.HasValue) { // Latency-only injection await Task.Delay(decision.InjectedLatency.Value, ct); } else if (decision.ShouldFail) { // First apply any latency if (decision.InjectedLatency.HasValue) { await Task.Delay(decision.InjectedLatency.Value, ct); } // Then throw the appropriate exception throw new ChaosInjectedException(decision); } } /// /// Creates a simple outage experiment config. /// public static ChaosExperimentConfig CreateOutageExperiment( string name, string initiatedBy, IReadOnlyList channelTypes, string? tenantId = null, TimeSpan? duration = null) { return new ChaosExperimentConfig { Name = name, InitiatedBy = initiatedBy, TenantId = tenantId, TargetChannelTypes = channelTypes, FaultType = ChaosFaultType.Outage, Duration = duration ?? TimeSpan.FromMinutes(5) }; } /// /// Creates a latency injection experiment config. /// public static ChaosExperimentConfig CreateLatencyExperiment( string name, string initiatedBy, IReadOnlyList channelTypes, TimeSpan minLatency, TimeSpan maxLatency, string? tenantId = null, TimeSpan? duration = null) { return new ChaosExperimentConfig { Name = name, InitiatedBy = initiatedBy, TenantId = tenantId, TargetChannelTypes = channelTypes, FaultType = ChaosFaultType.Latency, FaultConfig = new ChaosFaultConfig { MinLatency = minLatency, MaxLatency = maxLatency }, Duration = duration ?? TimeSpan.FromMinutes(5) }; } } /// /// Exception thrown when chaos is injected. /// public sealed class ChaosInjectedException : Exception { public ChaosDecision Decision { get; } public ChaosInjectedException(ChaosDecision decision) : base(decision.InjectedError ?? "Chaos fault injected") { Decision = decision; } }