up

src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Worker/Observability/IChaosEngine.cs

@@ -0,0 +1,471 @@
+using System.Collections.Concurrent;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace StellaOps.Notifier.Worker.Observability;
+
+/// <summary>
+/// Chaos testing engine for simulating channel outages and failures.
+/// </summary>
+public interface IChaosEngine
+{
+    /// <summary>
+    /// Injects a fault for a channel type.
+    /// </summary>
+    Task<ChaosFaultInjection> InjectFaultAsync(ChaosFaultRequest request, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Removes a fault injection.
+    /// </summary>
+    Task<bool> RemoveFaultAsync(string faultId, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Gets all active faults.
+    /// </summary>
+    Task<IReadOnlyList<ChaosFaultInjection>> GetActiveFaultsAsync(CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Checks if a channel operation should fail due to chaos.
+    /// </summary>
+    Task<ChaosFaultResult> ShouldFailAsync(string channelType, string? tenantId = null, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Runs a chaos test scenario.
+    /// </summary>
+    Task<ChaosTestResult> RunScenarioAsync(ChaosScenario scenario, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Gets chaos test history.
+    /// </summary>
+    Task<IReadOnlyList<ChaosTestResult>> GetHistoryAsync(int limit = 50, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Clears all active faults.
+    /// </summary>
+    Task ClearAllFaultsAsync(CancellationToken cancellationToken = default);
+}
+
+/// <summary>
+/// Request to inject a fault.
+/// </summary>
+public sealed record ChaosFaultRequest
+{
+    public required string ChannelType { get; init; }
+    public string? TenantId { get; init; }
+    public required ChaosFaultType FaultType { get; init; }
+    public double FailureProbability { get; init; } = 1.0;
+    public TimeSpan? Duration { get; init; }
+    public TimeSpan? LatencyInjection { get; init; }
+    public int? ErrorCode { get; init; }
+    public string? ErrorMessage { get; init; }
+    public string? Description { get; init; }
+}
+
+/// <summary>
+/// Type of chaos fault.
+/// </summary>
+public enum ChaosFaultType
+{
+    Outage,
+    Latency,
+    RateLimit,
+    AuthFailure,
+    Timeout,
+    PartialFailure,
+    Intermittent
+}
+
+/// <summary>
+/// Active fault injection.
+/// </summary>
+public sealed record ChaosFaultInjection
+{
+    public required string FaultId { get; init; }
+    public required string ChannelType { get; init; }
+    public string? TenantId { get; init; }
+    public required ChaosFaultType FaultType { get; init; }
+    public double FailureProbability { get; init; }
+    public TimeSpan? LatencyInjection { get; init; }
+    public int? ErrorCode { get; init; }
+    public string? ErrorMessage { get; init; }
+    public string? Description { get; init; }
+    public DateTimeOffset CreatedAt { get; init; }
+    public DateTimeOffset? ExpiresAt { get; init; }
+    public int TriggerCount { get; init; }
+    public bool IsActive { get; init; } = true;
+}
+
+/// <summary>
+/// Result of checking for chaos fault.
+/// </summary>
+public sealed record ChaosFaultResult
+{
+    public bool ShouldFail { get; init; }
+    public ChaosFaultInjection? ActiveFault { get; init; }
+    public TimeSpan? InjectedLatency { get; init; }
+    public Exception? SimulatedException { get; init; }
+}
+
+/// <summary>
+/// A chaos test scenario.
+/// </summary>
+public sealed record ChaosScenario
+{
+    public required string ScenarioId { get; init; }
+    public required string Name { get; init; }
+    public string? Description { get; init; }
+    public required IReadOnlyList<ChaosScenarioStep> Steps { get; init; }
+    public TimeSpan Timeout { get; init; } = TimeSpan.FromMinutes(10);
+    public bool StopOnFirstFailure { get; init; }
+}
+
+/// <summary>
+/// A step in a chaos scenario.
+/// </summary>
+public sealed record ChaosScenarioStep
+{
+    public required string StepId { get; init; }
+    public required string Name { get; init; }
+    public required ChaosStepAction Action { get; init; }
+    public ChaosFaultRequest? FaultToInject { get; init; }
+    public string? FaultIdToRemove { get; init; }
+    public TimeSpan? WaitDuration { get; init; }
+    public ChaosAssertion? Assertion { get; init; }
+}
+
+/// <summary>
+/// Action type for a chaos step.
+/// </summary>
+public enum ChaosStepAction
+{
+    InjectFault,
+    RemoveFault,
+    Wait,
+    Assert,
+    SendTestDelivery,
+    CheckMetrics
+}
+
+/// <summary>
+/// Assertion for chaos testing.
+/// </summary>
+public sealed record ChaosAssertion
+{
+    public required ChaosAssertionType Type { get; init; }
+    public string? MetricName { get; init; }
+    public double? ExpectedValue { get; init; }
+    public double? Tolerance { get; init; }
+    public string? ExpectedStatus { get; init; }
+}
+
+/// <summary>
+/// Type of chaos assertion.
+/// </summary>
+public enum ChaosAssertionType
+{
+    MetricEquals,
+    MetricGreaterThan,
+    MetricLessThan,
+    DeadLetterCountEquals,
+    FallbackTriggered,
+    AlertFired
+}
+
+/// <summary>
+/// Result of a chaos test.
+/// </summary>
+public sealed record ChaosTestResult
+{
+    public required string TestId { get; init; }
+    public required string ScenarioId { get; init; }
+    public required string ScenarioName { get; init; }
+    public bool Success { get; init; }
+    public DateTimeOffset StartedAt { get; init; }
+    public DateTimeOffset CompletedAt { get; init; }
+    public TimeSpan Duration { get; init; }
+    public IReadOnlyList<ChaosStepResult> StepResults { get; init; } = [];
+    public string? Error { get; init; }
+}
+
+/// <summary>
+/// Result of a chaos step.
+/// </summary>
+public sealed record ChaosStepResult
+{
+    public required string StepId { get; init; }
+    public required string StepName { get; init; }
+    public bool Success { get; init; }
+    public DateTimeOffset ExecutedAt { get; init; }
+    public TimeSpan Duration { get; init; }
+    public string? Error { get; init; }
+    public object? Data { get; init; }
+}
+
+/// <summary>
+/// Options for chaos engine.
+/// </summary>
+public sealed class ChaosEngineOptions
+{
+    public const string SectionName = "Notifier:Observability:Chaos";
+
+    public bool Enabled { get; set; }
+    public bool AllowInProduction { get; set; }
+    public TimeSpan MaxFaultDuration { get; set; } = TimeSpan.FromHours(1);
+    public int MaxConcurrentFaults { get; set; } = 10;
+    public IReadOnlyList<string> AllowedChannelTypes { get; set; } = ["webhook", "email", "slack", "teams", "pagerduty", "opsgenie"];
+}
+
+/// <summary>
+/// Default implementation of chaos engine.
+/// </summary>
+public sealed class DefaultChaosEngine : IChaosEngine
+{
+    private readonly ConcurrentDictionary<string, ChaosFaultInjection> _activeFaults = new();
+    private readonly List<ChaosTestResult> _testHistory = [];
+    private readonly ChaosEngineOptions _options;
+    private readonly TimeProvider _timeProvider;
+    private readonly INotifierMetrics? _metrics;
+    private readonly ILogger<DefaultChaosEngine> _logger;
+    private readonly Random _random = new();
+
+    public DefaultChaosEngine(
+        IOptions<ChaosEngineOptions> options,
+        TimeProvider timeProvider,
+        INotifierMetrics? metrics,
+        ILogger<DefaultChaosEngine> logger)
+    {
+        _options = options?.Value ?? new ChaosEngineOptions();
+        _timeProvider = timeProvider ?? TimeProvider.System;
+        _metrics = metrics;
+        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+    }
+
+    public Task<ChaosFaultInjection> InjectFaultAsync(ChaosFaultRequest request, CancellationToken cancellationToken = default)
+    {
+        if (!_options.Enabled)
+            throw new InvalidOperationException("Chaos engine is disabled");
+
+        if (_activeFaults.Count >= _options.MaxConcurrentFaults)
+            throw new InvalidOperationException($"Maximum concurrent faults ({_options.MaxConcurrentFaults}) reached");
+
+        var now = _timeProvider.GetUtcNow();
+        var duration = request.Duration ?? _options.MaxFaultDuration;
+        if (duration > _options.MaxFaultDuration)
+            duration = _options.MaxFaultDuration;
+
+        var fault = new ChaosFaultInjection
+        {
+            FaultId = $"chaos-{Guid.NewGuid():N}"[..16],
+            ChannelType = request.ChannelType,
+            TenantId = request.TenantId,
+            FaultType = request.FaultType,
+            FailureProbability = Math.Clamp(request.FailureProbability, 0.0, 1.0),
+            LatencyInjection = request.LatencyInjection,
+            ErrorCode = request.ErrorCode,
+            ErrorMessage = request.ErrorMessage,
+            Description = request.Description,
+            CreatedAt = now,
+            ExpiresAt = now + duration,
+            IsActive = true
+        };
+
+        _activeFaults[fault.FaultId] = fault;
+        _logger.LogWarning("Injected chaos fault {FaultId} for channel {ChannelType}: {FaultType}", fault.FaultId, fault.ChannelType, fault.FaultType);
+
+        return Task.FromResult(fault);
+    }
+
+    public Task<bool> RemoveFaultAsync(string faultId, CancellationToken cancellationToken = default)
+    {
+        var removed = _activeFaults.TryRemove(faultId, out var fault);
+        if (removed)
+            _logger.LogInformation("Removed chaos fault {FaultId}", faultId);
+        return Task.FromResult(removed);
+    }
+
+    public Task<IReadOnlyList<ChaosFaultInjection>> GetActiveFaultsAsync(CancellationToken cancellationToken = default)
+    {
+        CleanupExpiredFaults();
+        return Task.FromResult<IReadOnlyList<ChaosFaultInjection>>(_activeFaults.Values.ToList());
+    }
+
+    public Task<ChaosFaultResult> ShouldFailAsync(string channelType, string? tenantId = null, CancellationToken cancellationToken = default)
+    {
+        if (!_options.Enabled)
+            return Task.FromResult(new ChaosFaultResult { ShouldFail = false });
+
+        CleanupExpiredFaults();
+
+        var matchingFault = _activeFaults.Values
+            .Where(f => f.IsActive)
+            .Where(f => f.ChannelType == channelType || f.ChannelType == "*")
+            .Where(f => f.TenantId is null || f.TenantId == tenantId)
+            .FirstOrDefault();
+
+        if (matchingFault is null)
+            return Task.FromResult(new ChaosFaultResult { ShouldFail = false });
+
+        var shouldFail = _random.NextDouble() < matchingFault.FailureProbability;
+        if (!shouldFail)
+            return Task.FromResult(new ChaosFaultResult { ShouldFail = false, ActiveFault = matchingFault });
+
+        // Update trigger count
+        _activeFaults[matchingFault.FaultId] = matchingFault with { TriggerCount = matchingFault.TriggerCount + 1 };
+
+        var exception = matchingFault.FaultType switch
+        {
+            ChaosFaultType.Outage => new InvalidOperationException(matchingFault.ErrorMessage ?? "Channel outage (chaos)"),
+            ChaosFaultType.AuthFailure => new UnauthorizedAccessException(matchingFault.ErrorMessage ?? "Auth failure (chaos)"),
+            ChaosFaultType.Timeout => new TimeoutException(matchingFault.ErrorMessage ?? "Timeout (chaos)"),
+            ChaosFaultType.RateLimit => new InvalidOperationException(matchingFault.ErrorMessage ?? "Rate limited (chaos)"),
+            _ => new Exception(matchingFault.ErrorMessage ?? "Chaos fault")
+        };
+
+        return Task.FromResult(new ChaosFaultResult
+        {
+            ShouldFail = true,
+            ActiveFault = matchingFault,
+            InjectedLatency = matchingFault.LatencyInjection,
+            SimulatedException = exception
+        });
+    }
+
+    public async Task<ChaosTestResult> RunScenarioAsync(ChaosScenario scenario, CancellationToken cancellationToken = default)
+    {
+        if (!_options.Enabled)
+            throw new InvalidOperationException("Chaos engine is disabled");
+
+        var testId = $"test-{Guid.NewGuid():N}"[..16];
+        var startedAt = _timeProvider.GetUtcNow();
+        var stepResults = new List<ChaosStepResult>();
+        string? error = null;
+        var success = true;
+
+        _logger.LogInformation("Starting chaos scenario {ScenarioId}: {ScenarioName}", scenario.ScenarioId, scenario.Name);
+
+        using var timeoutCts = new CancellationTokenSource(scenario.Timeout);
+        using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCts.Token);
+
+        try
+        {
+            foreach (var step in scenario.Steps)
+            {
+                var stepStart = _timeProvider.GetUtcNow();
+                var stepResult = await ExecuteStepAsync(step, linkedCts.Token);
+                stepResults.Add(stepResult);
+
+                if (!stepResult.Success)
+                {
+                    success = false;
+                    if (scenario.StopOnFirstFailure)
+                    {
+                        error = $"Step '{step.Name}' failed: {stepResult.Error}";
+                        break;
+                    }
+                }
+            }
+        }
+        catch (OperationCanceledException) when (timeoutCts.IsCancellationRequested)
+        {
+            success = false;
+            error = "Scenario timed out";
+        }
+        catch (Exception ex)
+        {
+            success = false;
+            error = ex.Message;
+        }
+
+        var completedAt = _timeProvider.GetUtcNow();
+        var result = new ChaosTestResult
+        {
+            TestId = testId,
+            ScenarioId = scenario.ScenarioId,
+            ScenarioName = scenario.Name,
+            Success = success,
+            StartedAt = startedAt,
+            CompletedAt = completedAt,
+            Duration = completedAt - startedAt,
+            StepResults = stepResults,
+            Error = error
+        };
+
+        lock (_testHistory)
+        {
+            _testHistory.Add(result);
+            while (_testHistory.Count > 100) _testHistory.RemoveAt(0);
+        }
+
+        _logger.LogInformation("Chaos scenario {ScenarioId} completed: {Success}", scenario.ScenarioId, success ? "PASSED" : "FAILED");
+        return result;
+    }
+
+    private async Task<ChaosStepResult> ExecuteStepAsync(ChaosScenarioStep step, CancellationToken cancellationToken)
+    {
+        var executedAt = _timeProvider.GetUtcNow();
+        try
+        {
+            object? data = null;
+            switch (step.Action)
+            {
+                case ChaosStepAction.InjectFault when step.FaultToInject is not null:
+                    var fault = await InjectFaultAsync(step.FaultToInject, cancellationToken);
+                    data = fault;
+                    break;
+                case ChaosStepAction.RemoveFault when step.FaultIdToRemove is not null:
+                    await RemoveFaultAsync(step.FaultIdToRemove, cancellationToken);
+                    break;
+                case ChaosStepAction.Wait when step.WaitDuration.HasValue:
+                    await Task.Delay(step.WaitDuration.Value, cancellationToken);
+                    break;
+                case ChaosStepAction.Assert when step.Assertion is not null:
+                    var assertResult = EvaluateAssertion(step.Assertion);
+                    if (!assertResult.passed)
+                        return new ChaosStepResult { StepId = step.StepId, StepName = step.Name, Success = false, ExecutedAt = executedAt, Duration = _timeProvider.GetUtcNow() - executedAt, Error = assertResult.error };
+                    break;
+            }
+            return new ChaosStepResult { StepId = step.StepId, StepName = step.Name, Success = true, ExecutedAt = executedAt, Duration = _timeProvider.GetUtcNow() - executedAt, Data = data };
+        }
+        catch (Exception ex)
+        {
+            return new ChaosStepResult { StepId = step.StepId, StepName = step.Name, Success = false, ExecutedAt = executedAt, Duration = _timeProvider.GetUtcNow() - executedAt, Error = ex.Message };
+        }
+    }
+
+    private (bool passed, string? error) EvaluateAssertion(ChaosAssertion assertion)
+    {
+        // Simplified assertion evaluation - in production would query actual metrics
+        return assertion.Type switch
+        {
+            ChaosAssertionType.FallbackTriggered => (true, null),
+            ChaosAssertionType.AlertFired => (true, null),
+            _ => (true, null)
+        };
+    }
+
+    public Task<IReadOnlyList<ChaosTestResult>> GetHistoryAsync(int limit = 50, CancellationToken cancellationToken = default)
+    {
+        lock (_testHistory)
+        {
+            return Task.FromResult<IReadOnlyList<ChaosTestResult>>(_testHistory.TakeLast(limit).Reverse().ToList());
+        }
+    }
+
+    public Task ClearAllFaultsAsync(CancellationToken cancellationToken = default)
+    {
+        _activeFaults.Clear();
+        _logger.LogInformation("Cleared all chaos faults");
+        return Task.CompletedTask;
+    }
+
+    private void CleanupExpiredFaults()
+    {
+        var now = _timeProvider.GetUtcNow();
+        var expired = _activeFaults.Where(f => f.Value.ExpiresAt.HasValue && f.Value.ExpiresAt < now).Select(f => f.Key).ToList();
+        foreach (var id in expired)
+        {
+            _activeFaults.TryRemove(id, out _);
+            _logger.LogDebug("Expired chaos fault {FaultId}", id);
+        }
+    }
+}