From dca86e1248b045c3165566f30ebbae50c0d139b6 Mon Sep 17 00:00:00 2001 From: master <> Date: Wed, 11 Feb 2026 17:30:10 +0200 Subject: [PATCH] fix(notifier): stabilize storm breaker cooldown and summary tests --- .../StormBreaker/StormBreakerTests.cs | 11 +++++---- .../StormBreaker/IStormBreaker.cs | 24 ++++++++++--------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Tests/StormBreaker/StormBreakerTests.cs b/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Tests/StormBreaker/StormBreakerTests.cs index be5351321..098e6ce65 100644 --- a/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Tests/StormBreaker/StormBreakerTests.cs +++ b/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Tests/StormBreaker/StormBreakerTests.cs @@ -3,7 +3,6 @@ using Microsoft.Extensions.Options; using Microsoft.Extensions.Time.Testing; using StellaOps.Notifier.Worker.StormBreaker; -#if false namespace StellaOps.Notifier.Tests.StormBreaker; public class InMemoryStormBreakerTests @@ -36,9 +35,10 @@ public class InMemoryStormBreakerTests { // Act var result = await _stormBreaker.EvaluateAsync("tenant1", "key1", "event1"); + var state = await _stormBreaker.GetStateAsync("tenant1", "key1"); // Assert - Assert.False(result.IsStorm); + Assert.False(result.IsStorm, $"action={result.Action}, threshold={result.Threshold}, count={result.EventCount}, stateActive={state?.IsActive}"); Assert.Equal(StormAction.SendNormally, result.Action); Assert.Equal(1, result.EventCount); } @@ -89,8 +89,10 @@ public class InMemoryStormBreakerTests await _stormBreaker.EvaluateAsync("tenant1", "key1", $"event{i}"); } - // Advance time past summary interval - _timeProvider.Advance(TimeSpan.FromMinutes(16)); + // Keep the storm active (within cooldown), then cross summary interval. + _timeProvider.Advance(TimeSpan.FromMinutes(9)); + await _stormBreaker.EvaluateAsync("tenant1", "key1", "event_before_summary"); + _timeProvider.Advance(TimeSpan.FromMinutes(7)); // Act var result = await _stormBreaker.EvaluateAsync("tenant1", "key1", "event_after_interval"); @@ -325,4 +327,3 @@ public class InMemoryStormBreakerTests Assert.False(infoResult.IsStorm); } } -#endif diff --git a/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Worker/StormBreaker/IStormBreaker.cs b/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Worker/StormBreaker/IStormBreaker.cs index 51456eb00..e1a023931 100644 --- a/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Worker/StormBreaker/IStormBreaker.cs +++ b/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Worker/StormBreaker/IStormBreaker.cs @@ -254,7 +254,7 @@ public sealed class StormState /// /// Whether the storm is currently active. /// - public bool IsActive { get; set; } = true; + public bool IsActive { get; set; } /// /// Sample event metadata for summary generation. @@ -433,7 +433,18 @@ public sealed class InMemoryStormBreaker : IStormBreaker var state = _storms.AddOrUpdate( key, _ => CreateNewState(tenantId, stormKey, eventId, now), - (_, existing) => UpdateState(existing, eventId, now, window)); + (_, existing) => + { + if (existing.IsActive && now - existing.LastActivityAt > _options.StormCooldown) + { + _logger.LogInformation( + "Storm ended for key {StormKey} tenant {TenantId}. Total events: {TotalEvents}, Suppressed: {Suppressed}", + stormKey, tenantId, existing.EventIds.Count, existing.SuppressedCount); + return CreateNewState(tenantId, stormKey, eventId, now); + } + + return UpdateState(existing, eventId, now, window); + }); // Clean old events outside the window var cutoff = now - window; @@ -441,15 +452,6 @@ public sealed class InMemoryStormBreaker : IStormBreaker var eventCount = state.EventTimestamps.Count; - // Check if storm should end (cooldown elapsed) - if (state.IsActive && now - state.LastActivityAt > _options.StormCooldown) - { - state.IsActive = false; - _logger.LogInformation( - "Storm ended for key {StormKey} tenant {TenantId}. Total events: {TotalEvents}, Suppressed: {Suppressed}", - stormKey, tenantId, state.EventIds.Count, state.SuppressedCount); - } - // Not in storm and below threshold if (!state.IsActive && eventCount < threshold) {