fix(notifier): stabilize storm breaker cooldown and summary tests

This commit is contained in:
master
2026-02-11 17:30:10 +02:00
parent 6571c83bd4
commit dca86e1248
2 changed files with 19 additions and 16 deletions

View File

@@ -3,7 +3,6 @@ using Microsoft.Extensions.Options;
using Microsoft.Extensions.Time.Testing; using Microsoft.Extensions.Time.Testing;
using StellaOps.Notifier.Worker.StormBreaker; using StellaOps.Notifier.Worker.StormBreaker;
#if false
namespace StellaOps.Notifier.Tests.StormBreaker; namespace StellaOps.Notifier.Tests.StormBreaker;
public class InMemoryStormBreakerTests public class InMemoryStormBreakerTests
@@ -36,9 +35,10 @@ public class InMemoryStormBreakerTests
{ {
// Act // Act
var result = await _stormBreaker.EvaluateAsync("tenant1", "key1", "event1"); var result = await _stormBreaker.EvaluateAsync("tenant1", "key1", "event1");
var state = await _stormBreaker.GetStateAsync("tenant1", "key1");
// Assert // Assert
Assert.False(result.IsStorm); Assert.False(result.IsStorm, $"action={result.Action}, threshold={result.Threshold}, count={result.EventCount}, stateActive={state?.IsActive}");
Assert.Equal(StormAction.SendNormally, result.Action); Assert.Equal(StormAction.SendNormally, result.Action);
Assert.Equal(1, result.EventCount); Assert.Equal(1, result.EventCount);
} }
@@ -89,8 +89,10 @@ public class InMemoryStormBreakerTests
await _stormBreaker.EvaluateAsync("tenant1", "key1", $"event{i}"); await _stormBreaker.EvaluateAsync("tenant1", "key1", $"event{i}");
} }
// Advance time past summary interval // Keep the storm active (within cooldown), then cross summary interval.
_timeProvider.Advance(TimeSpan.FromMinutes(16)); _timeProvider.Advance(TimeSpan.FromMinutes(9));
await _stormBreaker.EvaluateAsync("tenant1", "key1", "event_before_summary");
_timeProvider.Advance(TimeSpan.FromMinutes(7));
// Act // Act
var result = await _stormBreaker.EvaluateAsync("tenant1", "key1", "event_after_interval"); var result = await _stormBreaker.EvaluateAsync("tenant1", "key1", "event_after_interval");
@@ -325,4 +327,3 @@ public class InMemoryStormBreakerTests
Assert.False(infoResult.IsStorm); Assert.False(infoResult.IsStorm);
} }
} }
#endif

View File

@@ -254,7 +254,7 @@ public sealed class StormState
/// <summary> /// <summary>
/// Whether the storm is currently active. /// Whether the storm is currently active.
/// </summary> /// </summary>
public bool IsActive { get; set; } = true; public bool IsActive { get; set; }
/// <summary> /// <summary>
/// Sample event metadata for summary generation. /// Sample event metadata for summary generation.
@@ -433,7 +433,18 @@ public sealed class InMemoryStormBreaker : IStormBreaker
var state = _storms.AddOrUpdate( var state = _storms.AddOrUpdate(
key, key,
_ => CreateNewState(tenantId, stormKey, eventId, now), _ => CreateNewState(tenantId, stormKey, eventId, now),
(_, existing) => UpdateState(existing, eventId, now, window)); (_, existing) =>
{
if (existing.IsActive && now - existing.LastActivityAt > _options.StormCooldown)
{
_logger.LogInformation(
"Storm ended for key {StormKey} tenant {TenantId}. Total events: {TotalEvents}, Suppressed: {Suppressed}",
stormKey, tenantId, existing.EventIds.Count, existing.SuppressedCount);
return CreateNewState(tenantId, stormKey, eventId, now);
}
return UpdateState(existing, eventId, now, window);
});
// Clean old events outside the window // Clean old events outside the window
var cutoff = now - window; var cutoff = now - window;
@@ -441,15 +452,6 @@ public sealed class InMemoryStormBreaker : IStormBreaker
var eventCount = state.EventTimestamps.Count; var eventCount = state.EventTimestamps.Count;
// Check if storm should end (cooldown elapsed)
if (state.IsActive && now - state.LastActivityAt > _options.StormCooldown)
{
state.IsActive = false;
_logger.LogInformation(
"Storm ended for key {StormKey} tenant {TenantId}. Total events: {TotalEvents}, Suppressed: {Suppressed}",
stormKey, tenantId, state.EventIds.Count, state.SuppressedCount);
}
// Not in storm and below threshold // Not in storm and below threshold
if (!state.IsActive && eventCount < threshold) if (!state.IsActive && eventCount < threshold)
{ {