stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

fix(notifier): stabilize storm breaker cooldown and summary tests

Browse Source
This commit is contained in:
master
2026-02-11 17:30:10 +02:00
parent 6571c83bd4
commit dca86e1248
2 changed files with 19 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Tests/StormBreaker/StormBreakerTests.cs
View File

@@ -3,7 +3,6 @@ using Microsoft.Extensions.Options;
using Microsoft.Extensions.Time.Testing;
using StellaOps.Notifier.Worker.StormBreaker;
#if false
namespace StellaOps.Notifier.Tests.StormBreaker;
public class InMemoryStormBreakerTests
@@ -36,9 +35,10 @@ public class InMemoryStormBreakerTests
{
// Act
var result = await _stormBreaker.EvaluateAsync("tenant1", "key1", "event1");
var state = await _stormBreaker.GetStateAsync("tenant1", "key1");
// Assert
Assert.False(result.IsStorm);
Assert.False(result.IsStorm, $"action={result.Action}, threshold={result.Threshold}, count={result.EventCount}, stateActive={state?.IsActive}");
Assert.Equal(StormAction.SendNormally, result.Action);
Assert.Equal(1, result.EventCount);
}
@@ -89,8 +89,10 @@ public class InMemoryStormBreakerTests
await _stormBreaker.EvaluateAsync("tenant1", "key1", $"event{i}");
}
// Advance time past summary interval
_timeProvider.Advance(TimeSpan.FromMinutes(16));
// Keep the storm active (within cooldown), then cross summary interval.
_timeProvider.Advance(TimeSpan.FromMinutes(9));
await _stormBreaker.EvaluateAsync("tenant1", "key1", "event_before_summary");
_timeProvider.Advance(TimeSpan.FromMinutes(7));
// Act
var result = await _stormBreaker.EvaluateAsync("tenant1", "key1", "event_after_interval");
@@ -325,4 +327,3 @@ public class InMemoryStormBreakerTests
Assert.False(infoResult.IsStorm);
}
}
#endif

24
src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Worker/StormBreaker/IStormBreaker.cs
View File

@@ -254,7 +254,7 @@ public sealed class StormState
/// <summary>
/// Whether the storm is currently active.
/// </summary>
public bool IsActive { get; set; } = true;
public bool IsActive { get; set; }
/// <summary>
/// Sample event metadata for summary generation.
@@ -433,7 +433,18 @@ public sealed class InMemoryStormBreaker : IStormBreaker
var state = _storms.AddOrUpdate(
key,
_ => CreateNewState(tenantId, stormKey, eventId, now),
(_, existing) => UpdateState(existing, eventId, now, window));
(_, existing) =>
{
if (existing.IsActive && now - existing.LastActivityAt > _options.StormCooldown)
{
_logger.LogInformation(
"Storm ended for key {StormKey} tenant {TenantId}. Total events: {TotalEvents}, Suppressed: {Suppressed}",
stormKey, tenantId, existing.EventIds.Count, existing.SuppressedCount);
return CreateNewState(tenantId, stormKey, eventId, now);
}
return UpdateState(existing, eventId, now, window);
});
// Clean old events outside the window
var cutoff = now - window;
@@ -441,15 +452,6 @@ public sealed class InMemoryStormBreaker : IStormBreaker
var eventCount = state.EventTimestamps.Count;
// Check if storm should end (cooldown elapsed)
if (state.IsActive && now - state.LastActivityAt > _options.StormCooldown)
{
state.IsActive = false;
_logger.LogInformation(
"Storm ended for key {StormKey} tenant {TenantId}. Total events: {TotalEvents}, Suppressed: {Suppressed}",
stormKey, tenantId, state.EventIds.Count, state.SuppressedCount);
}
// Not in storm and below threshold
if (!state.IsActive && eventCount < threshold)
{