fix(notifier): stabilize storm breaker cooldown and summary tests
This commit is contained in:
@@ -3,7 +3,6 @@ using Microsoft.Extensions.Options;
|
|||||||
using Microsoft.Extensions.Time.Testing;
|
using Microsoft.Extensions.Time.Testing;
|
||||||
using StellaOps.Notifier.Worker.StormBreaker;
|
using StellaOps.Notifier.Worker.StormBreaker;
|
||||||
|
|
||||||
#if false
|
|
||||||
namespace StellaOps.Notifier.Tests.StormBreaker;
|
namespace StellaOps.Notifier.Tests.StormBreaker;
|
||||||
|
|
||||||
public class InMemoryStormBreakerTests
|
public class InMemoryStormBreakerTests
|
||||||
@@ -36,9 +35,10 @@ public class InMemoryStormBreakerTests
|
|||||||
{
|
{
|
||||||
// Act
|
// Act
|
||||||
var result = await _stormBreaker.EvaluateAsync("tenant1", "key1", "event1");
|
var result = await _stormBreaker.EvaluateAsync("tenant1", "key1", "event1");
|
||||||
|
var state = await _stormBreaker.GetStateAsync("tenant1", "key1");
|
||||||
|
|
||||||
// Assert
|
// Assert
|
||||||
Assert.False(result.IsStorm);
|
Assert.False(result.IsStorm, $"action={result.Action}, threshold={result.Threshold}, count={result.EventCount}, stateActive={state?.IsActive}");
|
||||||
Assert.Equal(StormAction.SendNormally, result.Action);
|
Assert.Equal(StormAction.SendNormally, result.Action);
|
||||||
Assert.Equal(1, result.EventCount);
|
Assert.Equal(1, result.EventCount);
|
||||||
}
|
}
|
||||||
@@ -89,8 +89,10 @@ public class InMemoryStormBreakerTests
|
|||||||
await _stormBreaker.EvaluateAsync("tenant1", "key1", $"event{i}");
|
await _stormBreaker.EvaluateAsync("tenant1", "key1", $"event{i}");
|
||||||
}
|
}
|
||||||
|
|
||||||
// Advance time past summary interval
|
// Keep the storm active (within cooldown), then cross summary interval.
|
||||||
_timeProvider.Advance(TimeSpan.FromMinutes(16));
|
_timeProvider.Advance(TimeSpan.FromMinutes(9));
|
||||||
|
await _stormBreaker.EvaluateAsync("tenant1", "key1", "event_before_summary");
|
||||||
|
_timeProvider.Advance(TimeSpan.FromMinutes(7));
|
||||||
|
|
||||||
// Act
|
// Act
|
||||||
var result = await _stormBreaker.EvaluateAsync("tenant1", "key1", "event_after_interval");
|
var result = await _stormBreaker.EvaluateAsync("tenant1", "key1", "event_after_interval");
|
||||||
@@ -325,4 +327,3 @@ public class InMemoryStormBreakerTests
|
|||||||
Assert.False(infoResult.IsStorm);
|
Assert.False(infoResult.IsStorm);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endif
|
|
||||||
|
|||||||
@@ -254,7 +254,7 @@ public sealed class StormState
|
|||||||
/// <summary>
|
/// <summary>
|
||||||
/// Whether the storm is currently active.
|
/// Whether the storm is currently active.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public bool IsActive { get; set; } = true;
|
public bool IsActive { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Sample event metadata for summary generation.
|
/// Sample event metadata for summary generation.
|
||||||
@@ -433,7 +433,18 @@ public sealed class InMemoryStormBreaker : IStormBreaker
|
|||||||
var state = _storms.AddOrUpdate(
|
var state = _storms.AddOrUpdate(
|
||||||
key,
|
key,
|
||||||
_ => CreateNewState(tenantId, stormKey, eventId, now),
|
_ => CreateNewState(tenantId, stormKey, eventId, now),
|
||||||
(_, existing) => UpdateState(existing, eventId, now, window));
|
(_, existing) =>
|
||||||
|
{
|
||||||
|
if (existing.IsActive && now - existing.LastActivityAt > _options.StormCooldown)
|
||||||
|
{
|
||||||
|
_logger.LogInformation(
|
||||||
|
"Storm ended for key {StormKey} tenant {TenantId}. Total events: {TotalEvents}, Suppressed: {Suppressed}",
|
||||||
|
stormKey, tenantId, existing.EventIds.Count, existing.SuppressedCount);
|
||||||
|
return CreateNewState(tenantId, stormKey, eventId, now);
|
||||||
|
}
|
||||||
|
|
||||||
|
return UpdateState(existing, eventId, now, window);
|
||||||
|
});
|
||||||
|
|
||||||
// Clean old events outside the window
|
// Clean old events outside the window
|
||||||
var cutoff = now - window;
|
var cutoff = now - window;
|
||||||
@@ -441,15 +452,6 @@ public sealed class InMemoryStormBreaker : IStormBreaker
|
|||||||
|
|
||||||
var eventCount = state.EventTimestamps.Count;
|
var eventCount = state.EventTimestamps.Count;
|
||||||
|
|
||||||
// Check if storm should end (cooldown elapsed)
|
|
||||||
if (state.IsActive && now - state.LastActivityAt > _options.StormCooldown)
|
|
||||||
{
|
|
||||||
state.IsActive = false;
|
|
||||||
_logger.LogInformation(
|
|
||||||
"Storm ended for key {StormKey} tenant {TenantId}. Total events: {TotalEvents}, Suppressed: {Suppressed}",
|
|
||||||
stormKey, tenantId, state.EventIds.Count, state.SuppressedCount);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Not in storm and below threshold
|
// Not in storm and below threshold
|
||||||
if (!state.IsActive && eventCount < threshold)
|
if (!state.IsActive && eventCount < threshold)
|
||||||
{
|
{
|
||||||
|
|||||||
Reference in New Issue
Block a user