356 lines
12 KiB
C#
356 lines
12 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Linq;
|
|
using System.Threading.Tasks;
|
|
using FluentAssertions;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using StackExchange.Redis;
|
|
using StellaOps.Scheduler.Models;
|
|
using StellaOps.Scheduler.Queue.Redis;
|
|
using Testcontainers.Redis;
|
|
using Xunit;
|
|
|
|
|
|
using StellaOps.TestKit;
|
|
namespace StellaOps.Scheduler.Queue.Tests;
|
|
|
|
public sealed class RedisSchedulerQueueTests : IAsyncLifetime
|
|
{
|
|
private readonly RedisContainer _redis;
|
|
private string? _skipReason;
|
|
|
|
public RedisSchedulerQueueTests()
|
|
{
|
|
_redis = new RedisBuilder().Build();
|
|
}
|
|
|
|
public async ValueTask InitializeAsync()
|
|
{
|
|
try
|
|
{
|
|
await _redis.StartAsync();
|
|
}
|
|
catch (Exception ex) when (IsDockerUnavailable(ex))
|
|
{
|
|
_skipReason = $"Docker engine is not available for Redis-backed tests: {ex.Message}";
|
|
}
|
|
}
|
|
|
|
public async ValueTask DisposeAsync()
|
|
{
|
|
if (_skipReason is not null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
await _redis.DisposeAsync().AsTask();
|
|
}
|
|
|
|
[Trait("Category", TestCategories.Unit)]
|
|
[Fact]
|
|
public async Task PlannerQueue_EnqueueLeaseAck_RemovesMessage()
|
|
{
|
|
if (SkipIfUnavailable())
|
|
{
|
|
return;
|
|
}
|
|
|
|
var options = CreateOptions();
|
|
|
|
await using var queue = new RedisSchedulerPlannerQueue(
|
|
options,
|
|
options.Redis,
|
|
NullLogger<RedisSchedulerPlannerQueue>.Instance,
|
|
TimeProvider.System,
|
|
hlc: null,
|
|
connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
|
|
|
|
var message = TestData.CreatePlannerMessage();
|
|
|
|
var enqueue = await queue.EnqueueAsync(message);
|
|
enqueue.Deduplicated.Should().BeFalse();
|
|
|
|
var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("planner-1", batchSize: 5, options.DefaultLeaseDuration));
|
|
leases.Should().HaveCount(1);
|
|
|
|
var lease = leases[0];
|
|
lease.Message.Run.Id.Should().Be(message.Run.Id);
|
|
lease.TenantId.Should().Be(message.TenantId);
|
|
lease.ScheduleId.Should().Be(message.ScheduleId);
|
|
|
|
await lease.AcknowledgeAsync();
|
|
|
|
var afterAck = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("planner-1", 5, options.DefaultLeaseDuration));
|
|
afterAck.Should().BeEmpty();
|
|
}
|
|
|
|
[Trait("Category", TestCategories.Unit)]
|
|
[Fact]
|
|
public async Task RunnerQueue_Retry_IncrementsDeliveryAttempt()
|
|
{
|
|
if (SkipIfUnavailable())
|
|
{
|
|
return;
|
|
}
|
|
|
|
var options = CreateOptions();
|
|
options.RetryInitialBackoff = TimeSpan.Zero;
|
|
options.RetryMaxBackoff = TimeSpan.Zero;
|
|
|
|
await using var queue = new RedisSchedulerRunnerQueue(
|
|
options,
|
|
options.Redis,
|
|
NullLogger<RedisSchedulerRunnerQueue>.Instance,
|
|
TimeProvider.System,
|
|
hlc: null,
|
|
connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
|
|
|
|
var message = TestData.CreateRunnerMessage();
|
|
|
|
await queue.EnqueueAsync(message);
|
|
|
|
var firstLease = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("runner-1", batchSize: 1, options.DefaultLeaseDuration));
|
|
firstLease.Should().ContainSingle();
|
|
|
|
var lease = firstLease[0];
|
|
lease.Attempt.Should().Be(1);
|
|
|
|
await lease.ReleaseAsync(SchedulerQueueReleaseDisposition.Retry);
|
|
|
|
var secondLease = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("runner-1", batchSize: 1, options.DefaultLeaseDuration));
|
|
secondLease.Should().ContainSingle();
|
|
secondLease[0].Attempt.Should().Be(2);
|
|
}
|
|
|
|
[Trait("Category", TestCategories.Unit)]
|
|
[Fact]
|
|
public async Task PlannerQueue_ClaimExpired_ReassignsLease()
|
|
{
|
|
if (SkipIfUnavailable())
|
|
{
|
|
return;
|
|
}
|
|
|
|
var options = CreateOptions();
|
|
|
|
await using var queue = new RedisSchedulerPlannerQueue(
|
|
options,
|
|
options.Redis,
|
|
NullLogger<RedisSchedulerPlannerQueue>.Instance,
|
|
TimeProvider.System,
|
|
hlc: null,
|
|
connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
|
|
|
|
var message = TestData.CreatePlannerMessage();
|
|
await queue.EnqueueAsync(message);
|
|
|
|
var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("planner-a", 1, options.DefaultLeaseDuration));
|
|
leases.Should().ContainSingle();
|
|
|
|
await Task.Delay(50);
|
|
|
|
var reclaimed = await queue.ClaimExpiredAsync(new SchedulerQueueClaimOptions("planner-b", batchSize: 1, minIdleTime: TimeSpan.Zero));
|
|
reclaimed.Should().ContainSingle();
|
|
reclaimed[0].Consumer.Should().Be("planner-b");
|
|
reclaimed[0].RunId.Should().Be(message.Run.Id);
|
|
|
|
await reclaimed[0].AcknowledgeAsync();
|
|
}
|
|
|
|
[Trait("Category", TestCategories.Unit)]
|
|
[Fact]
|
|
public async Task PlannerQueue_RecordsDepthMetrics()
|
|
{
|
|
if (SkipIfUnavailable())
|
|
{
|
|
return;
|
|
}
|
|
|
|
var options = CreateOptions();
|
|
|
|
await using var queue = new RedisSchedulerPlannerQueue(
|
|
options,
|
|
options.Redis,
|
|
NullLogger<RedisSchedulerPlannerQueue>.Instance,
|
|
TimeProvider.System,
|
|
hlc: null,
|
|
connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
|
|
|
|
var message = TestData.CreatePlannerMessage();
|
|
|
|
await queue.EnqueueAsync(message);
|
|
|
|
var depths = SchedulerQueueMetrics.SnapshotDepths();
|
|
depths.TryGetValue(("redis", "planner"), out var plannerDepth)
|
|
.Should().BeTrue();
|
|
plannerDepth.Should().Be(1);
|
|
|
|
var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("planner-depth", 1, options.DefaultLeaseDuration));
|
|
leases.Should().ContainSingle();
|
|
await leases[0].AcknowledgeAsync();
|
|
|
|
depths = SchedulerQueueMetrics.SnapshotDepths();
|
|
depths.TryGetValue(("redis", "planner"), out plannerDepth).Should().BeTrue();
|
|
plannerDepth.Should().Be(0);
|
|
}
|
|
|
|
[Trait("Category", TestCategories.Unit)]
|
|
[Fact]
|
|
public async Task RunnerQueue_DropWhenDeadLetterDisabled()
|
|
{
|
|
if (SkipIfUnavailable())
|
|
{
|
|
return;
|
|
}
|
|
|
|
var options = CreateOptions();
|
|
options.MaxDeliveryAttempts = 1;
|
|
options.DeadLetterEnabled = false;
|
|
|
|
await using var queue = new RedisSchedulerRunnerQueue(
|
|
options,
|
|
options.Redis,
|
|
NullLogger<RedisSchedulerRunnerQueue>.Instance,
|
|
TimeProvider.System,
|
|
hlc: null,
|
|
connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
|
|
|
|
var message = TestData.CreateRunnerMessage();
|
|
await queue.EnqueueAsync(message);
|
|
|
|
var lease = (await queue.LeaseAsync(new SchedulerQueueLeaseRequest("runner-drop", 1, options.DefaultLeaseDuration)))[0];
|
|
|
|
await lease.ReleaseAsync(SchedulerQueueReleaseDisposition.Retry);
|
|
|
|
var depths = SchedulerQueueMetrics.SnapshotDepths();
|
|
depths.TryGetValue(("redis", "runner"), out var runnerDepth).Should().BeTrue();
|
|
runnerDepth.Should().Be(0);
|
|
}
|
|
|
|
private SchedulerQueueOptions CreateOptions()
|
|
{
|
|
var unique = Guid.NewGuid().ToString("N");
|
|
|
|
return new SchedulerQueueOptions
|
|
{
|
|
Kind = SchedulerQueueTransportKind.Redis,
|
|
DefaultLeaseDuration = TimeSpan.FromSeconds(2),
|
|
MaxDeliveryAttempts = 5,
|
|
RetryInitialBackoff = TimeSpan.FromMilliseconds(10),
|
|
RetryMaxBackoff = TimeSpan.FromMilliseconds(50),
|
|
Redis = new SchedulerRedisQueueOptions
|
|
{
|
|
ConnectionString = _redis.GetConnectionString(),
|
|
Database = 0,
|
|
InitializationTimeout = TimeSpan.FromSeconds(10),
|
|
Planner = new RedisSchedulerStreamOptions
|
|
{
|
|
Stream = $"scheduler:test:planner:{unique}",
|
|
ConsumerGroup = $"planner-consumers-{unique}",
|
|
DeadLetterStream = $"scheduler:test:planner:{unique}:dead",
|
|
IdempotencyKeyPrefix = $"scheduler:test:planner:{unique}:idemp:",
|
|
IdempotencyWindow = TimeSpan.FromMinutes(5)
|
|
},
|
|
Runner = new RedisSchedulerStreamOptions
|
|
{
|
|
Stream = $"scheduler:test:runner:{unique}",
|
|
ConsumerGroup = $"runner-consumers-{unique}",
|
|
DeadLetterStream = $"scheduler:test:runner:{unique}:dead",
|
|
IdempotencyKeyPrefix = $"scheduler:test:runner:{unique}:idemp:",
|
|
IdempotencyWindow = TimeSpan.FromMinutes(5)
|
|
}
|
|
}
|
|
};
|
|
}
|
|
|
|
private bool SkipIfUnavailable()
|
|
{
|
|
if (_skipReason is not null)
|
|
{
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
private static bool IsDockerUnavailable(Exception exception)
|
|
{
|
|
while (exception is AggregateException aggregate && aggregate.InnerException is not null)
|
|
{
|
|
exception = aggregate.InnerException;
|
|
}
|
|
|
|
return exception is TimeoutException
|
|
|| exception.GetType().Name.Contains("Docker", StringComparison.OrdinalIgnoreCase);
|
|
}
|
|
|
|
private static class TestData
|
|
{
|
|
public static PlannerQueueMessage CreatePlannerMessage()
|
|
{
|
|
var schedule = new Schedule(
|
|
id: "sch-test",
|
|
tenantId: "tenant-alpha",
|
|
name: "Test",
|
|
enabled: true,
|
|
cronExpression: "0 0 * * *",
|
|
timezone: "UTC",
|
|
mode: ScheduleMode.AnalysisOnly,
|
|
selection: new Selector(SelectorScope.AllImages, tenantId: "tenant-alpha"),
|
|
onlyIf: ScheduleOnlyIf.Default,
|
|
notify: ScheduleNotify.Default,
|
|
limits: ScheduleLimits.Default,
|
|
createdAt: DateTimeOffset.UtcNow,
|
|
createdBy: "tests",
|
|
updatedAt: DateTimeOffset.UtcNow,
|
|
updatedBy: "tests");
|
|
|
|
var run = new Run(
|
|
id: "run-test",
|
|
tenantId: "tenant-alpha",
|
|
trigger: RunTrigger.Manual,
|
|
state: RunState.Planning,
|
|
stats: RunStats.Empty,
|
|
createdAt: DateTimeOffset.UtcNow,
|
|
reason: RunReason.Empty,
|
|
scheduleId: schedule.Id);
|
|
|
|
var impactSet = new ImpactSet(
|
|
selector: new Selector(SelectorScope.AllImages, tenantId: "tenant-alpha"),
|
|
images: new[]
|
|
{
|
|
new ImpactImage(
|
|
imageDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
|
registry: "registry",
|
|
repository: "repo",
|
|
namespaces: new[] { "prod" },
|
|
tags: new[] { "latest" })
|
|
},
|
|
usageOnly: true,
|
|
generatedAt: DateTimeOffset.UtcNow,
|
|
total: 1);
|
|
|
|
return new PlannerQueueMessage(run, impactSet, schedule, correlationId: "corr-test");
|
|
}
|
|
|
|
public static RunnerSegmentQueueMessage CreateRunnerMessage()
|
|
{
|
|
return new RunnerSegmentQueueMessage(
|
|
segmentId: "segment-test",
|
|
runId: "run-test",
|
|
tenantId: "tenant-alpha",
|
|
imageDigests: new[]
|
|
{
|
|
"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
|
},
|
|
scheduleId: "sch-test",
|
|
ratePerSecond: 10,
|
|
usageOnly: true,
|
|
attributes: new Dictionary<string, string> { ["priority"] = "kev" },
|
|
correlationId: "corr-runner");
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
|