using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using FluentAssertions; using Microsoft.Extensions.Logging.Abstractions; using StackExchange.Redis; using StellaOps.Scheduler.Models; using StellaOps.Scheduler.Queue.Redis; using Testcontainers.Redis; using Xunit; using StellaOps.TestKit; namespace StellaOps.Scheduler.Queue.Tests; public sealed class RedisSchedulerQueueTests : IAsyncLifetime { private readonly RedisContainer _redis; private string? _skipReason; public RedisSchedulerQueueTests() { _redis = new RedisBuilder().Build(); } public async Task InitializeAsync() { try { await _redis.StartAsync(); } catch (Exception ex) when (IsDockerUnavailable(ex)) { _skipReason = $"Docker engine is not available for Redis-backed tests: {ex.Message}"; } } public async Task DisposeAsync() { if (_skipReason is not null) { return; } await _redis.DisposeAsync().AsTask(); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task PlannerQueue_EnqueueLeaseAck_RemovesMessage() { if (SkipIfUnavailable()) { return; } var options = CreateOptions(); await using var queue = new RedisSchedulerPlannerQueue( options, options.Redis, NullLogger.Instance, TimeProvider.System, async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false)); var message = TestData.CreatePlannerMessage(); var enqueue = await queue.EnqueueAsync(message); enqueue.Deduplicated.Should().BeFalse(); var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("planner-1", batchSize: 5, options.DefaultLeaseDuration)); leases.Should().HaveCount(1); var lease = leases[0]; lease.Message.Run.Id.Should().Be(message.Run.Id); lease.TenantId.Should().Be(message.TenantId); lease.ScheduleId.Should().Be(message.ScheduleId); await lease.AcknowledgeAsync(); var afterAck = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("planner-1", 5, options.DefaultLeaseDuration)); afterAck.Should().BeEmpty(); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task RunnerQueue_Retry_IncrementsDeliveryAttempt() { if (SkipIfUnavailable()) { return; } var options = CreateOptions(); options.RetryInitialBackoff = TimeSpan.Zero; options.RetryMaxBackoff = TimeSpan.Zero; await using var queue = new RedisSchedulerRunnerQueue( options, options.Redis, NullLogger.Instance, TimeProvider.System, async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false)); var message = TestData.CreateRunnerMessage(); await queue.EnqueueAsync(message); var firstLease = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("runner-1", batchSize: 1, options.DefaultLeaseDuration)); firstLease.Should().ContainSingle(); var lease = firstLease[0]; lease.Attempt.Should().Be(1); await lease.ReleaseAsync(SchedulerQueueReleaseDisposition.Retry); var secondLease = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("runner-1", batchSize: 1, options.DefaultLeaseDuration)); secondLease.Should().ContainSingle(); secondLease[0].Attempt.Should().Be(2); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task PlannerQueue_ClaimExpired_ReassignsLease() { if (SkipIfUnavailable()) { return; } var options = CreateOptions(); await using var queue = new RedisSchedulerPlannerQueue( options, options.Redis, NullLogger.Instance, TimeProvider.System, async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false)); var message = TestData.CreatePlannerMessage(); await queue.EnqueueAsync(message); var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("planner-a", 1, options.DefaultLeaseDuration)); leases.Should().ContainSingle(); await Task.Delay(50); var reclaimed = await queue.ClaimExpiredAsync(new SchedulerQueueClaimOptions("planner-b", batchSize: 1, minIdleTime: TimeSpan.Zero)); reclaimed.Should().ContainSingle(); reclaimed[0].Consumer.Should().Be("planner-b"); reclaimed[0].RunId.Should().Be(message.Run.Id); await reclaimed[0].AcknowledgeAsync(); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task PlannerQueue_RecordsDepthMetrics() { if (SkipIfUnavailable()) { return; } var options = CreateOptions(); await using var queue = new RedisSchedulerPlannerQueue( options, options.Redis, NullLogger.Instance, TimeProvider.System, async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false)); var message = TestData.CreatePlannerMessage(); await queue.EnqueueAsync(message); var depths = SchedulerQueueMetrics.SnapshotDepths(); depths.TryGetValue(("redis", "planner"), out var plannerDepth) .Should().BeTrue(); plannerDepth.Should().Be(1); var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("planner-depth", 1, options.DefaultLeaseDuration)); leases.Should().ContainSingle(); await leases[0].AcknowledgeAsync(); depths = SchedulerQueueMetrics.SnapshotDepths(); depths.TryGetValue(("redis", "planner"), out plannerDepth).Should().BeTrue(); plannerDepth.Should().Be(0); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task RunnerQueue_DropWhenDeadLetterDisabled() { if (SkipIfUnavailable()) { return; } var options = CreateOptions(); options.MaxDeliveryAttempts = 1; options.DeadLetterEnabled = false; await using var queue = new RedisSchedulerRunnerQueue( options, options.Redis, NullLogger.Instance, TimeProvider.System, async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false)); using StellaOps.TestKit; var message = TestData.CreateRunnerMessage(); await queue.EnqueueAsync(message); var lease = (await queue.LeaseAsync(new SchedulerQueueLeaseRequest("runner-drop", 1, options.DefaultLeaseDuration)))[0]; await lease.ReleaseAsync(SchedulerQueueReleaseDisposition.Retry); var depths = SchedulerQueueMetrics.SnapshotDepths(); depths.TryGetValue(("redis", "runner"), out var runnerDepth).Should().BeTrue(); runnerDepth.Should().Be(0); } private SchedulerQueueOptions CreateOptions() { var unique = Guid.NewGuid().ToString("N"); return new SchedulerQueueOptions { Kind = SchedulerQueueTransportKind.Redis, DefaultLeaseDuration = TimeSpan.FromSeconds(2), MaxDeliveryAttempts = 5, RetryInitialBackoff = TimeSpan.FromMilliseconds(10), RetryMaxBackoff = TimeSpan.FromMilliseconds(50), Redis = new SchedulerRedisQueueOptions { ConnectionString = _redis.GetConnectionString(), Database = 0, InitializationTimeout = TimeSpan.FromSeconds(10), Planner = new RedisSchedulerStreamOptions { Stream = $"scheduler:test:planner:{unique}", ConsumerGroup = $"planner-consumers-{unique}", DeadLetterStream = $"scheduler:test:planner:{unique}:dead", IdempotencyKeyPrefix = $"scheduler:test:planner:{unique}:idemp:", IdempotencyWindow = TimeSpan.FromMinutes(5) }, Runner = new RedisSchedulerStreamOptions { Stream = $"scheduler:test:runner:{unique}", ConsumerGroup = $"runner-consumers-{unique}", DeadLetterStream = $"scheduler:test:runner:{unique}:dead", IdempotencyKeyPrefix = $"scheduler:test:runner:{unique}:idemp:", IdempotencyWindow = TimeSpan.FromMinutes(5) } } }; } private bool SkipIfUnavailable() { if (_skipReason is not null) { return true; } return false; } private static bool IsDockerUnavailable(Exception exception) { while (exception is AggregateException aggregate && aggregate.InnerException is not null) { exception = aggregate.InnerException; } return exception is TimeoutException || exception.GetType().Name.Contains("Docker", StringComparison.OrdinalIgnoreCase); } private static class TestData { public static PlannerQueueMessage CreatePlannerMessage() { var schedule = new Schedule( id: "sch-test", tenantId: "tenant-alpha", name: "Test", enabled: true, cronExpression: "0 0 * * *", timezone: "UTC", mode: ScheduleMode.AnalysisOnly, selection: new Selector(SelectorScope.AllImages, tenantId: "tenant-alpha"), onlyIf: ScheduleOnlyIf.Default, notify: ScheduleNotify.Default, limits: ScheduleLimits.Default, createdAt: DateTimeOffset.UtcNow, createdBy: "tests", updatedAt: DateTimeOffset.UtcNow, updatedBy: "tests"); var run = new Run( id: "run-test", tenantId: "tenant-alpha", trigger: RunTrigger.Manual, state: RunState.Planning, stats: RunStats.Empty, createdAt: DateTimeOffset.UtcNow, reason: RunReason.Empty, scheduleId: schedule.Id); var impactSet = new ImpactSet( selector: new Selector(SelectorScope.AllImages, tenantId: "tenant-alpha"), images: new[] { new ImpactImage( imageDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", registry: "registry", repository: "repo", namespaces: new[] { "prod" }, tags: new[] { "latest" }) }, usageOnly: true, generatedAt: DateTimeOffset.UtcNow, total: 1); return new PlannerQueueMessage(run, impactSet, schedule, correlationId: "corr-test"); } public static RunnerSegmentQueueMessage CreateRunnerMessage() { return new RunnerSegmentQueueMessage( segmentId: "segment-test", runId: "run-test", tenantId: "tenant-alpha", imageDigests: new[] { "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" }, scheduleId: "sch-test", ratePerSecond: 10, usageOnly: true, attributes: new Dictionary { ["priority"] = "kev" }, correlationId: "corr-runner"); } } }