// // Copyright (c) StellaOps. Licensed under AGPL-3.0-or-later. // using System; using System.Collections.Generic; using System.Threading.Tasks; using FluentAssertions; using Microsoft.Extensions.Logging.Abstractions; using StackExchange.Redis; using StellaOps.HybridLogicalClock; using StellaOps.Scheduler.Models; using StellaOps.Scheduler.Queue.Redis; using StellaOps.TestKit; using Testcontainers.Redis; using Xunit; using HybridLogicalClockImpl = StellaOps.HybridLogicalClock.HybridLogicalClock; namespace StellaOps.Scheduler.Queue.Tests; /// /// Integration tests for HLC (Hybrid Logical Clock) integration with scheduler queues. /// [Trait("Category", TestCategories.Integration)] public sealed class HlcQueueIntegrationTests : IAsyncLifetime { private readonly RedisContainer _redis; private string? _skipReason; public HlcQueueIntegrationTests() { _redis = new RedisBuilder().Build(); } public async ValueTask InitializeAsync() { try { await _redis.StartAsync(); } catch (Exception ex) when (IsDockerUnavailable(ex)) { _skipReason = $"Docker engine is not available for Redis-backed tests: {ex.Message}"; } } public async ValueTask DisposeAsync() { if (_skipReason is not null) { return; } await _redis.DisposeAsync().AsTask(); } [Fact] public async Task PlannerQueue_WithHlc_LeasedMessageContainsHlcTimestamp() { if (SkipIfUnavailable()) { return; } var options = CreateOptions(); var hlc = new HybridLogicalClockImpl(TimeProvider.System, "test-node-1", new InMemoryHlcStateStore()); await using var queue = new RedisSchedulerPlannerQueue( options, options.Redis, NullLogger.Instance, TimeProvider.System, hlc, connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false)); var message = CreatePlannerMessage(); var enqueueResult = await queue.EnqueueAsync(message); enqueueResult.Deduplicated.Should().BeFalse(); var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("planner-hlc", batchSize: 1, options.DefaultLeaseDuration)); leases.Should().ContainSingle(); var lease = leases[0]; lease.HlcTimestamp.Should().NotBeNull("HLC timestamp should be present when HLC is configured"); lease.HlcTimestamp!.Value.NodeId.Should().Be("test-node-1"); lease.HlcTimestamp.Value.PhysicalTime.Should().BeGreaterThan(0); await lease.AcknowledgeAsync(); } [Fact] public async Task RunnerQueue_WithHlc_LeasedMessageContainsHlcTimestamp() { if (SkipIfUnavailable()) { return; } var options = CreateOptions(); var hlc = new HybridLogicalClockImpl(TimeProvider.System, "runner-node-1", new InMemoryHlcStateStore()); await using var queue = new RedisSchedulerRunnerQueue( options, options.Redis, NullLogger.Instance, TimeProvider.System, hlc, connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false)); var message = CreateRunnerMessage(); await queue.EnqueueAsync(message); var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("runner-hlc", batchSize: 1, options.DefaultLeaseDuration)); leases.Should().ContainSingle(); var lease = leases[0]; lease.HlcTimestamp.Should().NotBeNull("HLC timestamp should be present when HLC is configured"); lease.HlcTimestamp!.Value.NodeId.Should().Be("runner-node-1"); await lease.AcknowledgeAsync(); } [Fact] public async Task PlannerQueue_WithoutHlc_LeasedMessageHasNullTimestamp() { if (SkipIfUnavailable()) { return; } var options = CreateOptions(); // No HLC provided await using var queue = new RedisSchedulerPlannerQueue( options, options.Redis, NullLogger.Instance, TimeProvider.System, hlc: null, connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false)); var message = CreatePlannerMessage(); await queue.EnqueueAsync(message); var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("planner-no-hlc", batchSize: 1, options.DefaultLeaseDuration)); leases.Should().ContainSingle(); var lease = leases[0]; lease.HlcTimestamp.Should().BeNull("HLC timestamp should be null when HLC is not configured"); await lease.AcknowledgeAsync(); } [Fact] public async Task HlcTimestamp_IsMonotonicallyIncreasing_AcrossEnqueues() { if (SkipIfUnavailable()) { return; } var options = CreateOptions(); var hlc = new HybridLogicalClockImpl(TimeProvider.System, "monotonic-test", new InMemoryHlcStateStore()); await using var queue = new RedisSchedulerPlannerQueue( options, options.Redis, NullLogger.Instance, TimeProvider.System, hlc, connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false)); // Enqueue multiple messages var messages = new List(); for (int i = 0; i < 5; i++) { messages.Add(CreatePlannerMessage(suffix: i.ToString())); } foreach (var msg in messages) { await queue.EnqueueAsync(msg); } // Lease all messages var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("monotonic-consumer", batchSize: 10, options.DefaultLeaseDuration)); leases.Should().HaveCount(5); // Verify HLC timestamps are monotonically increasing HlcTimestamp? previousHlc = null; foreach (var lease in leases) { lease.HlcTimestamp.Should().NotBeNull(); if (previousHlc.HasValue) { var current = lease.HlcTimestamp!.Value; var prev = previousHlc.Value; // Current should be greater than previous (current > prev).Should().BeTrue( $"HLC {current} should be greater than {prev}"); } previousHlc = lease.HlcTimestamp; await lease.AcknowledgeAsync(); } } [Fact] public async Task HlcTimestamp_SortableString_ParsesCorrectly() { if (SkipIfUnavailable()) { return; } var options = CreateOptions(); var hlc = new HybridLogicalClockImpl(TimeProvider.System, "parse-test-node", new InMemoryHlcStateStore()); await using var queue = new RedisSchedulerPlannerQueue( options, options.Redis, NullLogger.Instance, TimeProvider.System, hlc, connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false)); var message = CreatePlannerMessage(); await queue.EnqueueAsync(message); var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("parse-consumer", batchSize: 1, options.DefaultLeaseDuration)); leases.Should().ContainSingle(); var lease = leases[0]; lease.HlcTimestamp.Should().NotBeNull(); // Verify round-trip through sortable string var hlcValue = lease.HlcTimestamp!.Value; var sortableString = hlcValue.ToSortableString(); HlcTimestamp.TryParse(sortableString, out var parsed).Should().BeTrue(); parsed.Should().Be(hlcValue); await lease.AcknowledgeAsync(); } [Fact] public async Task HlcTimestamp_DeterministicForSameInput_OnSameNode() { if (SkipIfUnavailable()) { return; } // This test verifies that HLC generates consistent timestamps // by checking that timestamps from the same node use the same node ID // and that logical counters increment correctly at same physical time var options = CreateOptions(); var hlc = new HybridLogicalClockImpl(TimeProvider.System, "determinism-node", new InMemoryHlcStateStore()); await using var queue = new RedisSchedulerPlannerQueue( options, options.Redis, NullLogger.Instance, TimeProvider.System, hlc, connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false)); // Enqueue rapidly to potentially hit same physical time var timestamps = new List(); for (int i = 0; i < 10; i++) { var message = CreatePlannerMessage(suffix: $"determinism-{i}"); await queue.EnqueueAsync(message); } var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("determinism-consumer", batchSize: 20, options.DefaultLeaseDuration)); leases.Should().HaveCount(10); foreach (var lease in leases) { lease.HlcTimestamp.Should().NotBeNull(); timestamps.Add(lease.HlcTimestamp!.Value); await lease.AcknowledgeAsync(); } // All timestamps should have same node ID foreach (var ts in timestamps) { ts.NodeId.Should().Be("determinism-node"); } // Verify strict ordering (no duplicates) for (int i = 1; i < timestamps.Count; i++) { (timestamps[i] > timestamps[i - 1]).Should().BeTrue( $"Timestamp {i} ({timestamps[i]}) should be greater than {i - 1} ({timestamps[i - 1]})"); } } private SchedulerQueueOptions CreateOptions() { var unique = Guid.NewGuid().ToString("N"); return new SchedulerQueueOptions { Kind = SchedulerQueueTransportKind.Redis, DefaultLeaseDuration = TimeSpan.FromSeconds(30), MaxDeliveryAttempts = 5, RetryInitialBackoff = TimeSpan.FromMilliseconds(10), RetryMaxBackoff = TimeSpan.FromMilliseconds(50), Redis = new SchedulerRedisQueueOptions { ConnectionString = _redis.GetConnectionString(), Database = 0, InitializationTimeout = TimeSpan.FromSeconds(10), Planner = new RedisSchedulerStreamOptions { Stream = $"scheduler:hlc-test:planner:{unique}", ConsumerGroup = $"planner-hlc-{unique}", DeadLetterStream = $"scheduler:hlc-test:planner:{unique}:dead", IdempotencyKeyPrefix = $"scheduler:hlc-test:planner:{unique}:idemp:", IdempotencyWindow = TimeSpan.FromMinutes(5) }, Runner = new RedisSchedulerStreamOptions { Stream = $"scheduler:hlc-test:runner:{unique}", ConsumerGroup = $"runner-hlc-{unique}", DeadLetterStream = $"scheduler:hlc-test:runner:{unique}:dead", IdempotencyKeyPrefix = $"scheduler:hlc-test:runner:{unique}:idemp:", IdempotencyWindow = TimeSpan.FromMinutes(5) } } }; } private bool SkipIfUnavailable() { if (_skipReason is not null) { return true; } return false; } private static bool IsDockerUnavailable(Exception exception) { while (exception is AggregateException aggregate && aggregate.InnerException is not null) { exception = aggregate.InnerException; } return exception is TimeoutException || exception.GetType().Name.Contains("Docker", StringComparison.OrdinalIgnoreCase); } private static PlannerQueueMessage CreatePlannerMessage(string suffix = "") { var id = string.IsNullOrEmpty(suffix) ? "run-hlc-test" : $"run-hlc-test-{suffix}"; var schedule = new Schedule( id: "sch-hlc-test", tenantId: "tenant-hlc", name: "HLC Test", enabled: true, cronExpression: "0 0 * * *", timezone: "UTC", mode: ScheduleMode.AnalysisOnly, selection: new Selector(SelectorScope.AllImages, tenantId: "tenant-hlc"), onlyIf: ScheduleOnlyIf.Default, notify: ScheduleNotify.Default, limits: ScheduleLimits.Default, createdAt: DateTimeOffset.UtcNow, createdBy: "tests", updatedAt: DateTimeOffset.UtcNow, updatedBy: "tests"); var run = new Run( id: id, tenantId: "tenant-hlc", trigger: RunTrigger.Manual, state: RunState.Planning, stats: RunStats.Empty, createdAt: DateTimeOffset.UtcNow, reason: RunReason.Empty, scheduleId: schedule.Id); var impactSet = new ImpactSet( selector: new Selector(SelectorScope.AllImages, tenantId: "tenant-hlc"), images: new[] { new ImpactImage( imageDigest: "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", registry: "registry", repository: "repo", namespaces: new[] { "prod" }, tags: new[] { "latest" }) }, usageOnly: true, generatedAt: DateTimeOffset.UtcNow, total: 1); return new PlannerQueueMessage(run, impactSet, schedule, correlationId: $"corr-hlc-{suffix}"); } private static RunnerSegmentQueueMessage CreateRunnerMessage() { return new RunnerSegmentQueueMessage( segmentId: "segment-hlc-test", runId: "run-hlc-test", tenantId: "tenant-hlc", imageDigests: new[] { "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" }, scheduleId: "sch-hlc-test", ratePerSecond: 10, usageOnly: true, attributes: new Dictionary { ["priority"] = "normal" }, correlationId: "corr-runner-hlc"); } }