save progress

2026-01-06 09:42:02 +02:00
parent 94d68bee8b
commit 37e11918e0
443 changed files with 85863 additions and 897 deletions
--- a/src/Scheduler/__Tests/StellaOps.Scheduler.Queue.Tests/HlcQueueIntegrationTests.cs
+++ b/src/Scheduler/__Tests/StellaOps.Scheduler.Queue.Tests/HlcQueueIntegrationTests.cs
@@ -0,0 +1,427 @@
+// <copyright file="HlcQueueIntegrationTests.cs" company="StellaOps">
+// Copyright (c) StellaOps. Licensed under AGPL-3.0-or-later.
+// </copyright>
+
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using FluentAssertions;
+using Microsoft.Extensions.Logging.Abstractions;
+using StackExchange.Redis;
+using StellaOps.HybridLogicalClock;
+using StellaOps.Scheduler.Models;
+using StellaOps.Scheduler.Queue.Redis;
+using StellaOps.TestKit;
+using Testcontainers.Redis;
+using Xunit;
+
+using HybridLogicalClockImpl = StellaOps.HybridLogicalClock.HybridLogicalClock;
+
+namespace StellaOps.Scheduler.Queue.Tests;
+
+/// <summary>
+/// Integration tests for HLC (Hybrid Logical Clock) integration with scheduler queues.
+/// </summary>
+[Trait("Category", TestCategories.Integration)]
+public sealed class HlcQueueIntegrationTests : IAsyncLifetime
+{
+    private readonly RedisContainer _redis;
+    private string? _skipReason;
+
+    public HlcQueueIntegrationTests()
+    {
+        _redis = new RedisBuilder().Build();
+    }
+
+    public async ValueTask InitializeAsync()
+    {
+        try
+        {
+            await _redis.StartAsync();
+        }
+        catch (Exception ex) when (IsDockerUnavailable(ex))
+        {
+            _skipReason = $"Docker engine is not available for Redis-backed tests: {ex.Message}";
+        }
+    }
+
+    public async ValueTask DisposeAsync()
+    {
+        if (_skipReason is not null)
+        {
+            return;
+        }
+
+        await _redis.DisposeAsync().AsTask();
+    }
+
+    [Fact]
+    public async Task PlannerQueue_WithHlc_LeasedMessageContainsHlcTimestamp()
+    {
+        if (SkipIfUnavailable())
+        {
+            return;
+        }
+
+        var options = CreateOptions();
+        var hlc = new HybridLogicalClockImpl(TimeProvider.System, "test-node-1", new InMemoryHlcStateStore());
+
+        await using var queue = new RedisSchedulerPlannerQueue(
+            options,
+            options.Redis,
+            NullLogger<RedisSchedulerPlannerQueue>.Instance,
+            TimeProvider.System,
+            hlc,
+            connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
+
+        var message = CreatePlannerMessage();
+
+        var enqueueResult = await queue.EnqueueAsync(message);
+        enqueueResult.Deduplicated.Should().BeFalse();
+
+        var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("planner-hlc", batchSize: 1, options.DefaultLeaseDuration));
+        leases.Should().ContainSingle();
+
+        var lease = leases[0];
+        lease.HlcTimestamp.Should().NotBeNull("HLC timestamp should be present when HLC is configured");
+        lease.HlcTimestamp!.Value.NodeId.Should().Be("test-node-1");
+        lease.HlcTimestamp.Value.PhysicalTime.Should().BeGreaterThan(0);
+
+        await lease.AcknowledgeAsync();
+    }
+
+    [Fact]
+    public async Task RunnerQueue_WithHlc_LeasedMessageContainsHlcTimestamp()
+    {
+        if (SkipIfUnavailable())
+        {
+            return;
+        }
+
+        var options = CreateOptions();
+        var hlc = new HybridLogicalClockImpl(TimeProvider.System, "runner-node-1", new InMemoryHlcStateStore());
+
+        await using var queue = new RedisSchedulerRunnerQueue(
+            options,
+            options.Redis,
+            NullLogger<RedisSchedulerRunnerQueue>.Instance,
+            TimeProvider.System,
+            hlc,
+            connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
+
+        var message = CreateRunnerMessage();
+
+        await queue.EnqueueAsync(message);
+
+        var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("runner-hlc", batchSize: 1, options.DefaultLeaseDuration));
+        leases.Should().ContainSingle();
+
+        var lease = leases[0];
+        lease.HlcTimestamp.Should().NotBeNull("HLC timestamp should be present when HLC is configured");
+        lease.HlcTimestamp!.Value.NodeId.Should().Be("runner-node-1");
+
+        await lease.AcknowledgeAsync();
+    }
+
+    [Fact]
+    public async Task PlannerQueue_WithoutHlc_LeasedMessageHasNullTimestamp()
+    {
+        if (SkipIfUnavailable())
+        {
+            return;
+        }
+
+        var options = CreateOptions();
+
+        // No HLC provided
+        await using var queue = new RedisSchedulerPlannerQueue(
+            options,
+            options.Redis,
+            NullLogger<RedisSchedulerPlannerQueue>.Instance,
+            TimeProvider.System,
+            hlc: null,
+            connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
+
+        var message = CreatePlannerMessage();
+        await queue.EnqueueAsync(message);
+
+        var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("planner-no-hlc", batchSize: 1, options.DefaultLeaseDuration));
+        leases.Should().ContainSingle();
+
+        var lease = leases[0];
+        lease.HlcTimestamp.Should().BeNull("HLC timestamp should be null when HLC is not configured");
+
+        await lease.AcknowledgeAsync();
+    }
+
+    [Fact]
+    public async Task HlcTimestamp_IsMonotonicallyIncreasing_AcrossEnqueues()
+    {
+        if (SkipIfUnavailable())
+        {
+            return;
+        }
+
+        var options = CreateOptions();
+        var hlc = new HybridLogicalClockImpl(TimeProvider.System, "monotonic-test", new InMemoryHlcStateStore());
+
+        await using var queue = new RedisSchedulerPlannerQueue(
+            options,
+            options.Redis,
+            NullLogger<RedisSchedulerPlannerQueue>.Instance,
+            TimeProvider.System,
+            hlc,
+            connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
+
+        // Enqueue multiple messages
+        var messages = new List<PlannerQueueMessage>();
+        for (int i = 0; i < 5; i++)
+        {
+            messages.Add(CreatePlannerMessage(suffix: i.ToString()));
+        }
+
+        foreach (var msg in messages)
+        {
+            await queue.EnqueueAsync(msg);
+        }
+
+        // Lease all messages
+        var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("monotonic-consumer", batchSize: 10, options.DefaultLeaseDuration));
+        leases.Should().HaveCount(5);
+
+        // Verify HLC timestamps are monotonically increasing
+        HlcTimestamp? previousHlc = null;
+        foreach (var lease in leases)
+        {
+            lease.HlcTimestamp.Should().NotBeNull();
+
+            if (previousHlc.HasValue)
+            {
+                var current = lease.HlcTimestamp!.Value;
+                var prev = previousHlc.Value;
+
+                // Current should be greater than previous
+                (current > prev).Should().BeTrue(
+                    $"HLC {current} should be greater than {prev}");
+            }
+
+            previousHlc = lease.HlcTimestamp;
+            await lease.AcknowledgeAsync();
+        }
+    }
+
+    [Fact]
+    public async Task HlcTimestamp_SortableString_ParsesCorrectly()
+    {
+        if (SkipIfUnavailable())
+        {
+            return;
+        }
+
+        var options = CreateOptions();
+        var hlc = new HybridLogicalClockImpl(TimeProvider.System, "parse-test-node", new InMemoryHlcStateStore());
+
+        await using var queue = new RedisSchedulerPlannerQueue(
+            options,
+            options.Redis,
+            NullLogger<RedisSchedulerPlannerQueue>.Instance,
+            TimeProvider.System,
+            hlc,
+            connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
+
+        var message = CreatePlannerMessage();
+        await queue.EnqueueAsync(message);
+
+        var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("parse-consumer", batchSize: 1, options.DefaultLeaseDuration));
+        leases.Should().ContainSingle();
+
+        var lease = leases[0];
+        lease.HlcTimestamp.Should().NotBeNull();
+
+        // Verify round-trip through sortable string
+        var hlcValue = lease.HlcTimestamp!.Value;
+        var sortableString = hlcValue.ToSortableString();
+
+        HlcTimestamp.TryParse(sortableString, out var parsed).Should().BeTrue();
+        parsed.Should().Be(hlcValue);
+
+        await lease.AcknowledgeAsync();
+    }
+
+    [Fact]
+    public async Task HlcTimestamp_DeterministicForSameInput_OnSameNode()
+    {
+        if (SkipIfUnavailable())
+        {
+            return;
+        }
+
+        // This test verifies that HLC generates consistent timestamps
+        // by checking that timestamps from the same node use the same node ID
+        // and that logical counters increment correctly at same physical time
+
+        var options = CreateOptions();
+        var hlc = new HybridLogicalClockImpl(TimeProvider.System, "determinism-node", new InMemoryHlcStateStore());
+
+        await using var queue = new RedisSchedulerPlannerQueue(
+            options,
+            options.Redis,
+            NullLogger<RedisSchedulerPlannerQueue>.Instance,
+            TimeProvider.System,
+            hlc,
+            connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
+
+        // Enqueue rapidly to potentially hit same physical time
+        var timestamps = new List<HlcTimestamp>();
+        for (int i = 0; i < 10; i++)
+        {
+            var message = CreatePlannerMessage(suffix: $"determinism-{i}");
+            await queue.EnqueueAsync(message);
+        }
+
+        var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("determinism-consumer", batchSize: 20, options.DefaultLeaseDuration));
+        leases.Should().HaveCount(10);
+
+        foreach (var lease in leases)
+        {
+            lease.HlcTimestamp.Should().NotBeNull();
+            timestamps.Add(lease.HlcTimestamp!.Value);
+            await lease.AcknowledgeAsync();
+        }
+
+        // All timestamps should have same node ID
+        foreach (var ts in timestamps)
+        {
+            ts.NodeId.Should().Be("determinism-node");
+        }
+
+        // Verify strict ordering (no duplicates)
+        for (int i = 1; i < timestamps.Count; i++)
+        {
+            (timestamps[i] > timestamps[i - 1]).Should().BeTrue(
+                $"Timestamp {i} ({timestamps[i]}) should be greater than {i - 1} ({timestamps[i - 1]})");
+        }
+    }
+
+    private SchedulerQueueOptions CreateOptions()
+    {
+        var unique = Guid.NewGuid().ToString("N");
+
+        return new SchedulerQueueOptions
+        {
+            Kind = SchedulerQueueTransportKind.Redis,
+            DefaultLeaseDuration = TimeSpan.FromSeconds(30),
+            MaxDeliveryAttempts = 5,
+            RetryInitialBackoff = TimeSpan.FromMilliseconds(10),
+            RetryMaxBackoff = TimeSpan.FromMilliseconds(50),
+            Redis = new SchedulerRedisQueueOptions
+            {
+                ConnectionString = _redis.GetConnectionString(),
+                Database = 0,
+                InitializationTimeout = TimeSpan.FromSeconds(10),
+                Planner = new RedisSchedulerStreamOptions
+                {
+                    Stream = $"scheduler:hlc-test:planner:{unique}",
+                    ConsumerGroup = $"planner-hlc-{unique}",
+                    DeadLetterStream = $"scheduler:hlc-test:planner:{unique}:dead",
+                    IdempotencyKeyPrefix = $"scheduler:hlc-test:planner:{unique}:idemp:",
+                    IdempotencyWindow = TimeSpan.FromMinutes(5)
+                },
+                Runner = new RedisSchedulerStreamOptions
+                {
+                    Stream = $"scheduler:hlc-test:runner:{unique}",
+                    ConsumerGroup = $"runner-hlc-{unique}",
+                    DeadLetterStream = $"scheduler:hlc-test:runner:{unique}:dead",
+                    IdempotencyKeyPrefix = $"scheduler:hlc-test:runner:{unique}:idemp:",
+                    IdempotencyWindow = TimeSpan.FromMinutes(5)
+                }
+            }
+        };
+    }
+
+    private bool SkipIfUnavailable()
+    {
+        if (_skipReason is not null)
+        {
+            return true;
+        }
+        return false;
+    }
+
+    private static bool IsDockerUnavailable(Exception exception)
+    {
+        while (exception is AggregateException aggregate && aggregate.InnerException is not null)
+        {
+            exception = aggregate.InnerException;
+        }
+
+        return exception is TimeoutException
+            || exception.GetType().Name.Contains("Docker", StringComparison.OrdinalIgnoreCase);
+    }
+
+    private static PlannerQueueMessage CreatePlannerMessage(string suffix = "")
+    {
+        var id = string.IsNullOrEmpty(suffix) ? "run-hlc-test" : $"run-hlc-test-{suffix}";
+
+        var schedule = new Schedule(
+            id: "sch-hlc-test",
+            tenantId: "tenant-hlc",
+            name: "HLC Test",
+            enabled: true,
+            cronExpression: "0 0 * * *",
+            timezone: "UTC",
+            mode: ScheduleMode.AnalysisOnly,
+            selection: new Selector(SelectorScope.AllImages, tenantId: "tenant-hlc"),
+            onlyIf: ScheduleOnlyIf.Default,
+            notify: ScheduleNotify.Default,
+            limits: ScheduleLimits.Default,
+            createdAt: DateTimeOffset.UtcNow,
+            createdBy: "tests",
+            updatedAt: DateTimeOffset.UtcNow,
+            updatedBy: "tests");
+
+        var run = new Run(
+            id: id,
+            tenantId: "tenant-hlc",
+            trigger: RunTrigger.Manual,
+            state: RunState.Planning,
+            stats: RunStats.Empty,
+            createdAt: DateTimeOffset.UtcNow,
+            reason: RunReason.Empty,
+            scheduleId: schedule.Id);
+
+        var impactSet = new ImpactSet(
+            selector: new Selector(SelectorScope.AllImages, tenantId: "tenant-hlc"),
+            images: new[]
+            {
+                new ImpactImage(
+                    imageDigest: "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",
+                    registry: "registry",
+                    repository: "repo",
+                    namespaces: new[] { "prod" },
+                    tags: new[] { "latest" })
+            },
+            usageOnly: true,
+            generatedAt: DateTimeOffset.UtcNow,
+            total: 1);
+
+        return new PlannerQueueMessage(run, impactSet, schedule, correlationId: $"corr-hlc-{suffix}");
+    }
+
+    private static RunnerSegmentQueueMessage CreateRunnerMessage()
+    {
+        return new RunnerSegmentQueueMessage(
+            segmentId: "segment-hlc-test",
+            runId: "run-hlc-test",
+            tenantId: "tenant-hlc",
+            imageDigests: new[]
+            {
+                "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"
+            },
+            scheduleId: "sch-hlc-test",
+            ratePerSecond: 10,
+            usageOnly: true,
+            attributes: new Dictionary<string, string> { ["priority"] = "normal" },
+            correlationId: "corr-runner-hlc");
+    }
+}
--- a/src/Scheduler/__Tests/StellaOps.Scheduler.Queue.Tests/RedisSchedulerQueueTests.cs
+++ b/src/Scheduler/__Tests/StellaOps.Scheduler.Queue.Tests/RedisSchedulerQueueTests.cs
@@ -62,7 +62,8 @@ public sealed class RedisSchedulerQueueTests : IAsyncLifetime
            options.Redis,
            NullLogger<RedisSchedulerPlannerQueue>.Instance,
            TimeProvider.System,
-            async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
+            hlc: null,
+            connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));

        var message = TestData.CreatePlannerMessage();

@@ -101,7 +102,8 @@ public sealed class RedisSchedulerQueueTests : IAsyncLifetime
            options.Redis,
            NullLogger<RedisSchedulerRunnerQueue>.Instance,
            TimeProvider.System,
-            async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
+            hlc: null,
+            connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));

        var message = TestData.CreateRunnerMessage();

@@ -136,7 +138,8 @@ public sealed class RedisSchedulerQueueTests : IAsyncLifetime
            options.Redis,
            NullLogger<RedisSchedulerPlannerQueue>.Instance,
            TimeProvider.System,
-            async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
+            hlc: null,
+            connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));

        var message = TestData.CreatePlannerMessage();
        await queue.EnqueueAsync(message);
@@ -170,7 +173,8 @@ public sealed class RedisSchedulerQueueTests : IAsyncLifetime
            options.Redis,
            NullLogger<RedisSchedulerPlannerQueue>.Instance,
            TimeProvider.System,
-            async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
+            hlc: null,
+            connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));

        var message = TestData.CreatePlannerMessage();

@@ -208,7 +212,8 @@ public sealed class RedisSchedulerQueueTests : IAsyncLifetime
            options.Redis,
            NullLogger<RedisSchedulerRunnerQueue>.Instance,
            TimeProvider.System,
-            async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
+            hlc: null,
+            connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));

        var message = TestData.CreateRunnerMessage();
        await queue.EnqueueAsync(message);