428 lines
15 KiB
C#
428 lines
15 KiB
C#
// <copyright file="HlcQueueIntegrationTests.cs" company="StellaOps">
|
|
// Copyright (c) StellaOps. Licensed under AGPL-3.0-or-later.
|
|
// </copyright>
|
|
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Threading.Tasks;
|
|
using FluentAssertions;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using StackExchange.Redis;
|
|
using StellaOps.HybridLogicalClock;
|
|
using StellaOps.Scheduler.Models;
|
|
using StellaOps.Scheduler.Queue.Redis;
|
|
using StellaOps.TestKit;
|
|
using Testcontainers.Redis;
|
|
using Xunit;
|
|
|
|
using HybridLogicalClockImpl = StellaOps.HybridLogicalClock.HybridLogicalClock;
|
|
|
|
namespace StellaOps.Scheduler.Queue.Tests;
|
|
|
|
/// <summary>
|
|
/// Integration tests for HLC (Hybrid Logical Clock) integration with scheduler queues.
|
|
/// </summary>
|
|
[Trait("Category", TestCategories.Integration)]
|
|
public sealed class HlcQueueIntegrationTests : IAsyncLifetime
|
|
{
|
|
private readonly RedisContainer _redis;
|
|
private string? _skipReason;
|
|
|
|
public HlcQueueIntegrationTests()
|
|
{
|
|
_redis = new RedisBuilder().Build();
|
|
}
|
|
|
|
public async ValueTask InitializeAsync()
|
|
{
|
|
try
|
|
{
|
|
await _redis.StartAsync();
|
|
}
|
|
catch (Exception ex) when (IsDockerUnavailable(ex))
|
|
{
|
|
_skipReason = $"Docker engine is not available for Redis-backed tests: {ex.Message}";
|
|
}
|
|
}
|
|
|
|
public async ValueTask DisposeAsync()
|
|
{
|
|
if (_skipReason is not null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
await _redis.DisposeAsync().AsTask();
|
|
}
|
|
|
|
[Fact]
|
|
public async Task PlannerQueue_WithHlc_LeasedMessageContainsHlcTimestamp()
|
|
{
|
|
if (SkipIfUnavailable())
|
|
{
|
|
return;
|
|
}
|
|
|
|
var options = CreateOptions();
|
|
var hlc = new HybridLogicalClockImpl(TimeProvider.System, "test-node-1", new InMemoryHlcStateStore());
|
|
|
|
await using var queue = new RedisSchedulerPlannerQueue(
|
|
options,
|
|
options.Redis,
|
|
NullLogger<RedisSchedulerPlannerQueue>.Instance,
|
|
TimeProvider.System,
|
|
hlc,
|
|
connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
|
|
|
|
var message = CreatePlannerMessage();
|
|
|
|
var enqueueResult = await queue.EnqueueAsync(message);
|
|
enqueueResult.Deduplicated.Should().BeFalse();
|
|
|
|
var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("planner-hlc", batchSize: 1, options.DefaultLeaseDuration));
|
|
leases.Should().ContainSingle();
|
|
|
|
var lease = leases[0];
|
|
lease.HlcTimestamp.Should().NotBeNull("HLC timestamp should be present when HLC is configured");
|
|
lease.HlcTimestamp!.Value.NodeId.Should().Be("test-node-1");
|
|
lease.HlcTimestamp.Value.PhysicalTime.Should().BeGreaterThan(0);
|
|
|
|
await lease.AcknowledgeAsync();
|
|
}
|
|
|
|
[Fact]
|
|
public async Task RunnerQueue_WithHlc_LeasedMessageContainsHlcTimestamp()
|
|
{
|
|
if (SkipIfUnavailable())
|
|
{
|
|
return;
|
|
}
|
|
|
|
var options = CreateOptions();
|
|
var hlc = new HybridLogicalClockImpl(TimeProvider.System, "runner-node-1", new InMemoryHlcStateStore());
|
|
|
|
await using var queue = new RedisSchedulerRunnerQueue(
|
|
options,
|
|
options.Redis,
|
|
NullLogger<RedisSchedulerRunnerQueue>.Instance,
|
|
TimeProvider.System,
|
|
hlc,
|
|
connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
|
|
|
|
var message = CreateRunnerMessage();
|
|
|
|
await queue.EnqueueAsync(message);
|
|
|
|
var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("runner-hlc", batchSize: 1, options.DefaultLeaseDuration));
|
|
leases.Should().ContainSingle();
|
|
|
|
var lease = leases[0];
|
|
lease.HlcTimestamp.Should().NotBeNull("HLC timestamp should be present when HLC is configured");
|
|
lease.HlcTimestamp!.Value.NodeId.Should().Be("runner-node-1");
|
|
|
|
await lease.AcknowledgeAsync();
|
|
}
|
|
|
|
[Fact]
|
|
public async Task PlannerQueue_WithoutHlc_LeasedMessageHasNullTimestamp()
|
|
{
|
|
if (SkipIfUnavailable())
|
|
{
|
|
return;
|
|
}
|
|
|
|
var options = CreateOptions();
|
|
|
|
// No HLC provided
|
|
await using var queue = new RedisSchedulerPlannerQueue(
|
|
options,
|
|
options.Redis,
|
|
NullLogger<RedisSchedulerPlannerQueue>.Instance,
|
|
TimeProvider.System,
|
|
hlc: null,
|
|
connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
|
|
|
|
var message = CreatePlannerMessage();
|
|
await queue.EnqueueAsync(message);
|
|
|
|
var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("planner-no-hlc", batchSize: 1, options.DefaultLeaseDuration));
|
|
leases.Should().ContainSingle();
|
|
|
|
var lease = leases[0];
|
|
lease.HlcTimestamp.Should().BeNull("HLC timestamp should be null when HLC is not configured");
|
|
|
|
await lease.AcknowledgeAsync();
|
|
}
|
|
|
|
[Fact]
|
|
public async Task HlcTimestamp_IsMonotonicallyIncreasing_AcrossEnqueues()
|
|
{
|
|
if (SkipIfUnavailable())
|
|
{
|
|
return;
|
|
}
|
|
|
|
var options = CreateOptions();
|
|
var hlc = new HybridLogicalClockImpl(TimeProvider.System, "monotonic-test", new InMemoryHlcStateStore());
|
|
|
|
await using var queue = new RedisSchedulerPlannerQueue(
|
|
options,
|
|
options.Redis,
|
|
NullLogger<RedisSchedulerPlannerQueue>.Instance,
|
|
TimeProvider.System,
|
|
hlc,
|
|
connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
|
|
|
|
// Enqueue multiple messages
|
|
var messages = new List<PlannerQueueMessage>();
|
|
for (int i = 0; i < 5; i++)
|
|
{
|
|
messages.Add(CreatePlannerMessage(suffix: i.ToString()));
|
|
}
|
|
|
|
foreach (var msg in messages)
|
|
{
|
|
await queue.EnqueueAsync(msg);
|
|
}
|
|
|
|
// Lease all messages
|
|
var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("monotonic-consumer", batchSize: 10, options.DefaultLeaseDuration));
|
|
leases.Should().HaveCount(5);
|
|
|
|
// Verify HLC timestamps are monotonically increasing
|
|
HlcTimestamp? previousHlc = null;
|
|
foreach (var lease in leases)
|
|
{
|
|
lease.HlcTimestamp.Should().NotBeNull();
|
|
|
|
if (previousHlc.HasValue)
|
|
{
|
|
var current = lease.HlcTimestamp!.Value;
|
|
var prev = previousHlc.Value;
|
|
|
|
// Current should be greater than previous
|
|
(current > prev).Should().BeTrue(
|
|
$"HLC {current} should be greater than {prev}");
|
|
}
|
|
|
|
previousHlc = lease.HlcTimestamp;
|
|
await lease.AcknowledgeAsync();
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task HlcTimestamp_SortableString_ParsesCorrectly()
|
|
{
|
|
if (SkipIfUnavailable())
|
|
{
|
|
return;
|
|
}
|
|
|
|
var options = CreateOptions();
|
|
var hlc = new HybridLogicalClockImpl(TimeProvider.System, "parse-test-node", new InMemoryHlcStateStore());
|
|
|
|
await using var queue = new RedisSchedulerPlannerQueue(
|
|
options,
|
|
options.Redis,
|
|
NullLogger<RedisSchedulerPlannerQueue>.Instance,
|
|
TimeProvider.System,
|
|
hlc,
|
|
connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
|
|
|
|
var message = CreatePlannerMessage();
|
|
await queue.EnqueueAsync(message);
|
|
|
|
var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("parse-consumer", batchSize: 1, options.DefaultLeaseDuration));
|
|
leases.Should().ContainSingle();
|
|
|
|
var lease = leases[0];
|
|
lease.HlcTimestamp.Should().NotBeNull();
|
|
|
|
// Verify round-trip through sortable string
|
|
var hlcValue = lease.HlcTimestamp!.Value;
|
|
var sortableString = hlcValue.ToSortableString();
|
|
|
|
HlcTimestamp.TryParse(sortableString, out var parsed).Should().BeTrue();
|
|
parsed.Should().Be(hlcValue);
|
|
|
|
await lease.AcknowledgeAsync();
|
|
}
|
|
|
|
[Fact]
|
|
public async Task HlcTimestamp_DeterministicForSameInput_OnSameNode()
|
|
{
|
|
if (SkipIfUnavailable())
|
|
{
|
|
return;
|
|
}
|
|
|
|
// This test verifies that HLC generates consistent timestamps
|
|
// by checking that timestamps from the same node use the same node ID
|
|
// and that logical counters increment correctly at same physical time
|
|
|
|
var options = CreateOptions();
|
|
var hlc = new HybridLogicalClockImpl(TimeProvider.System, "determinism-node", new InMemoryHlcStateStore());
|
|
|
|
await using var queue = new RedisSchedulerPlannerQueue(
|
|
options,
|
|
options.Redis,
|
|
NullLogger<RedisSchedulerPlannerQueue>.Instance,
|
|
TimeProvider.System,
|
|
hlc,
|
|
connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
|
|
|
|
// Enqueue rapidly to potentially hit same physical time
|
|
var timestamps = new List<HlcTimestamp>();
|
|
for (int i = 0; i < 10; i++)
|
|
{
|
|
var message = CreatePlannerMessage(suffix: $"determinism-{i}");
|
|
await queue.EnqueueAsync(message);
|
|
}
|
|
|
|
var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("determinism-consumer", batchSize: 20, options.DefaultLeaseDuration));
|
|
leases.Should().HaveCount(10);
|
|
|
|
foreach (var lease in leases)
|
|
{
|
|
lease.HlcTimestamp.Should().NotBeNull();
|
|
timestamps.Add(lease.HlcTimestamp!.Value);
|
|
await lease.AcknowledgeAsync();
|
|
}
|
|
|
|
// All timestamps should have same node ID
|
|
foreach (var ts in timestamps)
|
|
{
|
|
ts.NodeId.Should().Be("determinism-node");
|
|
}
|
|
|
|
// Verify strict ordering (no duplicates)
|
|
for (int i = 1; i < timestamps.Count; i++)
|
|
{
|
|
(timestamps[i] > timestamps[i - 1]).Should().BeTrue(
|
|
$"Timestamp {i} ({timestamps[i]}) should be greater than {i - 1} ({timestamps[i - 1]})");
|
|
}
|
|
}
|
|
|
|
private SchedulerQueueOptions CreateOptions()
|
|
{
|
|
var unique = Guid.NewGuid().ToString("N");
|
|
|
|
return new SchedulerQueueOptions
|
|
{
|
|
Kind = SchedulerQueueTransportKind.Redis,
|
|
DefaultLeaseDuration = TimeSpan.FromSeconds(30),
|
|
MaxDeliveryAttempts = 5,
|
|
RetryInitialBackoff = TimeSpan.FromMilliseconds(10),
|
|
RetryMaxBackoff = TimeSpan.FromMilliseconds(50),
|
|
Redis = new SchedulerRedisQueueOptions
|
|
{
|
|
ConnectionString = _redis.GetConnectionString(),
|
|
Database = 0,
|
|
InitializationTimeout = TimeSpan.FromSeconds(10),
|
|
Planner = new RedisSchedulerStreamOptions
|
|
{
|
|
Stream = $"scheduler:hlc-test:planner:{unique}",
|
|
ConsumerGroup = $"planner-hlc-{unique}",
|
|
DeadLetterStream = $"scheduler:hlc-test:planner:{unique}:dead",
|
|
IdempotencyKeyPrefix = $"scheduler:hlc-test:planner:{unique}:idemp:",
|
|
IdempotencyWindow = TimeSpan.FromMinutes(5)
|
|
},
|
|
Runner = new RedisSchedulerStreamOptions
|
|
{
|
|
Stream = $"scheduler:hlc-test:runner:{unique}",
|
|
ConsumerGroup = $"runner-hlc-{unique}",
|
|
DeadLetterStream = $"scheduler:hlc-test:runner:{unique}:dead",
|
|
IdempotencyKeyPrefix = $"scheduler:hlc-test:runner:{unique}:idemp:",
|
|
IdempotencyWindow = TimeSpan.FromMinutes(5)
|
|
}
|
|
}
|
|
};
|
|
}
|
|
|
|
private bool SkipIfUnavailable()
|
|
{
|
|
if (_skipReason is not null)
|
|
{
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
private static bool IsDockerUnavailable(Exception exception)
|
|
{
|
|
while (exception is AggregateException aggregate && aggregate.InnerException is not null)
|
|
{
|
|
exception = aggregate.InnerException;
|
|
}
|
|
|
|
return exception is TimeoutException
|
|
|| exception.GetType().Name.Contains("Docker", StringComparison.OrdinalIgnoreCase);
|
|
}
|
|
|
|
private static PlannerQueueMessage CreatePlannerMessage(string suffix = "")
|
|
{
|
|
var id = string.IsNullOrEmpty(suffix) ? "run-hlc-test" : $"run-hlc-test-{suffix}";
|
|
|
|
var schedule = new Schedule(
|
|
id: "sch-hlc-test",
|
|
tenantId: "tenant-hlc",
|
|
name: "HLC Test",
|
|
enabled: true,
|
|
cronExpression: "0 0 * * *",
|
|
timezone: "UTC",
|
|
mode: ScheduleMode.AnalysisOnly,
|
|
selection: new Selector(SelectorScope.AllImages, tenantId: "tenant-hlc"),
|
|
onlyIf: ScheduleOnlyIf.Default,
|
|
notify: ScheduleNotify.Default,
|
|
limits: ScheduleLimits.Default,
|
|
createdAt: DateTimeOffset.UtcNow,
|
|
createdBy: "tests",
|
|
updatedAt: DateTimeOffset.UtcNow,
|
|
updatedBy: "tests");
|
|
|
|
var run = new Run(
|
|
id: id,
|
|
tenantId: "tenant-hlc",
|
|
trigger: RunTrigger.Manual,
|
|
state: RunState.Planning,
|
|
stats: RunStats.Empty,
|
|
createdAt: DateTimeOffset.UtcNow,
|
|
reason: RunReason.Empty,
|
|
scheduleId: schedule.Id);
|
|
|
|
var impactSet = new ImpactSet(
|
|
selector: new Selector(SelectorScope.AllImages, tenantId: "tenant-hlc"),
|
|
images: new[]
|
|
{
|
|
new ImpactImage(
|
|
imageDigest: "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",
|
|
registry: "registry",
|
|
repository: "repo",
|
|
namespaces: new[] { "prod" },
|
|
tags: new[] { "latest" })
|
|
},
|
|
usageOnly: true,
|
|
generatedAt: DateTimeOffset.UtcNow,
|
|
total: 1);
|
|
|
|
return new PlannerQueueMessage(run, impactSet, schedule, correlationId: $"corr-hlc-{suffix}");
|
|
}
|
|
|
|
private static RunnerSegmentQueueMessage CreateRunnerMessage()
|
|
{
|
|
return new RunnerSegmentQueueMessage(
|
|
segmentId: "segment-hlc-test",
|
|
runId: "run-hlc-test",
|
|
tenantId: "tenant-hlc",
|
|
imageDigests: new[]
|
|
{
|
|
"sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"
|
|
},
|
|
scheduleId: "sch-hlc-test",
|
|
ratePerSecond: 10,
|
|
usageOnly: true,
|
|
attributes: new Dictionary<string, string> { ["priority"] = "normal" },
|
|
correlationId: "corr-runner-hlc");
|
|
}
|
|
}
|