sprints and audit work
This commit is contained in:
StellaOps Bot
@@ -8,20 +8,21 @@ using System.Threading.Tasks;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using StackExchange.Redis;
|
||||
using StellaOps.HybridLogicalClock;
|
||||
using StellaOps.Scheduler.Models;
|
||||
using StellaOps.Scheduler.Queue.Redis;
|
||||
using StellaOps.TestKit;
|
||||
using Testcontainers.Redis;
|
||||
using Xunit;
|
||||
|
||||
using HybridLogicalClockImpl = StellaOps.HybridLogicalClock.HybridLogicalClock;
|
||||
|
||||
namespace StellaOps.Scheduler.Queue.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for HLC (Hybrid Logical Clock) integration with scheduler queues.
|
||||
/// Integration tests for scheduler queues.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// HLC integration has been moved to the enqueue/dequeue services layer.
|
||||
/// These tests verify basic queue functionality.
|
||||
/// </remarks>
|
||||
[Trait("Category", TestCategories.Integration)]
|
||||
public sealed class HlcQueueIntegrationTests : IAsyncLifetime
|
||||
{
|
||||
@@ -56,7 +57,7 @@ public sealed class HlcQueueIntegrationTests : IAsyncLifetime
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PlannerQueue_WithHlc_LeasedMessageContainsHlcTimestamp()
|
||||
public async Task PlannerQueue_EnqueueAndLease_Works()
|
||||
{
|
||||
if (SkipIfUnavailable())
|
||||
{
|
||||
@@ -64,14 +65,12 @@ public sealed class HlcQueueIntegrationTests : IAsyncLifetime
|
||||
}
|
||||
|
||||
var options = CreateOptions();
|
||||
var hlc = new HybridLogicalClockImpl(TimeProvider.System, "test-node-1", new InMemoryHlcStateStore());
|
||||
|
||||
await using var queue = new RedisSchedulerPlannerQueue(
|
||||
options,
|
||||
options.Redis,
|
||||
NullLogger<RedisSchedulerPlannerQueue>.Instance,
|
||||
TimeProvider.System,
|
||||
hlc,
|
||||
connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
|
||||
|
||||
var message = CreatePlannerMessage();
|
||||
@@ -79,19 +78,17 @@ public sealed class HlcQueueIntegrationTests : IAsyncLifetime
|
||||
var enqueueResult = await queue.EnqueueAsync(message);
|
||||
enqueueResult.Deduplicated.Should().BeFalse();
|
||||
|
||||
var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("planner-hlc", batchSize: 1, options.DefaultLeaseDuration));
|
||||
var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("planner-test", batchSize: 1, options.DefaultLeaseDuration));
|
||||
leases.Should().ContainSingle();
|
||||
|
||||
var lease = leases[0];
|
||||
lease.HlcTimestamp.Should().NotBeNull("HLC timestamp should be present when HLC is configured");
|
||||
lease.HlcTimestamp!.Value.NodeId.Should().Be("test-node-1");
|
||||
lease.HlcTimestamp.Value.PhysicalTime.Should().BeGreaterThan(0);
|
||||
lease.Message.Run.Id.Should().Be(message.Run.Id);
|
||||
|
||||
await lease.AcknowledgeAsync();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RunnerQueue_WithHlc_LeasedMessageContainsHlcTimestamp()
|
||||
public async Task RunnerQueue_EnqueueAndLease_Works()
|
||||
{
|
||||
if (SkipIfUnavailable())
|
||||
{
|
||||
@@ -99,32 +96,29 @@ public sealed class HlcQueueIntegrationTests : IAsyncLifetime
|
||||
}
|
||||
|
||||
var options = CreateOptions();
|
||||
var hlc = new HybridLogicalClockImpl(TimeProvider.System, "runner-node-1", new InMemoryHlcStateStore());
|
||||
|
||||
await using var queue = new RedisSchedulerRunnerQueue(
|
||||
options,
|
||||
options.Redis,
|
||||
NullLogger<RedisSchedulerRunnerQueue>.Instance,
|
||||
TimeProvider.System,
|
||||
hlc,
|
||||
connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
|
||||
|
||||
var message = CreateRunnerMessage();
|
||||
|
||||
await queue.EnqueueAsync(message);
|
||||
|
||||
var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("runner-hlc", batchSize: 1, options.DefaultLeaseDuration));
|
||||
var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("runner-test", batchSize: 1, options.DefaultLeaseDuration));
|
||||
leases.Should().ContainSingle();
|
||||
|
||||
var lease = leases[0];
|
||||
lease.HlcTimestamp.Should().NotBeNull("HLC timestamp should be present when HLC is configured");
|
||||
lease.HlcTimestamp!.Value.NodeId.Should().Be("runner-node-1");
|
||||
lease.Message.SegmentId.Should().Be(message.SegmentId);
|
||||
|
||||
await lease.AcknowledgeAsync();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PlannerQueue_WithoutHlc_LeasedMessageHasNullTimestamp()
|
||||
public async Task PlannerQueue_MultipleMessages_AllLeased()
|
||||
{
|
||||
if (SkipIfUnavailable())
|
||||
{
|
||||
@@ -133,85 +127,32 @@ public sealed class HlcQueueIntegrationTests : IAsyncLifetime
|
||||
|
||||
var options = CreateOptions();
|
||||
|
||||
// No HLC provided
|
||||
await using var queue = new RedisSchedulerPlannerQueue(
|
||||
options,
|
||||
options.Redis,
|
||||
NullLogger<RedisSchedulerPlannerQueue>.Instance,
|
||||
TimeProvider.System,
|
||||
hlc: null,
|
||||
connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
|
||||
|
||||
var message = CreatePlannerMessage();
|
||||
await queue.EnqueueAsync(message);
|
||||
|
||||
var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("planner-no-hlc", batchSize: 1, options.DefaultLeaseDuration));
|
||||
leases.Should().ContainSingle();
|
||||
|
||||
var lease = leases[0];
|
||||
lease.HlcTimestamp.Should().BeNull("HLC timestamp should be null when HLC is not configured");
|
||||
|
||||
await lease.AcknowledgeAsync();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HlcTimestamp_IsMonotonicallyIncreasing_AcrossEnqueues()
|
||||
{
|
||||
if (SkipIfUnavailable())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var options = CreateOptions();
|
||||
var hlc = new HybridLogicalClockImpl(TimeProvider.System, "monotonic-test", new InMemoryHlcStateStore());
|
||||
|
||||
await using var queue = new RedisSchedulerPlannerQueue(
|
||||
options,
|
||||
options.Redis,
|
||||
NullLogger<RedisSchedulerPlannerQueue>.Instance,
|
||||
TimeProvider.System,
|
||||
hlc,
|
||||
connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
|
||||
|
||||
// Enqueue multiple messages
|
||||
var messages = new List<PlannerQueueMessage>();
|
||||
for (int i = 0; i < 5; i++)
|
||||
{
|
||||
messages.Add(CreatePlannerMessage(suffix: i.ToString()));
|
||||
}
|
||||
|
||||
foreach (var msg in messages)
|
||||
{
|
||||
var msg = CreatePlannerMessage(suffix: i.ToString());
|
||||
await queue.EnqueueAsync(msg);
|
||||
}
|
||||
|
||||
// Lease all messages
|
||||
var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("monotonic-consumer", batchSize: 10, options.DefaultLeaseDuration));
|
||||
var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("multi-consumer", batchSize: 10, options.DefaultLeaseDuration));
|
||||
leases.Should().HaveCount(5);
|
||||
|
||||
// Verify HLC timestamps are monotonically increasing
|
||||
HlcTimestamp? previousHlc = null;
|
||||
foreach (var lease in leases)
|
||||
{
|
||||
lease.HlcTimestamp.Should().NotBeNull();
|
||||
|
||||
if (previousHlc.HasValue)
|
||||
{
|
||||
var current = lease.HlcTimestamp!.Value;
|
||||
var prev = previousHlc.Value;
|
||||
|
||||
// Current should be greater than previous
|
||||
(current > prev).Should().BeTrue(
|
||||
$"HLC {current} should be greater than {prev}");
|
||||
}
|
||||
|
||||
previousHlc = lease.HlcTimestamp;
|
||||
await lease.AcknowledgeAsync();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HlcTimestamp_SortableString_ParsesCorrectly()
|
||||
public async Task PlannerQueue_Idempotency_DuplicatesAreDetected()
|
||||
{
|
||||
if (SkipIfUnavailable())
|
||||
{
|
||||
@@ -219,87 +160,66 @@ public sealed class HlcQueueIntegrationTests : IAsyncLifetime
|
||||
}
|
||||
|
||||
var options = CreateOptions();
|
||||
var hlc = new HybridLogicalClockImpl(TimeProvider.System, "parse-test-node", new InMemoryHlcStateStore());
|
||||
|
||||
await using var queue = new RedisSchedulerPlannerQueue(
|
||||
options,
|
||||
options.Redis,
|
||||
NullLogger<RedisSchedulerPlannerQueue>.Instance,
|
||||
TimeProvider.System,
|
||||
hlc,
|
||||
connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
|
||||
|
||||
var message = CreatePlannerMessage();
|
||||
await queue.EnqueueAsync(message);
|
||||
|
||||
var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("parse-consumer", batchSize: 1, options.DefaultLeaseDuration));
|
||||
// First enqueue
|
||||
var first = await queue.EnqueueAsync(message);
|
||||
first.Deduplicated.Should().BeFalse();
|
||||
|
||||
// Second enqueue with same message should be deduplicated
|
||||
var second = await queue.EnqueueAsync(message);
|
||||
second.Deduplicated.Should().BeTrue();
|
||||
|
||||
// Only one message should be leased
|
||||
var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("dedup-consumer", batchSize: 10, options.DefaultLeaseDuration));
|
||||
leases.Should().ContainSingle();
|
||||
|
||||
var lease = leases[0];
|
||||
lease.HlcTimestamp.Should().NotBeNull();
|
||||
|
||||
// Verify round-trip through sortable string
|
||||
var hlcValue = lease.HlcTimestamp!.Value;
|
||||
var sortableString = hlcValue.ToSortableString();
|
||||
|
||||
HlcTimestamp.TryParse(sortableString, out var parsed).Should().BeTrue();
|
||||
parsed.Should().Be(hlcValue);
|
||||
|
||||
await lease.AcknowledgeAsync();
|
||||
await leases[0].AcknowledgeAsync();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HlcTimestamp_DeterministicForSameInput_OnSameNode()
|
||||
public async Task RunnerQueue_Ordering_PreservedInLeases()
|
||||
{
|
||||
if (SkipIfUnavailable())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// This test verifies that HLC generates consistent timestamps
|
||||
// by checking that timestamps from the same node use the same node ID
|
||||
// and that logical counters increment correctly at same physical time
|
||||
|
||||
var options = CreateOptions();
|
||||
var hlc = new HybridLogicalClockImpl(TimeProvider.System, "determinism-node", new InMemoryHlcStateStore());
|
||||
|
||||
await using var queue = new RedisSchedulerPlannerQueue(
|
||||
await using var queue = new RedisSchedulerRunnerQueue(
|
||||
options,
|
||||
options.Redis,
|
||||
NullLogger<RedisSchedulerPlannerQueue>.Instance,
|
||||
NullLogger<RedisSchedulerRunnerQueue>.Instance,
|
||||
TimeProvider.System,
|
||||
hlc,
|
||||
connectionFactory: async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
|
||||
|
||||
// Enqueue rapidly to potentially hit same physical time
|
||||
var timestamps = new List<HlcTimestamp>();
|
||||
for (int i = 0; i < 10; i++)
|
||||
// Enqueue messages with sequential segment IDs
|
||||
var segmentIds = new List<string>();
|
||||
for (int i = 0; i < 5; i++)
|
||||
{
|
||||
var message = CreatePlannerMessage(suffix: $"determinism-{i}");
|
||||
await queue.EnqueueAsync(message);
|
||||
var segmentId = $"segment-order-{i:D3}";
|
||||
segmentIds.Add(segmentId);
|
||||
await queue.EnqueueAsync(CreateRunnerMessage(segmentId));
|
||||
}
|
||||
|
||||
var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("determinism-consumer", batchSize: 20, options.DefaultLeaseDuration));
|
||||
leases.Should().HaveCount(10);
|
||||
// Lease all messages
|
||||
var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("order-consumer", batchSize: 10, options.DefaultLeaseDuration));
|
||||
leases.Should().HaveCount(5);
|
||||
|
||||
foreach (var lease in leases)
|
||||
// Verify ordering is preserved
|
||||
for (int i = 0; i < leases.Count; i++)
|
||||
{
|
||||
lease.HlcTimestamp.Should().NotBeNull();
|
||||
timestamps.Add(lease.HlcTimestamp!.Value);
|
||||
await lease.AcknowledgeAsync();
|
||||
}
|
||||
|
||||
// All timestamps should have same node ID
|
||||
foreach (var ts in timestamps)
|
||||
{
|
||||
ts.NodeId.Should().Be("determinism-node");
|
||||
}
|
||||
|
||||
// Verify strict ordering (no duplicates)
|
||||
for (int i = 1; i < timestamps.Count; i++)
|
||||
{
|
||||
(timestamps[i] > timestamps[i - 1]).Should().BeTrue(
|
||||
$"Timestamp {i} ({timestamps[i]}) should be greater than {i - 1} ({timestamps[i - 1]})");
|
||||
leases[i].Message.SegmentId.Should().Be(segmentIds[i]);
|
||||
await leases[i].AcknowledgeAsync();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -321,18 +241,18 @@ public sealed class HlcQueueIntegrationTests : IAsyncLifetime
|
||||
InitializationTimeout = TimeSpan.FromSeconds(10),
|
||||
Planner = new RedisSchedulerStreamOptions
|
||||
{
|
||||
Stream = $"scheduler:hlc-test:planner:{unique}",
|
||||
ConsumerGroup = $"planner-hlc-{unique}",
|
||||
DeadLetterStream = $"scheduler:hlc-test:planner:{unique}:dead",
|
||||
IdempotencyKeyPrefix = $"scheduler:hlc-test:planner:{unique}:idemp:",
|
||||
Stream = $"scheduler:test:planner:{unique}",
|
||||
ConsumerGroup = $"planner-test-{unique}",
|
||||
DeadLetterStream = $"scheduler:test:planner:{unique}:dead",
|
||||
IdempotencyKeyPrefix = $"scheduler:test:planner:{unique}:idemp:",
|
||||
IdempotencyWindow = TimeSpan.FromMinutes(5)
|
||||
},
|
||||
Runner = new RedisSchedulerStreamOptions
|
||||
{
|
||||
Stream = $"scheduler:hlc-test:runner:{unique}",
|
||||
ConsumerGroup = $"runner-hlc-{unique}",
|
||||
DeadLetterStream = $"scheduler:hlc-test:runner:{unique}:dead",
|
||||
IdempotencyKeyPrefix = $"scheduler:hlc-test:runner:{unique}:idemp:",
|
||||
Stream = $"scheduler:test:runner:{unique}",
|
||||
ConsumerGroup = $"runner-test-{unique}",
|
||||
DeadLetterStream = $"scheduler:test:runner:{unique}:dead",
|
||||
IdempotencyKeyPrefix = $"scheduler:test:runner:{unique}:idemp:",
|
||||
IdempotencyWindow = TimeSpan.FromMinutes(5)
|
||||
}
|
||||
}
|
||||
@@ -361,17 +281,17 @@ public sealed class HlcQueueIntegrationTests : IAsyncLifetime
|
||||
|
||||
private static PlannerQueueMessage CreatePlannerMessage(string suffix = "")
|
||||
{
|
||||
var id = string.IsNullOrEmpty(suffix) ? "run-hlc-test" : $"run-hlc-test-{suffix}";
|
||||
var id = string.IsNullOrEmpty(suffix) ? "run-test" : $"run-test-{suffix}";
|
||||
|
||||
var schedule = new Schedule(
|
||||
id: "sch-hlc-test",
|
||||
tenantId: "tenant-hlc",
|
||||
name: "HLC Test",
|
||||
id: "sch-test",
|
||||
tenantId: "tenant-test",
|
||||
name: "Test",
|
||||
enabled: true,
|
||||
cronExpression: "0 0 * * *",
|
||||
timezone: "UTC",
|
||||
mode: ScheduleMode.AnalysisOnly,
|
||||
selection: new Selector(SelectorScope.AllImages, tenantId: "tenant-hlc"),
|
||||
selection: new Selector(SelectorScope.AllImages, tenantId: "tenant-test"),
|
||||
onlyIf: ScheduleOnlyIf.Default,
|
||||
notify: ScheduleNotify.Default,
|
||||
limits: ScheduleLimits.Default,
|
||||
@@ -382,7 +302,7 @@ public sealed class HlcQueueIntegrationTests : IAsyncLifetime
|
||||
|
||||
var run = new Run(
|
||||
id: id,
|
||||
tenantId: "tenant-hlc",
|
||||
tenantId: "tenant-test",
|
||||
trigger: RunTrigger.Manual,
|
||||
state: RunState.Planning,
|
||||
stats: RunStats.Empty,
|
||||
@@ -391,7 +311,7 @@ public sealed class HlcQueueIntegrationTests : IAsyncLifetime
|
||||
scheduleId: schedule.Id);
|
||||
|
||||
var impactSet = new ImpactSet(
|
||||
selector: new Selector(SelectorScope.AllImages, tenantId: "tenant-hlc"),
|
||||
selector: new Selector(SelectorScope.AllImages, tenantId: "tenant-test"),
|
||||
images: new[]
|
||||
{
|
||||
new ImpactImage(
|
||||
@@ -405,23 +325,23 @@ public sealed class HlcQueueIntegrationTests : IAsyncLifetime
|
||||
generatedAt: DateTimeOffset.UtcNow,
|
||||
total: 1);
|
||||
|
||||
return new PlannerQueueMessage(run, impactSet, schedule, correlationId: $"corr-hlc-{suffix}");
|
||||
return new PlannerQueueMessage(run, impactSet, schedule, correlationId: $"corr-{suffix}");
|
||||
}
|
||||
|
||||
private static RunnerSegmentQueueMessage CreateRunnerMessage()
|
||||
private static RunnerSegmentQueueMessage CreateRunnerMessage(string? segmentId = null)
|
||||
{
|
||||
return new RunnerSegmentQueueMessage(
|
||||
segmentId: "segment-hlc-test",
|
||||
runId: "run-hlc-test",
|
||||
tenantId: "tenant-hlc",
|
||||
segmentId: segmentId ?? "segment-test",
|
||||
runId: "run-test",
|
||||
tenantId: "tenant-test",
|
||||
imageDigests: new[]
|
||||
{
|
||||
"sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"
|
||||
},
|
||||
scheduleId: "sch-hlc-test",
|
||||
scheduleId: "sch-test",
|
||||
ratePerSecond: 10,
|
||||
usageOnly: true,
|
||||
attributes: new Dictionary<string, string> { ["priority"] = "normal" },
|
||||
correlationId: "corr-runner-hlc");
|
||||
correlationId: "corr-runner");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user