// ----------------------------------------------------------------------------- // AtLeastOnceDeliveryTests.cs // Sprint: SPRINT_5100_0010_0003 - Router + Messaging Test Implementation // Task: MESSAGING-5100-009 - At-least-once delivery with consumer idempotency // Description: Integration tests verifying at-least-once delivery semantics: // - Messages are never lost (guaranteed delivery) // - Consumer idempotency correctly handles duplicate deliveries // - Lease expiration triggers redelivery // - Simulated failures result in message redelivery // ----------------------------------------------------------------------------- using FluentAssertions; using StellaOps.Messaging; using StellaOps.Messaging.Abstractions; using StellaOps.Messaging.Transport.Valkey.Tests.Fixtures; using Xunit; using Xunit.Abstractions; namespace StellaOps.Messaging.Transport.Valkey.Tests; /// /// Tests for at-least-once delivery semantics with consumer idempotency. /// /// At-least-once delivery guarantees: /// 1. Every message sent is delivered at least once /// 2. Messages may be delivered multiple times (redelivery on failure) /// 3. Consumer idempotency handles duplicate deliveries /// 4. No message is ever lost, even under failure conditions /// [Collection(ValkeyIntegrationTestCollection.Name)] public sealed class AtLeastOnceDeliveryTests : IAsyncLifetime { private readonly ValkeyContainerFixture _fixture; private readonly ITestOutputHelper _output; private ValkeyConnectionFactory? _connectionFactory; private ValkeyIdempotencyStore? _idempotencyStore; public AtLeastOnceDeliveryTests(ValkeyContainerFixture fixture, ITestOutputHelper output) { _fixture = fixture; _output = output; } public Task InitializeAsync() { _connectionFactory = _fixture.CreateConnectionFactory(); _idempotencyStore = new ValkeyIdempotencyStore( _connectionFactory, $"test-consumer-{Guid.NewGuid():N}", null); return Task.CompletedTask; } public async Task DisposeAsync() { if (_connectionFactory is not null) { await _connectionFactory.DisposeAsync(); } } #region At-Least-Once Delivery Guarantee Tests [ValkeyIntegrationFact] public async Task AtLeastOnce_MessageSent_IsDeliveredAtLeastOnce() { // Arrange - Producer sends message var queue = CreateQueue(); var messageId = Guid.NewGuid(); var message = new TestMessage { Id = messageId, Content = "At-least-once test message" }; // Act - Send message var enqueueResult = await queue.EnqueueAsync(message); enqueueResult.Success.Should().BeTrue("message should be accepted by the queue"); // Act - Consumer receives message var leases = await queue.LeaseAsync(new LeaseRequest { BatchSize = 1 }); // Assert - Message is delivered leases.Should().HaveCount(1, "message must be delivered at least once"); leases[0].Message.Id.Should().Be(messageId); leases[0].Message.Content.Should().Be("At-least-once test message"); await leases[0].AcknowledgeAsync(); _output.WriteLine($"Message {messageId} delivered successfully"); } [ValkeyIntegrationFact] public async Task AtLeastOnce_UnacknowledgedLease_MessageRedelivered() { // Arrange - Create queue with short lease duration var queueOptions = _fixture.CreateQueueOptions(); queueOptions.DefaultLeaseDuration = TimeSpan.FromMilliseconds(200); var queue = CreateQueue(queueOptions); var messageId = Guid.NewGuid(); await queue.EnqueueAsync(new TestMessage { Id = messageId, Content = "Redelivery test" }); // Act - Lease message but don't acknowledge (simulating consumer crash) var firstLease = await queue.LeaseAsync(new LeaseRequest { BatchSize = 1 }); firstLease.Should().HaveCount(1); firstLease[0].Message.Id.Should().Be(messageId); // Don't acknowledge - simulate crash _output.WriteLine("Simulating consumer crash (not acknowledging message)"); // Wait for lease to expire await Task.Delay(500); // Act - Claim expired message (automatic redelivery) var redelivered = await queue.ClaimExpiredAsync(new ClaimRequest { BatchSize = 10, MinIdleTime = TimeSpan.FromMilliseconds(200), MinDeliveryAttempts = 1 }); // Assert - Message is redelivered redelivered.Should().HaveCount(1, "message must be redelivered after lease expiration"); redelivered[0].Message.Id.Should().Be(messageId); redelivered[0].Attempt.Should().BeGreaterThan(1, "this should be a redelivery"); await redelivered[0].AcknowledgeAsync(); _output.WriteLine($"Message {messageId} successfully redelivered on attempt {redelivered[0].Attempt}"); } [ValkeyIntegrationFact] public async Task AtLeastOnce_MultipleMessages_AllDelivered() { // Arrange var queue = CreateQueue(); const int messageCount = 100; var sentIds = new HashSet(); // Act - Send multiple messages for (int i = 0; i < messageCount; i++) { var id = Guid.NewGuid(); sentIds.Add(id); await queue.EnqueueAsync(new TestMessage { Id = id, Content = $"Message-{i}" }); } // Act - Receive all messages var receivedIds = new HashSet(); int remaining = messageCount; while (remaining > 0) { var leases = await queue.LeaseAsync(new LeaseRequest { BatchSize = 20 }); foreach (var lease in leases) { receivedIds.Add(lease.Message.Id); await lease.AcknowledgeAsync(); } remaining -= leases.Count; } // Assert - All messages delivered receivedIds.Should().BeEquivalentTo(sentIds, "all sent messages must be delivered"); _output.WriteLine($"All {messageCount} messages delivered successfully"); } [ValkeyIntegrationFact] public async Task AtLeastOnce_RetryAfterNack_MessageRedelivered() { // Arrange var queueOptions = _fixture.CreateQueueOptions(); queueOptions.RetryInitialBackoff = TimeSpan.Zero; // Immediate retry for test speed var queue = CreateQueue(queueOptions); var messageId = Guid.NewGuid(); await queue.EnqueueAsync(new TestMessage { Id = messageId, Content = "Retry test" }); // Act - First delivery, simulate processing failure with retry var firstLease = await queue.LeaseAsync(new LeaseRequest { BatchSize = 1 }); firstLease.Should().HaveCount(1); firstLease[0].Attempt.Should().Be(1); // Nack for retry await firstLease[0].ReleaseAsync(ReleaseDisposition.Retry); _output.WriteLine("Message nacked for retry"); // Brief delay for retry processing await Task.Delay(100); // Act - Second delivery after retry var secondLease = await queue.LeaseAsync(new LeaseRequest { BatchSize = 1 }); // Assert - Message is redelivered secondLease.Should().HaveCount(1, "message must be redelivered after nack"); secondLease[0].Message.Id.Should().Be(messageId); secondLease[0].Attempt.Should().Be(2, "this should be attempt 2"); await secondLease[0].AcknowledgeAsync(); _output.WriteLine($"Message {messageId} successfully processed on attempt 2"); } #endregion #region Consumer Idempotency Tests [ValkeyIntegrationFact] public async Task ConsumerIdempotency_DuplicateProcessing_DetectedAndSkipped() { // Arrange - Create a consumer with idempotency tracking var queue = CreateQueue(); var processedMessageIds = new HashSet(); var processingCount = new Dictionary(); var messageId = Guid.NewGuid(); await queue.EnqueueAsync(new TestMessage { Id = messageId, Content = "Idempotency test" }); // Act - Simulate receiving the message multiple times for (int delivery = 1; delivery <= 3; delivery++) { // Simulate message delivery (could be redelivery) var idempotencyKey = $"consumer-process:{messageId}"; var claimResult = await _idempotencyStore!.TryClaimAsync( idempotencyKey, messageId.ToString(), TimeSpan.FromMinutes(5)); if (claimResult.IsFirstClaim) { // First time processing this message processedMessageIds.Add(messageId); processingCount[messageId] = 1; _output.WriteLine($"Delivery {delivery}: First processing of message {messageId}"); } else { // Duplicate - skip processing processingCount[messageId] = processingCount.GetValueOrDefault(messageId) + 1; _output.WriteLine($"Delivery {delivery}: Duplicate detected, skipping message {messageId}"); } } // Assert - Message processed exactly once despite multiple deliveries processedMessageIds.Should().HaveCount(1); processingCount[messageId].Should().BeGreaterThan(1, "we simulated multiple deliveries"); // Cleanup var leases = await queue.LeaseAsync(new LeaseRequest { BatchSize = 1 }); if (leases.Count > 0) { await leases[0].AcknowledgeAsync(); } } [ValkeyIntegrationFact] public async Task ConsumerIdempotency_ConcurrentDuplicates_OnlyOneProcessed() { // Arrange var messageId = Guid.NewGuid(); var processedCount = 0; var duplicateCount = 0; var lockObject = new object(); // Simulate 10 concurrent consumers trying to process the same message var tasks = Enumerable.Range(1, 10).Select(async consumerId => { var idempotencyKey = $"concurrent-test:{messageId}"; var claimResult = await _idempotencyStore!.TryClaimAsync( idempotencyKey, $"consumer-{consumerId}", TimeSpan.FromMinutes(5)); lock (lockObject) { if (claimResult.IsFirstClaim) { processedCount++; _output.WriteLine($"Consumer {consumerId}: Processing message (first claim)"); } else { duplicateCount++; _output.WriteLine($"Consumer {consumerId}: Duplicate detected, existing value: {claimResult.ExistingValue}"); } } }); // Act await Task.WhenAll(tasks); // Assert - Exactly one consumer processed the message processedCount.Should().Be(1, "only one consumer should process the message"); duplicateCount.Should().Be(9, "9 consumers should detect duplicate"); _output.WriteLine($"Processed: {processedCount}, Duplicates: {duplicateCount}"); } [ValkeyIntegrationFact] public async Task ConsumerIdempotency_IdempotencyWindowExpires_ReprocessingAllowed() { // Arrange var messageId = Guid.NewGuid(); var shortWindow = TimeSpan.FromMilliseconds(200); var idempotencyKey = $"window-test:{messageId}"; // Act - First claim var firstClaim = await _idempotencyStore!.TryClaimAsync( idempotencyKey, "first-processor", shortWindow); firstClaim.IsFirstClaim.Should().BeTrue(); _output.WriteLine("First claim successful"); // Duplicate should be detected var duplicateClaim = await _idempotencyStore!.TryClaimAsync( idempotencyKey, "duplicate-processor", shortWindow); duplicateClaim.IsDuplicate.Should().BeTrue(); _output.WriteLine("Duplicate detected as expected"); // Wait for window to expire await Task.Delay(500); // Act - After expiration, claim should succeed again var afterExpiration = await _idempotencyStore!.TryClaimAsync( idempotencyKey, "new-processor", shortWindow); // Assert - Reprocessing allowed after window expiration afterExpiration.IsFirstClaim.Should().BeTrue( "after idempotency window expires, message can be reprocessed"); _output.WriteLine("After window expiration, new claim succeeded"); } [ValkeyIntegrationFact] public async Task ConsumerIdempotency_DifferentMessages_IndependentProcessing() { // Arrange - Three different messages var messageIds = Enumerable.Range(1, 3).Select(_ => Guid.NewGuid()).ToList(); var processedIds = new List(); // Act - Process each message (simulating first-time delivery) foreach (var messageId in messageIds) { var idempotencyKey = $"different-msg-test:{messageId}"; var claimResult = await _idempotencyStore!.TryClaimAsync( idempotencyKey, messageId.ToString(), TimeSpan.FromMinutes(5)); if (claimResult.IsFirstClaim) { processedIds.Add(messageId); } } // Assert - All different messages processed processedIds.Should().BeEquivalentTo(messageIds); _output.WriteLine($"All {messageIds.Count} different messages processed independently"); } #endregion #region End-to-End At-Least-Once with Idempotency Tests [ValkeyIntegrationFact] public async Task EndToEnd_AtLeastOnceWithIdempotency_NoDuplicateProcessing() { // Arrange var queueOptions = _fixture.CreateQueueOptions(); queueOptions.DefaultLeaseDuration = TimeSpan.FromMilliseconds(200); var queue = CreateQueue(queueOptions); var messageId = Guid.NewGuid(); var processedIds = new HashSet(); var deliveryCount = 0; await queue.EnqueueAsync(new TestMessage { Id = messageId, Content = "E2E test" }); // Act - Consumer with idempotency-aware processing // Simulate: first delivery - lease but crash, second delivery - process successfully // First delivery (crash simulation - don't ack) var firstLease = await queue.LeaseAsync(new LeaseRequest { BatchSize = 1 }); firstLease.Should().HaveCount(1); deliveryCount++; // Attempt to claim for processing var firstClaim = await _idempotencyStore!.TryClaimAsync( $"e2e-test:{firstLease[0].Message.Id}", firstLease[0].MessageId, TimeSpan.FromMinutes(5)); if (firstClaim.IsFirstClaim) { processedIds.Add(firstLease[0].Message.Id); } // Simulate crash - don't acknowledge _output.WriteLine("First delivery: Processing started but consumer crashed"); // Wait for lease expiration await Task.Delay(500); // Claim expired message (redelivery) var redelivered = await queue.ClaimExpiredAsync(new ClaimRequest { BatchSize = 1, MinIdleTime = TimeSpan.FromMilliseconds(200), MinDeliveryAttempts = 1 }); if (redelivered.Count > 0) { deliveryCount++; // Attempt to claim again (should be duplicate) var secondClaim = await _idempotencyStore!.TryClaimAsync( $"e2e-test:{redelivered[0].Message.Id}", redelivered[0].MessageId, TimeSpan.FromMinutes(5)); if (secondClaim.IsFirstClaim) { processedIds.Add(redelivered[0].Message.Id); } else { _output.WriteLine($"Second delivery: Duplicate detected, skipping processing"); } // This time, acknowledge await redelivered[0].AcknowledgeAsync(); _output.WriteLine("Second delivery: Message acknowledged"); } // Assert processedIds.Should().HaveCount(1, "message should be processed exactly once"); deliveryCount.Should().BeGreaterThan(1, "message should be delivered at least twice (crash + redelivery)"); _output.WriteLine($"Total deliveries: {deliveryCount}, Unique processing: {processedIds.Count}"); } [ValkeyIntegrationFact] public async Task EndToEnd_BulkMessages_AtLeastOnceWithIdempotency() { // Arrange var queue = CreateQueue(); const int messageCount = 50; var processedIds = new ConcurrentHashSet(); var deliveryAttempts = new Dictionary(); // Send messages var sentIds = new List(); for (int i = 0; i < messageCount; i++) { var id = Guid.NewGuid(); sentIds.Add(id); await queue.EnqueueAsync(new TestMessage { Id = id, Content = $"Bulk-{i}" }); } // Act - Process all messages with idempotency int remaining = messageCount; while (remaining > 0) { var leases = await queue.LeaseAsync(new LeaseRequest { BatchSize = 10 }); if (leases.Count == 0) break; foreach (var lease in leases) { var msgId = lease.Message.Id; deliveryAttempts[msgId] = deliveryAttempts.GetValueOrDefault(msgId) + 1; // Check idempotency before processing var claim = await _idempotencyStore!.TryClaimAsync( $"bulk-test:{msgId}", lease.MessageId, TimeSpan.FromMinutes(5)); if (claim.IsFirstClaim) { processedIds.Add(msgId); } await lease.AcknowledgeAsync(); } remaining -= leases.Count; } // Assert - All messages processed exactly once processedIds.Count.Should().Be(messageCount, "all messages should be processed"); sentIds.Should().BeEquivalentTo(processedIds.ToList(), "all sent messages should be processed"); _output.WriteLine($"Processed {processedIds.Count}/{messageCount} messages with idempotency"); } #endregion #region Edge Cases [ValkeyIntegrationFact] public async Task EdgeCase_IdempotencyStore_ExtendWindow() { // Arrange var messageId = Guid.NewGuid(); var idempotencyKey = $"extend-test:{messageId}"; var shortWindow = TimeSpan.FromSeconds(1); // Act - Claim with short window var claim = await _idempotencyStore!.TryClaimAsync( idempotencyKey, "original-value", shortWindow); claim.IsFirstClaim.Should().BeTrue(); // Extend the window var extended = await _idempotencyStore!.ExtendAsync( idempotencyKey, TimeSpan.FromMinutes(5)); // Assert - Window extended extended.Should().BeTrue(); // Duplicate should still be detected after original window would have expired await Task.Delay(1500); var afterOriginalExpiry = await _idempotencyStore!.TryClaimAsync( idempotencyKey, "new-value", shortWindow); afterOriginalExpiry.IsDuplicate.Should().BeTrue( "window was extended, so duplicate should still be detected"); _output.WriteLine("Window extension verified - duplicate detected after original expiry"); } [ValkeyIntegrationFact] public async Task EdgeCase_IdempotencyStore_Release() { // Arrange var messageId = Guid.NewGuid(); var idempotencyKey = $"release-test:{messageId}"; // Claim the key var firstClaim = await _idempotencyStore!.TryClaimAsync( idempotencyKey, "first-value", TimeSpan.FromMinutes(5)); firstClaim.IsFirstClaim.Should().BeTrue(); // Duplicate should be detected var duplicate = await _idempotencyStore!.TryClaimAsync( idempotencyKey, "duplicate-value", TimeSpan.FromMinutes(5)); duplicate.IsDuplicate.Should().BeTrue(); // Act - Release the key var released = await _idempotencyStore!.ReleaseAsync(idempotencyKey); released.Should().BeTrue(); // Assert - After release, key can be claimed again var afterRelease = await _idempotencyStore!.TryClaimAsync( idempotencyKey, "new-value", TimeSpan.FromMinutes(5)); afterRelease.IsFirstClaim.Should().BeTrue( "after release, key should be claimable again"); _output.WriteLine("Release verified - key claimable after release"); } [ValkeyIntegrationFact] public async Task EdgeCase_IdempotencyStore_Exists() { // Arrange var messageId = Guid.NewGuid(); var idempotencyKey = $"exists-test:{messageId}"; // Act - Check before claiming var existsBefore = await _idempotencyStore!.ExistsAsync(idempotencyKey); existsBefore.Should().BeFalse(); // Claim await _idempotencyStore!.TryClaimAsync(idempotencyKey, "value", TimeSpan.FromMinutes(5)); // Check after claiming var existsAfter = await _idempotencyStore!.ExistsAsync(idempotencyKey); existsAfter.Should().BeTrue(); _output.WriteLine("Exists check verified"); } [ValkeyIntegrationFact] public async Task EdgeCase_IdempotencyStore_Get() { // Arrange var messageId = Guid.NewGuid(); var idempotencyKey = $"get-test:{messageId}"; var storedValue = "stored-processor-id"; // Act - Get before claiming var valueBefore = await _idempotencyStore!.GetAsync(idempotencyKey); valueBefore.Should().BeNull(); // Claim await _idempotencyStore!.TryClaimAsync(idempotencyKey, storedValue, TimeSpan.FromMinutes(5)); // Get after claiming var valueAfter = await _idempotencyStore!.GetAsync(idempotencyKey); // Assert valueAfter.Should().Be(storedValue); _output.WriteLine($"Get verified - stored value: {valueAfter}"); } #endregion #region Helpers private ValkeyMessageQueue CreateQueue( MessageQueueOptions? queueOptions = null) where TMessage : class { queueOptions ??= _fixture.CreateQueueOptions(); var transportOptions = _fixture.CreateOptions(); return new ValkeyMessageQueue( _connectionFactory!, queueOptions, transportOptions, _fixture.GetLogger>()); } #endregion #region Test Types public sealed class TestMessage { public Guid Id { get; set; } public string? Content { get; set; } } /// /// Thread-safe hash set for concurrent test scenarios. /// private sealed class ConcurrentHashSet where T : notnull { private readonly HashSet _set = new(); private readonly object _lock = new(); public bool Add(T item) { lock (_lock) return _set.Add(item); } public int Count { get { lock (_lock) return _set.Count; } } public List ToList() { lock (_lock) return _set.ToList(); } } #endregion }