// -----------------------------------------------------------------------------
// AtLeastOnceDeliveryTests.cs
// Sprint: SPRINT_5100_0010_0003 - Router + Messaging Test Implementation
// Task: MESSAGING-5100-009 - At-least-once delivery with consumer idempotency
// Description: Integration tests verifying at-least-once delivery semantics:
// - Messages are never lost (guaranteed delivery)
// - Consumer idempotency correctly handles duplicate deliveries
// - Lease expiration triggers redelivery
// - Simulated failures result in message redelivery
// -----------------------------------------------------------------------------
using FluentAssertions;
using StellaOps.Messaging;
using StellaOps.Messaging.Abstractions;
using StellaOps.Messaging.Transport.Valkey.Tests.Fixtures;
using Xunit;
using Xunit.Abstractions;
namespace StellaOps.Messaging.Transport.Valkey.Tests;
///
/// Tests for at-least-once delivery semantics with consumer idempotency.
///
/// At-least-once delivery guarantees:
/// 1. Every message sent is delivered at least once
/// 2. Messages may be delivered multiple times (redelivery on failure)
/// 3. Consumer idempotency handles duplicate deliveries
/// 4. No message is ever lost, even under failure conditions
///
[Collection(ValkeyIntegrationTestCollection.Name)]
public sealed class AtLeastOnceDeliveryTests : IAsyncLifetime
{
private readonly ValkeyContainerFixture _fixture;
private readonly ITestOutputHelper _output;
private ValkeyConnectionFactory? _connectionFactory;
private ValkeyIdempotencyStore? _idempotencyStore;
public AtLeastOnceDeliveryTests(ValkeyContainerFixture fixture, ITestOutputHelper output)
{
_fixture = fixture;
_output = output;
}
public Task InitializeAsync()
{
_connectionFactory = _fixture.CreateConnectionFactory();
_idempotencyStore = new ValkeyIdempotencyStore(
_connectionFactory,
$"test-consumer-{Guid.NewGuid():N}",
null);
return Task.CompletedTask;
}
public async Task DisposeAsync()
{
if (_connectionFactory is not null)
{
await _connectionFactory.DisposeAsync();
}
}
#region At-Least-Once Delivery Guarantee Tests
[ValkeyIntegrationFact]
public async Task AtLeastOnce_MessageSent_IsDeliveredAtLeastOnce()
{
// Arrange - Producer sends message
var queue = CreateQueue();
var messageId = Guid.NewGuid();
var message = new TestMessage
{
Id = messageId,
Content = "At-least-once test message"
};
// Act - Send message
var enqueueResult = await queue.EnqueueAsync(message);
enqueueResult.Success.Should().BeTrue("message should be accepted by the queue");
// Act - Consumer receives message
var leases = await queue.LeaseAsync(new LeaseRequest { BatchSize = 1 });
// Assert - Message is delivered
leases.Should().HaveCount(1, "message must be delivered at least once");
leases[0].Message.Id.Should().Be(messageId);
leases[0].Message.Content.Should().Be("At-least-once test message");
await leases[0].AcknowledgeAsync();
_output.WriteLine($"Message {messageId} delivered successfully");
}
[ValkeyIntegrationFact]
public async Task AtLeastOnce_UnacknowledgedLease_MessageRedelivered()
{
// Arrange - Create queue with short lease duration
var queueOptions = _fixture.CreateQueueOptions();
queueOptions.DefaultLeaseDuration = TimeSpan.FromMilliseconds(200);
var queue = CreateQueue(queueOptions);
var messageId = Guid.NewGuid();
await queue.EnqueueAsync(new TestMessage { Id = messageId, Content = "Redelivery test" });
// Act - Lease message but don't acknowledge (simulating consumer crash)
var firstLease = await queue.LeaseAsync(new LeaseRequest { BatchSize = 1 });
firstLease.Should().HaveCount(1);
firstLease[0].Message.Id.Should().Be(messageId);
// Don't acknowledge - simulate crash
_output.WriteLine("Simulating consumer crash (not acknowledging message)");
// Wait for lease to expire
await Task.Delay(500);
// Act - Claim expired message (automatic redelivery)
var redelivered = await queue.ClaimExpiredAsync(new ClaimRequest
{
BatchSize = 10,
MinIdleTime = TimeSpan.FromMilliseconds(200),
MinDeliveryAttempts = 1
});
// Assert - Message is redelivered
redelivered.Should().HaveCount(1, "message must be redelivered after lease expiration");
redelivered[0].Message.Id.Should().Be(messageId);
redelivered[0].Attempt.Should().BeGreaterThan(1, "this should be a redelivery");
await redelivered[0].AcknowledgeAsync();
_output.WriteLine($"Message {messageId} successfully redelivered on attempt {redelivered[0].Attempt}");
}
[ValkeyIntegrationFact]
public async Task AtLeastOnce_MultipleMessages_AllDelivered()
{
// Arrange
var queue = CreateQueue();
const int messageCount = 100;
var sentIds = new HashSet();
// Act - Send multiple messages
for (int i = 0; i < messageCount; i++)
{
var id = Guid.NewGuid();
sentIds.Add(id);
await queue.EnqueueAsync(new TestMessage { Id = id, Content = $"Message-{i}" });
}
// Act - Receive all messages
var receivedIds = new HashSet();
int remaining = messageCount;
while (remaining > 0)
{
var leases = await queue.LeaseAsync(new LeaseRequest { BatchSize = 20 });
foreach (var lease in leases)
{
receivedIds.Add(lease.Message.Id);
await lease.AcknowledgeAsync();
}
remaining -= leases.Count;
}
// Assert - All messages delivered
receivedIds.Should().BeEquivalentTo(sentIds, "all sent messages must be delivered");
_output.WriteLine($"All {messageCount} messages delivered successfully");
}
[ValkeyIntegrationFact]
public async Task AtLeastOnce_RetryAfterNack_MessageRedelivered()
{
// Arrange
var queueOptions = _fixture.CreateQueueOptions();
queueOptions.RetryInitialBackoff = TimeSpan.Zero; // Immediate retry for test speed
var queue = CreateQueue(queueOptions);
var messageId = Guid.NewGuid();
await queue.EnqueueAsync(new TestMessage { Id = messageId, Content = "Retry test" });
// Act - First delivery, simulate processing failure with retry
var firstLease = await queue.LeaseAsync(new LeaseRequest { BatchSize = 1 });
firstLease.Should().HaveCount(1);
firstLease[0].Attempt.Should().Be(1);
// Nack for retry
await firstLease[0].ReleaseAsync(ReleaseDisposition.Retry);
_output.WriteLine("Message nacked for retry");
// Brief delay for retry processing
await Task.Delay(100);
// Act - Second delivery after retry
var secondLease = await queue.LeaseAsync(new LeaseRequest { BatchSize = 1 });
// Assert - Message is redelivered
secondLease.Should().HaveCount(1, "message must be redelivered after nack");
secondLease[0].Message.Id.Should().Be(messageId);
secondLease[0].Attempt.Should().Be(2, "this should be attempt 2");
await secondLease[0].AcknowledgeAsync();
_output.WriteLine($"Message {messageId} successfully processed on attempt 2");
}
#endregion
#region Consumer Idempotency Tests
[ValkeyIntegrationFact]
public async Task ConsumerIdempotency_DuplicateProcessing_DetectedAndSkipped()
{
// Arrange - Create a consumer with idempotency tracking
var queue = CreateQueue();
var processedMessageIds = new HashSet();
var processingCount = new Dictionary();
var messageId = Guid.NewGuid();
await queue.EnqueueAsync(new TestMessage { Id = messageId, Content = "Idempotency test" });
// Act - Simulate receiving the message multiple times
for (int delivery = 1; delivery <= 3; delivery++)
{
// Simulate message delivery (could be redelivery)
var idempotencyKey = $"consumer-process:{messageId}";
var claimResult = await _idempotencyStore!.TryClaimAsync(
idempotencyKey,
messageId.ToString(),
TimeSpan.FromMinutes(5));
if (claimResult.IsFirstClaim)
{
// First time processing this message
processedMessageIds.Add(messageId);
processingCount[messageId] = 1;
_output.WriteLine($"Delivery {delivery}: First processing of message {messageId}");
}
else
{
// Duplicate - skip processing
processingCount[messageId] = processingCount.GetValueOrDefault(messageId) + 1;
_output.WriteLine($"Delivery {delivery}: Duplicate detected, skipping message {messageId}");
}
}
// Assert - Message processed exactly once despite multiple deliveries
processedMessageIds.Should().HaveCount(1);
processingCount[messageId].Should().BeGreaterThan(1, "we simulated multiple deliveries");
// Cleanup
var leases = await queue.LeaseAsync(new LeaseRequest { BatchSize = 1 });
if (leases.Count > 0)
{
await leases[0].AcknowledgeAsync();
}
}
[ValkeyIntegrationFact]
public async Task ConsumerIdempotency_ConcurrentDuplicates_OnlyOneProcessed()
{
// Arrange
var messageId = Guid.NewGuid();
var processedCount = 0;
var duplicateCount = 0;
var lockObject = new object();
// Simulate 10 concurrent consumers trying to process the same message
var tasks = Enumerable.Range(1, 10).Select(async consumerId =>
{
var idempotencyKey = $"concurrent-test:{messageId}";
var claimResult = await _idempotencyStore!.TryClaimAsync(
idempotencyKey,
$"consumer-{consumerId}",
TimeSpan.FromMinutes(5));
lock (lockObject)
{
if (claimResult.IsFirstClaim)
{
processedCount++;
_output.WriteLine($"Consumer {consumerId}: Processing message (first claim)");
}
else
{
duplicateCount++;
_output.WriteLine($"Consumer {consumerId}: Duplicate detected, existing value: {claimResult.ExistingValue}");
}
}
});
// Act
await Task.WhenAll(tasks);
// Assert - Exactly one consumer processed the message
processedCount.Should().Be(1, "only one consumer should process the message");
duplicateCount.Should().Be(9, "9 consumers should detect duplicate");
_output.WriteLine($"Processed: {processedCount}, Duplicates: {duplicateCount}");
}
[ValkeyIntegrationFact]
public async Task ConsumerIdempotency_IdempotencyWindowExpires_ReprocessingAllowed()
{
// Arrange
var messageId = Guid.NewGuid();
var shortWindow = TimeSpan.FromMilliseconds(200);
var idempotencyKey = $"window-test:{messageId}";
// Act - First claim
var firstClaim = await _idempotencyStore!.TryClaimAsync(
idempotencyKey,
"first-processor",
shortWindow);
firstClaim.IsFirstClaim.Should().BeTrue();
_output.WriteLine("First claim successful");
// Duplicate should be detected
var duplicateClaim = await _idempotencyStore!.TryClaimAsync(
idempotencyKey,
"duplicate-processor",
shortWindow);
duplicateClaim.IsDuplicate.Should().BeTrue();
_output.WriteLine("Duplicate detected as expected");
// Wait for window to expire
await Task.Delay(500);
// Act - After expiration, claim should succeed again
var afterExpiration = await _idempotencyStore!.TryClaimAsync(
idempotencyKey,
"new-processor",
shortWindow);
// Assert - Reprocessing allowed after window expiration
afterExpiration.IsFirstClaim.Should().BeTrue(
"after idempotency window expires, message can be reprocessed");
_output.WriteLine("After window expiration, new claim succeeded");
}
[ValkeyIntegrationFact]
public async Task ConsumerIdempotency_DifferentMessages_IndependentProcessing()
{
// Arrange - Three different messages
var messageIds = Enumerable.Range(1, 3).Select(_ => Guid.NewGuid()).ToList();
var processedIds = new List();
// Act - Process each message (simulating first-time delivery)
foreach (var messageId in messageIds)
{
var idempotencyKey = $"different-msg-test:{messageId}";
var claimResult = await _idempotencyStore!.TryClaimAsync(
idempotencyKey,
messageId.ToString(),
TimeSpan.FromMinutes(5));
if (claimResult.IsFirstClaim)
{
processedIds.Add(messageId);
}
}
// Assert - All different messages processed
processedIds.Should().BeEquivalentTo(messageIds);
_output.WriteLine($"All {messageIds.Count} different messages processed independently");
}
#endregion
#region End-to-End At-Least-Once with Idempotency Tests
[ValkeyIntegrationFact]
public async Task EndToEnd_AtLeastOnceWithIdempotency_NoDuplicateProcessing()
{
// Arrange
var queueOptions = _fixture.CreateQueueOptions();
queueOptions.DefaultLeaseDuration = TimeSpan.FromMilliseconds(200);
var queue = CreateQueue(queueOptions);
var messageId = Guid.NewGuid();
var processedIds = new HashSet();
var deliveryCount = 0;
await queue.EnqueueAsync(new TestMessage { Id = messageId, Content = "E2E test" });
// Act - Consumer with idempotency-aware processing
// Simulate: first delivery - lease but crash, second delivery - process successfully
// First delivery (crash simulation - don't ack)
var firstLease = await queue.LeaseAsync(new LeaseRequest { BatchSize = 1 });
firstLease.Should().HaveCount(1);
deliveryCount++;
// Attempt to claim for processing
var firstClaim = await _idempotencyStore!.TryClaimAsync(
$"e2e-test:{firstLease[0].Message.Id}",
firstLease[0].MessageId,
TimeSpan.FromMinutes(5));
if (firstClaim.IsFirstClaim)
{
processedIds.Add(firstLease[0].Message.Id);
}
// Simulate crash - don't acknowledge
_output.WriteLine("First delivery: Processing started but consumer crashed");
// Wait for lease expiration
await Task.Delay(500);
// Claim expired message (redelivery)
var redelivered = await queue.ClaimExpiredAsync(new ClaimRequest
{
BatchSize = 1,
MinIdleTime = TimeSpan.FromMilliseconds(200),
MinDeliveryAttempts = 1
});
if (redelivered.Count > 0)
{
deliveryCount++;
// Attempt to claim again (should be duplicate)
var secondClaim = await _idempotencyStore!.TryClaimAsync(
$"e2e-test:{redelivered[0].Message.Id}",
redelivered[0].MessageId,
TimeSpan.FromMinutes(5));
if (secondClaim.IsFirstClaim)
{
processedIds.Add(redelivered[0].Message.Id);
}
else
{
_output.WriteLine($"Second delivery: Duplicate detected, skipping processing");
}
// This time, acknowledge
await redelivered[0].AcknowledgeAsync();
_output.WriteLine("Second delivery: Message acknowledged");
}
// Assert
processedIds.Should().HaveCount(1, "message should be processed exactly once");
deliveryCount.Should().BeGreaterThan(1, "message should be delivered at least twice (crash + redelivery)");
_output.WriteLine($"Total deliveries: {deliveryCount}, Unique processing: {processedIds.Count}");
}
[ValkeyIntegrationFact]
public async Task EndToEnd_BulkMessages_AtLeastOnceWithIdempotency()
{
// Arrange
var queue = CreateQueue();
const int messageCount = 50;
var processedIds = new ConcurrentHashSet();
var deliveryAttempts = new Dictionary();
// Send messages
var sentIds = new List();
for (int i = 0; i < messageCount; i++)
{
var id = Guid.NewGuid();
sentIds.Add(id);
await queue.EnqueueAsync(new TestMessage { Id = id, Content = $"Bulk-{i}" });
}
// Act - Process all messages with idempotency
int remaining = messageCount;
while (remaining > 0)
{
var leases = await queue.LeaseAsync(new LeaseRequest { BatchSize = 10 });
if (leases.Count == 0) break;
foreach (var lease in leases)
{
var msgId = lease.Message.Id;
deliveryAttempts[msgId] = deliveryAttempts.GetValueOrDefault(msgId) + 1;
// Check idempotency before processing
var claim = await _idempotencyStore!.TryClaimAsync(
$"bulk-test:{msgId}",
lease.MessageId,
TimeSpan.FromMinutes(5));
if (claim.IsFirstClaim)
{
processedIds.Add(msgId);
}
await lease.AcknowledgeAsync();
}
remaining -= leases.Count;
}
// Assert - All messages processed exactly once
processedIds.Count.Should().Be(messageCount, "all messages should be processed");
sentIds.Should().BeEquivalentTo(processedIds.ToList(), "all sent messages should be processed");
_output.WriteLine($"Processed {processedIds.Count}/{messageCount} messages with idempotency");
}
#endregion
#region Edge Cases
[ValkeyIntegrationFact]
public async Task EdgeCase_IdempotencyStore_ExtendWindow()
{
// Arrange
var messageId = Guid.NewGuid();
var idempotencyKey = $"extend-test:{messageId}";
var shortWindow = TimeSpan.FromSeconds(1);
// Act - Claim with short window
var claim = await _idempotencyStore!.TryClaimAsync(
idempotencyKey,
"original-value",
shortWindow);
claim.IsFirstClaim.Should().BeTrue();
// Extend the window
var extended = await _idempotencyStore!.ExtendAsync(
idempotencyKey,
TimeSpan.FromMinutes(5));
// Assert - Window extended
extended.Should().BeTrue();
// Duplicate should still be detected after original window would have expired
await Task.Delay(1500);
var afterOriginalExpiry = await _idempotencyStore!.TryClaimAsync(
idempotencyKey,
"new-value",
shortWindow);
afterOriginalExpiry.IsDuplicate.Should().BeTrue(
"window was extended, so duplicate should still be detected");
_output.WriteLine("Window extension verified - duplicate detected after original expiry");
}
[ValkeyIntegrationFact]
public async Task EdgeCase_IdempotencyStore_Release()
{
// Arrange
var messageId = Guid.NewGuid();
var idempotencyKey = $"release-test:{messageId}";
// Claim the key
var firstClaim = await _idempotencyStore!.TryClaimAsync(
idempotencyKey,
"first-value",
TimeSpan.FromMinutes(5));
firstClaim.IsFirstClaim.Should().BeTrue();
// Duplicate should be detected
var duplicate = await _idempotencyStore!.TryClaimAsync(
idempotencyKey,
"duplicate-value",
TimeSpan.FromMinutes(5));
duplicate.IsDuplicate.Should().BeTrue();
// Act - Release the key
var released = await _idempotencyStore!.ReleaseAsync(idempotencyKey);
released.Should().BeTrue();
// Assert - After release, key can be claimed again
var afterRelease = await _idempotencyStore!.TryClaimAsync(
idempotencyKey,
"new-value",
TimeSpan.FromMinutes(5));
afterRelease.IsFirstClaim.Should().BeTrue(
"after release, key should be claimable again");
_output.WriteLine("Release verified - key claimable after release");
}
[ValkeyIntegrationFact]
public async Task EdgeCase_IdempotencyStore_Exists()
{
// Arrange
var messageId = Guid.NewGuid();
var idempotencyKey = $"exists-test:{messageId}";
// Act - Check before claiming
var existsBefore = await _idempotencyStore!.ExistsAsync(idempotencyKey);
existsBefore.Should().BeFalse();
// Claim
await _idempotencyStore!.TryClaimAsync(idempotencyKey, "value", TimeSpan.FromMinutes(5));
// Check after claiming
var existsAfter = await _idempotencyStore!.ExistsAsync(idempotencyKey);
existsAfter.Should().BeTrue();
_output.WriteLine("Exists check verified");
}
[ValkeyIntegrationFact]
public async Task EdgeCase_IdempotencyStore_Get()
{
// Arrange
var messageId = Guid.NewGuid();
var idempotencyKey = $"get-test:{messageId}";
var storedValue = "stored-processor-id";
// Act - Get before claiming
var valueBefore = await _idempotencyStore!.GetAsync(idempotencyKey);
valueBefore.Should().BeNull();
// Claim
await _idempotencyStore!.TryClaimAsync(idempotencyKey, storedValue, TimeSpan.FromMinutes(5));
// Get after claiming
var valueAfter = await _idempotencyStore!.GetAsync(idempotencyKey);
// Assert
valueAfter.Should().Be(storedValue);
_output.WriteLine($"Get verified - stored value: {valueAfter}");
}
#endregion
#region Helpers
private ValkeyMessageQueue CreateQueue(
MessageQueueOptions? queueOptions = null)
where TMessage : class
{
queueOptions ??= _fixture.CreateQueueOptions();
var transportOptions = _fixture.CreateOptions();
return new ValkeyMessageQueue(
_connectionFactory!,
queueOptions,
transportOptions,
_fixture.GetLogger>());
}
#endregion
#region Test Types
public sealed class TestMessage
{
public Guid Id { get; set; }
public string? Content { get; set; }
}
///
/// Thread-safe hash set for concurrent test scenarios.
///
private sealed class ConcurrentHashSet where T : notnull
{
private readonly HashSet _set = new();
private readonly object _lock = new();
public bool Add(T item)
{
lock (_lock) return _set.Add(item);
}
public int Count
{
get { lock (_lock) return _set.Count; }
}
public List ToList()
{
lock (_lock) return _set.ToList();
}
}
#endregion
}