Restructure solution layout by module

This commit is contained in:
master
2025-10-28 15:10:40 +02:00
parent 95daa159c4
commit d870da18ce
4103 changed files with 192899 additions and 187024 deletions

View File

@@ -0,0 +1,4 @@
# StellaOps.Notify.Queue — Agent Charter
## Mission
Provide event & delivery queues for Notify per `docs/ARCHITECTURE_NOTIFY.md`.

View File

@@ -0,0 +1,80 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using NATS.Client.JetStream;
namespace StellaOps.Notify.Queue.Nats;
internal sealed class NatsNotifyDeliveryLease : INotifyQueueLease<NotifyDeliveryQueueMessage>
{
private readonly NatsNotifyDeliveryQueue _queue;
private readonly NatsJSMsg<byte[]> _message;
private int _completed;
internal NatsNotifyDeliveryLease(
NatsNotifyDeliveryQueue queue,
NatsJSMsg<byte[]> message,
string messageId,
NotifyDeliveryQueueMessage payload,
int attempt,
string consumer,
DateTimeOffset enqueuedAt,
DateTimeOffset leaseExpiresAt,
string idempotencyKey)
{
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
_message = message;
MessageId = messageId ?? throw new ArgumentNullException(nameof(messageId));
Message = payload ?? throw new ArgumentNullException(nameof(payload));
Attempt = attempt;
Consumer = consumer ?? throw new ArgumentNullException(nameof(consumer));
EnqueuedAt = enqueuedAt;
LeaseExpiresAt = leaseExpiresAt;
IdempotencyKey = idempotencyKey ?? payload.IdempotencyKey;
}
public string MessageId { get; }
public int Attempt { get; internal set; }
public DateTimeOffset EnqueuedAt { get; }
public DateTimeOffset LeaseExpiresAt { get; private set; }
public string Consumer { get; }
public string Stream => Message.Stream;
public string TenantId => Message.TenantId;
public string? PartitionKey => Message.PartitionKey;
public string IdempotencyKey { get; }
public string? TraceId => Message.TraceId;
public IReadOnlyDictionary<string, string> Attributes => Message.Attributes;
public NotifyDeliveryQueueMessage Message { get; }
internal NatsJSMsg<byte[]> RawMessage => _message;
public Task AcknowledgeAsync(CancellationToken cancellationToken = default)
=> _queue.AcknowledgeAsync(this, cancellationToken);
public Task RenewAsync(TimeSpan leaseDuration, CancellationToken cancellationToken = default)
=> _queue.RenewLeaseAsync(this, leaseDuration, cancellationToken);
public Task ReleaseAsync(NotifyQueueReleaseDisposition disposition, CancellationToken cancellationToken = default)
=> _queue.ReleaseAsync(this, disposition, cancellationToken);
public Task DeadLetterAsync(string reason, CancellationToken cancellationToken = default)
=> _queue.DeadLetterAsync(this, reason, cancellationToken);
internal bool TryBeginCompletion()
=> Interlocked.CompareExchange(ref _completed, 1, 0) == 0;
internal void RefreshLease(DateTimeOffset expiresAt)
=> LeaseExpiresAt = expiresAt;
}

View File

@@ -0,0 +1,697 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Globalization;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using NATS.Client.Core;
using NATS.Client.JetStream;
using NATS.Client.JetStream.Models;
using StellaOps.Notify.Models;
namespace StellaOps.Notify.Queue.Nats;
internal sealed class NatsNotifyDeliveryQueue : INotifyDeliveryQueue, IAsyncDisposable
{
private const string TransportName = "nats";
private static readonly INatsSerializer<byte[]> PayloadSerializer = NatsRawSerializer<byte[]>.Default;
private readonly NotifyDeliveryQueueOptions _queueOptions;
private readonly NotifyNatsDeliveryQueueOptions _options;
private readonly ILogger<NatsNotifyDeliveryQueue> _logger;
private readonly TimeProvider _timeProvider;
private readonly SemaphoreSlim _connectionGate = new(1, 1);
private readonly Func<NatsOpts, CancellationToken, ValueTask<NatsConnection>> _connectionFactory;
private NatsConnection? _connection;
private NatsJSContext? _jsContext;
private INatsJSConsumer? _consumer;
private bool _disposed;
public NatsNotifyDeliveryQueue(
NotifyDeliveryQueueOptions queueOptions,
NotifyNatsDeliveryQueueOptions options,
ILogger<NatsNotifyDeliveryQueue> logger,
TimeProvider timeProvider,
Func<NatsOpts, CancellationToken, ValueTask<NatsConnection>>? connectionFactory = null)
{
_queueOptions = queueOptions ?? throw new ArgumentNullException(nameof(queueOptions));
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
_connectionFactory = connectionFactory ?? ((opts, token) => new ValueTask<NatsConnection>(new NatsConnection(opts)));
if (string.IsNullOrWhiteSpace(_options.Url))
{
throw new InvalidOperationException("NATS connection URL must be configured for the Notify delivery queue.");
}
if (string.IsNullOrWhiteSpace(_options.Stream) || string.IsNullOrWhiteSpace(_options.Subject))
{
throw new InvalidOperationException("NATS stream and subject must be configured for the Notify delivery queue.");
}
}
public async ValueTask<NotifyQueueEnqueueResult> PublishAsync(
NotifyDeliveryQueueMessage message,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(message);
var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false);
await EnsureStreamAndConsumerAsync(js, cancellationToken).ConfigureAwait(false);
await EnsureDeadLetterStreamAsync(js, cancellationToken).ConfigureAwait(false);
var payload = Encoding.UTF8.GetBytes(NotifyCanonicalJsonSerializer.Serialize(message.Delivery));
var headers = BuildHeaders(message);
var publishOpts = new NatsJSPubOpts
{
MsgId = message.IdempotencyKey,
RetryAttempts = 0
};
var ack = await js.PublishAsync(
_options.Subject,
payload,
PayloadSerializer,
publishOpts,
headers,
cancellationToken)
.ConfigureAwait(false);
if (ack.Duplicate)
{
NotifyQueueMetrics.RecordDeduplicated(TransportName, _options.Stream);
_logger.LogDebug(
"Duplicate Notify delivery enqueue detected for delivery {DeliveryId}.",
message.Delivery.DeliveryId);
return new NotifyQueueEnqueueResult(ack.Seq.ToString(), true);
}
NotifyQueueMetrics.RecordEnqueued(TransportName, _options.Stream);
_logger.LogDebug(
"Enqueued Notify delivery {DeliveryId} into NATS stream {Stream} (sequence {Sequence}).",
message.Delivery.DeliveryId,
ack.Stream,
ack.Seq);
return new NotifyQueueEnqueueResult(ack.Seq.ToString(), false);
}
public async ValueTask<IReadOnlyList<INotifyQueueLease<NotifyDeliveryQueueMessage>>> LeaseAsync(
NotifyQueueLeaseRequest request,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(request);
var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false);
var consumer = await EnsureStreamAndConsumerAsync(js, cancellationToken).ConfigureAwait(false);
var fetchOpts = new NatsJSFetchOpts
{
MaxMsgs = request.BatchSize,
Expires = request.LeaseDuration,
IdleHeartbeat = _options.IdleHeartbeat
};
var now = _timeProvider.GetUtcNow();
var leases = new List<INotifyQueueLease<NotifyDeliveryQueueMessage>>(request.BatchSize);
await foreach (var msg in consumer.FetchAsync(PayloadSerializer, fetchOpts, cancellationToken).ConfigureAwait(false))
{
var lease = CreateLease(msg, request.Consumer, now, request.LeaseDuration);
if (lease is null)
{
await msg.AckAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
continue;
}
leases.Add(lease);
}
return leases;
}
public async ValueTask<IReadOnlyList<INotifyQueueLease<NotifyDeliveryQueueMessage>>> ClaimExpiredAsync(
NotifyQueueClaimOptions options,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(options);
var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false);
var consumer = await EnsureStreamAndConsumerAsync(js, cancellationToken).ConfigureAwait(false);
var fetchOpts = new NatsJSFetchOpts
{
MaxMsgs = options.BatchSize,
Expires = options.MinIdleTime,
IdleHeartbeat = _options.IdleHeartbeat
};
var now = _timeProvider.GetUtcNow();
var leases = new List<INotifyQueueLease<NotifyDeliveryQueueMessage>>(options.BatchSize);
await foreach (var msg in consumer.FetchAsync(PayloadSerializer, fetchOpts, cancellationToken).ConfigureAwait(false))
{
var deliveries = (int)(msg.Metadata?.NumDelivered ?? 1);
if (deliveries <= 1)
{
await msg.NakAsync(new AckOpts(), TimeSpan.Zero, cancellationToken).ConfigureAwait(false);
continue;
}
var lease = CreateLease(msg, options.ClaimantConsumer, now, _queueOptions.DefaultLeaseDuration);
if (lease is null)
{
await msg.AckAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
continue;
}
leases.Add(lease);
}
return leases;
}
public async ValueTask DisposeAsync()
{
if (_disposed)
{
return;
}
_disposed = true;
if (_connection is not null)
{
await _connection.DisposeAsync().ConfigureAwait(false);
}
_connectionGate.Dispose();
GC.SuppressFinalize(this);
}
internal async Task AcknowledgeAsync(
NatsNotifyDeliveryLease lease,
CancellationToken cancellationToken)
{
if (!lease.TryBeginCompletion())
{
return;
}
await lease.RawMessage.AckAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
NotifyQueueMetrics.RecordAck(TransportName, _options.Stream);
_logger.LogDebug(
"Acknowledged Notify delivery {DeliveryId} (sequence {Sequence}).",
lease.Message.Delivery.DeliveryId,
lease.MessageId);
}
internal async Task RenewLeaseAsync(
NatsNotifyDeliveryLease lease,
TimeSpan leaseDuration,
CancellationToken cancellationToken)
{
await lease.RawMessage.AckProgressAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
var expires = _timeProvider.GetUtcNow().Add(leaseDuration);
lease.RefreshLease(expires);
_logger.LogDebug(
"Renewed NATS lease for Notify delivery {DeliveryId} until {Expires:u}.",
lease.Message.Delivery.DeliveryId,
expires);
}
internal async Task ReleaseAsync(
NatsNotifyDeliveryLease lease,
NotifyQueueReleaseDisposition disposition,
CancellationToken cancellationToken)
{
if (disposition == NotifyQueueReleaseDisposition.Retry
&& lease.Attempt >= _queueOptions.MaxDeliveryAttempts)
{
_logger.LogWarning(
"Notify delivery {DeliveryId} reached max delivery attempts ({Attempts}); moving to dead-letter stream.",
lease.Message.Delivery.DeliveryId,
lease.Attempt);
await DeadLetterAsync(
lease,
$"max-delivery-attempts:{lease.Attempt}",
cancellationToken).ConfigureAwait(false);
return;
}
if (!lease.TryBeginCompletion())
{
return;
}
if (disposition == NotifyQueueReleaseDisposition.Retry)
{
var delay = CalculateBackoff(lease.Attempt);
await lease.RawMessage.NakAsync(new AckOpts(), delay, cancellationToken).ConfigureAwait(false);
NotifyQueueMetrics.RecordRetry(TransportName, _options.Stream);
_logger.LogInformation(
"Scheduled Notify delivery {DeliveryId} for retry with delay {Delay} (attempt {Attempt}).",
lease.Message.Delivery.DeliveryId,
delay,
lease.Attempt);
}
else
{
await lease.RawMessage.AckTerminateAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
NotifyQueueMetrics.RecordAck(TransportName, _options.Stream);
_logger.LogInformation(
"Abandoned Notify delivery {DeliveryId} after {Attempt} attempt(s).",
lease.Message.Delivery.DeliveryId,
lease.Attempt);
}
}
internal async Task DeadLetterAsync(
NatsNotifyDeliveryLease lease,
string reason,
CancellationToken cancellationToken)
{
if (!lease.TryBeginCompletion())
{
return;
}
await lease.RawMessage.AckTerminateAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false);
await EnsureDeadLetterStreamAsync(js, cancellationToken).ConfigureAwait(false);
var payload = Encoding.UTF8.GetBytes(NotifyCanonicalJsonSerializer.Serialize(lease.Message.Delivery));
var headers = BuildDeadLetterHeaders(lease, reason);
await js.PublishAsync(
_options.DeadLetterSubject,
payload,
PayloadSerializer,
new NatsJSPubOpts(),
headers,
cancellationToken)
.ConfigureAwait(false);
NotifyQueueMetrics.RecordDeadLetter(TransportName, _options.DeadLetterStream);
_logger.LogError(
"Dead-lettered Notify delivery {DeliveryId} (attempt {Attempt}): {Reason}",
lease.Message.Delivery.DeliveryId,
lease.Attempt,
reason);
}
internal async Task PingAsync(CancellationToken cancellationToken)
{
var connection = await EnsureConnectionAsync(cancellationToken).ConfigureAwait(false);
await connection.PingAsync(cancellationToken).ConfigureAwait(false);
}
private async Task<NatsJSContext> GetJetStreamAsync(CancellationToken cancellationToken)
{
if (_jsContext is not null)
{
return _jsContext;
}
var connection = await EnsureConnectionAsync(cancellationToken).ConfigureAwait(false);
await _connectionGate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
_jsContext ??= new NatsJSContext(connection);
return _jsContext;
}
finally
{
_connectionGate.Release();
}
}
private async ValueTask<INatsJSConsumer> EnsureStreamAndConsumerAsync(
NatsJSContext js,
CancellationToken cancellationToken)
{
if (_consumer is not null)
{
return _consumer;
}
await _connectionGate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_consumer is not null)
{
return _consumer;
}
await EnsureStreamAsync(js, cancellationToken).ConfigureAwait(false);
await EnsureDeadLetterStreamAsync(js, cancellationToken).ConfigureAwait(false);
var consumerConfig = new ConsumerConfig
{
DurableName = _options.DurableConsumer,
AckPolicy = ConsumerConfigAckPolicy.Explicit,
ReplayPolicy = ConsumerConfigReplayPolicy.Instant,
DeliverPolicy = ConsumerConfigDeliverPolicy.All,
AckWait = ToNanoseconds(_options.AckWait),
MaxAckPending = _options.MaxAckPending,
MaxDeliver = Math.Max(1, _queueOptions.MaxDeliveryAttempts),
FilterSubjects = new[] { _options.Subject }
};
try
{
_consumer = await js.CreateConsumerAsync(
_options.Stream,
consumerConfig,
cancellationToken)
.ConfigureAwait(false);
}
catch (NatsJSApiException apiEx)
{
_logger.LogDebug(
apiEx,
"CreateConsumerAsync failed with code {Code}; attempting to fetch existing durable consumer {Durable}.",
apiEx.Error?.Code,
_options.DurableConsumer);
_consumer = await js.GetConsumerAsync(
_options.Stream,
_options.DurableConsumer,
cancellationToken)
.ConfigureAwait(false);
}
return _consumer;
}
finally
{
_connectionGate.Release();
}
}
private async Task<NatsConnection> EnsureConnectionAsync(CancellationToken cancellationToken)
{
if (_connection is not null)
{
return _connection;
}
await _connectionGate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_connection is not null)
{
return _connection;
}
var opts = new NatsOpts
{
Url = _options.Url!,
Name = "stellaops-notify-delivery",
CommandTimeout = TimeSpan.FromSeconds(10),
RequestTimeout = TimeSpan.FromSeconds(20),
PingInterval = TimeSpan.FromSeconds(30)
};
_connection = await _connectionFactory(opts, cancellationToken).ConfigureAwait(false);
await _connection.ConnectAsync().ConfigureAwait(false);
return _connection;
}
finally
{
_connectionGate.Release();
}
}
private async Task EnsureStreamAsync(NatsJSContext js, CancellationToken cancellationToken)
{
try
{
await js.GetStreamAsync(_options.Stream, cancellationToken: cancellationToken).ConfigureAwait(false);
}
catch (NatsJSApiException ex) when (ex.Error?.Code == 404)
{
var config = new StreamConfig(name: _options.Stream, subjects: new[] { _options.Subject })
{
Retention = StreamConfigRetention.Workqueue,
Storage = StreamConfigStorage.File,
MaxConsumers = -1,
MaxMsgs = -1,
MaxBytes = -1
};
await js.CreateStreamAsync(config, cancellationToken).ConfigureAwait(false);
_logger.LogInformation("Created NATS Notify delivery stream {Stream} ({Subject}).", _options.Stream, _options.Subject);
}
}
private async Task EnsureDeadLetterStreamAsync(NatsJSContext js, CancellationToken cancellationToken)
{
try
{
await js.GetStreamAsync(_options.DeadLetterStream, cancellationToken: cancellationToken).ConfigureAwait(false);
}
catch (NatsJSApiException ex) when (ex.Error?.Code == 404)
{
var config = new StreamConfig(name: _options.DeadLetterStream, subjects: new[] { _options.DeadLetterSubject })
{
Retention = StreamConfigRetention.Workqueue,
Storage = StreamConfigStorage.File,
MaxConsumers = -1,
MaxMsgs = -1,
MaxBytes = -1
};
await js.CreateStreamAsync(config, cancellationToken).ConfigureAwait(false);
_logger.LogInformation("Created NATS Notify delivery dead-letter stream {Stream} ({Subject}).", _options.DeadLetterStream, _options.DeadLetterSubject);
}
}
private NatsNotifyDeliveryLease? CreateLease(
NatsJSMsg<byte[]> message,
string consumer,
DateTimeOffset now,
TimeSpan leaseDuration)
{
var payloadBytes = message.Data ?? Array.Empty<byte>();
if (payloadBytes.Length == 0)
{
return null;
}
NotifyDelivery delivery;
try
{
var json = Encoding.UTF8.GetString(payloadBytes);
delivery = NotifyCanonicalJsonSerializer.Deserialize<NotifyDelivery>(json);
}
catch (Exception ex)
{
_logger.LogWarning(
ex,
"Failed to deserialize Notify delivery payload for NATS message {Sequence}.",
message.Metadata?.Sequence.Stream);
return null;
}
var headers = message.Headers ?? new NatsHeaders();
var deliveryId = TryGetHeader(headers, NotifyQueueFields.DeliveryId) ?? delivery.DeliveryId;
var channelId = TryGetHeader(headers, NotifyQueueFields.ChannelId);
var channelTypeRaw = TryGetHeader(headers, NotifyQueueFields.ChannelType);
if (channelId is null || channelTypeRaw is null)
{
return null;
}
if (!Enum.TryParse<NotifyChannelType>(channelTypeRaw, ignoreCase: true, out var channelType))
{
_logger.LogWarning("Unknown channel type '{ChannelType}' for delivery {DeliveryId}.", channelTypeRaw, deliveryId);
return null;
}
var traceId = TryGetHeader(headers, NotifyQueueFields.TraceId);
var partitionKey = TryGetHeader(headers, NotifyQueueFields.PartitionKey) ?? channelId;
var idempotencyKey = TryGetHeader(headers, NotifyQueueFields.IdempotencyKey) ?? delivery.DeliveryId;
var enqueuedAt = TryGetHeader(headers, NotifyQueueFields.EnqueuedAt) is { } enqueuedRaw
&& long.TryParse(enqueuedRaw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var unix)
? DateTimeOffset.FromUnixTimeMilliseconds(unix)
: now;
var attempt = TryGetHeader(headers, NotifyQueueFields.Attempt) is { } attemptRaw
&& int.TryParse(attemptRaw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedAttempt)
? parsedAttempt
: 1;
if (message.Metadata?.NumDelivered is ulong delivered && delivered > 0)
{
var deliveredInt = delivered > int.MaxValue ? int.MaxValue : (int)delivered;
if (deliveredInt > attempt)
{
attempt = deliveredInt;
}
}
var attributes = ExtractAttributes(headers);
var leaseExpires = now.Add(leaseDuration);
var messageId = message.Metadata?.Sequence.Stream.ToString() ?? Guid.NewGuid().ToString("n");
var queueMessage = new NotifyDeliveryQueueMessage(
delivery,
channelId,
channelType,
_options.Subject,
traceId,
attributes);
return new NatsNotifyDeliveryLease(
this,
message,
messageId,
queueMessage,
attempt,
consumer,
enqueuedAt,
leaseExpires,
idempotencyKey);
}
private NatsHeaders BuildHeaders(NotifyDeliveryQueueMessage message)
{
var headers = new NatsHeaders
{
{ NotifyQueueFields.DeliveryId, message.Delivery.DeliveryId },
{ NotifyQueueFields.ChannelId, message.ChannelId },
{ NotifyQueueFields.ChannelType, message.ChannelType.ToString() },
{ NotifyQueueFields.Tenant, message.Delivery.TenantId },
{ NotifyQueueFields.Attempt, "1" },
{ NotifyQueueFields.EnqueuedAt, _timeProvider.GetUtcNow().ToUnixTimeMilliseconds().ToString(CultureInfo.InvariantCulture) },
{ NotifyQueueFields.IdempotencyKey, message.IdempotencyKey },
{ NotifyQueueFields.PartitionKey, message.PartitionKey }
};
if (!string.IsNullOrWhiteSpace(message.TraceId))
{
headers.Add(NotifyQueueFields.TraceId, message.TraceId!);
}
foreach (var kvp in message.Attributes)
{
headers.Add(NotifyQueueFields.AttributePrefix + kvp.Key, kvp.Value);
}
return headers;
}
private NatsHeaders BuildDeadLetterHeaders(NatsNotifyDeliveryLease lease, string reason)
{
var headers = new NatsHeaders
{
{ NotifyQueueFields.DeliveryId, lease.Message.Delivery.DeliveryId },
{ NotifyQueueFields.ChannelId, lease.Message.ChannelId },
{ NotifyQueueFields.ChannelType, lease.Message.ChannelType.ToString() },
{ NotifyQueueFields.Tenant, lease.Message.Delivery.TenantId },
{ NotifyQueueFields.Attempt, lease.Attempt.ToString(CultureInfo.InvariantCulture) },
{ NotifyQueueFields.IdempotencyKey, lease.Message.IdempotencyKey },
{ "deadletter-reason", reason }
};
if (!string.IsNullOrWhiteSpace(lease.Message.TraceId))
{
headers.Add(NotifyQueueFields.TraceId, lease.Message.TraceId!);
}
foreach (var kvp in lease.Message.Attributes)
{
headers.Add(NotifyQueueFields.AttributePrefix + kvp.Key, kvp.Value);
}
return headers;
}
private static string? TryGetHeader(NatsHeaders headers, string key)
{
if (headers.TryGetValue(key, out var values) && values.Count > 0)
{
var value = values[0];
return string.IsNullOrWhiteSpace(value) ? null : value;
}
return null;
}
private static IReadOnlyDictionary<string, string> ExtractAttributes(NatsHeaders headers)
{
var attributes = new Dictionary<string, string>(StringComparer.Ordinal);
foreach (var key in headers.Keys)
{
if (!key.StartsWith(NotifyQueueFields.AttributePrefix, StringComparison.Ordinal))
{
continue;
}
if (headers.TryGetValue(key, out var values) && values.Count > 0)
{
attributes[key[NotifyQueueFields.AttributePrefix.Length..]] = values[0]!;
}
}
return attributes.Count == 0
? EmptyReadOnlyDictionary<string, string>.Instance
: new ReadOnlyDictionary<string, string>(attributes);
}
private TimeSpan CalculateBackoff(int attempt)
{
var initial = _queueOptions.RetryInitialBackoff > TimeSpan.Zero
? _queueOptions.RetryInitialBackoff
: _options.RetryDelay;
if (initial <= TimeSpan.Zero)
{
return TimeSpan.Zero;
}
if (attempt <= 1)
{
return initial;
}
var max = _queueOptions.RetryMaxBackoff > TimeSpan.Zero
? _queueOptions.RetryMaxBackoff
: initial;
var exponent = attempt - 1;
var scaledTicks = initial.Ticks * Math.Pow(2, exponent - 1);
var cappedTicks = Math.Min(max.Ticks, scaledTicks);
var resultTicks = Math.Max(initial.Ticks, (long)cappedTicks);
return TimeSpan.FromTicks(resultTicks);
}
private static long ToNanoseconds(TimeSpan value)
=> value <= TimeSpan.Zero ? 0 : value.Ticks * 100L;
private static class EmptyReadOnlyDictionary<TKey, TValue>
where TKey : notnull
{
public static readonly IReadOnlyDictionary<TKey, TValue> Instance =
new ReadOnlyDictionary<TKey, TValue>(new Dictionary<TKey, TValue>(0, EqualityComparer<TKey>.Default));
}
}

View File

@@ -0,0 +1,83 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using NATS.Client.JetStream;
namespace StellaOps.Notify.Queue.Nats;
internal sealed class NatsNotifyEventLease : INotifyQueueLease<NotifyQueueEventMessage>
{
private readonly NatsNotifyEventQueue _queue;
private readonly NatsJSMsg<byte[]> _message;
private int _completed;
internal NatsNotifyEventLease(
NatsNotifyEventQueue queue,
NatsJSMsg<byte[]> message,
string messageId,
NotifyQueueEventMessage payload,
int attempt,
string consumer,
DateTimeOffset enqueuedAt,
DateTimeOffset leaseExpiresAt)
{
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
if (EqualityComparer<NatsJSMsg<byte[]>>.Default.Equals(message, default))
{
throw new ArgumentException("Message must be provided.", nameof(message));
}
_message = message;
MessageId = messageId ?? throw new ArgumentNullException(nameof(messageId));
Message = payload ?? throw new ArgumentNullException(nameof(payload));
Attempt = attempt;
Consumer = consumer ?? throw new ArgumentNullException(nameof(consumer));
EnqueuedAt = enqueuedAt;
LeaseExpiresAt = leaseExpiresAt;
}
public string MessageId { get; }
public int Attempt { get; internal set; }
public DateTimeOffset EnqueuedAt { get; }
public DateTimeOffset LeaseExpiresAt { get; private set; }
public string Consumer { get; }
public string Stream => Message.Stream;
public string TenantId => Message.TenantId;
public string? PartitionKey => Message.PartitionKey;
public string IdempotencyKey => Message.IdempotencyKey;
public string? TraceId => Message.TraceId;
public IReadOnlyDictionary<string, string> Attributes => Message.Attributes;
public NotifyQueueEventMessage Message { get; }
internal NatsJSMsg<byte[]> RawMessage => _message;
public Task AcknowledgeAsync(CancellationToken cancellationToken = default)
=> _queue.AcknowledgeAsync(this, cancellationToken);
public Task RenewAsync(TimeSpan leaseDuration, CancellationToken cancellationToken = default)
=> _queue.RenewLeaseAsync(this, leaseDuration, cancellationToken);
public Task ReleaseAsync(NotifyQueueReleaseDisposition disposition, CancellationToken cancellationToken = default)
=> _queue.ReleaseAsync(this, disposition, cancellationToken);
public Task DeadLetterAsync(string reason, CancellationToken cancellationToken = default)
=> _queue.DeadLetterAsync(this, reason, cancellationToken);
internal bool TryBeginCompletion()
=> Interlocked.CompareExchange(ref _completed, 1, 0) == 0;
internal void RefreshLease(DateTimeOffset expiresAt)
=> LeaseExpiresAt = expiresAt;
}

View File

@@ -0,0 +1,698 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Globalization;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using NATS.Client.Core;
using NATS.Client.JetStream;
using NATS.Client.JetStream.Models;
using StellaOps.Notify.Models;
namespace StellaOps.Notify.Queue.Nats;
internal sealed class NatsNotifyEventQueue : INotifyEventQueue, IAsyncDisposable
{
private const string TransportName = "nats";
private static readonly INatsSerializer<byte[]> PayloadSerializer = NatsRawSerializer<byte[]>.Default;
private readonly NotifyEventQueueOptions _queueOptions;
private readonly NotifyNatsEventQueueOptions _options;
private readonly ILogger<NatsNotifyEventQueue> _logger;
private readonly TimeProvider _timeProvider;
private readonly SemaphoreSlim _connectionGate = new(1, 1);
private readonly Func<NatsOpts, CancellationToken, ValueTask<NatsConnection>> _connectionFactory;
private NatsConnection? _connection;
private NatsJSContext? _jsContext;
private INatsJSConsumer? _consumer;
private bool _disposed;
public NatsNotifyEventQueue(
NotifyEventQueueOptions queueOptions,
NotifyNatsEventQueueOptions options,
ILogger<NatsNotifyEventQueue> logger,
TimeProvider timeProvider,
Func<NatsOpts, CancellationToken, ValueTask<NatsConnection>>? connectionFactory = null)
{
_queueOptions = queueOptions ?? throw new ArgumentNullException(nameof(queueOptions));
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
_connectionFactory = connectionFactory ?? ((opts, cancellationToken) => new ValueTask<NatsConnection>(new NatsConnection(opts)));
if (string.IsNullOrWhiteSpace(_options.Url))
{
throw new InvalidOperationException("NATS connection URL must be configured for the Notify event queue.");
}
if (string.IsNullOrWhiteSpace(_options.Stream) || string.IsNullOrWhiteSpace(_options.Subject))
{
throw new InvalidOperationException("NATS stream and subject must be configured for the Notify event queue.");
}
}
public async ValueTask<NotifyQueueEnqueueResult> PublishAsync(
NotifyQueueEventMessage message,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(message);
var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false);
await EnsureStreamAndConsumerAsync(js, cancellationToken).ConfigureAwait(false);
await EnsureDeadLetterStreamAsync(js, cancellationToken).ConfigureAwait(false);
var idempotencyKey = string.IsNullOrWhiteSpace(message.IdempotencyKey)
? message.Event.EventId.ToString("N")
: message.IdempotencyKey;
var payload = Encoding.UTF8.GetBytes(NotifyCanonicalJsonSerializer.Serialize(message.Event));
var headers = BuildHeaders(message, idempotencyKey);
var publishOpts = new NatsJSPubOpts
{
MsgId = idempotencyKey,
RetryAttempts = 0
};
var ack = await js.PublishAsync(
_options.Subject,
payload,
PayloadSerializer,
publishOpts,
headers,
cancellationToken)
.ConfigureAwait(false);
if (ack.Duplicate)
{
_logger.LogDebug(
"Duplicate Notify event enqueue detected for idempotency token {Token}.",
idempotencyKey);
NotifyQueueMetrics.RecordDeduplicated(TransportName, _options.Stream);
return new NotifyQueueEnqueueResult(ack.Seq.ToString(), true);
}
NotifyQueueMetrics.RecordEnqueued(TransportName, _options.Stream);
_logger.LogDebug(
"Enqueued Notify event {EventId} into NATS stream {Stream} (sequence {Sequence}).",
message.Event.EventId,
ack.Stream,
ack.Seq);
return new NotifyQueueEnqueueResult(ack.Seq.ToString(), false);
}
public async ValueTask<IReadOnlyList<INotifyQueueLease<NotifyQueueEventMessage>>> LeaseAsync(
NotifyQueueLeaseRequest request,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(request);
var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false);
var consumer = await EnsureStreamAndConsumerAsync(js, cancellationToken).ConfigureAwait(false);
var fetchOpts = new NatsJSFetchOpts
{
MaxMsgs = request.BatchSize,
Expires = request.LeaseDuration,
IdleHeartbeat = _options.IdleHeartbeat
};
var now = _timeProvider.GetUtcNow();
var leases = new List<INotifyQueueLease<NotifyQueueEventMessage>>(request.BatchSize);
await foreach (var msg in consumer.FetchAsync(PayloadSerializer, fetchOpts, cancellationToken).ConfigureAwait(false))
{
var lease = CreateLease(msg, request.Consumer, now, request.LeaseDuration);
if (lease is null)
{
await msg.AckAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
continue;
}
leases.Add(lease);
}
return leases;
}
public async ValueTask<IReadOnlyList<INotifyQueueLease<NotifyQueueEventMessage>>> ClaimExpiredAsync(
NotifyQueueClaimOptions options,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(options);
var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false);
var consumer = await EnsureStreamAndConsumerAsync(js, cancellationToken).ConfigureAwait(false);
var fetchOpts = new NatsJSFetchOpts
{
MaxMsgs = options.BatchSize,
Expires = options.MinIdleTime,
IdleHeartbeat = _options.IdleHeartbeat
};
var now = _timeProvider.GetUtcNow();
var leases = new List<INotifyQueueLease<NotifyQueueEventMessage>>(options.BatchSize);
await foreach (var msg in consumer.FetchAsync(PayloadSerializer, fetchOpts, cancellationToken).ConfigureAwait(false))
{
var deliveries = (int)(msg.Metadata?.NumDelivered ?? 1);
if (deliveries <= 1)
{
await msg.NakAsync(new AckOpts(), TimeSpan.Zero, cancellationToken).ConfigureAwait(false);
continue;
}
var lease = CreateLease(msg, options.ClaimantConsumer, now, _queueOptions.DefaultLeaseDuration);
if (lease is null)
{
await msg.AckAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
continue;
}
leases.Add(lease);
}
return leases;
}
public async ValueTask DisposeAsync()
{
if (_disposed)
{
return;
}
_disposed = true;
if (_connection is not null)
{
await _connection.DisposeAsync().ConfigureAwait(false);
}
_connectionGate.Dispose();
GC.SuppressFinalize(this);
}
internal async Task AcknowledgeAsync(
NatsNotifyEventLease lease,
CancellationToken cancellationToken)
{
if (!lease.TryBeginCompletion())
{
return;
}
await lease.RawMessage.AckAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
NotifyQueueMetrics.RecordAck(TransportName, _options.Stream);
_logger.LogDebug(
"Acknowledged Notify event {EventId} (sequence {Sequence}).",
lease.Message.Event.EventId,
lease.MessageId);
}
internal async Task RenewLeaseAsync(
NatsNotifyEventLease lease,
TimeSpan leaseDuration,
CancellationToken cancellationToken)
{
await lease.RawMessage.AckProgressAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
var expires = _timeProvider.GetUtcNow().Add(leaseDuration);
lease.RefreshLease(expires);
_logger.LogDebug(
"Renewed NATS lease for Notify event {EventId} until {Expires:u}.",
lease.Message.Event.EventId,
expires);
}
internal async Task ReleaseAsync(
NatsNotifyEventLease lease,
NotifyQueueReleaseDisposition disposition,
CancellationToken cancellationToken)
{
if (disposition == NotifyQueueReleaseDisposition.Retry
&& lease.Attempt >= _queueOptions.MaxDeliveryAttempts)
{
_logger.LogWarning(
"Notify event {EventId} reached max delivery attempts ({Attempts}); moving to dead-letter stream.",
lease.Message.Event.EventId,
lease.Attempt);
await DeadLetterAsync(
lease,
$"max-delivery-attempts:{lease.Attempt}",
cancellationToken).ConfigureAwait(false);
return;
}
if (!lease.TryBeginCompletion())
{
return;
}
if (disposition == NotifyQueueReleaseDisposition.Retry)
{
var delay = CalculateBackoff(lease.Attempt);
await lease.RawMessage.NakAsync(new AckOpts(), delay, cancellationToken).ConfigureAwait(false);
NotifyQueueMetrics.RecordRetry(TransportName, _options.Stream);
_logger.LogInformation(
"Scheduled Notify event {EventId} for retry with delay {Delay} (attempt {Attempt}).",
lease.Message.Event.EventId,
delay,
lease.Attempt);
}
else
{
await lease.RawMessage.AckTerminateAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
NotifyQueueMetrics.RecordAck(TransportName, _options.Stream);
_logger.LogInformation(
"Abandoned Notify event {EventId} after {Attempt} attempt(s).",
lease.Message.Event.EventId,
lease.Attempt);
}
}
internal async Task DeadLetterAsync(
NatsNotifyEventLease lease,
string reason,
CancellationToken cancellationToken)
{
if (!lease.TryBeginCompletion())
{
return;
}
await lease.RawMessage.AckTerminateAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false);
await EnsureDeadLetterStreamAsync(js, cancellationToken).ConfigureAwait(false);
var headers = BuildDeadLetterHeaders(lease, reason);
var payload = Encoding.UTF8.GetBytes(NotifyCanonicalJsonSerializer.Serialize(lease.Message.Event));
await js.PublishAsync(
_options.DeadLetterSubject,
payload,
PayloadSerializer,
new NatsJSPubOpts(),
headers,
cancellationToken)
.ConfigureAwait(false);
NotifyQueueMetrics.RecordDeadLetter(TransportName, _options.DeadLetterStream);
_logger.LogError(
"Dead-lettered Notify event {EventId} (attempt {Attempt}): {Reason}",
lease.Message.Event.EventId,
lease.Attempt,
reason);
}
internal async Task PingAsync(CancellationToken cancellationToken)
{
var connection = await EnsureConnectionAsync(cancellationToken).ConfigureAwait(false);
await connection.PingAsync(cancellationToken).ConfigureAwait(false);
}
private async Task<NatsJSContext> GetJetStreamAsync(CancellationToken cancellationToken)
{
if (_jsContext is not null)
{
return _jsContext;
}
var connection = await EnsureConnectionAsync(cancellationToken).ConfigureAwait(false);
await _connectionGate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
_jsContext ??= new NatsJSContext(connection);
return _jsContext;
}
finally
{
_connectionGate.Release();
}
}
private async ValueTask<INatsJSConsumer> EnsureStreamAndConsumerAsync(
NatsJSContext js,
CancellationToken cancellationToken)
{
if (_consumer is not null)
{
return _consumer;
}
await _connectionGate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_consumer is not null)
{
return _consumer;
}
await EnsureStreamAsync(js, cancellationToken).ConfigureAwait(false);
await EnsureDeadLetterStreamAsync(js, cancellationToken).ConfigureAwait(false);
var consumerConfig = new ConsumerConfig
{
DurableName = _options.DurableConsumer,
AckPolicy = ConsumerConfigAckPolicy.Explicit,
ReplayPolicy = ConsumerConfigReplayPolicy.Instant,
DeliverPolicy = ConsumerConfigDeliverPolicy.All,
AckWait = ToNanoseconds(_options.AckWait),
MaxAckPending = _options.MaxAckPending,
MaxDeliver = Math.Max(1, _queueOptions.MaxDeliveryAttempts),
FilterSubjects = new[] { _options.Subject }
};
try
{
_consumer = await js.CreateConsumerAsync(
_options.Stream,
consumerConfig,
cancellationToken)
.ConfigureAwait(false);
}
catch (NatsJSApiException apiEx)
{
_logger.LogDebug(
apiEx,
"CreateConsumerAsync failed with code {Code}; attempting to fetch existing durable consumer {Durable}.",
apiEx.Error?.Code,
_options.DurableConsumer);
_consumer = await js.GetConsumerAsync(
_options.Stream,
_options.DurableConsumer,
cancellationToken)
.ConfigureAwait(false);
}
return _consumer;
}
finally
{
_connectionGate.Release();
}
}
private async Task<NatsConnection> EnsureConnectionAsync(CancellationToken cancellationToken)
{
if (_connection is not null)
{
return _connection;
}
await _connectionGate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_connection is not null)
{
return _connection;
}
var opts = new NatsOpts
{
Url = _options.Url!,
Name = "stellaops-notify-queue",
CommandTimeout = TimeSpan.FromSeconds(10),
RequestTimeout = TimeSpan.FromSeconds(20),
PingInterval = TimeSpan.FromSeconds(30)
};
_connection = await _connectionFactory(opts, cancellationToken).ConfigureAwait(false);
await _connection.ConnectAsync().ConfigureAwait(false);
return _connection;
}
finally
{
_connectionGate.Release();
}
}
private async Task EnsureStreamAsync(NatsJSContext js, CancellationToken cancellationToken)
{
try
{
await js.GetStreamAsync(_options.Stream, cancellationToken: cancellationToken).ConfigureAwait(false);
}
catch (NatsJSApiException ex) when (ex.Error?.Code == 404)
{
var config = new StreamConfig(name: _options.Stream, subjects: new[] { _options.Subject })
{
Retention = StreamConfigRetention.Workqueue,
Storage = StreamConfigStorage.File,
MaxConsumers = -1,
MaxMsgs = -1,
MaxBytes = -1
};
await js.CreateStreamAsync(config, cancellationToken).ConfigureAwait(false);
_logger.LogInformation("Created NATS Notify stream {Stream} ({Subject}).", _options.Stream, _options.Subject);
}
}
private async Task EnsureDeadLetterStreamAsync(NatsJSContext js, CancellationToken cancellationToken)
{
try
{
await js.GetStreamAsync(_options.DeadLetterStream, cancellationToken: cancellationToken).ConfigureAwait(false);
}
catch (NatsJSApiException ex) when (ex.Error?.Code == 404)
{
var config = new StreamConfig(name: _options.DeadLetterStream, subjects: new[] { _options.DeadLetterSubject })
{
Retention = StreamConfigRetention.Workqueue,
Storage = StreamConfigStorage.File,
MaxConsumers = -1,
MaxMsgs = -1,
MaxBytes = -1
};
await js.CreateStreamAsync(config, cancellationToken).ConfigureAwait(false);
_logger.LogInformation("Created NATS Notify dead-letter stream {Stream} ({Subject}).", _options.DeadLetterStream, _options.DeadLetterSubject);
}
}
private NatsNotifyEventLease? CreateLease(
NatsJSMsg<byte[]> message,
string consumer,
DateTimeOffset now,
TimeSpan leaseDuration)
{
var payloadBytes = message.Data ?? Array.Empty<byte>();
if (payloadBytes.Length == 0)
{
return null;
}
NotifyEvent notifyEvent;
try
{
var json = Encoding.UTF8.GetString(payloadBytes);
notifyEvent = NotifyCanonicalJsonSerializer.Deserialize<NotifyEvent>(json);
}
catch (Exception ex)
{
_logger.LogWarning(
ex,
"Failed to deserialize Notify event payload for NATS message {Sequence}.",
message.Metadata?.Sequence.Stream);
return null;
}
var headers = message.Headers ?? new NatsHeaders();
var idempotencyKey = TryGetHeader(headers, NotifyQueueFields.IdempotencyKey)
?? notifyEvent.EventId.ToString("N");
var partitionKey = TryGetHeader(headers, NotifyQueueFields.PartitionKey);
var traceId = TryGetHeader(headers, NotifyQueueFields.TraceId);
var enqueuedAt = TryGetHeader(headers, NotifyQueueFields.EnqueuedAt) is { } enqueuedRaw
&& long.TryParse(enqueuedRaw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var unix)
? DateTimeOffset.FromUnixTimeMilliseconds(unix)
: now;
var attempt = TryGetHeader(headers, NotifyQueueFields.Attempt) is { } attemptRaw
&& int.TryParse(attemptRaw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedAttempt)
? parsedAttempt
: 1;
if (message.Metadata?.NumDelivered is ulong delivered && delivered > 0)
{
var deliveredInt = delivered > int.MaxValue ? int.MaxValue : (int)delivered;
if (deliveredInt > attempt)
{
attempt = deliveredInt;
}
}
var attributes = ExtractAttributes(headers);
var leaseExpires = now.Add(leaseDuration);
var messageId = message.Metadata?.Sequence.Stream.ToString() ?? Guid.NewGuid().ToString("n");
var queueMessage = new NotifyQueueEventMessage(
notifyEvent,
_options.Subject,
idempotencyKey,
partitionKey,
traceId,
attributes);
return new NatsNotifyEventLease(
this,
message,
messageId,
queueMessage,
attempt,
consumer,
enqueuedAt,
leaseExpires);
}
private NatsHeaders BuildHeaders(NotifyQueueEventMessage message, string idempotencyKey)
{
var headers = new NatsHeaders
{
{ NotifyQueueFields.EventId, message.Event.EventId.ToString("D") },
{ NotifyQueueFields.Tenant, message.TenantId },
{ NotifyQueueFields.Kind, message.Event.Kind },
{ NotifyQueueFields.Attempt, "1" },
{ NotifyQueueFields.EnqueuedAt, _timeProvider.GetUtcNow().ToUnixTimeMilliseconds().ToString(CultureInfo.InvariantCulture) },
{ NotifyQueueFields.IdempotencyKey, idempotencyKey }
};
if (!string.IsNullOrWhiteSpace(message.TraceId))
{
headers.Add(NotifyQueueFields.TraceId, message.TraceId!);
}
if (!string.IsNullOrWhiteSpace(message.PartitionKey))
{
headers.Add(NotifyQueueFields.PartitionKey, message.PartitionKey!);
}
foreach (var kvp in message.Attributes)
{
headers.Add(NotifyQueueFields.AttributePrefix + kvp.Key, kvp.Value);
}
return headers;
}
private NatsHeaders BuildDeadLetterHeaders(NatsNotifyEventLease lease, string reason)
{
var headers = new NatsHeaders
{
{ NotifyQueueFields.EventId, lease.Message.Event.EventId.ToString("D") },
{ NotifyQueueFields.Tenant, lease.Message.TenantId },
{ NotifyQueueFields.Kind, lease.Message.Event.Kind },
{ NotifyQueueFields.Attempt, lease.Attempt.ToString(CultureInfo.InvariantCulture) },
{ NotifyQueueFields.IdempotencyKey, lease.Message.IdempotencyKey },
{ "deadletter-reason", reason }
};
if (!string.IsNullOrWhiteSpace(lease.Message.TraceId))
{
headers.Add(NotifyQueueFields.TraceId, lease.Message.TraceId!);
}
if (!string.IsNullOrWhiteSpace(lease.Message.PartitionKey))
{
headers.Add(NotifyQueueFields.PartitionKey, lease.Message.PartitionKey!);
}
foreach (var kvp in lease.Message.Attributes)
{
headers.Add(NotifyQueueFields.AttributePrefix + kvp.Key, kvp.Value);
}
return headers;
}
private static string? TryGetHeader(NatsHeaders headers, string key)
{
if (headers.TryGetValue(key, out var values) && values.Count > 0)
{
var value = values[0];
return string.IsNullOrWhiteSpace(value) ? null : value;
}
return null;
}
private static IReadOnlyDictionary<string, string> ExtractAttributes(NatsHeaders headers)
{
var attributes = new Dictionary<string, string>(StringComparer.Ordinal);
foreach (var key in headers.Keys)
{
if (!key.StartsWith(NotifyQueueFields.AttributePrefix, StringComparison.Ordinal))
{
continue;
}
if (headers.TryGetValue(key, out var values) && values.Count > 0)
{
attributes[key[NotifyQueueFields.AttributePrefix.Length..]] = values[0]!;
}
}
return attributes.Count == 0
? EmptyReadOnlyDictionary<string, string>.Instance
: new ReadOnlyDictionary<string, string>(attributes);
}
private TimeSpan CalculateBackoff(int attempt)
{
var initial = _queueOptions.RetryInitialBackoff > TimeSpan.Zero
? _queueOptions.RetryInitialBackoff
: _options.RetryDelay;
if (initial <= TimeSpan.Zero)
{
return TimeSpan.Zero;
}
if (attempt <= 1)
{
return initial;
}
var max = _queueOptions.RetryMaxBackoff > TimeSpan.Zero
? _queueOptions.RetryMaxBackoff
: initial;
var exponent = attempt - 1;
var scaledTicks = initial.Ticks * Math.Pow(2, exponent - 1);
var cappedTicks = Math.Min(max.Ticks, scaledTicks);
var resultTicks = Math.Max(initial.Ticks, (long)cappedTicks);
return TimeSpan.FromTicks(resultTicks);
}
private static long ToNanoseconds(TimeSpan value)
=> value <= TimeSpan.Zero ? 0 : value.Ticks * 100L;
private static class EmptyReadOnlyDictionary<TKey, TValue>
where TKey : notnull
{
public static readonly IReadOnlyDictionary<TKey, TValue> Instance =
new ReadOnlyDictionary<TKey, TValue>(new Dictionary<TKey, TValue>(0, EqualityComparer<TKey>.Default));
}
}

View File

@@ -0,0 +1,55 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Logging;
using StellaOps.Notify.Queue.Nats;
using StellaOps.Notify.Queue.Redis;
namespace StellaOps.Notify.Queue;
public sealed class NotifyDeliveryQueueHealthCheck : IHealthCheck
{
private readonly INotifyDeliveryQueue _queue;
private readonly ILogger<NotifyDeliveryQueueHealthCheck> _logger;
public NotifyDeliveryQueueHealthCheck(
INotifyDeliveryQueue queue,
ILogger<NotifyDeliveryQueueHealthCheck> logger)
{
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<HealthCheckResult> CheckHealthAsync(
HealthCheckContext context,
CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
switch (_queue)
{
case RedisNotifyDeliveryQueue redisQueue:
await redisQueue.PingAsync(cancellationToken).ConfigureAwait(false);
return HealthCheckResult.Healthy("Redis Notify delivery queue reachable.");
case NatsNotifyDeliveryQueue natsQueue:
await natsQueue.PingAsync(cancellationToken).ConfigureAwait(false);
return HealthCheckResult.Healthy("NATS Notify delivery queue reachable.");
default:
return HealthCheckResult.Healthy("Notify delivery queue transport without dedicated ping returned healthy.");
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Notify delivery queue health check failed.");
return new HealthCheckResult(
context.Registration.FailureStatus,
"Notify delivery queue transport unreachable.",
ex);
}
}
}

View File

@@ -0,0 +1,69 @@
using System;
namespace StellaOps.Notify.Queue;
/// <summary>
/// Configuration options for the Notify delivery queue abstraction.
/// </summary>
public sealed class NotifyDeliveryQueueOptions
{
public NotifyQueueTransportKind Transport { get; set; } = NotifyQueueTransportKind.Redis;
public NotifyRedisDeliveryQueueOptions Redis { get; set; } = new();
public NotifyNatsDeliveryQueueOptions Nats { get; set; } = new();
public TimeSpan DefaultLeaseDuration { get; set; } = TimeSpan.FromMinutes(5);
public int MaxDeliveryAttempts { get; set; } = 5;
public TimeSpan RetryInitialBackoff { get; set; } = TimeSpan.FromSeconds(5);
public TimeSpan RetryMaxBackoff { get; set; } = TimeSpan.FromMinutes(2);
public TimeSpan ClaimIdleThreshold { get; set; } = TimeSpan.FromMinutes(5);
}
public sealed class NotifyRedisDeliveryQueueOptions
{
public string? ConnectionString { get; set; }
public int? Database { get; set; }
public TimeSpan InitializationTimeout { get; set; } = TimeSpan.FromSeconds(30);
public string StreamName { get; set; } = "notify:deliveries";
public string ConsumerGroup { get; set; } = "notify-deliveries";
public string IdempotencyKeyPrefix { get; set; } = "notify:deliveries:idemp:";
public int? ApproximateMaxLength { get; set; }
public string DeadLetterStreamName { get; set; } = "notify:deliveries:dead";
public TimeSpan DeadLetterRetention { get; set; } = TimeSpan.FromDays(7);
}
public sealed class NotifyNatsDeliveryQueueOptions
{
public string? Url { get; set; }
public string Stream { get; set; } = "NOTIFY_DELIVERIES";
public string Subject { get; set; } = "notify.deliveries";
public string DurableConsumer { get; set; } = "notify-deliveries";
public string DeadLetterStream { get; set; } = "NOTIFY_DELIVERIES_DEAD";
public string DeadLetterSubject { get; set; } = "notify.deliveries.dead";
public int MaxAckPending { get; set; } = 128;
public TimeSpan AckWait { get; set; } = TimeSpan.FromMinutes(5);
public TimeSpan RetryDelay { get; set; } = TimeSpan.FromSeconds(10);
public TimeSpan IdleHeartbeat { get; set; } = TimeSpan.FromSeconds(30);
}

View File

@@ -0,0 +1,177 @@
using System;
using System.Collections.Generic;
namespace StellaOps.Notify.Queue;
/// <summary>
/// Configuration options for the Notify event queue abstraction.
/// </summary>
public sealed class NotifyEventQueueOptions
{
/// <summary>
/// Transport backing the queue.
/// </summary>
public NotifyQueueTransportKind Transport { get; set; } = NotifyQueueTransportKind.Redis;
/// <summary>
/// Redis-specific configuration.
/// </summary>
public NotifyRedisEventQueueOptions Redis { get; set; } = new();
/// <summary>
/// NATS JetStream-specific configuration.
/// </summary>
public NotifyNatsEventQueueOptions Nats { get; set; } = new();
/// <summary>
/// Default lease duration to use when consumers do not specify one explicitly.
/// </summary>
public TimeSpan DefaultLeaseDuration { get; set; } = TimeSpan.FromMinutes(5);
/// <summary>
/// Maximum number of deliveries before a message should be considered failed.
/// </summary>
public int MaxDeliveryAttempts { get; set; } = 5;
/// <summary>
/// Initial retry backoff applied when a message is released for retry.
/// </summary>
public TimeSpan RetryInitialBackoff { get; set; } = TimeSpan.FromSeconds(5);
/// <summary>
/// Cap applied to exponential retry backoff.
/// </summary>
public TimeSpan RetryMaxBackoff { get; set; } = TimeSpan.FromMinutes(2);
/// <summary>
/// Minimum idle window before a pending message becomes eligible for claim.
/// </summary>
public TimeSpan ClaimIdleThreshold { get; set; } = TimeSpan.FromMinutes(5);
}
/// <summary>
/// Redis transport options for the Notify event queue.
/// </summary>
public sealed class NotifyRedisEventQueueOptions
{
private IReadOnlyList<NotifyRedisEventStreamOptions> _streams = new List<NotifyRedisEventStreamOptions>
{
NotifyRedisEventStreamOptions.ForDefaultStream()
};
/// <summary>
/// Connection string for the Redis instance.
/// </summary>
public string? ConnectionString { get; set; }
/// <summary>
/// Optional logical database to select when connecting.
/// </summary>
public int? Database { get; set; }
/// <summary>
/// Time allowed for initial connection/consumer-group creation.
/// </summary>
public TimeSpan InitializationTimeout { get; set; } = TimeSpan.FromSeconds(30);
/// <summary>
/// TTL applied to idempotency keys stored alongside events.
/// </summary>
public TimeSpan IdempotencyWindow { get; set; } = TimeSpan.FromHours(12);
/// <summary>
/// Streams consumed by Notify. Ordering is preserved during leasing.
/// </summary>
public IReadOnlyList<NotifyRedisEventStreamOptions> Streams
{
get => _streams;
set => _streams = value is null || value.Count == 0
? new List<NotifyRedisEventStreamOptions> { NotifyRedisEventStreamOptions.ForDefaultStream() }
: value;
}
}
/// <summary>
/// Per-Redis-stream options for the Notify event queue.
/// </summary>
public sealed class NotifyRedisEventStreamOptions
{
/// <summary>
/// Name of the Redis stream containing events.
/// </summary>
public string Stream { get; set; } = "notify:events";
/// <summary>
/// Consumer group used by Notify workers.
/// </summary>
public string ConsumerGroup { get; set; } = "notify-workers";
/// <summary>
/// Prefix used when storing idempotency keys in Redis.
/// </summary>
public string IdempotencyKeyPrefix { get; set; } = "notify:events:idemp:";
/// <summary>
/// Approximate maximum length for the stream; when set Redis will trim entries.
/// </summary>
public int? ApproximateMaxLength { get; set; }
public static NotifyRedisEventStreamOptions ForDefaultStream()
=> new();
}
/// <summary>
/// NATS JetStream options for the Notify event queue.
/// </summary>
public sealed class NotifyNatsEventQueueOptions
{
/// <summary>
/// URL for the JetStream-enabled NATS cluster.
/// </summary>
public string? Url { get; set; }
/// <summary>
/// Stream name carrying Notify events.
/// </summary>
public string Stream { get; set; } = "NOTIFY_EVENTS";
/// <summary>
/// Subject that producers publish Notify events to.
/// </summary>
public string Subject { get; set; } = "notify.events";
/// <summary>
/// Durable consumer identifier for Notify workers.
/// </summary>
public string DurableConsumer { get; set; } = "notify-workers";
/// <summary>
/// Dead-letter stream name used when deliveries exhaust retry budget.
/// </summary>
public string DeadLetterStream { get; set; } = "NOTIFY_EVENTS_DEAD";
/// <summary>
/// Subject used for dead-letter publications.
/// </summary>
public string DeadLetterSubject { get; set; } = "notify.events.dead";
/// <summary>
/// Maximum pending messages before backpressure is applied.
/// </summary>
public int MaxAckPending { get; set; } = 256;
/// <summary>
/// Visibility timeout applied to leased events.
/// </summary>
public TimeSpan AckWait { get; set; } = TimeSpan.FromMinutes(5);
/// <summary>
/// Delay applied when releasing a message for retry.
/// </summary>
public TimeSpan RetryDelay { get; set; } = TimeSpan.FromSeconds(10);
/// <summary>
/// Idle heartbeat emitted by the server to detect consumer disconnects.
/// </summary>
public TimeSpan IdleHeartbeat { get; set; } = TimeSpan.FromSeconds(30);
}

View File

@@ -0,0 +1,231 @@
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Notify.Models;
namespace StellaOps.Notify.Queue;
/// <summary>
/// Message queued for Notify event processing.
/// </summary>
public sealed class NotifyQueueEventMessage
{
private readonly NotifyEvent _event;
private readonly IReadOnlyDictionary<string, string> _attributes;
public NotifyQueueEventMessage(
NotifyEvent @event,
string stream,
string? idempotencyKey = null,
string? partitionKey = null,
string? traceId = null,
IReadOnlyDictionary<string, string>? attributes = null)
{
_event = @event ?? throw new ArgumentNullException(nameof(@event));
if (string.IsNullOrWhiteSpace(stream))
{
throw new ArgumentException("Stream must be provided.", nameof(stream));
}
Stream = stream;
IdempotencyKey = string.IsNullOrWhiteSpace(idempotencyKey)
? @event.EventId.ToString("N")
: idempotencyKey!;
PartitionKey = string.IsNullOrWhiteSpace(partitionKey) ? null : partitionKey.Trim();
TraceId = string.IsNullOrWhiteSpace(traceId) ? null : traceId.Trim();
_attributes = attributes is null
? EmptyReadOnlyDictionary<string, string>.Instance
: new ReadOnlyDictionary<string, string>(new Dictionary<string, string>(attributes, StringComparer.Ordinal));
}
public NotifyEvent Event => _event;
public string Stream { get; }
public string IdempotencyKey { get; }
public string TenantId => _event.Tenant;
public string? PartitionKey { get; }
public string? TraceId { get; }
public IReadOnlyDictionary<string, string> Attributes => _attributes;
}
/// <summary>
/// Message queued for channel delivery execution.
/// </summary>
public sealed class NotifyDeliveryQueueMessage
{
public const string DefaultStream = "notify:deliveries";
private readonly IReadOnlyDictionary<string, string> _attributes;
public NotifyDeliveryQueueMessage(
NotifyDelivery delivery,
string channelId,
NotifyChannelType channelType,
string? stream = null,
string? traceId = null,
IReadOnlyDictionary<string, string>? attributes = null)
{
Delivery = delivery ?? throw new ArgumentNullException(nameof(delivery));
ChannelId = NotifyValidation.EnsureNotNullOrWhiteSpace(channelId, nameof(channelId));
ChannelType = channelType;
Stream = string.IsNullOrWhiteSpace(stream) ? DefaultStream : stream!.Trim();
TraceId = string.IsNullOrWhiteSpace(traceId) ? null : traceId.Trim();
_attributes = attributes is null
? EmptyReadOnlyDictionary<string, string>.Instance
: new ReadOnlyDictionary<string, string>(new Dictionary<string, string>(attributes, StringComparer.Ordinal));
}
public NotifyDelivery Delivery { get; }
public string ChannelId { get; }
public NotifyChannelType ChannelType { get; }
public string Stream { get; }
public string? TraceId { get; }
public string TenantId => Delivery.TenantId;
public string IdempotencyKey => Delivery.DeliveryId;
public string PartitionKey => ChannelId;
public IReadOnlyDictionary<string, string> Attributes => _attributes;
}
public readonly record struct NotifyQueueEnqueueResult(string MessageId, bool Deduplicated);
public sealed class NotifyQueueLeaseRequest
{
public NotifyQueueLeaseRequest(string consumer, int batchSize, TimeSpan leaseDuration)
{
if (string.IsNullOrWhiteSpace(consumer))
{
throw new ArgumentException("Consumer must be provided.", nameof(consumer));
}
if (batchSize <= 0)
{
throw new ArgumentOutOfRangeException(nameof(batchSize), batchSize, "Batch size must be positive.");
}
if (leaseDuration <= TimeSpan.Zero)
{
throw new ArgumentOutOfRangeException(nameof(leaseDuration), leaseDuration, "Lease duration must be positive.");
}
Consumer = consumer;
BatchSize = batchSize;
LeaseDuration = leaseDuration;
}
public string Consumer { get; }
public int BatchSize { get; }
public TimeSpan LeaseDuration { get; }
}
public sealed class NotifyQueueClaimOptions
{
public NotifyQueueClaimOptions(string claimantConsumer, int batchSize, TimeSpan minIdleTime)
{
if (string.IsNullOrWhiteSpace(claimantConsumer))
{
throw new ArgumentException("Consumer must be provided.", nameof(claimantConsumer));
}
if (batchSize <= 0)
{
throw new ArgumentOutOfRangeException(nameof(batchSize), batchSize, "Batch size must be positive.");
}
if (minIdleTime < TimeSpan.Zero)
{
throw new ArgumentOutOfRangeException(nameof(minIdleTime), minIdleTime, "Minimum idle time cannot be negative.");
}
ClaimantConsumer = claimantConsumer;
BatchSize = batchSize;
MinIdleTime = minIdleTime;
}
public string ClaimantConsumer { get; }
public int BatchSize { get; }
public TimeSpan MinIdleTime { get; }
}
public enum NotifyQueueReleaseDisposition
{
Retry,
Abandon
}
public interface INotifyQueue<TMessage>
{
ValueTask<NotifyQueueEnqueueResult> PublishAsync(TMessage message, CancellationToken cancellationToken = default);
ValueTask<IReadOnlyList<INotifyQueueLease<TMessage>>> LeaseAsync(NotifyQueueLeaseRequest request, CancellationToken cancellationToken = default);
ValueTask<IReadOnlyList<INotifyQueueLease<TMessage>>> ClaimExpiredAsync(NotifyQueueClaimOptions options, CancellationToken cancellationToken = default);
}
public interface INotifyQueueLease<out TMessage>
{
string MessageId { get; }
int Attempt { get; }
DateTimeOffset EnqueuedAt { get; }
DateTimeOffset LeaseExpiresAt { get; }
string Consumer { get; }
string Stream { get; }
string TenantId { get; }
string? PartitionKey { get; }
string IdempotencyKey { get; }
string? TraceId { get; }
IReadOnlyDictionary<string, string> Attributes { get; }
TMessage Message { get; }
Task AcknowledgeAsync(CancellationToken cancellationToken = default);
Task RenewAsync(TimeSpan leaseDuration, CancellationToken cancellationToken = default);
Task ReleaseAsync(NotifyQueueReleaseDisposition disposition, CancellationToken cancellationToken = default);
Task DeadLetterAsync(string reason, CancellationToken cancellationToken = default);
}
public interface INotifyEventQueue : INotifyQueue<NotifyQueueEventMessage>
{
}
public interface INotifyDeliveryQueue : INotifyQueue<NotifyDeliveryQueueMessage>
{
}
internal static class EmptyReadOnlyDictionary<TKey, TValue>
where TKey : notnull
{
public static readonly IReadOnlyDictionary<TKey, TValue> Instance =
new ReadOnlyDictionary<TKey, TValue>(new Dictionary<TKey, TValue>(0, EqualityComparer<TKey>.Default));
}

View File

@@ -0,0 +1,18 @@
namespace StellaOps.Notify.Queue;
internal static class NotifyQueueFields
{
public const string Payload = "payload";
public const string EventId = "eventId";
public const string DeliveryId = "deliveryId";
public const string Tenant = "tenant";
public const string Kind = "kind";
public const string Attempt = "attempt";
public const string EnqueuedAt = "enqueuedAt";
public const string TraceId = "traceId";
public const string PartitionKey = "partitionKey";
public const string ChannelId = "channelId";
public const string ChannelType = "channelType";
public const string IdempotencyKey = "idempotency";
public const string AttributePrefix = "attr:";
}

View File

@@ -0,0 +1,55 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Logging;
using StellaOps.Notify.Queue.Nats;
using StellaOps.Notify.Queue.Redis;
namespace StellaOps.Notify.Queue;
public sealed class NotifyQueueHealthCheck : IHealthCheck
{
private readonly INotifyEventQueue _queue;
private readonly ILogger<NotifyQueueHealthCheck> _logger;
public NotifyQueueHealthCheck(
INotifyEventQueue queue,
ILogger<NotifyQueueHealthCheck> logger)
{
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<HealthCheckResult> CheckHealthAsync(
HealthCheckContext context,
CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
switch (_queue)
{
case RedisNotifyEventQueue redisQueue:
await redisQueue.PingAsync(cancellationToken).ConfigureAwait(false);
return HealthCheckResult.Healthy("Redis Notify queue reachable.");
case NatsNotifyEventQueue natsQueue:
await natsQueue.PingAsync(cancellationToken).ConfigureAwait(false);
return HealthCheckResult.Healthy("NATS Notify queue reachable.");
default:
return HealthCheckResult.Healthy("Notify queue transport without dedicated ping returned healthy.");
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Notify queue health check failed.");
return new HealthCheckResult(
context.Registration.FailureStatus,
"Notify queue transport unreachable.",
ex);
}
}
}

View File

@@ -0,0 +1,39 @@
using System.Collections.Generic;
using System.Diagnostics.Metrics;
namespace StellaOps.Notify.Queue;
internal static class NotifyQueueMetrics
{
private const string TransportTag = "transport";
private const string StreamTag = "stream";
private static readonly Meter Meter = new("StellaOps.Notify.Queue");
private static readonly Counter<long> EnqueuedCounter = Meter.CreateCounter<long>("notify_queue_enqueued_total");
private static readonly Counter<long> DeduplicatedCounter = Meter.CreateCounter<long>("notify_queue_deduplicated_total");
private static readonly Counter<long> AckCounter = Meter.CreateCounter<long>("notify_queue_ack_total");
private static readonly Counter<long> RetryCounter = Meter.CreateCounter<long>("notify_queue_retry_total");
private static readonly Counter<long> DeadLetterCounter = Meter.CreateCounter<long>("notify_queue_deadletter_total");
public static void RecordEnqueued(string transport, string stream)
=> EnqueuedCounter.Add(1, BuildTags(transport, stream));
public static void RecordDeduplicated(string transport, string stream)
=> DeduplicatedCounter.Add(1, BuildTags(transport, stream));
public static void RecordAck(string transport, string stream)
=> AckCounter.Add(1, BuildTags(transport, stream));
public static void RecordRetry(string transport, string stream)
=> RetryCounter.Add(1, BuildTags(transport, stream));
public static void RecordDeadLetter(string transport, string stream)
=> DeadLetterCounter.Add(1, BuildTags(transport, stream));
private static KeyValuePair<string, object?>[] BuildTags(string transport, string stream)
=> new[]
{
new KeyValuePair<string, object?>(TransportTag, transport),
new KeyValuePair<string, object?>(StreamTag, stream)
};
}

View File

@@ -0,0 +1,146 @@
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Logging;
using StellaOps.Notify.Queue.Nats;
using StellaOps.Notify.Queue.Redis;
namespace StellaOps.Notify.Queue;
public static class NotifyQueueServiceCollectionExtensions
{
public static IServiceCollection AddNotifyEventQueue(
this IServiceCollection services,
IConfiguration configuration,
string sectionName = "notify:queue")
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
var eventOptions = new NotifyEventQueueOptions();
configuration.GetSection(sectionName).Bind(eventOptions);
services.TryAddSingleton(TimeProvider.System);
services.AddSingleton(eventOptions);
services.AddSingleton<INotifyEventQueue>(sp =>
{
var loggerFactory = sp.GetRequiredService<ILoggerFactory>();
var timeProvider = sp.GetService<TimeProvider>() ?? TimeProvider.System;
var opts = sp.GetRequiredService<NotifyEventQueueOptions>();
return opts.Transport switch
{
NotifyQueueTransportKind.Redis => new RedisNotifyEventQueue(
opts,
opts.Redis,
loggerFactory.CreateLogger<RedisNotifyEventQueue>(),
timeProvider),
NotifyQueueTransportKind.Nats => new NatsNotifyEventQueue(
opts,
opts.Nats,
loggerFactory.CreateLogger<NatsNotifyEventQueue>(),
timeProvider),
_ => throw new InvalidOperationException($"Unsupported Notify queue transport kind '{opts.Transport}'.")
};
});
services.AddSingleton<NotifyQueueHealthCheck>();
return services;
}
public static IServiceCollection AddNotifyDeliveryQueue(
this IServiceCollection services,
IConfiguration configuration,
string sectionName = "notify:deliveryQueue")
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
var deliveryOptions = new NotifyDeliveryQueueOptions();
configuration.GetSection(sectionName).Bind(deliveryOptions);
services.AddSingleton(deliveryOptions);
services.AddSingleton<INotifyDeliveryQueue>(sp =>
{
var loggerFactory = sp.GetRequiredService<ILoggerFactory>();
var timeProvider = sp.GetService<TimeProvider>() ?? TimeProvider.System;
var opts = sp.GetRequiredService<NotifyDeliveryQueueOptions>();
var eventOpts = sp.GetService<NotifyEventQueueOptions>();
ApplyDeliveryFallbacks(opts, eventOpts);
return opts.Transport switch
{
NotifyQueueTransportKind.Redis => new RedisNotifyDeliveryQueue(
opts,
opts.Redis,
loggerFactory.CreateLogger<RedisNotifyDeliveryQueue>(),
timeProvider),
NotifyQueueTransportKind.Nats => new NatsNotifyDeliveryQueue(
opts,
opts.Nats,
loggerFactory.CreateLogger<NatsNotifyDeliveryQueue>(),
timeProvider),
_ => throw new InvalidOperationException($"Unsupported Notify delivery queue transport kind '{opts.Transport}'.")
};
});
services.AddSingleton<NotifyDeliveryQueueHealthCheck>();
return services;
}
public static IHealthChecksBuilder AddNotifyQueueHealthCheck(
this IHealthChecksBuilder builder)
{
ArgumentNullException.ThrowIfNull(builder);
builder.Services.TryAddSingleton<NotifyQueueHealthCheck>();
builder.AddCheck<NotifyQueueHealthCheck>(
name: "notify-queue",
failureStatus: HealthStatus.Unhealthy,
tags: new[] { "notify", "queue" });
return builder;
}
public static IHealthChecksBuilder AddNotifyDeliveryQueueHealthCheck(
this IHealthChecksBuilder builder)
{
ArgumentNullException.ThrowIfNull(builder);
builder.Services.TryAddSingleton<NotifyDeliveryQueueHealthCheck>();
builder.AddCheck<NotifyDeliveryQueueHealthCheck>(
name: "notify-delivery-queue",
failureStatus: HealthStatus.Unhealthy,
tags: new[] { "notify", "queue", "delivery" });
return builder;
}
private static void ApplyDeliveryFallbacks(
NotifyDeliveryQueueOptions deliveryOptions,
NotifyEventQueueOptions? eventOptions)
{
if (eventOptions is null)
{
return;
}
if (string.IsNullOrWhiteSpace(deliveryOptions.Redis.ConnectionString))
{
deliveryOptions.Redis.ConnectionString = eventOptions.Redis.ConnectionString;
deliveryOptions.Redis.Database ??= eventOptions.Redis.Database;
}
if (string.IsNullOrWhiteSpace(deliveryOptions.Nats.Url))
{
deliveryOptions.Nats.Url = eventOptions.Nats.Url;
}
}
}

View File

@@ -0,0 +1,10 @@
namespace StellaOps.Notify.Queue;
/// <summary>
/// Supported transports for the Notify event queue.
/// </summary>
public enum NotifyQueueTransportKind
{
Redis,
Nats
}

View File

@@ -0,0 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Notify.Queue.Tests")]

View File

@@ -0,0 +1,76 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Notify.Queue.Redis;
internal sealed class RedisNotifyDeliveryLease : INotifyQueueLease<NotifyDeliveryQueueMessage>
{
private readonly RedisNotifyDeliveryQueue _queue;
private int _completed;
internal RedisNotifyDeliveryLease(
RedisNotifyDeliveryQueue queue,
string messageId,
NotifyDeliveryQueueMessage message,
int attempt,
DateTimeOffset enqueuedAt,
DateTimeOffset leaseExpiresAt,
string consumer,
string? idempotencyKey,
string partitionKey)
{
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
MessageId = messageId ?? throw new ArgumentNullException(nameof(messageId));
Message = message ?? throw new ArgumentNullException(nameof(message));
Attempt = attempt;
EnqueuedAt = enqueuedAt;
LeaseExpiresAt = leaseExpiresAt;
Consumer = consumer ?? throw new ArgumentNullException(nameof(consumer));
IdempotencyKey = idempotencyKey ?? message.IdempotencyKey;
PartitionKey = partitionKey ?? message.ChannelId;
}
public string MessageId { get; }
public int Attempt { get; internal set; }
public DateTimeOffset EnqueuedAt { get; }
public DateTimeOffset LeaseExpiresAt { get; private set; }
public string Consumer { get; }
public string Stream => Message.Stream;
public string TenantId => Message.TenantId;
public string PartitionKey { get; }
public string IdempotencyKey { get; }
public string? TraceId => Message.TraceId;
public IReadOnlyDictionary<string, string> Attributes => Message.Attributes;
public NotifyDeliveryQueueMessage Message { get; }
public Task AcknowledgeAsync(CancellationToken cancellationToken = default)
=> _queue.AcknowledgeAsync(this, cancellationToken);
public Task RenewAsync(TimeSpan leaseDuration, CancellationToken cancellationToken = default)
=> _queue.RenewLeaseAsync(this, leaseDuration, cancellationToken);
public Task ReleaseAsync(NotifyQueueReleaseDisposition disposition, CancellationToken cancellationToken = default)
=> _queue.ReleaseAsync(this, disposition, cancellationToken);
public Task DeadLetterAsync(string reason, CancellationToken cancellationToken = default)
=> _queue.DeadLetterAsync(this, reason, cancellationToken);
internal bool TryBeginCompletion()
=> Interlocked.CompareExchange(ref _completed, 1, 0) == 0;
internal void RefreshLease(DateTimeOffset expiresAt)
=> LeaseExpiresAt = expiresAt;
}

View File

@@ -0,0 +1,788 @@
using System;
using System.Buffers;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Globalization;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StackExchange.Redis;
using StellaOps.Notify.Models;
namespace StellaOps.Notify.Queue.Redis;
internal sealed class RedisNotifyDeliveryQueue : INotifyDeliveryQueue, IAsyncDisposable
{
private const string TransportName = "redis";
private readonly NotifyDeliveryQueueOptions _options;
private readonly NotifyRedisDeliveryQueueOptions _redisOptions;
private readonly ILogger<RedisNotifyDeliveryQueue> _logger;
private readonly TimeProvider _timeProvider;
private readonly Func<ConfigurationOptions, Task<IConnectionMultiplexer>> _connectionFactory;
private readonly SemaphoreSlim _connectionLock = new(1, 1);
private readonly SemaphoreSlim _groupLock = new(1, 1);
private readonly ConcurrentDictionary<string, bool> _streamInitialized = new(StringComparer.Ordinal);
private IConnectionMultiplexer? _connection;
private bool _disposed;
public RedisNotifyDeliveryQueue(
NotifyDeliveryQueueOptions options,
NotifyRedisDeliveryQueueOptions redisOptions,
ILogger<RedisNotifyDeliveryQueue> logger,
TimeProvider timeProvider,
Func<ConfigurationOptions, Task<IConnectionMultiplexer>>? connectionFactory = null)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
_redisOptions = redisOptions ?? throw new ArgumentNullException(nameof(redisOptions));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
_connectionFactory = connectionFactory ?? (async config =>
{
var connection = await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false);
return (IConnectionMultiplexer)connection;
});
if (string.IsNullOrWhiteSpace(_redisOptions.ConnectionString))
{
throw new InvalidOperationException("Redis connection string must be configured for the Notify delivery queue.");
}
}
public async ValueTask<NotifyQueueEnqueueResult> PublishAsync(
NotifyDeliveryQueueMessage message,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(message);
cancellationToken.ThrowIfCancellationRequested();
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
await EnsureConsumerGroupAsync(db, cancellationToken).ConfigureAwait(false);
var now = _timeProvider.GetUtcNow();
var attempt = 1;
var entries = BuildEntries(message, now, attempt);
var messageId = await AddToStreamAsync(
db,
_redisOptions.StreamName,
entries)
.ConfigureAwait(false);
var idempotencyKey = BuildIdempotencyKey(message.IdempotencyKey);
var stored = await db.StringSetAsync(
idempotencyKey,
messageId,
when: When.NotExists,
expiry: _options.ClaimIdleThreshold)
.ConfigureAwait(false);
if (!stored)
{
await db.StreamDeleteAsync(
_redisOptions.StreamName,
new RedisValue[] { messageId })
.ConfigureAwait(false);
var existing = await db.StringGetAsync(idempotencyKey).ConfigureAwait(false);
var duplicateId = existing.IsNullOrEmpty ? messageId : existing;
NotifyQueueMetrics.RecordDeduplicated(TransportName, _redisOptions.StreamName);
_logger.LogDebug(
"Duplicate Notify delivery enqueue detected for delivery {DeliveryId}.",
message.Delivery.DeliveryId);
return new NotifyQueueEnqueueResult(duplicateId.ToString()!, true);
}
NotifyQueueMetrics.RecordEnqueued(TransportName, _redisOptions.StreamName);
_logger.LogDebug(
"Enqueued Notify delivery {DeliveryId} (channel {ChannelId}) into stream {Stream}.",
message.Delivery.DeliveryId,
message.ChannelId,
_redisOptions.StreamName);
return new NotifyQueueEnqueueResult(messageId.ToString()!, false);
}
public async ValueTask<IReadOnlyList<INotifyQueueLease<NotifyDeliveryQueueMessage>>> LeaseAsync(
NotifyQueueLeaseRequest request,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(request);
cancellationToken.ThrowIfCancellationRequested();
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
await EnsureConsumerGroupAsync(db, cancellationToken).ConfigureAwait(false);
var entries = await db.StreamReadGroupAsync(
_redisOptions.StreamName,
_redisOptions.ConsumerGroup,
request.Consumer,
StreamPosition.NewMessages,
request.BatchSize)
.ConfigureAwait(false);
if (entries is null || entries.Length == 0)
{
return Array.Empty<INotifyQueueLease<NotifyDeliveryQueueMessage>>();
}
var now = _timeProvider.GetUtcNow();
var leases = new List<INotifyQueueLease<NotifyDeliveryQueueMessage>>(entries.Length);
foreach (var entry in entries)
{
var lease = TryMapLease(entry, request.Consumer, now, request.LeaseDuration, attemptOverride: null);
if (lease is null)
{
await AckPoisonAsync(db, entry.Id).ConfigureAwait(false);
continue;
}
leases.Add(lease);
}
return leases;
}
public async ValueTask<IReadOnlyList<INotifyQueueLease<NotifyDeliveryQueueMessage>>> ClaimExpiredAsync(
NotifyQueueClaimOptions options,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(options);
cancellationToken.ThrowIfCancellationRequested();
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
await EnsureConsumerGroupAsync(db, cancellationToken).ConfigureAwait(false);
var pending = await db.StreamPendingMessagesAsync(
_redisOptions.StreamName,
_redisOptions.ConsumerGroup,
options.BatchSize,
RedisValue.Null,
(long)options.MinIdleTime.TotalMilliseconds)
.ConfigureAwait(false);
if (pending is null || pending.Length == 0)
{
return Array.Empty<INotifyQueueLease<NotifyDeliveryQueueMessage>>();
}
var eligible = pending
.Where(p => p.IdleTimeInMilliseconds >= options.MinIdleTime.TotalMilliseconds)
.ToArray();
if (eligible.Length == 0)
{
return Array.Empty<INotifyQueueLease<NotifyDeliveryQueueMessage>>();
}
var messageIds = eligible
.Select(static p => (RedisValue)p.MessageId)
.ToArray();
var entries = await db.StreamClaimAsync(
_redisOptions.StreamName,
_redisOptions.ConsumerGroup,
options.ClaimantConsumer,
0,
messageIds)
.ConfigureAwait(false);
if (entries is null || entries.Length == 0)
{
return Array.Empty<INotifyQueueLease<NotifyDeliveryQueueMessage>>();
}
var now = _timeProvider.GetUtcNow();
var attemptLookup = eligible
.Where(static info => !info.MessageId.IsNullOrEmpty)
.ToDictionary(
info => info.MessageId!.ToString(),
info => (int)Math.Max(1, info.DeliveryCount),
StringComparer.Ordinal);
var leases = new List<INotifyQueueLease<NotifyDeliveryQueueMessage>>(entries.Length);
foreach (var entry in entries)
{
attemptLookup.TryGetValue(entry.Id.ToString(), out var attempt);
var lease = TryMapLease(entry, options.ClaimantConsumer, now, _options.DefaultLeaseDuration, attempt == 0 ? null : attempt);
if (lease is null)
{
await AckPoisonAsync(db, entry.Id).ConfigureAwait(false);
continue;
}
leases.Add(lease);
}
return leases;
}
public async ValueTask DisposeAsync()
{
if (_disposed)
{
return;
}
_disposed = true;
if (_connection is not null)
{
await _connection.CloseAsync().ConfigureAwait(false);
_connection.Dispose();
}
_connectionLock.Dispose();
_groupLock.Dispose();
GC.SuppressFinalize(this);
}
internal async Task AcknowledgeAsync(
RedisNotifyDeliveryLease lease,
CancellationToken cancellationToken)
{
if (!lease.TryBeginCompletion())
{
return;
}
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
await db.StreamAcknowledgeAsync(
_redisOptions.StreamName,
_redisOptions.ConsumerGroup,
new RedisValue[] { lease.MessageId })
.ConfigureAwait(false);
await db.StreamDeleteAsync(
_redisOptions.StreamName,
new RedisValue[] { lease.MessageId })
.ConfigureAwait(false);
NotifyQueueMetrics.RecordAck(TransportName, _redisOptions.StreamName);
_logger.LogDebug(
"Acknowledged Notify delivery {DeliveryId} (message {MessageId}).",
lease.Message.Delivery.DeliveryId,
lease.MessageId);
}
internal async Task RenewLeaseAsync(
RedisNotifyDeliveryLease lease,
TimeSpan leaseDuration,
CancellationToken cancellationToken)
{
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
await db.StreamClaimAsync(
_redisOptions.StreamName,
_redisOptions.ConsumerGroup,
lease.Consumer,
0,
new RedisValue[] { lease.MessageId })
.ConfigureAwait(false);
var expires = _timeProvider.GetUtcNow().Add(leaseDuration);
lease.RefreshLease(expires);
_logger.LogDebug(
"Renewed Notify delivery lease {DeliveryId} until {Expires:u}.",
lease.Message.Delivery.DeliveryId,
expires);
}
internal async Task ReleaseAsync(
RedisNotifyDeliveryLease lease,
NotifyQueueReleaseDisposition disposition,
CancellationToken cancellationToken)
{
if (disposition == NotifyQueueReleaseDisposition.Retry
&& lease.Attempt >= _options.MaxDeliveryAttempts)
{
_logger.LogWarning(
"Notify delivery {DeliveryId} reached max delivery attempts ({Attempts}); moving to dead-letter stream.",
lease.Message.Delivery.DeliveryId,
lease.Attempt);
await DeadLetterAsync(
lease,
$"max-delivery-attempts:{lease.Attempt}",
cancellationToken).ConfigureAwait(false);
return;
}
if (!lease.TryBeginCompletion())
{
return;
}
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
await db.StreamAcknowledgeAsync(
_redisOptions.StreamName,
_redisOptions.ConsumerGroup,
new RedisValue[] { lease.MessageId })
.ConfigureAwait(false);
await db.StreamDeleteAsync(
_redisOptions.StreamName,
new RedisValue[] { lease.MessageId })
.ConfigureAwait(false);
if (disposition == NotifyQueueReleaseDisposition.Retry)
{
NotifyQueueMetrics.RecordRetry(TransportName, _redisOptions.StreamName);
var delay = CalculateBackoff(lease.Attempt);
if (delay > TimeSpan.Zero)
{
try
{
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
}
catch (TaskCanceledException)
{
return;
}
}
var now = _timeProvider.GetUtcNow();
var entries = BuildEntries(lease.Message, now, lease.Attempt + 1);
await AddToStreamAsync(
db,
_redisOptions.StreamName,
entries)
.ConfigureAwait(false);
NotifyQueueMetrics.RecordEnqueued(TransportName, _redisOptions.StreamName);
_logger.LogInformation(
"Retrying Notify delivery {DeliveryId} (attempt {Attempt}).",
lease.Message.Delivery.DeliveryId,
lease.Attempt + 1);
}
else
{
NotifyQueueMetrics.RecordAck(TransportName, _redisOptions.StreamName);
_logger.LogInformation(
"Abandoned Notify delivery {DeliveryId} after {Attempt} attempt(s).",
lease.Message.Delivery.DeliveryId,
lease.Attempt);
}
}
internal async Task DeadLetterAsync(
RedisNotifyDeliveryLease lease,
string reason,
CancellationToken cancellationToken)
{
if (!lease.TryBeginCompletion())
{
return;
}
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
await db.StreamAcknowledgeAsync(
_redisOptions.StreamName,
_redisOptions.ConsumerGroup,
new RedisValue[] { lease.MessageId })
.ConfigureAwait(false);
await db.StreamDeleteAsync(
_redisOptions.StreamName,
new RedisValue[] { lease.MessageId })
.ConfigureAwait(false);
await EnsureDeadLetterStreamAsync(db, cancellationToken).ConfigureAwait(false);
var entries = BuildDeadLetterEntries(lease, reason);
await AddToStreamAsync(
db,
_redisOptions.DeadLetterStreamName,
entries)
.ConfigureAwait(false);
NotifyQueueMetrics.RecordDeadLetter(TransportName, _redisOptions.DeadLetterStreamName);
_logger.LogError(
"Dead-lettered Notify delivery {DeliveryId} (attempt {Attempt}): {Reason}",
lease.Message.Delivery.DeliveryId,
lease.Attempt,
reason);
}
internal async ValueTask PingAsync(CancellationToken cancellationToken)
{
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
_ = await db.PingAsync().ConfigureAwait(false);
}
private async Task<IDatabase> GetDatabaseAsync(CancellationToken cancellationToken)
{
if (_connection is { IsConnected: true })
{
return _connection.GetDatabase(_redisOptions.Database ?? -1);
}
await _connectionLock.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_connection is { IsConnected: true })
{
return _connection.GetDatabase(_redisOptions.Database ?? -1);
}
var configuration = ConfigurationOptions.Parse(_redisOptions.ConnectionString!);
configuration.AbortOnConnectFail = false;
if (_redisOptions.Database.HasValue)
{
configuration.DefaultDatabase = _redisOptions.Database.Value;
}
using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
timeoutCts.CancelAfter(_redisOptions.InitializationTimeout);
_connection = await _connectionFactory(configuration).WaitAsync(timeoutCts.Token).ConfigureAwait(false);
return _connection.GetDatabase(_redisOptions.Database ?? -1);
}
finally
{
_connectionLock.Release();
}
}
private async Task EnsureConsumerGroupAsync(
IDatabase database,
CancellationToken cancellationToken)
{
if (_streamInitialized.ContainsKey(_redisOptions.StreamName))
{
return;
}
await _groupLock.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_streamInitialized.ContainsKey(_redisOptions.StreamName))
{
return;
}
try
{
await database.StreamCreateConsumerGroupAsync(
_redisOptions.StreamName,
_redisOptions.ConsumerGroup,
StreamPosition.Beginning,
createStream: true)
.ConfigureAwait(false);
}
catch (RedisServerException ex) when (ex.Message.Contains("BUSYGROUP", StringComparison.OrdinalIgnoreCase))
{
// group already exists
}
_streamInitialized[_redisOptions.StreamName] = true;
}
finally
{
_groupLock.Release();
}
}
private async Task EnsureDeadLetterStreamAsync(
IDatabase database,
CancellationToken cancellationToken)
{
if (_streamInitialized.ContainsKey(_redisOptions.DeadLetterStreamName))
{
return;
}
await _groupLock.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_streamInitialized.ContainsKey(_redisOptions.DeadLetterStreamName))
{
return;
}
try
{
await database.StreamCreateConsumerGroupAsync(
_redisOptions.DeadLetterStreamName,
_redisOptions.ConsumerGroup,
StreamPosition.Beginning,
createStream: true)
.ConfigureAwait(false);
}
catch (RedisServerException ex) when (ex.Message.Contains("BUSYGROUP", StringComparison.OrdinalIgnoreCase))
{
// ignore
}
_streamInitialized[_redisOptions.DeadLetterStreamName] = true;
}
finally
{
_groupLock.Release();
}
}
private NameValueEntry[] BuildEntries(
NotifyDeliveryQueueMessage message,
DateTimeOffset enqueuedAt,
int attempt)
{
var json = NotifyCanonicalJsonSerializer.Serialize(message.Delivery);
var attributeCount = message.Attributes.Count;
var entries = ArrayPool<NameValueEntry>.Shared.Rent(8 + attributeCount);
var index = 0;
entries[index++] = new NameValueEntry(NotifyQueueFields.Payload, json);
entries[index++] = new NameValueEntry(NotifyQueueFields.DeliveryId, message.Delivery.DeliveryId);
entries[index++] = new NameValueEntry(NotifyQueueFields.ChannelId, message.ChannelId);
entries[index++] = new NameValueEntry(NotifyQueueFields.ChannelType, message.ChannelType.ToString());
entries[index++] = new NameValueEntry(NotifyQueueFields.Tenant, message.Delivery.TenantId);
entries[index++] = new NameValueEntry(NotifyQueueFields.Attempt, attempt);
entries[index++] = new NameValueEntry(NotifyQueueFields.EnqueuedAt, enqueuedAt.ToUnixTimeMilliseconds());
entries[index++] = new NameValueEntry(NotifyQueueFields.IdempotencyKey, message.IdempotencyKey);
entries[index++] = new NameValueEntry(NotifyQueueFields.TraceId, message.TraceId ?? string.Empty);
entries[index++] = new NameValueEntry(NotifyQueueFields.PartitionKey, message.PartitionKey);
if (attributeCount > 0)
{
foreach (var kvp in message.Attributes)
{
entries[index++] = new NameValueEntry(
NotifyQueueFields.AttributePrefix + kvp.Key,
kvp.Value);
}
}
return entries.AsSpan(0, index).ToArray();
}
private NameValueEntry[] BuildDeadLetterEntries(RedisNotifyDeliveryLease lease, string reason)
{
var json = NotifyCanonicalJsonSerializer.Serialize(lease.Message.Delivery);
var attributes = lease.Message.Attributes;
var attributeCount = attributes.Count;
var entries = ArrayPool<NameValueEntry>.Shared.Rent(9 + attributeCount);
var index = 0;
entries[index++] = new NameValueEntry(NotifyQueueFields.Payload, json);
entries[index++] = new NameValueEntry(NotifyQueueFields.DeliveryId, lease.Message.Delivery.DeliveryId);
entries[index++] = new NameValueEntry(NotifyQueueFields.ChannelId, lease.Message.ChannelId);
entries[index++] = new NameValueEntry(NotifyQueueFields.ChannelType, lease.Message.ChannelType.ToString());
entries[index++] = new NameValueEntry(NotifyQueueFields.Tenant, lease.Message.Delivery.TenantId);
entries[index++] = new NameValueEntry(NotifyQueueFields.Attempt, lease.Attempt);
entries[index++] = new NameValueEntry(NotifyQueueFields.IdempotencyKey, lease.Message.IdempotencyKey);
entries[index++] = new NameValueEntry("deadletter-reason", reason);
entries[index++] = new NameValueEntry(NotifyQueueFields.TraceId, lease.Message.TraceId ?? string.Empty);
foreach (var kvp in attributes)
{
entries[index++] = new NameValueEntry(
NotifyQueueFields.AttributePrefix + kvp.Key,
kvp.Value);
}
return entries.AsSpan(0, index).ToArray();
}
private RedisNotifyDeliveryLease? TryMapLease(
StreamEntry entry,
string consumer,
DateTimeOffset now,
TimeSpan leaseDuration,
int? attemptOverride)
{
if (entry.Values is null || entry.Values.Length == 0)
{
return null;
}
string? payload = null;
string? deliveryId = null;
string? channelId = null;
string? channelTypeRaw = null;
string? traceId = null;
string? idempotency = null;
string? partitionKey = null;
long? enqueuedAtUnix = null;
var attempt = attemptOverride ?? 1;
var attributes = new Dictionary<string, string>(StringComparer.Ordinal);
foreach (var value in entry.Values)
{
var name = value.Name.ToString();
var data = value.Value;
if (name.Equals(NotifyQueueFields.Payload, StringComparison.Ordinal))
{
payload = data.ToString();
}
else if (name.Equals(NotifyQueueFields.DeliveryId, StringComparison.Ordinal))
{
deliveryId = data.ToString();
}
else if (name.Equals(NotifyQueueFields.ChannelId, StringComparison.Ordinal))
{
channelId = data.ToString();
}
else if (name.Equals(NotifyQueueFields.ChannelType, StringComparison.Ordinal))
{
channelTypeRaw = data.ToString();
}
else if (name.Equals(NotifyQueueFields.Attempt, StringComparison.Ordinal))
{
if (int.TryParse(data.ToString(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed))
{
attempt = Math.Max(parsed, attempt);
}
}
else if (name.Equals(NotifyQueueFields.EnqueuedAt, StringComparison.Ordinal))
{
if (long.TryParse(data.ToString(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var unix))
{
enqueuedAtUnix = unix;
}
}
else if (name.Equals(NotifyQueueFields.IdempotencyKey, StringComparison.Ordinal))
{
idempotency = data.ToString();
}
else if (name.Equals(NotifyQueueFields.TraceId, StringComparison.Ordinal))
{
var text = data.ToString();
traceId = string.IsNullOrWhiteSpace(text) ? null : text;
}
else if (name.Equals(NotifyQueueFields.PartitionKey, StringComparison.Ordinal))
{
partitionKey = data.ToString();
}
else if (name.StartsWith(NotifyQueueFields.AttributePrefix, StringComparison.Ordinal))
{
attributes[name[NotifyQueueFields.AttributePrefix.Length..]] = data.ToString();
}
}
if (payload is null || deliveryId is null || channelId is null || channelTypeRaw is null)
{
return null;
}
NotifyDelivery delivery;
try
{
delivery = NotifyCanonicalJsonSerializer.Deserialize<NotifyDelivery>(payload);
}
catch (Exception ex)
{
_logger.LogWarning(
ex,
"Failed to deserialize Notify delivery payload for entry {EntryId}.",
entry.Id.ToString());
return null;
}
if (!Enum.TryParse<NotifyChannelType>(channelTypeRaw, ignoreCase: true, out var channelType))
{
_logger.LogWarning(
"Unknown channel type '{ChannelType}' for delivery {DeliveryId}; acknowledging as poison.",
channelTypeRaw,
deliveryId);
return null;
}
var attributeView = attributes.Count == 0
? EmptyReadOnlyDictionary<string, string>.Instance
: new ReadOnlyDictionary<string, string>(attributes);
var enqueuedAt = enqueuedAtUnix is null
? now
: DateTimeOffset.FromUnixTimeMilliseconds(enqueuedAtUnix.Value);
var message = new NotifyDeliveryQueueMessage(
delivery,
channelId,
channelType,
_redisOptions.StreamName,
traceId,
attributeView);
var leaseExpires = now.Add(leaseDuration);
return new RedisNotifyDeliveryLease(
this,
entry.Id.ToString(),
message,
attempt,
enqueuedAt,
leaseExpires,
consumer,
idempotency,
partitionKey ?? channelId);
}
private async Task AckPoisonAsync(IDatabase database, RedisValue messageId)
{
await database.StreamAcknowledgeAsync(
_redisOptions.StreamName,
_redisOptions.ConsumerGroup,
new RedisValue[] { messageId })
.ConfigureAwait(false);
await database.StreamDeleteAsync(
_redisOptions.StreamName,
new RedisValue[] { messageId })
.ConfigureAwait(false);
}
private static async Task<RedisValue> AddToStreamAsync(
IDatabase database,
string stream,
IReadOnlyList<NameValueEntry> entries)
{
return await database.StreamAddAsync(
stream,
entries.ToArray())
.ConfigureAwait(false);
}
private string BuildIdempotencyKey(string token)
=> string.Concat(_redisOptions.IdempotencyKeyPrefix, token);
private TimeSpan CalculateBackoff(int attempt)
{
var initial = _options.RetryInitialBackoff > TimeSpan.Zero
? _options.RetryInitialBackoff
: TimeSpan.FromSeconds(1);
if (initial <= TimeSpan.Zero)
{
return TimeSpan.Zero;
}
if (attempt <= 1)
{
return initial;
}
var max = _options.RetryMaxBackoff > TimeSpan.Zero
? _options.RetryMaxBackoff
: initial;
var exponent = attempt - 1;
var scaledTicks = initial.Ticks * Math.Pow(2, exponent - 1);
var cappedTicks = Math.Min(max.Ticks, scaledTicks);
var resultTicks = Math.Max(initial.Ticks, (long)cappedTicks);
return TimeSpan.FromTicks(resultTicks);
}
}

View File

@@ -0,0 +1,76 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Notify.Queue.Redis;
internal sealed class RedisNotifyEventLease : INotifyQueueLease<NotifyQueueEventMessage>
{
private readonly RedisNotifyEventQueue _queue;
private int _completed;
internal RedisNotifyEventLease(
RedisNotifyEventQueue queue,
NotifyRedisEventStreamOptions streamOptions,
string messageId,
NotifyQueueEventMessage message,
int attempt,
string consumer,
DateTimeOffset enqueuedAt,
DateTimeOffset leaseExpiresAt)
{
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
StreamOptions = streamOptions ?? throw new ArgumentNullException(nameof(streamOptions));
MessageId = messageId ?? throw new ArgumentNullException(nameof(messageId));
Message = message ?? throw new ArgumentNullException(nameof(message));
Attempt = attempt;
Consumer = consumer ?? throw new ArgumentNullException(nameof(consumer));
EnqueuedAt = enqueuedAt;
LeaseExpiresAt = leaseExpiresAt;
}
internal NotifyRedisEventStreamOptions StreamOptions { get; }
public string MessageId { get; }
public int Attempt { get; }
public DateTimeOffset EnqueuedAt { get; }
public DateTimeOffset LeaseExpiresAt { get; private set; }
public string Consumer { get; }
public string Stream => StreamOptions.Stream;
public string TenantId => Message.TenantId;
public string? PartitionKey => Message.PartitionKey;
public string IdempotencyKey => Message.IdempotencyKey;
public string? TraceId => Message.TraceId;
public IReadOnlyDictionary<string, string> Attributes => Message.Attributes;
public NotifyQueueEventMessage Message { get; }
public Task AcknowledgeAsync(CancellationToken cancellationToken = default)
=> _queue.AcknowledgeAsync(this, cancellationToken);
public Task RenewAsync(TimeSpan leaseDuration, CancellationToken cancellationToken = default)
=> _queue.RenewLeaseAsync(this, leaseDuration, cancellationToken);
public Task ReleaseAsync(NotifyQueueReleaseDisposition disposition, CancellationToken cancellationToken = default)
=> _queue.ReleaseAsync(this, disposition, cancellationToken);
public Task DeadLetterAsync(string reason, CancellationToken cancellationToken = default)
=> _queue.DeadLetterAsync(this, reason, cancellationToken);
internal bool TryBeginCompletion()
=> Interlocked.CompareExchange(ref _completed, 1, 0) == 0;
internal void RefreshLease(DateTimeOffset expiresAt)
=> LeaseExpiresAt = expiresAt;
}

View File

@@ -0,0 +1,655 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Globalization;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StackExchange.Redis;
using StellaOps.Notify.Models;
namespace StellaOps.Notify.Queue.Redis;
internal sealed class RedisNotifyEventQueue : INotifyEventQueue, IAsyncDisposable
{
private const string TransportName = "redis";
private readonly NotifyEventQueueOptions _options;
private readonly NotifyRedisEventQueueOptions _redisOptions;
private readonly ILogger<RedisNotifyEventQueue> _logger;
private readonly TimeProvider _timeProvider;
private readonly Func<ConfigurationOptions, Task<IConnectionMultiplexer>> _connectionFactory;
private readonly SemaphoreSlim _connectionLock = new(1, 1);
private readonly SemaphoreSlim _groupInitLock = new(1, 1);
private readonly IReadOnlyDictionary<string, NotifyRedisEventStreamOptions> _streamsByName;
private readonly ConcurrentDictionary<string, bool> _initializedStreams = new(StringComparer.Ordinal);
private IConnectionMultiplexer? _connection;
private bool _disposed;
public RedisNotifyEventQueue(
NotifyEventQueueOptions options,
NotifyRedisEventQueueOptions redisOptions,
ILogger<RedisNotifyEventQueue> logger,
TimeProvider timeProvider,
Func<ConfigurationOptions, Task<IConnectionMultiplexer>>? connectionFactory = null)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
_redisOptions = redisOptions ?? throw new ArgumentNullException(nameof(redisOptions));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
_connectionFactory = connectionFactory ?? (async config =>
{
var connection = await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false);
return (IConnectionMultiplexer)connection;
});
if (string.IsNullOrWhiteSpace(_redisOptions.ConnectionString))
{
throw new InvalidOperationException("Redis connection string must be configured for Notify event queue.");
}
_streamsByName = _redisOptions.Streams.ToDictionary(
stream => stream.Stream,
stream => stream,
StringComparer.Ordinal);
}
public async ValueTask<NotifyQueueEnqueueResult> PublishAsync(
NotifyQueueEventMessage message,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(message);
cancellationToken.ThrowIfCancellationRequested();
var streamOptions = GetStreamOptions(message.Stream);
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
await EnsureStreamInitializedAsync(db, streamOptions, cancellationToken).ConfigureAwait(false);
var now = _timeProvider.GetUtcNow();
var entries = BuildEntries(message, now, attempt: 1);
var messageId = await AddToStreamAsync(
db,
streamOptions,
entries)
.ConfigureAwait(false);
var idempotencyToken = string.IsNullOrWhiteSpace(message.IdempotencyKey)
? message.Event.EventId.ToString("N")
: message.IdempotencyKey;
var idempotencyKey = streamOptions.IdempotencyKeyPrefix + idempotencyToken;
var stored = await db.StringSetAsync(
idempotencyKey,
messageId,
when: When.NotExists,
expiry: _redisOptions.IdempotencyWindow)
.ConfigureAwait(false);
if (!stored)
{
await db.StreamDeleteAsync(
streamOptions.Stream,
new RedisValue[] { messageId })
.ConfigureAwait(false);
var existing = await db.StringGetAsync(idempotencyKey).ConfigureAwait(false);
var duplicateId = existing.IsNullOrEmpty ? messageId : existing;
_logger.LogDebug(
"Duplicate Notify event enqueue detected for idempotency token {Token}; returning existing stream id {StreamId}.",
idempotencyToken,
duplicateId.ToString());
NotifyQueueMetrics.RecordDeduplicated(TransportName, streamOptions.Stream);
return new NotifyQueueEnqueueResult(duplicateId.ToString()!, true);
}
NotifyQueueMetrics.RecordEnqueued(TransportName, streamOptions.Stream);
_logger.LogDebug(
"Enqueued Notify event {EventId} for tenant {Tenant} on stream {Stream} (id {StreamId}).",
message.Event.EventId,
message.TenantId,
streamOptions.Stream,
messageId.ToString());
return new NotifyQueueEnqueueResult(messageId.ToString()!, false);
}
public async ValueTask<IReadOnlyList<INotifyQueueLease<NotifyQueueEventMessage>>> LeaseAsync(
NotifyQueueLeaseRequest request,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(request);
cancellationToken.ThrowIfCancellationRequested();
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
var now = _timeProvider.GetUtcNow();
var leases = new List<INotifyQueueLease<NotifyQueueEventMessage>>(request.BatchSize);
foreach (var streamOptions in _streamsByName.Values)
{
await EnsureStreamInitializedAsync(db, streamOptions, cancellationToken).ConfigureAwait(false);
var remaining = request.BatchSize - leases.Count;
if (remaining <= 0)
{
break;
}
var entries = await db.StreamReadGroupAsync(
streamOptions.Stream,
streamOptions.ConsumerGroup,
request.Consumer,
StreamPosition.NewMessages,
remaining)
.ConfigureAwait(false);
if (entries is null || entries.Length == 0)
{
continue;
}
foreach (var entry in entries)
{
var lease = TryMapLease(
streamOptions,
entry,
request.Consumer,
now,
request.LeaseDuration,
attemptOverride: null);
if (lease is null)
{
await AckPoisonAsync(db, streamOptions, entry.Id).ConfigureAwait(false);
continue;
}
leases.Add(lease);
if (leases.Count >= request.BatchSize)
{
break;
}
}
}
return leases;
}
public async ValueTask<IReadOnlyList<INotifyQueueLease<NotifyQueueEventMessage>>> ClaimExpiredAsync(
NotifyQueueClaimOptions options,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(options);
cancellationToken.ThrowIfCancellationRequested();
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
var now = _timeProvider.GetUtcNow();
var leases = new List<INotifyQueueLease<NotifyQueueEventMessage>>(options.BatchSize);
foreach (var streamOptions in _streamsByName.Values)
{
await EnsureStreamInitializedAsync(db, streamOptions, cancellationToken).ConfigureAwait(false);
var pending = await db.StreamPendingMessagesAsync(
streamOptions.Stream,
streamOptions.ConsumerGroup,
options.BatchSize,
RedisValue.Null,
(long)options.MinIdleTime.TotalMilliseconds)
.ConfigureAwait(false);
if (pending is null || pending.Length == 0)
{
continue;
}
var eligible = pending
.Where(p => p.IdleTimeInMilliseconds >= options.MinIdleTime.TotalMilliseconds)
.ToArray();
if (eligible.Length == 0)
{
continue;
}
var messageIds = eligible
.Select(static p => (RedisValue)p.MessageId)
.ToArray();
var entries = await db.StreamClaimAsync(
streamOptions.Stream,
streamOptions.ConsumerGroup,
options.ClaimantConsumer,
0,
messageIds)
.ConfigureAwait(false);
if (entries is null || entries.Length == 0)
{
continue;
}
var attemptById = eligible
.Where(static info => !info.MessageId.IsNullOrEmpty)
.ToDictionary(
info => info.MessageId!.ToString(),
info => (int)Math.Max(1, info.DeliveryCount),
StringComparer.Ordinal);
foreach (var entry in entries)
{
var entryId = entry.Id.ToString();
attemptById.TryGetValue(entryId, out var attempt);
var lease = TryMapLease(
streamOptions,
entry,
options.ClaimantConsumer,
now,
_options.DefaultLeaseDuration,
attempt == 0 ? null : attempt);
if (lease is null)
{
await AckPoisonAsync(db, streamOptions, entry.Id).ConfigureAwait(false);
continue;
}
leases.Add(lease);
if (leases.Count >= options.BatchSize)
{
return leases;
}
}
}
return leases;
}
public async ValueTask DisposeAsync()
{
if (_disposed)
{
return;
}
_disposed = true;
if (_connection is not null)
{
await _connection.CloseAsync();
_connection.Dispose();
}
_connectionLock.Dispose();
_groupInitLock.Dispose();
GC.SuppressFinalize(this);
}
internal async Task AcknowledgeAsync(
RedisNotifyEventLease lease,
CancellationToken cancellationToken)
{
if (!lease.TryBeginCompletion())
{
return;
}
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
var streamOptions = lease.StreamOptions;
await db.StreamAcknowledgeAsync(
streamOptions.Stream,
streamOptions.ConsumerGroup,
new RedisValue[] { lease.MessageId })
.ConfigureAwait(false);
await db.StreamDeleteAsync(
streamOptions.Stream,
new RedisValue[] { lease.MessageId })
.ConfigureAwait(false);
NotifyQueueMetrics.RecordAck(TransportName, streamOptions.Stream);
_logger.LogDebug(
"Acknowledged Notify event {EventId} on consumer {Consumer} (stream {Stream}, id {MessageId}).",
lease.Message.Event.EventId,
lease.Consumer,
streamOptions.Stream,
lease.MessageId);
}
internal async Task RenewLeaseAsync(
RedisNotifyEventLease lease,
TimeSpan leaseDuration,
CancellationToken cancellationToken)
{
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
var streamOptions = lease.StreamOptions;
await db.StreamClaimAsync(
streamOptions.Stream,
streamOptions.ConsumerGroup,
lease.Consumer,
0,
new RedisValue[] { lease.MessageId })
.ConfigureAwait(false);
var expires = _timeProvider.GetUtcNow().Add(leaseDuration);
lease.RefreshLease(expires);
_logger.LogDebug(
"Renewed Notify event lease for {EventId} until {Expires:u}.",
lease.Message.Event.EventId,
expires);
}
internal Task ReleaseAsync(
RedisNotifyEventLease lease,
NotifyQueueReleaseDisposition disposition,
CancellationToken cancellationToken)
=> Task.FromException(new NotSupportedException("Retry/abandon is not supported for Notify event streams."));
internal async Task DeadLetterAsync(
RedisNotifyEventLease lease,
string reason,
CancellationToken cancellationToken)
{
if (!lease.TryBeginCompletion())
{
return;
}
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
var streamOptions = lease.StreamOptions;
await db.StreamAcknowledgeAsync(
streamOptions.Stream,
streamOptions.ConsumerGroup,
new RedisValue[] { lease.MessageId })
.ConfigureAwait(false);
await db.StreamDeleteAsync(
streamOptions.Stream,
new RedisValue[] { lease.MessageId })
.ConfigureAwait(false);
_logger.LogWarning(
"Dead-lettered Notify event {EventId} on stream {Stream} with reason '{Reason}'.",
lease.Message.Event.EventId,
streamOptions.Stream,
reason);
}
internal async ValueTask PingAsync(CancellationToken cancellationToken)
{
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
_ = await db.PingAsync().ConfigureAwait(false);
}
private NotifyRedisEventStreamOptions GetStreamOptions(string stream)
{
if (!_streamsByName.TryGetValue(stream, out var options))
{
throw new InvalidOperationException($"Stream '{stream}' is not configured for the Notify event queue.");
}
return options;
}
private async Task<IDatabase> GetDatabaseAsync(CancellationToken cancellationToken)
{
if (_connection is { IsConnected: true })
{
return _connection.GetDatabase(_redisOptions.Database ?? -1);
}
await _connectionLock.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_connection is { IsConnected: true })
{
return _connection.GetDatabase(_redisOptions.Database ?? -1);
}
var configuration = ConfigurationOptions.Parse(_redisOptions.ConnectionString!);
configuration.AbortOnConnectFail = false;
if (_redisOptions.Database.HasValue)
{
configuration.DefaultDatabase = _redisOptions.Database;
}
using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
timeoutCts.CancelAfter(_redisOptions.InitializationTimeout);
_connection = await _connectionFactory(configuration).WaitAsync(timeoutCts.Token).ConfigureAwait(false);
return _connection.GetDatabase(_redisOptions.Database ?? -1);
}
finally
{
_connectionLock.Release();
}
}
private async Task EnsureStreamInitializedAsync(
IDatabase database,
NotifyRedisEventStreamOptions streamOptions,
CancellationToken cancellationToken)
{
if (_initializedStreams.ContainsKey(streamOptions.Stream))
{
return;
}
await _groupInitLock.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_initializedStreams.ContainsKey(streamOptions.Stream))
{
return;
}
try
{
await database.StreamCreateConsumerGroupAsync(
streamOptions.Stream,
streamOptions.ConsumerGroup,
StreamPosition.Beginning,
createStream: true)
.ConfigureAwait(false);
}
catch (RedisServerException ex) when (ex.Message.Contains("BUSYGROUP", StringComparison.OrdinalIgnoreCase))
{
// Consumer group already exists — nothing to do.
}
_initializedStreams[streamOptions.Stream] = true;
}
finally
{
_groupInitLock.Release();
}
}
private static async Task<RedisValue> AddToStreamAsync(
IDatabase database,
NotifyRedisEventStreamOptions streamOptions,
IReadOnlyList<NameValueEntry> entries)
{
return await database.StreamAddAsync(
streamOptions.Stream,
entries.ToArray(),
maxLength: streamOptions.ApproximateMaxLength,
useApproximateMaxLength: streamOptions.ApproximateMaxLength is not null)
.ConfigureAwait(false);
}
private IReadOnlyList<NameValueEntry> BuildEntries(
NotifyQueueEventMessage message,
DateTimeOffset enqueuedAt,
int attempt)
{
var payload = NotifyCanonicalJsonSerializer.Serialize(message.Event);
var entries = new List<NameValueEntry>(8 + message.Attributes.Count)
{
new(NotifyQueueFields.Payload, payload),
new(NotifyQueueFields.EventId, message.Event.EventId.ToString("D")),
new(NotifyQueueFields.Tenant, message.TenantId),
new(NotifyQueueFields.Kind, message.Event.Kind),
new(NotifyQueueFields.Attempt, attempt),
new(NotifyQueueFields.EnqueuedAt, enqueuedAt.ToUnixTimeMilliseconds()),
new(NotifyQueueFields.IdempotencyKey, message.IdempotencyKey),
new(NotifyQueueFields.PartitionKey, message.PartitionKey ?? string.Empty),
new(NotifyQueueFields.TraceId, message.TraceId ?? string.Empty)
};
foreach (var kvp in message.Attributes)
{
entries.Add(new NameValueEntry(
NotifyQueueFields.AttributePrefix + kvp.Key,
kvp.Value));
}
return entries;
}
private RedisNotifyEventLease? TryMapLease(
NotifyRedisEventStreamOptions streamOptions,
StreamEntry entry,
string consumer,
DateTimeOffset now,
TimeSpan leaseDuration,
int? attemptOverride)
{
if (entry.Values is null || entry.Values.Length == 0)
{
return null;
}
string? payloadJson = null;
string? eventIdRaw = null;
long? enqueuedAtUnix = null;
string? idempotency = null;
string? partitionKey = null;
string? traceId = null;
var attempt = attemptOverride ?? 1;
var attributes = new Dictionary<string, string>(StringComparer.Ordinal);
foreach (var field in entry.Values)
{
var name = field.Name.ToString();
var value = field.Value;
if (name.Equals(NotifyQueueFields.Payload, StringComparison.Ordinal))
{
payloadJson = value.ToString();
}
else if (name.Equals(NotifyQueueFields.EventId, StringComparison.Ordinal))
{
eventIdRaw = value.ToString();
}
else if (name.Equals(NotifyQueueFields.Attempt, StringComparison.Ordinal))
{
if (int.TryParse(value.ToString(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed))
{
attempt = Math.Max(parsed, attempt);
}
}
else if (name.Equals(NotifyQueueFields.EnqueuedAt, StringComparison.Ordinal))
{
if (long.TryParse(value.ToString(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var unix))
{
enqueuedAtUnix = unix;
}
}
else if (name.Equals(NotifyQueueFields.IdempotencyKey, StringComparison.Ordinal))
{
var text = value.ToString();
idempotency = string.IsNullOrWhiteSpace(text) ? null : text;
}
else if (name.Equals(NotifyQueueFields.PartitionKey, StringComparison.Ordinal))
{
var text = value.ToString();
partitionKey = string.IsNullOrWhiteSpace(text) ? null : text;
}
else if (name.Equals(NotifyQueueFields.TraceId, StringComparison.Ordinal))
{
var text = value.ToString();
traceId = string.IsNullOrWhiteSpace(text) ? null : text;
}
else if (name.StartsWith(NotifyQueueFields.AttributePrefix, StringComparison.Ordinal))
{
var key = name[NotifyQueueFields.AttributePrefix.Length..];
attributes[key] = value.ToString();
}
}
if (payloadJson is null || enqueuedAtUnix is null)
{
return null;
}
NotifyEvent notifyEvent;
try
{
notifyEvent = NotifyCanonicalJsonSerializer.Deserialize<NotifyEvent>(payloadJson);
}
catch (Exception ex)
{
_logger.LogWarning(
ex,
"Failed to deserialize Notify event payload for stream {Stream} entry {EntryId}.",
streamOptions.Stream,
entry.Id.ToString());
return null;
}
var attributeView = attributes.Count == 0
? EmptyReadOnlyDictionary<string, string>.Instance
: new ReadOnlyDictionary<string, string>(attributes);
var message = new NotifyQueueEventMessage(
notifyEvent,
streamOptions.Stream,
idempotencyKey: idempotency ?? notifyEvent.EventId.ToString("N"),
partitionKey: partitionKey,
traceId: traceId,
attributes: attributeView);
var enqueuedAt = DateTimeOffset.FromUnixTimeMilliseconds(enqueuedAtUnix.Value);
var leaseExpiresAt = now.Add(leaseDuration);
return new RedisNotifyEventLease(
this,
streamOptions,
entry.Id.ToString(),
message,
attempt,
consumer,
enqueuedAt,
leaseExpiresAt);
}
private async Task AckPoisonAsync(
IDatabase database,
NotifyRedisEventStreamOptions streamOptions,
RedisValue messageId)
{
await database.StreamAcknowledgeAsync(
streamOptions.Stream,
streamOptions.ConsumerGroup,
new RedisValue[] { messageId })
.ConfigureAwait(false);
await database.StreamDeleteAsync(
streamOptions.Stream,
new RedisValue[] { messageId })
.ConfigureAwait(false);
}
}

View File

@@ -0,0 +1,23 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks.Abstractions" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="NATS.Client.Core" Version="2.0.0" />
<PackageReference Include="NATS.Client.JetStream" Version="2.0.0" />
<PackageReference Include="StackExchange.Redis" Version="2.7.33" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Notify.Models\StellaOps.Notify.Models.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,2 @@
# Notify Queue Task Board (Sprint 15)
> Archived 2025-10-26 — queue infrastructure maintained in `src/Notifier/StellaOps.Notifier` (Sprints 3840).