Restructure solution layout by module
This commit is contained in:
master
@@ -0,0 +1,26 @@
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StellaOps.Scheduler.Queue.Redis;
|
||||
|
||||
internal interface IRedisSchedulerQueuePayload<TMessage>
|
||||
{
|
||||
string QueueName { get; }
|
||||
|
||||
string GetIdempotencyKey(TMessage message);
|
||||
|
||||
string Serialize(TMessage message);
|
||||
|
||||
TMessage Deserialize(string payload);
|
||||
|
||||
string GetRunId(TMessage message);
|
||||
|
||||
string GetTenantId(TMessage message);
|
||||
|
||||
string? GetScheduleId(TMessage message);
|
||||
|
||||
string? GetSegmentId(TMessage message);
|
||||
|
||||
string? GetCorrelationId(TMessage message);
|
||||
|
||||
IReadOnlyDictionary<string, string>? GetAttributes(TMessage message);
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StackExchange.Redis;
|
||||
using StellaOps.Scheduler.Models;
|
||||
|
||||
namespace StellaOps.Scheduler.Queue.Redis;
|
||||
|
||||
internal sealed class RedisSchedulerPlannerQueue
|
||||
: RedisSchedulerQueueBase<PlannerQueueMessage>, ISchedulerPlannerQueue
|
||||
{
|
||||
public RedisSchedulerPlannerQueue(
|
||||
SchedulerQueueOptions queueOptions,
|
||||
SchedulerRedisQueueOptions redisOptions,
|
||||
ILogger<RedisSchedulerPlannerQueue> logger,
|
||||
TimeProvider timeProvider,
|
||||
Func<ConfigurationOptions, Task<IConnectionMultiplexer>>? connectionFactory = null)
|
||||
: base(
|
||||
queueOptions,
|
||||
redisOptions,
|
||||
redisOptions.Planner,
|
||||
PlannerPayload.Instance,
|
||||
logger,
|
||||
timeProvider,
|
||||
connectionFactory)
|
||||
{
|
||||
}
|
||||
|
||||
private sealed class PlannerPayload : IRedisSchedulerQueuePayload<PlannerQueueMessage>
|
||||
{
|
||||
public static PlannerPayload Instance { get; } = new();
|
||||
|
||||
public string QueueName => "planner";
|
||||
|
||||
public string GetIdempotencyKey(PlannerQueueMessage message)
|
||||
=> message.IdempotencyKey;
|
||||
|
||||
public string Serialize(PlannerQueueMessage message)
|
||||
=> CanonicalJsonSerializer.Serialize(message);
|
||||
|
||||
public PlannerQueueMessage Deserialize(string payload)
|
||||
=> CanonicalJsonSerializer.Deserialize<PlannerQueueMessage>(payload);
|
||||
|
||||
public string GetRunId(PlannerQueueMessage message)
|
||||
=> message.Run.Id;
|
||||
|
||||
public string GetTenantId(PlannerQueueMessage message)
|
||||
=> message.Run.TenantId;
|
||||
|
||||
public string? GetScheduleId(PlannerQueueMessage message)
|
||||
=> message.ScheduleId;
|
||||
|
||||
public string? GetSegmentId(PlannerQueueMessage message)
|
||||
=> null;
|
||||
|
||||
public string? GetCorrelationId(PlannerQueueMessage message)
|
||||
=> message.CorrelationId;
|
||||
|
||||
public IReadOnlyDictionary<string, string>? GetAttributes(PlannerQueueMessage message)
|
||||
=> null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,805 @@
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.ObjectModel;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StackExchange.Redis;
|
||||
|
||||
namespace StellaOps.Scheduler.Queue.Redis;
|
||||
|
||||
internal abstract class RedisSchedulerQueueBase<TMessage> : ISchedulerQueue<TMessage>, IAsyncDisposable, ISchedulerQueueTransportDiagnostics
|
||||
{
|
||||
private const string TransportName = "redis";
|
||||
|
||||
private readonly SchedulerQueueOptions _queueOptions;
|
||||
private readonly SchedulerRedisQueueOptions _redisOptions;
|
||||
private readonly RedisSchedulerStreamOptions _streamOptions;
|
||||
private readonly IRedisSchedulerQueuePayload<TMessage> _payload;
|
||||
private readonly ILogger _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly Func<ConfigurationOptions, Task<IConnectionMultiplexer>> _connectionFactory;
|
||||
private readonly SemaphoreSlim _connectionLock = new(1, 1);
|
||||
private readonly SemaphoreSlim _groupInitLock = new(1, 1);
|
||||
private long _approximateDepth;
|
||||
|
||||
private IConnectionMultiplexer? _connection;
|
||||
private volatile bool _groupInitialized;
|
||||
private bool _disposed;
|
||||
|
||||
protected RedisSchedulerQueueBase(
|
||||
SchedulerQueueOptions queueOptions,
|
||||
SchedulerRedisQueueOptions redisOptions,
|
||||
RedisSchedulerStreamOptions streamOptions,
|
||||
IRedisSchedulerQueuePayload<TMessage> payload,
|
||||
ILogger logger,
|
||||
TimeProvider timeProvider,
|
||||
Func<ConfigurationOptions, Task<IConnectionMultiplexer>>? connectionFactory = null)
|
||||
{
|
||||
_queueOptions = queueOptions ?? throw new ArgumentNullException(nameof(queueOptions));
|
||||
_redisOptions = redisOptions ?? throw new ArgumentNullException(nameof(redisOptions));
|
||||
_streamOptions = streamOptions ?? throw new ArgumentNullException(nameof(streamOptions));
|
||||
_payload = payload ?? throw new ArgumentNullException(nameof(payload));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
_connectionFactory = connectionFactory ?? (config => Task.FromResult<IConnectionMultiplexer>(ConnectionMultiplexer.Connect(config)));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(_redisOptions.ConnectionString))
|
||||
{
|
||||
throw new InvalidOperationException("Redis connection string must be configured for the scheduler queue.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(_streamOptions.Stream))
|
||||
{
|
||||
throw new InvalidOperationException("Redis stream name must be configured for the scheduler queue.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(_streamOptions.ConsumerGroup))
|
||||
{
|
||||
throw new InvalidOperationException("Redis consumer group must be configured for the scheduler queue.");
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<SchedulerQueueEnqueueResult> EnqueueAsync(
|
||||
TMessage message,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(message);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var database = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
||||
await EnsureConsumerGroupAsync(database, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var attempt = 1;
|
||||
var entries = BuildEntries(message, now, attempt);
|
||||
|
||||
var messageId = await AddToStreamAsync(
|
||||
database,
|
||||
_streamOptions.Stream,
|
||||
entries,
|
||||
_streamOptions.ApproximateMaxLength,
|
||||
_streamOptions.ApproximateMaxLength is not null)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var idempotencyKey = BuildIdempotencyKey(_payload.GetIdempotencyKey(message));
|
||||
var stored = await database.StringSetAsync(
|
||||
idempotencyKey,
|
||||
messageId,
|
||||
when: When.NotExists,
|
||||
expiry: _streamOptions.IdempotencyWindow)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (!stored)
|
||||
{
|
||||
await database.StreamDeleteAsync(_streamOptions.Stream, new RedisValue[] { messageId }).ConfigureAwait(false);
|
||||
|
||||
var existing = await database.StringGetAsync(idempotencyKey).ConfigureAwait(false);
|
||||
var reusable = existing.IsNullOrEmpty ? messageId : existing;
|
||||
|
||||
SchedulerQueueMetrics.RecordDeduplicated(TransportName, _payload.QueueName);
|
||||
_logger.LogDebug(
|
||||
"Duplicate enqueue detected for scheduler queue {Queue} with key {Key}; returning existing stream id {StreamId}.",
|
||||
_payload.QueueName,
|
||||
idempotencyKey,
|
||||
reusable.ToString());
|
||||
|
||||
PublishDepth();
|
||||
return new SchedulerQueueEnqueueResult(reusable.ToString(), true);
|
||||
}
|
||||
|
||||
SchedulerQueueMetrics.RecordEnqueued(TransportName, _payload.QueueName);
|
||||
_logger.LogDebug(
|
||||
"Enqueued {Queue} message into {Stream} with id {StreamId}.",
|
||||
_payload.QueueName,
|
||||
_streamOptions.Stream,
|
||||
messageId.ToString());
|
||||
|
||||
IncrementDepth();
|
||||
return new SchedulerQueueEnqueueResult(messageId.ToString(), false);
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<ISchedulerQueueLease<TMessage>>> LeaseAsync(
|
||||
SchedulerQueueLeaseRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var database = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
||||
await EnsureConsumerGroupAsync(database, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var entries = await database.StreamReadGroupAsync(
|
||||
_streamOptions.Stream,
|
||||
_streamOptions.ConsumerGroup,
|
||||
request.Consumer,
|
||||
position: ">",
|
||||
count: request.BatchSize,
|
||||
flags: CommandFlags.None)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (entries is null || entries.Length == 0)
|
||||
{
|
||||
PublishDepth();
|
||||
return Array.Empty<ISchedulerQueueLease<TMessage>>();
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var leases = new List<ISchedulerQueueLease<TMessage>>(entries.Length);
|
||||
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
var lease = TryMapLease(entry, request.Consumer, now, request.LeaseDuration, attemptOverride: null);
|
||||
if (lease is null)
|
||||
{
|
||||
await HandlePoisonEntryAsync(database, entry.Id).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
leases.Add(lease);
|
||||
}
|
||||
|
||||
PublishDepth();
|
||||
return leases;
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<ISchedulerQueueLease<TMessage>>> ClaimExpiredAsync(
|
||||
SchedulerQueueClaimOptions options,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var database = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
||||
await EnsureConsumerGroupAsync(database, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var pending = await database.StreamPendingMessagesAsync(
|
||||
_streamOptions.Stream,
|
||||
_streamOptions.ConsumerGroup,
|
||||
options.BatchSize,
|
||||
RedisValue.Null,
|
||||
(long)options.MinIdleTime.TotalMilliseconds)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (pending is null || pending.Length == 0)
|
||||
{
|
||||
return Array.Empty<ISchedulerQueueLease<TMessage>>();
|
||||
}
|
||||
|
||||
var eligible = pending
|
||||
.Where(info => info.IdleTimeInMilliseconds >= options.MinIdleTime.TotalMilliseconds)
|
||||
.ToArray();
|
||||
|
||||
if (eligible.Length == 0)
|
||||
{
|
||||
return Array.Empty<ISchedulerQueueLease<TMessage>>();
|
||||
}
|
||||
|
||||
var messageIds = eligible
|
||||
.Select(info => (RedisValue)info.MessageId)
|
||||
.ToArray();
|
||||
|
||||
var claimed = await database.StreamClaimAsync(
|
||||
_streamOptions.Stream,
|
||||
_streamOptions.ConsumerGroup,
|
||||
options.ClaimantConsumer,
|
||||
0,
|
||||
messageIds,
|
||||
CommandFlags.None)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (claimed is null || claimed.Length == 0)
|
||||
{
|
||||
PublishDepth();
|
||||
return Array.Empty<ISchedulerQueueLease<TMessage>>();
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var attemptLookup = eligible.ToDictionary(
|
||||
info => info.MessageId.IsNullOrEmpty ? string.Empty : info.MessageId.ToString(),
|
||||
info => (int)Math.Max(1, info.DeliveryCount),
|
||||
StringComparer.Ordinal);
|
||||
|
||||
var leases = new List<ISchedulerQueueLease<TMessage>>(claimed.Length);
|
||||
foreach (var entry in claimed)
|
||||
{
|
||||
var entryId = entry.Id.ToString();
|
||||
attemptLookup.TryGetValue(entryId, out var attempt);
|
||||
|
||||
var lease = TryMapLease(
|
||||
entry,
|
||||
options.ClaimantConsumer,
|
||||
now,
|
||||
_queueOptions.DefaultLeaseDuration,
|
||||
attemptOverride: attempt);
|
||||
|
||||
if (lease is null)
|
||||
{
|
||||
await HandlePoisonEntryAsync(database, entry.Id).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
leases.Add(lease);
|
||||
}
|
||||
|
||||
PublishDepth();
|
||||
return leases;
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_disposed = true;
|
||||
|
||||
if (_connection is not null)
|
||||
{
|
||||
await _connection.CloseAsync();
|
||||
_connection.Dispose();
|
||||
}
|
||||
|
||||
_connectionLock.Dispose();
|
||||
_groupInitLock.Dispose();
|
||||
SchedulerQueueMetrics.RemoveDepth(TransportName, _payload.QueueName);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
internal async Task AcknowledgeAsync(
|
||||
RedisSchedulerQueueLease<TMessage> lease,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!lease.TryBeginCompletion())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var database = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await database.StreamAcknowledgeAsync(
|
||||
_streamOptions.Stream,
|
||||
_streamOptions.ConsumerGroup,
|
||||
new RedisValue[] { lease.MessageId })
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await database.StreamDeleteAsync(
|
||||
_streamOptions.Stream,
|
||||
new RedisValue[] { lease.MessageId })
|
||||
.ConfigureAwait(false);
|
||||
|
||||
SchedulerQueueMetrics.RecordAck(TransportName, _payload.QueueName);
|
||||
DecrementDepth();
|
||||
}
|
||||
|
||||
internal async Task RenewLeaseAsync(
|
||||
RedisSchedulerQueueLease<TMessage> lease,
|
||||
TimeSpan leaseDuration,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var database = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await database.StreamClaimAsync(
|
||||
_streamOptions.Stream,
|
||||
_streamOptions.ConsumerGroup,
|
||||
lease.Consumer,
|
||||
0,
|
||||
new RedisValue[] { lease.MessageId },
|
||||
CommandFlags.None)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var expires = _timeProvider.GetUtcNow().Add(leaseDuration);
|
||||
lease.RefreshLease(expires);
|
||||
}
|
||||
|
||||
internal async Task ReleaseAsync(
|
||||
RedisSchedulerQueueLease<TMessage> lease,
|
||||
SchedulerQueueReleaseDisposition disposition,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (disposition == SchedulerQueueReleaseDisposition.Retry
|
||||
&& lease.Attempt >= _queueOptions.MaxDeliveryAttempts)
|
||||
{
|
||||
await DeadLetterAsync(
|
||||
lease,
|
||||
$"max-delivery-attempts:{lease.Attempt}",
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!lease.TryBeginCompletion())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var database = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await database.StreamAcknowledgeAsync(
|
||||
_streamOptions.Stream,
|
||||
_streamOptions.ConsumerGroup,
|
||||
new RedisValue[] { lease.MessageId })
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await database.StreamDeleteAsync(
|
||||
_streamOptions.Stream,
|
||||
new RedisValue[] { lease.MessageId })
|
||||
.ConfigureAwait(false);
|
||||
|
||||
SchedulerQueueMetrics.RecordAck(TransportName, _payload.QueueName);
|
||||
DecrementDepth();
|
||||
|
||||
if (disposition == SchedulerQueueReleaseDisposition.Retry)
|
||||
{
|
||||
SchedulerQueueMetrics.RecordRetry(TransportName, _payload.QueueName);
|
||||
|
||||
lease.IncrementAttempt();
|
||||
|
||||
var backoff = CalculateBackoff(lease.Attempt);
|
||||
if (backoff > TimeSpan.Zero)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.Delay(backoff, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (TaskCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var entries = BuildEntries(lease.Message, now, lease.Attempt);
|
||||
|
||||
await AddToStreamAsync(
|
||||
database,
|
||||
_streamOptions.Stream,
|
||||
entries,
|
||||
_streamOptions.ApproximateMaxLength,
|
||||
_streamOptions.ApproximateMaxLength is not null)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
SchedulerQueueMetrics.RecordEnqueued(TransportName, _payload.QueueName);
|
||||
IncrementDepth();
|
||||
}
|
||||
}
|
||||
|
||||
internal async Task DeadLetterAsync(
|
||||
RedisSchedulerQueueLease<TMessage> lease,
|
||||
string reason,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!lease.TryBeginCompletion())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var database = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await database.StreamAcknowledgeAsync(
|
||||
_streamOptions.Stream,
|
||||
_streamOptions.ConsumerGroup,
|
||||
new RedisValue[] { lease.MessageId })
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await database.StreamDeleteAsync(
|
||||
_streamOptions.Stream,
|
||||
new RedisValue[] { lease.MessageId })
|
||||
.ConfigureAwait(false);
|
||||
|
||||
DecrementDepth();
|
||||
|
||||
if (!_queueOptions.DeadLetterEnabled)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Dropped {Queue} message {MessageId} after {Attempt} attempt(s); dead-letter disabled. Reason: {Reason}",
|
||||
_payload.QueueName,
|
||||
lease.MessageId,
|
||||
lease.Attempt,
|
||||
reason);
|
||||
return;
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var entries = BuildEntries(lease.Message, now, lease.Attempt);
|
||||
|
||||
await AddToStreamAsync(
|
||||
database,
|
||||
_streamOptions.DeadLetterStream,
|
||||
entries,
|
||||
null,
|
||||
false)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
SchedulerQueueMetrics.RecordDeadLetter(TransportName, _payload.QueueName);
|
||||
_logger.LogError(
|
||||
"Dead-lettered {Queue} message {MessageId} after {Attempt} attempt(s): {Reason}",
|
||||
_payload.QueueName,
|
||||
lease.MessageId,
|
||||
lease.Attempt,
|
||||
reason);
|
||||
}
|
||||
|
||||
public async ValueTask PingAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var database = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
||||
await database.ExecuteAsync("PING").ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private string BuildIdempotencyKey(string key)
|
||||
=> string.Concat(_streamOptions.IdempotencyKeyPrefix, key);
|
||||
|
||||
private TimeSpan CalculateBackoff(int attempt)
|
||||
{
|
||||
if (attempt <= 1)
|
||||
{
|
||||
return _queueOptions.RetryInitialBackoff > TimeSpan.Zero
|
||||
? _queueOptions.RetryInitialBackoff
|
||||
: TimeSpan.Zero;
|
||||
}
|
||||
|
||||
var initial = _queueOptions.RetryInitialBackoff > TimeSpan.Zero
|
||||
? _queueOptions.RetryInitialBackoff
|
||||
: TimeSpan.Zero;
|
||||
|
||||
if (initial <= TimeSpan.Zero)
|
||||
{
|
||||
return TimeSpan.Zero;
|
||||
}
|
||||
|
||||
var max = _queueOptions.RetryMaxBackoff > TimeSpan.Zero
|
||||
? _queueOptions.RetryMaxBackoff
|
||||
: initial;
|
||||
|
||||
var exponent = attempt - 1;
|
||||
var scaledTicks = initial.Ticks * Math.Pow(2, exponent - 1);
|
||||
var cappedTicks = Math.Min(max.Ticks, scaledTicks);
|
||||
|
||||
return TimeSpan.FromTicks((long)Math.Max(initial.Ticks, cappedTicks));
|
||||
}
|
||||
|
||||
private async ValueTask<IDatabase> GetDatabaseAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_connection is not null)
|
||||
{
|
||||
return _connection.GetDatabase(_redisOptions.Database ?? -1);
|
||||
}
|
||||
|
||||
await _connectionLock.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_connection is null)
|
||||
{
|
||||
var config = ConfigurationOptions.Parse(_redisOptions.ConnectionString!);
|
||||
config.AbortOnConnectFail = false;
|
||||
config.ConnectTimeout = (int)_redisOptions.InitializationTimeout.TotalMilliseconds;
|
||||
config.ConnectRetry = 3;
|
||||
|
||||
if (_redisOptions.Database is not null)
|
||||
{
|
||||
config.DefaultDatabase = _redisOptions.Database;
|
||||
}
|
||||
|
||||
_connection = await _connectionFactory(config).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_connectionLock.Release();
|
||||
}
|
||||
|
||||
return _connection.GetDatabase(_redisOptions.Database ?? -1);
|
||||
}
|
||||
|
||||
private async Task EnsureConsumerGroupAsync(
|
||||
IDatabase database,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (_groupInitialized)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _groupInitLock.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_groupInitialized)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await database.StreamCreateConsumerGroupAsync(
|
||||
_streamOptions.Stream,
|
||||
_streamOptions.ConsumerGroup,
|
||||
StreamPosition.Beginning,
|
||||
createStream: true)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (RedisServerException ex) when (ex.Message.Contains("BUSYGROUP", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
// Group already exists.
|
||||
}
|
||||
|
||||
_groupInitialized = true;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_groupInitLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private NameValueEntry[] BuildEntries(
|
||||
TMessage message,
|
||||
DateTimeOffset enqueuedAt,
|
||||
int attempt)
|
||||
{
|
||||
var attributes = _payload.GetAttributes(message);
|
||||
var attributeCount = attributes?.Count ?? 0;
|
||||
var entries = ArrayPool<NameValueEntry>.Shared.Rent(10 + attributeCount);
|
||||
var index = 0;
|
||||
|
||||
entries[index++] = new NameValueEntry(SchedulerQueueFields.QueueKind, _payload.QueueName);
|
||||
entries[index++] = new NameValueEntry(SchedulerQueueFields.RunId, _payload.GetRunId(message));
|
||||
entries[index++] = new NameValueEntry(SchedulerQueueFields.TenantId, _payload.GetTenantId(message));
|
||||
|
||||
var scheduleId = _payload.GetScheduleId(message);
|
||||
if (!string.IsNullOrWhiteSpace(scheduleId))
|
||||
{
|
||||
entries[index++] = new NameValueEntry(SchedulerQueueFields.ScheduleId, scheduleId);
|
||||
}
|
||||
|
||||
var segmentId = _payload.GetSegmentId(message);
|
||||
if (!string.IsNullOrWhiteSpace(segmentId))
|
||||
{
|
||||
entries[index++] = new NameValueEntry(SchedulerQueueFields.SegmentId, segmentId);
|
||||
}
|
||||
|
||||
var correlationId = _payload.GetCorrelationId(message);
|
||||
if (!string.IsNullOrWhiteSpace(correlationId))
|
||||
{
|
||||
entries[index++] = new NameValueEntry(SchedulerQueueFields.CorrelationId, correlationId);
|
||||
}
|
||||
|
||||
entries[index++] = new NameValueEntry(SchedulerQueueFields.IdempotencyKey, _payload.GetIdempotencyKey(message));
|
||||
entries[index++] = new NameValueEntry(SchedulerQueueFields.Attempt, attempt);
|
||||
entries[index++] = new NameValueEntry(SchedulerQueueFields.EnqueuedAt, enqueuedAt.ToUnixTimeMilliseconds());
|
||||
entries[index++] = new NameValueEntry(SchedulerQueueFields.Payload, _payload.Serialize(message));
|
||||
|
||||
if (attributeCount > 0 && attributes is not null)
|
||||
{
|
||||
foreach (var kvp in attributes)
|
||||
{
|
||||
entries[index++] = new NameValueEntry(
|
||||
SchedulerQueueFields.AttributePrefix + kvp.Key,
|
||||
kvp.Value);
|
||||
}
|
||||
}
|
||||
|
||||
var result = entries.AsSpan(0, index).ToArray();
|
||||
ArrayPool<NameValueEntry>.Shared.Return(entries, clearArray: true);
|
||||
return result;
|
||||
}
|
||||
|
||||
private RedisSchedulerQueueLease<TMessage>? TryMapLease(
|
||||
StreamEntry entry,
|
||||
string consumer,
|
||||
DateTimeOffset now,
|
||||
TimeSpan leaseDuration,
|
||||
int? attemptOverride)
|
||||
{
|
||||
if (entry.Values is null || entry.Values.Length == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
string? payload = null;
|
||||
string? runId = null;
|
||||
string? tenantId = null;
|
||||
string? scheduleId = null;
|
||||
string? segmentId = null;
|
||||
string? correlationId = null;
|
||||
string? idempotencyKey = null;
|
||||
long? enqueuedAtUnix = null;
|
||||
var attempt = attemptOverride ?? 1;
|
||||
var attributes = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var field in entry.Values)
|
||||
{
|
||||
var name = field.Name.ToString();
|
||||
var value = field.Value;
|
||||
|
||||
if (name.Equals(SchedulerQueueFields.Payload, StringComparison.Ordinal))
|
||||
{
|
||||
payload = value.ToString();
|
||||
}
|
||||
else if (name.Equals(SchedulerQueueFields.RunId, StringComparison.Ordinal))
|
||||
{
|
||||
runId = value.ToString();
|
||||
}
|
||||
else if (name.Equals(SchedulerQueueFields.TenantId, StringComparison.Ordinal))
|
||||
{
|
||||
tenantId = value.ToString();
|
||||
}
|
||||
else if (name.Equals(SchedulerQueueFields.ScheduleId, StringComparison.Ordinal))
|
||||
{
|
||||
scheduleId = NormalizeOptional(value.ToString());
|
||||
}
|
||||
else if (name.Equals(SchedulerQueueFields.SegmentId, StringComparison.Ordinal))
|
||||
{
|
||||
segmentId = NormalizeOptional(value.ToString());
|
||||
}
|
||||
else if (name.Equals(SchedulerQueueFields.CorrelationId, StringComparison.Ordinal))
|
||||
{
|
||||
correlationId = NormalizeOptional(value.ToString());
|
||||
}
|
||||
else if (name.Equals(SchedulerQueueFields.IdempotencyKey, StringComparison.Ordinal))
|
||||
{
|
||||
idempotencyKey = value.ToString();
|
||||
}
|
||||
else if (name.Equals(SchedulerQueueFields.EnqueuedAt, StringComparison.Ordinal))
|
||||
{
|
||||
if (long.TryParse(value.ToString(), out var unixMs))
|
||||
{
|
||||
enqueuedAtUnix = unixMs;
|
||||
}
|
||||
}
|
||||
else if (name.Equals(SchedulerQueueFields.Attempt, StringComparison.Ordinal))
|
||||
{
|
||||
if (int.TryParse(value.ToString(), out var parsedAttempt))
|
||||
{
|
||||
attempt = attemptOverride.HasValue
|
||||
? Math.Max(attemptOverride.Value, parsedAttempt)
|
||||
: Math.Max(1, parsedAttempt);
|
||||
}
|
||||
}
|
||||
else if (name.StartsWith(SchedulerQueueFields.AttributePrefix, StringComparison.Ordinal))
|
||||
{
|
||||
var key = name[SchedulerQueueFields.AttributePrefix.Length..];
|
||||
attributes[key] = value.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
if (payload is null || runId is null || tenantId is null || enqueuedAtUnix is null || idempotencyKey is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var message = _payload.Deserialize(payload);
|
||||
var enqueuedAt = DateTimeOffset.FromUnixTimeMilliseconds(enqueuedAtUnix.Value);
|
||||
var leaseExpires = now.Add(leaseDuration);
|
||||
|
||||
IReadOnlyDictionary<string, string> attributeView = attributes.Count == 0
|
||||
? EmptyReadOnlyDictionary<string, string>.Instance
|
||||
: new ReadOnlyDictionary<string, string>(attributes);
|
||||
|
||||
return new RedisSchedulerQueueLease<TMessage>(
|
||||
this,
|
||||
entry.Id.ToString(),
|
||||
idempotencyKey,
|
||||
runId,
|
||||
tenantId,
|
||||
scheduleId,
|
||||
segmentId,
|
||||
correlationId,
|
||||
attributeView,
|
||||
message,
|
||||
attempt,
|
||||
enqueuedAt,
|
||||
leaseExpires,
|
||||
consumer);
|
||||
}
|
||||
|
||||
private async Task HandlePoisonEntryAsync(IDatabase database, RedisValue entryId)
|
||||
{
|
||||
await database.StreamAcknowledgeAsync(
|
||||
_streamOptions.Stream,
|
||||
_streamOptions.ConsumerGroup,
|
||||
new RedisValue[] { entryId })
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await database.StreamDeleteAsync(
|
||||
_streamOptions.Stream,
|
||||
new RedisValue[] { entryId })
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<RedisValue> AddToStreamAsync(
|
||||
IDatabase database,
|
||||
RedisKey stream,
|
||||
NameValueEntry[] entries,
|
||||
int? maxLength,
|
||||
bool useApproximateLength)
|
||||
{
|
||||
var capacity = 4 + (entries.Length * 2);
|
||||
var args = new List<object>(capacity)
|
||||
{
|
||||
stream
|
||||
};
|
||||
|
||||
if (maxLength.HasValue)
|
||||
{
|
||||
args.Add("MAXLEN");
|
||||
if (useApproximateLength)
|
||||
{
|
||||
args.Add("~");
|
||||
}
|
||||
|
||||
args.Add(maxLength.Value);
|
||||
}
|
||||
|
||||
args.Add("*");
|
||||
|
||||
for (var i = 0; i < entries.Length; i++)
|
||||
{
|
||||
args.Add(entries[i].Name);
|
||||
args.Add(entries[i].Value);
|
||||
}
|
||||
|
||||
var result = await database.ExecuteAsync("XADD", args.ToArray()).ConfigureAwait(false);
|
||||
return (RedisValue)result!;
|
||||
}
|
||||
|
||||
private void IncrementDepth()
|
||||
{
|
||||
var depth = Interlocked.Increment(ref _approximateDepth);
|
||||
SchedulerQueueMetrics.RecordDepth(TransportName, _payload.QueueName, depth);
|
||||
}
|
||||
|
||||
private void DecrementDepth()
|
||||
{
|
||||
var depth = Interlocked.Decrement(ref _approximateDepth);
|
||||
if (depth < 0)
|
||||
{
|
||||
depth = Interlocked.Exchange(ref _approximateDepth, 0);
|
||||
}
|
||||
|
||||
SchedulerQueueMetrics.RecordDepth(TransportName, _payload.QueueName, depth);
|
||||
}
|
||||
|
||||
private void PublishDepth()
|
||||
{
|
||||
var depth = Volatile.Read(ref _approximateDepth);
|
||||
SchedulerQueueMetrics.RecordDepth(TransportName, _payload.QueueName, depth);
|
||||
}
|
||||
|
||||
private static string? NormalizeOptional(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
private sealed class EmptyReadOnlyDictionary<TKey, TValue>
|
||||
where TKey : notnull
|
||||
{
|
||||
public static readonly IReadOnlyDictionary<TKey, TValue> Instance =
|
||||
new ReadOnlyDictionary<TKey, TValue>(new Dictionary<TKey, TValue>(0, EqualityComparer<TKey>.Default));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StellaOps.Scheduler.Queue.Redis;
|
||||
|
||||
internal sealed class RedisSchedulerQueueLease<TMessage> : ISchedulerQueueLease<TMessage>
|
||||
{
|
||||
private readonly RedisSchedulerQueueBase<TMessage> _queue;
|
||||
private int _completed;
|
||||
|
||||
internal RedisSchedulerQueueLease(
|
||||
RedisSchedulerQueueBase<TMessage> queue,
|
||||
string messageId,
|
||||
string idempotencyKey,
|
||||
string runId,
|
||||
string tenantId,
|
||||
string? scheduleId,
|
||||
string? segmentId,
|
||||
string? correlationId,
|
||||
IReadOnlyDictionary<string, string> attributes,
|
||||
TMessage message,
|
||||
int attempt,
|
||||
DateTimeOffset enqueuedAt,
|
||||
DateTimeOffset leaseExpiresAt,
|
||||
string consumer)
|
||||
{
|
||||
_queue = queue;
|
||||
MessageId = messageId;
|
||||
IdempotencyKey = idempotencyKey;
|
||||
RunId = runId;
|
||||
TenantId = tenantId;
|
||||
ScheduleId = scheduleId;
|
||||
SegmentId = segmentId;
|
||||
CorrelationId = correlationId;
|
||||
Attributes = attributes;
|
||||
Message = message;
|
||||
Attempt = attempt;
|
||||
EnqueuedAt = enqueuedAt;
|
||||
LeaseExpiresAt = leaseExpiresAt;
|
||||
Consumer = consumer;
|
||||
}
|
||||
|
||||
public string MessageId { get; }
|
||||
|
||||
public string IdempotencyKey { get; }
|
||||
|
||||
public string RunId { get; }
|
||||
|
||||
public string TenantId { get; }
|
||||
|
||||
public string? ScheduleId { get; }
|
||||
|
||||
public string? SegmentId { get; }
|
||||
|
||||
public string? CorrelationId { get; }
|
||||
|
||||
public IReadOnlyDictionary<string, string> Attributes { get; }
|
||||
|
||||
public TMessage Message { get; }
|
||||
|
||||
public int Attempt { get; private set; }
|
||||
|
||||
public DateTimeOffset EnqueuedAt { get; }
|
||||
|
||||
public DateTimeOffset LeaseExpiresAt { get; private set; }
|
||||
|
||||
public string Consumer { get; }
|
||||
|
||||
public Task AcknowledgeAsync(CancellationToken cancellationToken = default)
|
||||
=> _queue.AcknowledgeAsync(this, cancellationToken);
|
||||
|
||||
public Task RenewAsync(TimeSpan leaseDuration, CancellationToken cancellationToken = default)
|
||||
=> _queue.RenewLeaseAsync(this, leaseDuration, cancellationToken);
|
||||
|
||||
public Task ReleaseAsync(SchedulerQueueReleaseDisposition disposition, CancellationToken cancellationToken = default)
|
||||
=> _queue.ReleaseAsync(this, disposition, cancellationToken);
|
||||
|
||||
public Task DeadLetterAsync(string reason, CancellationToken cancellationToken = default)
|
||||
=> _queue.DeadLetterAsync(this, reason, cancellationToken);
|
||||
|
||||
internal bool TryBeginCompletion()
|
||||
=> Interlocked.CompareExchange(ref _completed, 1, 0) == 0;
|
||||
|
||||
internal void RefreshLease(DateTimeOffset expiresAt)
|
||||
=> LeaseExpiresAt = expiresAt;
|
||||
|
||||
internal void IncrementAttempt()
|
||||
=> Attempt++;
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StackExchange.Redis;
|
||||
using StellaOps.Scheduler.Models;
|
||||
|
||||
namespace StellaOps.Scheduler.Queue.Redis;
|
||||
|
||||
internal sealed class RedisSchedulerRunnerQueue
|
||||
: RedisSchedulerQueueBase<RunnerSegmentQueueMessage>, ISchedulerRunnerQueue
|
||||
{
|
||||
public RedisSchedulerRunnerQueue(
|
||||
SchedulerQueueOptions queueOptions,
|
||||
SchedulerRedisQueueOptions redisOptions,
|
||||
ILogger<RedisSchedulerRunnerQueue> logger,
|
||||
TimeProvider timeProvider,
|
||||
Func<ConfigurationOptions, Task<IConnectionMultiplexer>>? connectionFactory = null)
|
||||
: base(
|
||||
queueOptions,
|
||||
redisOptions,
|
||||
redisOptions.Runner,
|
||||
RunnerPayload.Instance,
|
||||
logger,
|
||||
timeProvider,
|
||||
connectionFactory)
|
||||
{
|
||||
}
|
||||
|
||||
private sealed class RunnerPayload : IRedisSchedulerQueuePayload<RunnerSegmentQueueMessage>
|
||||
{
|
||||
public static RunnerPayload Instance { get; } = new();
|
||||
|
||||
public string QueueName => "runner";
|
||||
|
||||
public string GetIdempotencyKey(RunnerSegmentQueueMessage message)
|
||||
=> message.IdempotencyKey;
|
||||
|
||||
public string Serialize(RunnerSegmentQueueMessage message)
|
||||
=> CanonicalJsonSerializer.Serialize(message);
|
||||
|
||||
public RunnerSegmentQueueMessage Deserialize(string payload)
|
||||
=> CanonicalJsonSerializer.Deserialize<RunnerSegmentQueueMessage>(payload);
|
||||
|
||||
public string GetRunId(RunnerSegmentQueueMessage message)
|
||||
=> message.RunId;
|
||||
|
||||
public string GetTenantId(RunnerSegmentQueueMessage message)
|
||||
=> message.TenantId;
|
||||
|
||||
public string? GetScheduleId(RunnerSegmentQueueMessage message)
|
||||
=> message.ScheduleId;
|
||||
|
||||
public string? GetSegmentId(RunnerSegmentQueueMessage message)
|
||||
=> message.SegmentId;
|
||||
|
||||
public string? GetCorrelationId(RunnerSegmentQueueMessage message)
|
||||
=> message.CorrelationId;
|
||||
|
||||
public IReadOnlyDictionary<string, string>? GetAttributes(RunnerSegmentQueueMessage message)
|
||||
{
|
||||
if (message.Attributes.Count == 0 && message.ImageDigests.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
// Ensure digests remain accessible without deserializing the entire payload.
|
||||
var map = new Dictionary<string, string>(message.Attributes, StringComparer.Ordinal);
|
||||
map["imageDigestCount"] = message.ImageDigests.Count.ToString();
|
||||
|
||||
// populate first few digests for quick inspection (bounded)
|
||||
var take = Math.Min(message.ImageDigests.Count, 5);
|
||||
for (var i = 0; i < take; i++)
|
||||
{
|
||||
map[$"digest{i}"] = message.ImageDigests[i];
|
||||
}
|
||||
|
||||
if (message.RatePerSecond.HasValue)
|
||||
{
|
||||
map["ratePerSecond"] = message.RatePerSecond.Value.ToString();
|
||||
}
|
||||
|
||||
map["usageOnly"] = message.UsageOnly ? "true" : "false";
|
||||
|
||||
return map;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user