using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using StackExchange.Redis; using StellaOps.TimelineIndexer.Core.Abstractions; using StellaOps.TimelineIndexer.Core.Models; using StellaOps.TimelineIndexer.Infrastructure.Options; using System.Runtime.CompilerServices; namespace StellaOps.TimelineIndexer.Infrastructure.Subscriptions; /// /// Redis Stream subscriber that reads orchestrator events and yields timeline envelopes. /// public sealed class RedisTimelineEventSubscriber : ITimelineEventSubscriber, IAsyncDisposable { private readonly IOptions _options; private readonly TimelineEnvelopeParser _parser; private readonly ILogger _logger; private ConnectionMultiplexer? _connection; public RedisTimelineEventSubscriber( IOptions options, TimelineEnvelopeParser parser, ILogger logger) { _options = options ?? throw new ArgumentNullException(nameof(options)); _parser = parser ?? throw new ArgumentNullException(nameof(parser)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } public async IAsyncEnumerable SubscribeAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) { var cfg = _options.Value.Redis; if (!cfg.Enabled) { yield break; } _connection = await ConnectionMultiplexer.ConnectAsync(cfg.ConnectionString); var db = _connection.GetDatabase(); await EnsureGroupAsync(db, cfg, cancellationToken).ConfigureAwait(false); while (!cancellationToken.IsCancellationRequested) { StreamEntry[] entries; try { entries = await db.StreamReadGroupAsync( cfg.Stream, cfg.ConsumerGroup, cfg.ConsumerName, ">", count: cfg.MaxBatchSize, flags: CommandFlags.DemandMaster).ConfigureAwait(false); } catch (RedisServerException ex) when (ex.Message.Contains("NOGROUP", StringComparison.OrdinalIgnoreCase)) { await EnsureGroupAsync(db, cfg, cancellationToken).ConfigureAwait(false); continue; } if (entries.Length == 0) { await Task.Delay(cfg.PollIntervalMilliseconds, cancellationToken).ConfigureAwait(false); continue; } foreach (var entry in entries) { if (!TryGetValue(entry, cfg.ValueField, out var json)) { _logger.LogWarning("Redis entry {EntryId} missing expected field {Field}", entry.Id, cfg.ValueField); await db.StreamAcknowledgeAsync(cfg.Stream, cfg.ConsumerGroup, entry.Id).ConfigureAwait(false); continue; } if (_parser.TryParse(json!, out var envelope, out var reason)) { yield return envelope; } else { _logger.LogWarning("Redis entry {EntryId} dropped: {Reason}", entry.Id, reason); } await db.StreamAcknowledgeAsync(cfg.Stream, cfg.ConsumerGroup, entry.Id).ConfigureAwait(false); } } } private static async Task EnsureGroupAsync(IDatabase db, RedisIngestionOptions cfg, CancellationToken cancellationToken) { try { await db.StreamCreateConsumerGroupAsync(cfg.Stream, cfg.ConsumerGroup, "$", true).ConfigureAwait(false); } catch (RedisServerException ex) when (ex.Message.Contains("BUSYGROUP", StringComparison.OrdinalIgnoreCase)) { // Group already exists; nothing to do. } } private static bool TryGetValue(in StreamEntry entry, string fieldName, out string? value) { foreach (var nv in entry.Values) { if (string.Equals(nv.Name, fieldName, StringComparison.OrdinalIgnoreCase)) { value = nv.Value.HasValue ? nv.Value.ToString() : null; return true; } } value = null; return false; } public async ValueTask DisposeAsync() { if (_connection is not null) { await _connection.DisposeAsync().ConfigureAwait(false); } } }