git.stella-ops.org/src/Timeline/__Libraries/StellaOps.TimelineIndexer.Infrastructure/Subscriptions/RedisTimelineEventSubscriber.cs

using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StackExchange.Redis;
using StellaOps.TimelineIndexer.Core.Abstractions;
using StellaOps.TimelineIndexer.Core.Models;
using StellaOps.TimelineIndexer.Infrastructure.Options;
using System.Runtime.CompilerServices;

namespace StellaOps.TimelineIndexer.Infrastructure.Subscriptions;

/// <summary>
/// Redis Stream subscriber that reads orchestrator events and yields timeline envelopes.
/// </summary>
public sealed class RedisTimelineEventSubscriber : ITimelineEventSubscriber, IAsyncDisposable
{
    private readonly IOptions<TimelineIngestionOptions> _options;
    private readonly TimelineEnvelopeParser _parser;
    private readonly ILogger<RedisTimelineEventSubscriber> _logger;
    private ConnectionMultiplexer? _connection;

    public RedisTimelineEventSubscriber(
        IOptions<TimelineIngestionOptions> options,
        TimelineEnvelopeParser parser,
        ILogger<RedisTimelineEventSubscriber> logger)
    {
        _options = options ?? throw new ArgumentNullException(nameof(options));
        _parser = parser ?? throw new ArgumentNullException(nameof(parser));
        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
    }

    public async IAsyncEnumerable<TimelineEventEnvelope> SubscribeAsync([EnumeratorCancellation] CancellationToken cancellationToken = default)
    {
        var cfg = _options.Value.Redis;
        if (!cfg.Enabled)
        {
            yield break;
        }

        _connection = await ConnectionMultiplexer.ConnectAsync(cfg.ConnectionString);
        var db = _connection.GetDatabase();

        await EnsureGroupAsync(db, cfg, cancellationToken).ConfigureAwait(false);

        while (!cancellationToken.IsCancellationRequested)
        {
            StreamEntry[] entries;
            try
            {
                entries = await db.StreamReadGroupAsync(
                    cfg.Stream,
                    cfg.ConsumerGroup,
                    cfg.ConsumerName,
                    ">",
                    count: cfg.MaxBatchSize,
                    flags: CommandFlags.DemandMaster).ConfigureAwait(false);
            }
            catch (RedisServerException ex) when (ex.Message.Contains("NOGROUP", StringComparison.OrdinalIgnoreCase))
            {
                await EnsureGroupAsync(db, cfg, cancellationToken).ConfigureAwait(false);
                continue;
            }

            if (entries.Length == 0)
            {
                await Task.Delay(cfg.PollIntervalMilliseconds, cancellationToken).ConfigureAwait(false);
                continue;
            }

            foreach (var entry in entries)
            {
                if (!TryGetValue(entry, cfg.ValueField, out var json))
                {
                    _logger.LogWarning("Redis entry {EntryId} missing expected field {Field}", entry.Id, cfg.ValueField);
                    await db.StreamAcknowledgeAsync(cfg.Stream, cfg.ConsumerGroup, entry.Id).ConfigureAwait(false);
                    continue;
                }

                if (_parser.TryParse(json!, out var envelope, out var reason))
                {
                    yield return envelope;
                }
                else
                {
                    _logger.LogWarning("Redis entry {EntryId} dropped: {Reason}", entry.Id, reason);
                }

                await db.StreamAcknowledgeAsync(cfg.Stream, cfg.ConsumerGroup, entry.Id).ConfigureAwait(false);
            }
        }
    }

    private static async Task EnsureGroupAsync(IDatabase db, RedisIngestionOptions cfg, CancellationToken cancellationToken)
    {
        try
        {
            await db.StreamCreateConsumerGroupAsync(cfg.Stream, cfg.ConsumerGroup, "$", true).ConfigureAwait(false);
        }
        catch (RedisServerException ex) when (ex.Message.Contains("BUSYGROUP", StringComparison.OrdinalIgnoreCase))
        {
            // Group already exists; nothing to do.
        }
    }

    private static bool TryGetValue(in StreamEntry entry, string fieldName, out string? value)
    {
        foreach (var nv in entry.Values)
        {
            if (string.Equals(nv.Name, fieldName, StringComparison.OrdinalIgnoreCase))
            {
                value = nv.Value.HasValue ? nv.Value.ToString() : null;
                return true;
            }
        }

        value = null;
        return false;
    }

    public async ValueTask DisposeAsync()
    {
        if (_connection is not null)
        {
            await _connection.DisposeAsync().ConfigureAwait(false);
        }
    }
}