audit remarks work

src/Tools/NotifySmokeCheck/InternalsVisibleTo.cs

@@ -0,0 +1,3 @@
+using System.Runtime.CompilerServices;
+
+[assembly: InternalsVisibleTo("NotifySmokeCheck.Tests")]

src/Tools/NotifySmokeCheck/NotifySmokeCheckApp.cs

@@ -0,0 +1,21 @@
+namespace StellaOps.Tools.NotifySmokeCheck;
+
+public static class NotifySmokeCheckApp
+{
+    public static async Task<int> RunAsync(string[] args)
+    {
+        try
+        {
+            var options = NotifySmokeOptions.FromEnvironment(Environment.GetEnvironmentVariable);
+            var runner = new NotifySmokeCheckRunner(options, Console.WriteLine, Console.Error.WriteLine);
+            await runner.RunAsync(CancellationToken.None).ConfigureAwait(false);
+            Console.WriteLine("[OK] Notify smoke validation completed successfully.");
+            return 0;
+        }
+        catch (Exception ex)
+        {
+            Console.Error.WriteLine($"[FAIL] {ex.Message}");
+            return 1;
+        }
+    }
+}

src/Tools/NotifySmokeCheck/NotifySmokeCheckRunner.cs

@@ -0,0 +1,482 @@
+using System.Globalization;
+using System.Net;
+using System.Net.Http.Headers;
+using System.Text.Json;
+using StackExchange.Redis;
+
+namespace StellaOps.Tools.NotifySmokeCheck;
+
+public sealed record NotifyDeliveryOptions(
+    Uri BaseUri,
+    string Token,
+    string Tenant,
+    string TenantHeader,
+    TimeSpan Timeout,
+    int Limit);
+
+public sealed record NotifySmokeOptions(
+    string RedisDsn,
+    string RedisStream,
+    IReadOnlyList<string> ExpectedKinds,
+    TimeSpan Lookback,
+    int StreamPageSize,
+    int StreamMaxEntries,
+    int RetryAttempts,
+    TimeSpan RetryDelay,
+    NotifyDeliveryOptions Delivery,
+    TimeProvider TimeProvider)
+{
+    public static NotifySmokeOptions FromEnvironment(Func<string, string?> getEnv)
+    {
+        string RequireEnv(string name)
+        {
+            var value = getEnv(name);
+            if (string.IsNullOrWhiteSpace(value))
+            {
+                throw new InvalidOperationException($"Environment variable '{name}' is required for Notify smoke validation.");
+            }
+
+            return value;
+        }
+
+        var redisDsn = RequireEnv("NOTIFY_SMOKE_REDIS_DSN");
+        var redisStream = getEnv("NOTIFY_SMOKE_STREAM");
+        if (string.IsNullOrWhiteSpace(redisStream))
+        {
+            redisStream = "stella.events";
+        }
+
+        var expectedKindsEnv = RequireEnv("NOTIFY_SMOKE_EXPECT_KINDS");
+        var expectedKinds = expectedKindsEnv
+            .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
+            .Select(kind => kind.ToLowerInvariant())
+            .Distinct(StringComparer.Ordinal)
+            .ToArray();
+        if (expectedKinds.Length == 0)
+        {
+            throw new InvalidOperationException("Expected at least one event kind in NOTIFY_SMOKE_EXPECT_KINDS.");
+        }
+
+        var lookbackMinutesEnv = RequireEnv("NOTIFY_SMOKE_LOOKBACK_MINUTES");
+        if (!double.TryParse(lookbackMinutesEnv, NumberStyles.Any, CultureInfo.InvariantCulture, out var lookbackMinutes))
+        {
+            throw new InvalidOperationException("NOTIFY_SMOKE_LOOKBACK_MINUTES must be numeric.");
+        }
+
+        if (lookbackMinutes <= 0)
+        {
+            throw new InvalidOperationException("NOTIFY_SMOKE_LOOKBACK_MINUTES must be greater than zero.");
+        }
+
+        var streamPageSize = ParseInt(getEnv("NOTIFY_SMOKE_STREAM_PAGE_SIZE"), 500, min: 50);
+        var streamMaxEntries = ParseInt(getEnv("NOTIFY_SMOKE_STREAM_MAX_ENTRIES"), 2000, min: streamPageSize);
+        if (streamMaxEntries < streamPageSize)
+        {
+            streamMaxEntries = streamPageSize;
+        }
+
+        var retryAttempts = ParseInt(getEnv("NOTIFY_SMOKE_RETRY_ATTEMPTS"), 3, min: 1, max: 10);
+        var retryDelayMs = ParseInt(getEnv("NOTIFY_SMOKE_RETRY_DELAY_MS"), 250, min: 50, max: 2000);
+
+        var baseUrlRaw = RequireEnv("NOTIFY_SMOKE_NOTIFY_BASEURL").TrimEnd('/');
+        if (!Uri.TryCreate(baseUrlRaw, UriKind.Absolute, out var baseUri))
+        {
+            throw new InvalidOperationException("NOTIFY_SMOKE_NOTIFY_BASEURL must be an absolute URL.");
+        }
+
+        var deliveryToken = RequireEnv("NOTIFY_SMOKE_NOTIFY_TOKEN");
+        var deliveryTenant = RequireEnv("NOTIFY_SMOKE_NOTIFY_TENANT");
+        var tenantHeader = getEnv("NOTIFY_SMOKE_NOTIFY_TENANT_HEADER");
+        if (string.IsNullOrWhiteSpace(tenantHeader))
+        {
+            tenantHeader = "X-StellaOps-Tenant";
+        }
+
+        var timeoutSeconds = ParseInt(getEnv("NOTIFY_SMOKE_NOTIFY_TIMEOUT_SECONDS"), 30, min: 5, max: 120);
+        var limit = ParseInt(getEnv("NOTIFY_SMOKE_NOTIFY_LIMIT"), 200, min: 50, max: 2000);
+
+        var fixedTimeEnv = getEnv("NOTIFY_SMOKE_FIXED_TIME");
+        var timeProvider = ResolveTimeProvider(fixedTimeEnv);
+
+        return new NotifySmokeOptions(
+            RedisDsn: redisDsn,
+            RedisStream: redisStream,
+            ExpectedKinds: expectedKinds,
+            Lookback: TimeSpan.FromMinutes(lookbackMinutes),
+            StreamPageSize: streamPageSize,
+            StreamMaxEntries: streamMaxEntries,
+            RetryAttempts: retryAttempts,
+            RetryDelay: TimeSpan.FromMilliseconds(retryDelayMs),
+            Delivery: new NotifyDeliveryOptions(
+                BaseUri: baseUri,
+                Token: deliveryToken,
+                Tenant: deliveryTenant,
+                TenantHeader: tenantHeader,
+                Timeout: TimeSpan.FromSeconds(timeoutSeconds),
+                Limit: limit),
+            TimeProvider: timeProvider);
+    }
+
+    private static int ParseInt(string? value, int fallback, int min = 0, int max = int.MaxValue)
+    {
+        if (string.IsNullOrWhiteSpace(value))
+        {
+            return fallback;
+        }
+
+        if (!int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed))
+        {
+            return fallback;
+        }
+
+        if (parsed < min)
+        {
+            return min;
+        }
+
+        return parsed > max ? max : parsed;
+    }
+
+    private static TimeProvider ResolveTimeProvider(string? fixedTimeEnv)
+    {
+        if (string.IsNullOrWhiteSpace(fixedTimeEnv))
+        {
+            return TimeProvider.System;
+        }
+
+        if (!DateTimeOffset.TryParse(fixedTimeEnv, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var fixedTime))
+        {
+            throw new InvalidOperationException("NOTIFY_SMOKE_FIXED_TIME must be an ISO-8601 timestamp.");
+        }
+
+        return new FixedTimeProvider(fixedTime);
+    }
+}
+
+public sealed record NotifyDeliveryRecord(string Kind, string? Status);
+
+public sealed class NotifySmokeCheckRunner
+{
+    private readonly NotifySmokeOptions _options;
+    private readonly Action<string> _info;
+    private readonly Action<string> _error;
+
+    public NotifySmokeCheckRunner(NotifySmokeOptions options, Action<string>? info = null, Action<string>? error = null)
+    {
+        _options = options;
+        _info = info ?? (_ => { });
+        _error = error ?? (_ => { });
+    }
+
+    public async Task RunAsync(CancellationToken cancellationToken)
+    {
+        var now = _options.TimeProvider.GetUtcNow();
+        var sinceThreshold = now - _options.Lookback;
+
+        _info($"[INFO] Checking Redis stream '{_options.RedisStream}' for kinds [{string.Join(", ", _options.ExpectedKinds)}] within the last {_options.Lookback.TotalMinutes:F1} minutes.");
+
+        var redisConfig = ConfigurationOptions.Parse(_options.RedisDsn);
+        redisConfig.AbortOnConnectFail = false;
+
+        await using var redisConnection = await ConnectWithRetriesAsync(redisConfig, cancellationToken).ConfigureAwait(false);
+        var database = redisConnection.GetDatabase();
+
+        var recentEntries = await ReadRecentStreamEntriesAsync(database, _options.RedisStream, sinceThreshold, cancellationToken).ConfigureAwait(false);
+        Ensure(recentEntries.Count > 0, $"No Redis events newer than {sinceThreshold:u} located in stream '{_options.RedisStream}'.");
+
+        var missingKinds = FindMissingKinds(recentEntries, _options.ExpectedKinds);
+        Ensure(missingKinds.Count == 0, $"Missing expected Redis events for kinds: {string.Join(", ", missingKinds)}");
+
+        _info("[INFO] Redis event stream contains the expected scanner events.");
+
+        var deliveriesUrl = BuildDeliveriesUrl(_options.Delivery.BaseUri, sinceThreshold, _options.Delivery.Limit);
+        _info($"[INFO] Querying Notify deliveries via {deliveriesUrl}.");
+
+        using var httpClient = BuildHttpClient(_options.Delivery);
+        using var response = await GetWithRetriesAsync(httpClient, deliveriesUrl, cancellationToken).ConfigureAwait(false);
+
+        if (!response.IsSuccessStatusCode)
+        {
+            var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
+            throw new InvalidOperationException($"Notify deliveries request failed with {(int)response.StatusCode} {response.ReasonPhrase}: {body}");
+        }
+
+        var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
+        Ensure(!string.IsNullOrWhiteSpace(json), "Notify deliveries response body was empty.");
+
+        var deliveries = ParseDeliveries(json);
+        Ensure(deliveries.Count > 0, "Notify deliveries response did not return any records.");
+
+        var missingDeliveryKinds = FindMissingDeliveryKinds(deliveries, _options.ExpectedKinds);
+        Ensure(missingDeliveryKinds.Count == 0, $"Notify deliveries missing successful records for kinds: {string.Join(", ", missingDeliveryKinds)}");
+
+        _info("[INFO] Notify deliveries include the expected scanner events.");
+    }
+
+    internal static IReadOnlyList<NotifyDeliveryRecord> ParseDeliveries(string json)
+    {
+        using var document = JsonDocument.Parse(json);
+        var root = document.RootElement;
+
+        IEnumerable<JsonElement> EnumerateDeliveries(JsonElement element)
+        {
+            return element.ValueKind switch
+            {
+                JsonValueKind.Array => element.EnumerateArray(),
+                JsonValueKind.Object when element.TryGetProperty("items", out var items) && items.ValueKind == JsonValueKind.Array => items.EnumerateArray(),
+                _ => throw new InvalidOperationException("Notify deliveries response was not an array or did not contain an 'items' collection.")
+            };
+        }
+
+        var deliveries = new List<NotifyDeliveryRecord>();
+        foreach (var delivery in EnumerateDeliveries(root))
+        {
+            var kind = delivery.TryGetProperty("kind", out var kindProperty) ? kindProperty.GetString() : null;
+            if (string.IsNullOrWhiteSpace(kind))
+            {
+                continue;
+            }
+
+            var status = delivery.TryGetProperty("status", out var statusProperty) ? statusProperty.GetString() : null;
+            deliveries.Add(new NotifyDeliveryRecord(kind, status));
+        }
+
+        return deliveries;
+    }
+
+    internal static IReadOnlyList<string> FindMissingDeliveryKinds(IReadOnlyList<NotifyDeliveryRecord> deliveries, IReadOnlyList<string> expectedKinds)
+    {
+        var missingKinds = new List<string>();
+        foreach (var kind in expectedKinds)
+        {
+            var found = deliveries.Any(delivery =>
+                string.Equals(delivery.Kind, kind, StringComparison.OrdinalIgnoreCase) &&
+                !string.Equals(delivery.Status, "failed", StringComparison.OrdinalIgnoreCase));
+
+            if (!found)
+            {
+                missingKinds.Add(kind);
+            }
+        }
+
+        return missingKinds;
+    }
+
+    internal static IReadOnlyList<string> FindMissingKinds(IReadOnlyList<StreamEntry> entries, IReadOnlyList<string> expectedKinds)
+    {
+        var missingKinds = new List<string>();
+        foreach (var kind in expectedKinds)
+        {
+            var match = entries.FirstOrDefault(entry =>
+            {
+                var entryKind = GetField(entry, "kind");
+                return entryKind is not null && string.Equals(entryKind, kind, StringComparison.OrdinalIgnoreCase);
+            });
+
+            if (match.Equals(default(StreamEntry)))
+            {
+                missingKinds.Add(kind);
+            }
+        }
+
+        return missingKinds;
+    }
+
+    private async Task<IReadOnlyList<StreamEntry>> ReadRecentStreamEntriesAsync(IDatabase database, string stream, DateTimeOffset sinceThreshold, CancellationToken cancellationToken)
+    {
+        var recentEntries = new List<StreamEntry>();
+        var scannedEntries = 0;
+        RedisValue maxId = "+";
+        var reachedThreshold = false;
+
+        while (scannedEntries < _options.StreamMaxEntries && !reachedThreshold)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            var batchSize = Math.Min(_options.StreamPageSize, _options.StreamMaxEntries - scannedEntries);
+            var batch = await ReadStreamBatchAsync(database, stream, maxId, batchSize, cancellationToken).ConfigureAwait(false);
+            if (batch.Length == 0)
+            {
+                break;
+            }
+
+            foreach (var entry in batch)
+            {
+                scannedEntries++;
+                if (TryGetStreamTimestamp(entry, out var entryTimestamp))
+                {
+                    if (entryTimestamp >= sinceThreshold)
+                    {
+                        recentEntries.Add(entry);
+                    }
+                    else
+                    {
+                        reachedThreshold = true;
+                        break;
+                    }
+                }
+                else
+                {
+                    _error($"[WARN] Unable to parse stream entry id '{entry.Id}'.");
+                }
+            }
+
+            maxId = $"({batch[^1].Id}";
+        }
+
+        if (scannedEntries >= _options.StreamMaxEntries && !reachedThreshold)
+        {
+            _error($"[WARN] Reached stream scan limit ({_options.StreamMaxEntries}) before lookback threshold {sinceThreshold:u}.");
+        }
+
+        return recentEntries;
+    }
+
+    private async Task<StreamEntry[]> ReadStreamBatchAsync(IDatabase database, string stream, RedisValue maxId, int batchSize, CancellationToken cancellationToken)
+    {
+        for (var attempt = 1; attempt <= _options.RetryAttempts; attempt++)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            try
+            {
+                return await database.StreamRangeAsync(stream, "-", maxId, batchSize, Order.Descending).ConfigureAwait(false);
+            }
+            catch (Exception ex) when (attempt < _options.RetryAttempts)
+            {
+                _error($"[WARN] Redis stream range attempt {attempt} failed: {ex.Message}");
+                await Task.Delay(_options.RetryDelay, cancellationToken).ConfigureAwait(false);
+            }
+        }
+
+        return await database.StreamRangeAsync(stream, "-", maxId, batchSize, Order.Descending).ConfigureAwait(false);
+    }
+
+    internal static bool TryGetStreamTimestamp(StreamEntry entry, out DateTimeOffset timestamp)
+    {
+        var id = entry.Id.ToString();
+        var dash = id.IndexOf('-', StringComparison.Ordinal);
+        if (dash <= 0)
+        {
+            timestamp = default;
+            return false;
+        }
+
+        if (!long.TryParse(id[..dash], NumberStyles.Integer, CultureInfo.InvariantCulture, out var millis))
+        {
+            timestamp = default;
+            return false;
+        }
+
+        timestamp = DateTimeOffset.FromUnixTimeMilliseconds(millis);
+        return true;
+    }
+
+    private static string? GetField(StreamEntry entry, string fieldName)
+    {
+        foreach (var pair in entry.Values)
+        {
+            if (string.Equals(pair.Name, fieldName, StringComparison.OrdinalIgnoreCase))
+            {
+                return pair.Value.ToString();
+            }
+        }
+
+        return null;
+    }
+
+    private async Task<ConnectionMultiplexer> ConnectWithRetriesAsync(ConfigurationOptions options, CancellationToken cancellationToken)
+    {
+        for (var attempt = 1; attempt <= _options.RetryAttempts; attempt++)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            try
+            {
+                return await ConnectionMultiplexer.ConnectAsync(options).ConfigureAwait(false);
+            }
+            catch (Exception ex) when (attempt < _options.RetryAttempts)
+            {
+                _error($"[WARN] Redis connection attempt {attempt} failed: {ex.Message}");
+                await Task.Delay(_options.RetryDelay, cancellationToken).ConfigureAwait(false);
+            }
+        }
+
+        return await ConnectionMultiplexer.ConnectAsync(options).ConfigureAwait(false);
+    }
+
+    private HttpClient BuildHttpClient(NotifyDeliveryOptions delivery)
+    {
+        var httpClient = new HttpClient
+        {
+            Timeout = delivery.Timeout,
+        };
+
+        httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", delivery.Token);
+        httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
+        httpClient.DefaultRequestHeaders.Add(delivery.TenantHeader, delivery.Tenant);
+
+        return httpClient;
+    }
+
+    private async Task<HttpResponseMessage> GetWithRetriesAsync(HttpClient httpClient, Uri url, CancellationToken cancellationToken)
+    {
+        for (var attempt = 1; attempt <= _options.RetryAttempts; attempt++)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            var response = await httpClient.GetAsync(url, cancellationToken).ConfigureAwait(false);
+            if (!ShouldRetry(response.StatusCode) || attempt == _options.RetryAttempts)
+            {
+                return response;
+            }
+
+            _error($"[WARN] Notify deliveries attempt {attempt} returned {(int)response.StatusCode}. Retrying after {_options.RetryDelay.TotalMilliseconds:F0} ms.");
+            response.Dispose();
+            await Task.Delay(_options.RetryDelay, cancellationToken).ConfigureAwait(false);
+        }
+
+        return await httpClient.GetAsync(url, cancellationToken).ConfigureAwait(false);
+    }
+
+    private static bool ShouldRetry(HttpStatusCode statusCode)
+        => statusCode == HttpStatusCode.RequestTimeout
+           || statusCode == (HttpStatusCode)429
+           || (int)statusCode >= 500;
+
+    private static Uri BuildDeliveriesUrl(Uri baseUri, DateTimeOffset sinceThreshold, int limit)
+    {
+        var sinceQuery = Uri.EscapeDataString(sinceThreshold.ToString("O", CultureInfo.InvariantCulture));
+        var builder = new UriBuilder(baseUri)
+        {
+            Path = "/api/v1/deliveries",
+            Query = $"since={sinceQuery}&limit={limit}"
+        };
+
+        return builder.Uri;
+    }
+
+    private static void Ensure(bool condition, string message)
+    {
+        if (!condition)
+        {
+            throw new InvalidOperationException(message);
+        }
+    }
+}
+
+internal sealed class FixedTimeProvider : TimeProvider
+{
+    private readonly DateTimeOffset _fixedTime;
+    private readonly long _timestamp;
+
+    public FixedTimeProvider(DateTimeOffset fixedTime)
+    {
+        _fixedTime = fixedTime;
+        _timestamp = fixedTime.UtcTicks;
+    }
+
+    public override DateTimeOffset GetUtcNow() => _fixedTime;
+
+    public override long GetTimestamp() => _timestamp;
+}

src/Tools/NotifySmokeCheck/Program.cs

@@ -1,198 +1,3 @@
-using System.Globalization;
-using System.Net.Http.Headers;
-using System.Linq;
-using System.Text.Json;
-using StackExchange.Redis;
+using StellaOps.Tools.NotifySmokeCheck;
 
-static string RequireEnv(string name)
-{
-    var value = Environment.GetEnvironmentVariable(name);
-    if (string.IsNullOrWhiteSpace(value))
-    {
-        throw new InvalidOperationException($"Environment variable '{name}' is required for Notify smoke validation.");
-    }
-
-    return value;
-}
-
-static string? GetField(StreamEntry entry, string fieldName)
-{
-    foreach (var pair in entry.Values)
-    {
-        if (string.Equals(pair.Name, fieldName, StringComparison.OrdinalIgnoreCase))
-        {
-            return pair.Value.ToString();
-        }
-    }
-
-    return null;
-}
-
-static void Ensure(bool condition, string message)
-{
-    if (!condition)
-    {
-        throw new InvalidOperationException(message);
-    }
-}
-
-var redisDsn = RequireEnv("NOTIFY_SMOKE_REDIS_DSN");
-var redisStream = Environment.GetEnvironmentVariable("NOTIFY_SMOKE_STREAM");
-if (string.IsNullOrWhiteSpace(redisStream))
-{
-    redisStream = "stella.events";
-}
-
-var expectedKindsEnv = RequireEnv("NOTIFY_SMOKE_EXPECT_KINDS");
-
-var expectedKinds = expectedKindsEnv
-    .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
-    .Select(kind => kind.ToLowerInvariant())
-    .Distinct()
-    .ToArray();
-Ensure(expectedKinds.Length > 0, "Expected at least one event kind in NOTIFY_SMOKE_EXPECT_KINDS.");
-
-var lookbackMinutesEnv = RequireEnv("NOTIFY_SMOKE_LOOKBACK_MINUTES");
-if (!double.TryParse(lookbackMinutesEnv, NumberStyles.Any, CultureInfo.InvariantCulture, out var lookbackMinutes))
-{
-    throw new InvalidOperationException("NOTIFY_SMOKE_LOOKBACK_MINUTES must be numeric.");
-}
-Ensure(lookbackMinutes > 0, "NOTIFY_SMOKE_LOOKBACK_MINUTES must be greater than zero.");
-
-var now = DateTimeOffset.UtcNow;
-var sinceThreshold = now - TimeSpan.FromMinutes(Math.Max(1, lookbackMinutes));
-
-Console.WriteLine($"ℹ️  Checking Redis stream '{redisStream}' for kinds [{string.Join(", ", expectedKinds)}] within the last {lookbackMinutes:F1} minutes.");
-
-var redisConfig = ConfigurationOptions.Parse(redisDsn);
-redisConfig.AbortOnConnectFail = false;
-
-await using var redisConnection = await ConnectionMultiplexer.ConnectAsync(redisConfig);
-var database = redisConnection.GetDatabase();
-
-var streamEntries = await database.StreamRangeAsync(redisStream, "-", "+", count: 200);
-if (streamEntries.Length > 1)
-{
-    Array.Reverse(streamEntries);
-}
-Ensure(streamEntries.Length > 0, $"Redis stream '{redisStream}' is empty.");
-
-var recentEntries = new List<StreamEntry>();
-foreach (var entry in streamEntries)
-{
-    var timestampText = GetField(entry, "ts");
-    if (timestampText is null)
-    {
-        continue;
-    }
-
-    if (!DateTimeOffset.TryParse(timestampText, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var entryTimestamp))
-    {
-        continue;
-    }
-
-    if (entryTimestamp >= sinceThreshold)
-    {
-        recentEntries.Add(entry);
-    }
-}
-
-Ensure(recentEntries.Count > 0, $"No Redis events newer than {sinceThreshold:u} located in stream '{redisStream}'.");
-
-var missingKinds = new List<string>();
-foreach (var kind in expectedKinds)
-{
-    var match = recentEntries.FirstOrDefault(entry =>
-    {
-        var entryKind = GetField(entry, "kind")?.ToLowerInvariant();
-        return entryKind == kind;
-    });
-
-    if (match.Equals(default(StreamEntry)))
-    {
-        missingKinds.Add(kind);
-    }
-}
-
-Ensure(missingKinds.Count == 0, $"Missing expected Redis events for kinds: {string.Join(", ", missingKinds)}");
-
-Console.WriteLine("✅ Redis event stream contains the expected scanner events.");
-
-var notifyBaseUrl = RequireEnv("NOTIFY_SMOKE_NOTIFY_BASEURL").TrimEnd('/');
-var notifyToken = RequireEnv("NOTIFY_SMOKE_NOTIFY_TOKEN");
-var notifyTenant = RequireEnv("NOTIFY_SMOKE_NOTIFY_TENANT");
-var notifyTenantHeader = Environment.GetEnvironmentVariable("NOTIFY_SMOKE_NOTIFY_TENANT_HEADER");
-if (string.IsNullOrWhiteSpace(notifyTenantHeader))
-{
-    notifyTenantHeader = "X-StellaOps-Tenant";
-}
-
-var notifyTimeoutSeconds = 30;
-var notifyTimeoutEnv = Environment.GetEnvironmentVariable("NOTIFY_SMOKE_NOTIFY_TIMEOUT_SECONDS");
-if (!string.IsNullOrWhiteSpace(notifyTimeoutEnv) && int.TryParse(notifyTimeoutEnv, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedTimeout))
-{
-    notifyTimeoutSeconds = Math.Max(5, parsedTimeout);
-}
-
-using var httpClient = new HttpClient
-{
-    Timeout = TimeSpan.FromSeconds(notifyTimeoutSeconds),
-};
-
-httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", notifyToken);
-httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
-httpClient.DefaultRequestHeaders.Add(notifyTenantHeader, notifyTenant);
-
-var sinceQuery = Uri.EscapeDataString(sinceThreshold.ToString("O", CultureInfo.InvariantCulture));
-var deliveriesUrl = $"{notifyBaseUrl}/api/v1/deliveries?since={sinceQuery}&limit=200";
-
-Console.WriteLine($"ℹ️  Querying Notify deliveries via {deliveriesUrl}.");
-
-using var response = await httpClient.GetAsync(deliveriesUrl);
-if (!response.IsSuccessStatusCode)
-{
-    var body = await response.Content.ReadAsStringAsync();
-    throw new InvalidOperationException($"Notify deliveries request failed with {(int)response.StatusCode} {response.ReasonPhrase}: {body}");
-}
-
-var json = await response.Content.ReadAsStringAsync();
-if (string.IsNullOrWhiteSpace(json))
-{
-    throw new InvalidOperationException("Notify deliveries response body was empty.");
-}
-
-using var document = JsonDocument.Parse(json);
-var root = document.RootElement;
-
-IEnumerable<JsonElement> EnumerateDeliveries(JsonElement element)
-{
-    return element.ValueKind switch
-    {
-        JsonValueKind.Array => element.EnumerateArray(),
-        JsonValueKind.Object when element.TryGetProperty("items", out var items) && items.ValueKind == JsonValueKind.Array => items.EnumerateArray(),
-        _ => throw new InvalidOperationException("Notify deliveries response was not an array or did not contain an 'items' collection.")
-    };
-}
-
-var deliveries = EnumerateDeliveries(root).ToArray();
-Ensure(deliveries.Length > 0, "Notify deliveries response did not return any records.");
-
-var missingDeliveryKinds = new List<string>();
-foreach (var kind in expectedKinds)
-{
-    var found = deliveries.Any(delivery =>
-        delivery.TryGetProperty("kind", out var kindProperty) &&
-        kindProperty.GetString()?.Equals(kind, StringComparison.OrdinalIgnoreCase) == true &&
-        delivery.TryGetProperty("status", out var statusProperty) &&
-        !string.Equals(statusProperty.GetString(), "failed", StringComparison.OrdinalIgnoreCase));
-
-    if (!found)
-    {
-        missingDeliveryKinds.Add(kind);
-    }
-}
-
-Ensure(missingDeliveryKinds.Count == 0, $"Notify deliveries missing successful records for kinds: {string.Join(", ", missingDeliveryKinds)}");
-
-Console.WriteLine("✅ Notify deliveries include the expected scanner events.");
-Console.WriteLine("🎉 Notify smoke validation completed successfully.");
+return await NotifySmokeCheckApp.RunAsync(args);