Retry microservice startup and validate async Valkey connects

2026-03-12 13:12:54 +02:00
parent 509b97a1a7
commit 19b9c90a8d
4 changed files with 252 additions and 6 deletions
--- a/src/Router/__Tests/StellaOps.Messaging.Transport.Valkey.Tests/ValkeyConnectionFactoryTests.cs
+++ b/src/Router/__Tests/StellaOps.Messaging.Transport.Valkey.Tests/ValkeyConnectionFactoryTests.cs
@@ -0,0 +1,36 @@
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using Moq;
+using StackExchange.Redis;
+
+namespace StellaOps.Messaging.Transport.Valkey.Tests;
+
+public sealed class ValkeyConnectionFactoryTests
+{
+    [Fact]
+    public async Task GetConnectionAsync_Throws_WhenFactoryReturnsDisconnectedConnection()
+    {
+        var multiplexer = new Mock<IConnectionMultiplexer>();
+        multiplexer.SetupGet(connection => connection.IsConnected).Returns(false);
+        multiplexer.Setup(connection => connection.CloseAsync(It.IsAny<bool>())).Returns(Task.CompletedTask);
+
+        var options = Options.Create(new ValkeyTransportOptions
+        {
+            ConnectionString = "cache.stella-ops.local:6379"
+        });
+
+        await using var factory = new ValkeyConnectionFactory(
+            options,
+            NullLogger<ValkeyConnectionFactory>.Instance,
+            _ => Task.FromResult(multiplexer.Object));
+
+        var act = () => factory.GetConnectionAsync(CancellationToken.None).AsTask();
+
+        await act.Should()
+            .ThrowAsync<RedisConnectionException>()
+            .WithMessage("*cache.stella-ops.local:6379*");
+
+        multiplexer.Verify(connection => connection.CloseAsync(It.IsAny<bool>()), Times.Once);
+        multiplexer.Verify(connection => connection.Dispose(), Times.Once);
+    }
+}
--- a/src/Router/__Tests/StellaOps.Microservice.Tests/MicroserviceHostedServiceTests.cs
+++ b/src/Router/__Tests/StellaOps.Microservice.Tests/MicroserviceHostedServiceTests.cs
@@ -0,0 +1,148 @@
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using Moq;
+using StellaOps.TestKit;
+
+namespace StellaOps.Microservice.Tests;
+
+public sealed class MicroserviceHostedServiceTests
+{
+    [Trait("Category", TestCategories.Unit)]
+    [Fact]
+    public async Task StartAsync_RetriesTransientFailures_WithoutStoppingHost()
+    {
+        var attempts = 0;
+        var connected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+
+        var connectionManager = new Mock<IRouterConnectionManager>();
+        connectionManager
+            .Setup(manager => manager.StartAsync(It.IsAny<CancellationToken>()))
+            .Returns<CancellationToken>(_ =>
+            {
+                var attempt = Interlocked.Increment(ref attempts);
+                if (attempt < 3)
+                {
+                    throw new InvalidOperationException($"transient-{attempt}");
+                }
+
+                connected.TrySetResult();
+                return Task.CompletedTask;
+            });
+        connectionManager
+            .Setup(manager => manager.StopAsync(It.IsAny<CancellationToken>()))
+            .Returns(Task.CompletedTask);
+
+        var lifetime = new TestHostApplicationLifetime();
+        var options = Options.Create(new StellaMicroserviceOptions
+        {
+            ServiceName = "test-service",
+            Version = "1.0.0",
+            Region = "local",
+            ReconnectBackoffInitial = TimeSpan.FromMilliseconds(10),
+            ReconnectBackoffMax = TimeSpan.FromMilliseconds(20)
+        });
+
+        var service = new MicroserviceHostedService(
+            connectionManager.Object,
+            lifetime,
+            NullLogger<MicroserviceHostedService>.Instance,
+            options);
+
+        await service.StartAsync(CancellationToken.None);
+        lifetime.SignalStarted();
+
+        await connected.Task.WaitAsync(TimeSpan.FromSeconds(1));
+        await service.StopAsync(CancellationToken.None);
+
+        attempts.Should().BeGreaterThanOrEqualTo(3);
+        lifetime.StopRequested.Should().BeFalse();
+        connectionManager.Verify(manager => manager.StopAsync(It.IsAny<CancellationToken>()), Times.AtLeast(3));
+    }
+
+    [Trait("Category", TestCategories.Unit)]
+    [Fact]
+    public async Task StopAsync_CancelsRetryLoop_WhenStartupNeverSucceeds()
+    {
+        var attempts = 0;
+        var secondAttemptObserved = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+
+        var connectionManager = new Mock<IRouterConnectionManager>();
+        connectionManager
+            .Setup(manager => manager.StartAsync(It.IsAny<CancellationToken>()))
+            .Returns<CancellationToken>(_ =>
+            {
+                if (Interlocked.Increment(ref attempts) >= 2)
+                {
+                    secondAttemptObserved.TrySetResult();
+                }
+
+                throw new InvalidOperationException("router unavailable");
+            });
+        connectionManager
+            .Setup(manager => manager.StopAsync(It.IsAny<CancellationToken>()))
+            .Returns(Task.CompletedTask);
+
+        var lifetime = new TestHostApplicationLifetime();
+        var options = Options.Create(new StellaMicroserviceOptions
+        {
+            ServiceName = "test-service",
+            Version = "1.0.0",
+            Region = "local",
+            ReconnectBackoffInitial = TimeSpan.FromMilliseconds(10),
+            ReconnectBackoffMax = TimeSpan.FromMilliseconds(20)
+        });
+
+        var service = new MicroserviceHostedService(
+            connectionManager.Object,
+            lifetime,
+            NullLogger<MicroserviceHostedService>.Instance,
+            options);
+
+        await service.StartAsync(CancellationToken.None);
+        lifetime.SignalStarted();
+
+        await secondAttemptObserved.Task.WaitAsync(TimeSpan.FromSeconds(1));
+        await service.StopAsync(new CancellationTokenSource(TimeSpan.FromSeconds(1)).Token);
+
+        lifetime.StopRequested.Should().BeFalse();
+        attempts.Should().BeGreaterThanOrEqualTo(2);
+    }
+
+    private sealed class TestHostApplicationLifetime : IHostApplicationLifetime
+    {
+        private readonly CancellationTokenSource _started = new();
+        private readonly CancellationTokenSource _stopping = new();
+        private readonly CancellationTokenSource _stopped = new();
+
+        public CancellationToken ApplicationStarted => _started.Token;
+
+        public CancellationToken ApplicationStopping => _stopping.Token;
+
+        public CancellationToken ApplicationStopped => _stopped.Token;
+
+        public bool StopRequested { get; private set; }
+
+        public void StopApplication()
+        {
+            StopRequested = true;
+            if (!_stopping.IsCancellationRequested)
+            {
+                _stopping.Cancel();
+            }
+
+            if (!_stopped.IsCancellationRequested)
+            {
+                _stopped.Cancel();
+            }
+        }
+
+        public void SignalStarted()
+        {
+            if (!_started.IsCancellationRequested)
+            {
+                _started.Cancel();
+            }
+        }
+    }
+}