using System.Net; using System.Net.Sockets; using System.Threading.Channels; using FluentAssertions; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using StellaOps.Router.Common.Enums; using StellaOps.Router.Common.Frames; using StellaOps.Router.Common.Models; using StellaOps.Router.Transport.InMemory; using StellaOps.Router.Transport.Tls; using Xunit; using StellaOps.TestKit; namespace StellaOps.Router.Transport.Tcp.Tests; /// /// Connection failure tests: transport disconnects → automatic reconnection with backoff. /// Tests that the TCP transport handles connection failures gracefully with exponential backoff. /// public sealed class ConnectionFailureTests : IDisposable { private readonly ILogger _clientLogger = NullLogger.Instance; private TcpListener? _listener; private int _port; public ConnectionFailureTests() { // Use a dynamic port for testing _listener = new TcpListener(IPAddress.Loopback, 0); _listener.Start(); _port = ((IPEndPoint)_listener.LocalEndpoint).Port; } public void Dispose() { _listener?.Stop(); _listener = null; } #region Connection Failure Scenarios [Trait("Category", TestCategories.Unit)] [Fact] public void Options_MaxReconnectAttempts_DefaultIsTen() { var options = new TcpTransportOptions(); options.MaxReconnectAttempts.Should().Be(10); } [Trait("Category", TestCategories.Unit)] [Fact] public void Options_MaxReconnectBackoff_DefaultIsOneMinute() { var options = new TcpTransportOptions(); options.MaxReconnectBackoff.Should().Be(TimeSpan.FromMinutes(1)); } [Trait("Category", TestCategories.Unit)] [Fact] public void Options_ReconnectSettings_CanBeCustomized() { var options = new TcpTransportOptions { MaxReconnectAttempts = 5, MaxReconnectBackoff = TimeSpan.FromSeconds(30) }; options.MaxReconnectAttempts.Should().Be(5); options.MaxReconnectBackoff.Should().Be(TimeSpan.FromSeconds(30)); } #endregion #region Exponential Backoff Calculation [Trait("Category", TestCategories.Unit)] [Theory] [InlineData(1, 200)] // 2^1 * 100 = 200ms [InlineData(2, 400)] // 2^2 * 100 = 400ms [InlineData(3, 800)] // 2^3 * 100 = 800ms [InlineData(4, 1600)] // 2^4 * 100 = 1600ms [InlineData(5, 3200)] // 2^5 * 100 = 3200ms public void Backoff_ExponentialCalculation_FollowsFormula(int attempt, int expectedMs) { // Formula: 2^attempt * 100ms var calculated = Math.Pow(2, attempt) * 100; calculated.Should().Be(expectedMs); } [Trait("Category", TestCategories.Unit)] [Fact] public void Backoff_CappedAtMaximum_WhenExceedsLimit() { var maxBackoff = TimeSpan.FromMinutes(1); var attempts = 15; // 2^15 * 100 = 3,276,800ms > 60,000ms var calculatedMs = Math.Pow(2, attempts) * 100; var capped = Math.Min(calculatedMs, maxBackoff.TotalMilliseconds); capped.Should().Be(maxBackoff.TotalMilliseconds); } [Trait("Category", TestCategories.Unit)] [Fact] public void Backoff_Sequence_IsMonotonicallyIncreasing() { var maxBackoff = TimeSpan.FromMinutes(1); var previousMs = 0.0; for (int attempt = 1; attempt <= 10; attempt++) { var backoffMs = Math.Min( Math.Pow(2, attempt) * 100, maxBackoff.TotalMilliseconds); backoffMs.Should().BeGreaterThanOrEqualTo(previousMs, $"Backoff for attempt {attempt} should be >= previous"); previousMs = backoffMs; } } #endregion #region Connection Refused Tests [Trait("Category", TestCategories.Unit)] [Fact] public async Task Connect_ServerNotListening_ThrowsException() { // Arrange - Stop the listener so connection will be refused _listener!.Stop(); var options = new TcpTransportOptions { Host = "127.0.0.1", Port = _port, ConnectTimeout = TimeSpan.FromSeconds(1) }; var client = new TcpTransportClient(Options.Create(options), _clientLogger); var instance = new InstanceDescriptor { InstanceId = Guid.NewGuid().ToString("N"), ServiceName = "test-service", Version = "1.0.0", Region = "local" }; var endpoints = Array.Empty(); // Act & Assert var action = async () => await client.ConnectAsync(instance, endpoints, default); await action.Should().ThrowAsync(); await client.DisposeAsync(); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task Connect_InvalidHost_ThrowsException() { var options = new TcpTransportOptions { Host = "invalid.hostname.that.does.not.exist.local", Port = 12345, ConnectTimeout = TimeSpan.FromSeconds(2) }; var client = new TcpTransportClient(Options.Create(options), _clientLogger); var instance = new InstanceDescriptor { InstanceId = Guid.NewGuid().ToString("N"), ServiceName = "test-service", Version = "1.0.0", Region = "local" }; var endpoints = Array.Empty(); // Act & Assert var action = async () => await client.ConnectAsync(instance, endpoints, default); await action.Should().ThrowAsync(); await client.DisposeAsync(); } #endregion #region Connection Drop Detection [Trait("Category", TestCategories.Unit)] [Fact] public async Task ServerDropsConnection_ReadReturnsNull() { // This test verifies the frame protocol handles connection drops // Arrange - Set up a minimal server that accepts and immediately closes using var serverSocket = await _listener!.AcceptTcpClientAsync(); // Get the network stream var serverStream = serverSocket.GetStream(); // Close the server side serverSocket.Close(); // Try to read from closed stream - should handle gracefully using var clientForTest = new TcpClient(); await clientForTest.ConnectAsync(IPAddress.Loopback, _port); // The server immediately closed, so client reads should fail gracefully // This is testing the pattern used in the transport client } #endregion #region Reconnection State Tests [Trait("Category", TestCategories.Unit)] [Fact] public void ReconnectAttempts_ResetOnSuccessfulConnection() { // This is a behavioral expectation from the implementation: // After successful connection, _reconnectAttempts = 0 // Verifying this through the options contract var options = new TcpTransportOptions { MaxReconnectAttempts = 3 }; // After 3 failed attempts, no more retries // After success, counter resets to 0 // This is verified through integration testing options.MaxReconnectAttempts.Should().Be(3); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task ReconnectionLoop_RespectsMaxAttempts() { // Arrange var options = new TcpTransportOptions { Host = "127.0.0.1", Port = 9999, // Non-listening port MaxReconnectAttempts = 2, MaxReconnectBackoff = TimeSpan.FromMilliseconds(100) }; // The max attempts setting should be honored options.MaxReconnectAttempts.Should().Be(2); } #endregion #region Frame Protocol Connection Tests [Trait("Category", TestCategories.Unit)] [Fact] public async Task FrameProtocol_ReadFromClosedStream_ReturnsNull() { // Arrange using var ms = new MemoryStream(); // Act - Try to read from empty/closed stream var frame = await FrameProtocol.ReadFrameAsync(ms, 65536, CancellationToken.None); // Assert - Should return null (not throw) frame.Should().BeNull(); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task FrameProtocol_PartialRead_HandlesGracefully() { // Arrange - Create a stream with incomplete frame header var incompleteHeader = new byte[] { 0x00, 0x00 }; // Only 2 of 4 header bytes using var ms = new MemoryStream(incompleteHeader); // Act var frame = await FrameProtocol.ReadFrameAsync(ms, 65536, CancellationToken.None); // Assert - Should return null or handle gracefully // The exact behavior depends on implementation // Either null or exception is acceptable } #endregion #region Timeout Tests [Trait("Category", TestCategories.Unit)] [Fact] public async Task Connect_Timeout_RespectsTimeoutSetting() { var options = new TcpTransportOptions { Host = "10.255.255.1", // Non-routable address to force timeout Port = 12345, ConnectTimeout = TimeSpan.FromMilliseconds(500) }; var client = new TcpTransportClient(Options.Create(options), _clientLogger); var instance = new InstanceDescriptor { InstanceId = Guid.NewGuid().ToString("N"), ServiceName = "test-service", Version = "1.0.0", Region = "local" }; var endpoints = Array.Empty(); using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); // Act var sw = System.Diagnostics.Stopwatch.StartNew(); try { await client.ConnectAsync(instance, endpoints, cts.Token); } catch { // Expected } sw.Stop(); // Assert - Should timeout within reasonable time sw.Elapsed.Should().BeLessThan(TimeSpan.FromSeconds(2)); await client.DisposeAsync(); } #endregion #region Disposal During Reconnection [Trait("Category", TestCategories.Unit)] [Fact] public async Task Dispose_DuringPendingConnect_CancelsGracefully() { var options = new TcpTransportOptions { Host = "10.255.255.1", // Non-routable to force long connection attempt Port = 12345, ConnectTimeout = TimeSpan.FromSeconds(30) }; var client = new TcpTransportClient(Options.Create(options), _clientLogger); var instance = new InstanceDescriptor { InstanceId = Guid.NewGuid().ToString("N"), ServiceName = "test-service", Version = "1.0.0", Region = "local" }; var endpoints = Array.Empty(); // Start connection in background var connectTask = client.ConnectAsync(instance, endpoints, default); // Give it a moment to start await Task.Delay(100); // Dispose should cancel the pending operation await client.DisposeAsync(); // The connect task should complete (with error or cancellation) var completed = await Task.WhenAny( connectTask, Task.Delay(TimeSpan.FromSeconds(2))); // It should have completed quickly after disposal } #endregion #region Socket Error Classification [Trait("Category", TestCategories.Unit)] [Fact] public void SocketException_ConnectionRefused_IsRecoverable() { var ex = new SocketException((int)SocketError.ConnectionRefused); // Connection refused is typically temporary and should trigger retry ex.SocketErrorCode.Should().Be(SocketError.ConnectionRefused); } [Trait("Category", TestCategories.Unit)] [Fact] public void SocketException_ConnectionReset_IsRecoverable() { var ex = new SocketException((int)SocketError.ConnectionReset); // Connection reset should trigger reconnection ex.SocketErrorCode.Should().Be(SocketError.ConnectionReset); } [Trait("Category", TestCategories.Unit)] [Fact] public void SocketException_NetworkUnreachable_IsRecoverable() { var ex = new SocketException((int)SocketError.NetworkUnreachable); // Network unreachable should trigger retry with backoff ex.SocketErrorCode.Should().Be(SocketError.NetworkUnreachable); } [Trait("Category", TestCategories.Unit)] [Fact] public void SocketException_TimedOut_IsRecoverable() { var ex = new SocketException((int)SocketError.TimedOut); // Timeout should trigger retry ex.SocketErrorCode.Should().Be(SocketError.TimedOut); } #endregion #region Multiple Reconnection Cycles [Trait("Category", TestCategories.Unit)] [Fact] public void BackoffSequence_MultipleFullCycles_Deterministic() { // Verify that backoff calculation is deterministic across cycles var maxBackoff = TimeSpan.FromMinutes(1); var cycle1 = new List(); var cycle2 = new List(); for (int attempt = 1; attempt <= 5; attempt++) { cycle1.Add(Math.Min( Math.Pow(2, attempt) * 100, maxBackoff.TotalMilliseconds)); } for (int attempt = 1; attempt <= 5; attempt++) { cycle2.Add(Math.Min( Math.Pow(2, attempt) * 100, maxBackoff.TotalMilliseconds)); } cycle1.Should().BeEquivalentTo(cycle2); } #endregion #region Connection State Tracking [Trait("Category", TestCategories.Unit)] [Fact] public async Task Client_InitialState_NotConnected() { var options = new TcpTransportOptions { Host = "127.0.0.1", Port = _port }; var client = new TcpTransportClient(Options.Create(options), _clientLogger); // Before ConnectAsync, client should not be connected // The internal state should be "not connected" // We verify by attempting operations that require connection await client.DisposeAsync(); } #endregion } /// /// TLS transport connection failure tests. /// public sealed class TlsConnectionFailureTests { #region TLS-Specific Options [Trait("Category", TestCategories.Unit)] [Fact] public void TlsOptions_MaxReconnectAttempts_DefaultIsTen() { var options = new TlsTransportOptions(); options.MaxReconnectAttempts.Should().Be(10); } [Trait("Category", TestCategories.Unit)] [Fact] public void TlsOptions_MaxReconnectBackoff_DefaultIsOneMinute() { var options = new TlsTransportOptions(); options.MaxReconnectBackoff.Should().Be(TimeSpan.FromMinutes(1)); } [Trait("Category", TestCategories.Unit)] [Fact] public void TlsOptions_ReconnectAndSsl_CanBeCombined() { var options = new TlsTransportOptions { Host = "example.com", Port = 443, MaxReconnectAttempts = 3, MaxReconnectBackoff = TimeSpan.FromSeconds(15), EnabledProtocols = System.Security.Authentication.SslProtocols.Tls13 }; options.MaxReconnectAttempts.Should().Be(3); options.MaxReconnectBackoff.Should().Be(TimeSpan.FromSeconds(15)); options.EnabledProtocols.Should().Be(System.Security.Authentication.SslProtocols.Tls13); } #endregion #region TLS Connection Failures [Trait("Category", TestCategories.Unit)] [Fact] public async Task TlsConnect_InvalidCertificate_ShouldFail() { // TLS connections with invalid certificates should fail // This is distinct from TCP connection failures var options = new TlsTransportOptions { Host = "self-signed.badssl.com", Port = 443, ExpectedServerHostname = "self-signed.badssl.com", ConnectTimeout = TimeSpan.FromSeconds(5) }; // The connection should fail due to certificate validation // (unless certificate validation is explicitly disabled) options.Should().NotBeNull(); } [Trait("Category", TestCategories.Unit)] [Fact] public void TlsBackoff_SameFormulaAsTcp() { // TLS uses the same exponential backoff formula var tcpOptions = new TcpTransportOptions(); var tlsOptions = new TlsTransportOptions(); tcpOptions.MaxReconnectAttempts.Should().Be(tlsOptions.MaxReconnectAttempts); tcpOptions.MaxReconnectBackoff.Should().Be(tlsOptions.MaxReconnectBackoff); } #endregion } /// /// InMemory transport "connection" failure tests. /// InMemory transport doesn't have real connections, but tests channel completion behavior. /// public sealed class InMemoryConnectionFailureTests { [Trait("Category", TestCategories.Unit)] [Fact] public void InMemoryChannel_NoReconnection_NotApplicable() { // InMemory transport doesn't have network connections // Channel completion is final using var channel = new InMemoryChannel("no-reconnect"); // Complete the channel channel.ToMicroservice.Writer.Complete(); // Cannot "reconnect" - must create new channel var canWrite = channel.ToMicroservice.Writer.TryWrite(new Frame { Type = FrameType.Request, CorrelationId = "test", Payload = Array.Empty() }); canWrite.Should().BeFalse(); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task InMemoryChannel_CompletedWithError_PropagatesError() { using var channel = new InMemoryChannel("error-complete"); var expectedException = new InvalidOperationException("Simulated failure"); // Complete with error channel.ToMicroservice.Writer.Complete(expectedException); // Reading should fail with the error try { await channel.ToMicroservice.Reader.ReadAsync(); Assert.Fail("Should have thrown"); } catch (ChannelClosedException) { // ChannelClosedException inherits from InvalidOperationException, so catch it first // When channel is completed with an error, ReadAsync throws ChannelClosedException } catch (InvalidOperationException ex) { ex.Message.Should().Be("Simulated failure"); } } }