git.stella-ops.org/docs/router/29-Step.md

Step 29: Integration Testing & CI

Overview

This final step establishes the comprehensive integration testing framework and CI/CD pipeline for the Stella Router. It ensures all components work together correctly in realistic deployment scenarios and provides automated quality gates for every change.

Goals

1. Create integration test suites covering all component interactions
2. Implement performance benchmarks with regression detection
3. Configure CI/CD pipelines for automated testing
4. Establish deployment validation tests
5. Create chaos testing for resilience verification

Integration Test Architecture

┌─────────────────────────────────────────────────────────────────────────────┐
│                        Integration Test Layers                               │
├─────────────────────────────────────────────────────────────────────────────┤
│                                                                              │
│  ┌─────────────────────────────────────────────────────────────────────┐    │
│  │                         E2E Tests                                    │    │
│  │  Full deployment simulation with external services                   │    │
│  └─────────────────────────────────────────────────────────────────────┘    │
│                                    │                                         │
│  ┌─────────────────────────────────────────────────────────────────────┐    │
│  │                    Component Integration Tests                       │    │
│  │  Gateway + Transport + Microservice + Security                       │    │
│  └─────────────────────────────────────────────────────────────────────┘    │
│                                    │                                         │
│  ┌─────────────────────────────────────────────────────────────────────┐    │
│  │                      Contract Tests                                  │    │
│  │  API contracts, Protocol compatibility                               │    │
│  └─────────────────────────────────────────────────────────────────────┘    │
│                                    │                                         │
│  ┌─────────────────────────────────────────────────────────────────────┐    │
│  │                       Unit Tests                                     │    │
│  │  Individual component tests (covered in previous steps)              │    │
│  └─────────────────────────────────────────────────────────────────────┘    │
│                                                                              │
└─────────────────────────────────────────────────────────────────────────────┘

Component Integration Tests

Test Infrastructure

// Tests/Integration/StellaOps.Router.Integration.Tests/Infrastructure/IntegrationTestBase.cs
namespace StellaOps.Router.Integration.Tests.Infrastructure;

public abstract class IntegrationTestBase : IAsyncLifetime
{
    protected IHost GatewayHost { get; private set; } = null!;
    protected IHost MicroserviceHost { get; private set; } = null!;
    protected HttpClient HttpClient { get; private set; } = null!;
    protected ITransportClient TransportClient { get; private set; } = null!;

    protected virtual int GatewayHttpPort => 15000 + Random.Shared.Next(1000);
    protected virtual int GatewayTransportPort => 19000 + Random.Shared.Next(1000);
    protected virtual int MicroservicePort => 19500 + Random.Shared.Next(1000);

    protected virtual void ConfigureGateway(WebApplicationBuilder builder) { }
    protected virtual void ConfigureMicroservice(StellaMicroserviceBuilder builder) { }

    public async Task InitializeAsync()
    {
        // Start microservice first
        MicroserviceHost = await CreateMicroserviceHostAsync();
        await MicroserviceHost.StartAsync();

        // Then start gateway
        GatewayHost = await CreateGatewayHostAsync();
        await GatewayHost.StartAsync();

        // Create clients
        HttpClient = new HttpClient
        {
            BaseAddress = new Uri($"http://localhost:{GatewayHttpPort}")
        };

        TransportClient = GatewayHost.Services.GetRequiredService<ITransportClient>();

        // Wait for services to be ready
        await WaitForReadyAsync();
    }

    public async Task DisposeAsync()
    {
        HttpClient?.Dispose();

        if (GatewayHost != null)
        {
            await GatewayHost.StopAsync();
            GatewayHost.Dispose();
        }

        if (MicroserviceHost != null)
        {
            await MicroserviceHost.StopAsync();
            MicroserviceHost.Dispose();
        }
    }

    private async Task<IHost> CreateGatewayHostAsync()
    {
        var builder = WebApplication.CreateBuilder();

        builder.WebHost.UseUrls($"http://localhost:{GatewayHttpPort}");

        builder.Services.AddStellaRouter(options =>
        {
            options.Routes = GetRouteConfiguration();
            options.Transport.DefaultPort = MicroservicePort;
        });

        builder.Services.AddStellaJwtValidation(options =>
        {
            options.Issuer = "https://test.auth.local";
            options.Audience = "stella-router-test";
            options.SigningKey = TestKeys.SymmetricKey;
        });

        builder.Services.AddStellaRateLimiting(options =>
        {
            options.DefaultLimits = new RateLimitConfiguration
            {
                RequestsPerMinute = 1000,
                WindowSize = TimeSpan.FromMinutes(1)
            };
        });

        ConfigureGateway(builder);

        var app = builder.Build();

        app.UseRouting();
        app.UseStellaRouter();
        app.MapStellaHealthChecks();

        return app;
    }

    private async Task<IHost> CreateMicroserviceHostAsync()
    {
        var builder = StellaMicroservice.CreateBuilder();

        builder.UseTransport("tcp", options =>
        {
            options.Host = "127.0.0.1";
            options.Port = MicroservicePort;
        });

        builder.AddHandler<TestUserHandler>();
        builder.AddHandler<TestProductHandler>();
        builder.AddHandler<TestOrderHandler>();

        ConfigureMicroservice(builder);

        return builder.Build();
    }

    protected virtual List<RouteConfiguration> GetRouteConfiguration()
    {
        return new List<RouteConfiguration>
        {
            new()
            {
                Path = "/users/**",
                Method = "*",
                Handler = "microservice",
                Target = "users-service",
                RequiredClaims = new[] { "user:read" }
            },
            new()
            {
                Path = "/products/**",
                Method = "*",
                Handler = "microservice",
                Target = "products-service"
            },
            new()
            {
                Path = "/orders/**",
                Method = "*",
                Handler = "microservice",
                Target = "orders-service",
                RequiredClaims = new[] { "order:access" },
                RateLimitKey = "user_id",
                RateLimitRequests = 100
            }
        };
    }

    private async Task WaitForReadyAsync()
    {
        var timeout = TimeSpan.FromSeconds(30);
        var sw = Stopwatch.StartNew();

        while (sw.Elapsed < timeout)
        {
            try
            {
                var response = await HttpClient.GetAsync("/health/ready");
                if (response.IsSuccessStatusCode)
                    return;
            }
            catch
            {
                // Keep trying
            }

            await Task.Delay(100);
        }

        throw new TimeoutException("Services failed to become ready");
    }

    protected HttpClient CreateAuthenticatedClient(Dictionary<string, object>? claims = null)
    {
        var token = TestTokenGenerator.Generate(claims ?? new Dictionary<string, object>
        {
            ["sub"] = "test-user",
            ["user:read"] = true,
            ["user:write"] = true
        });

        var client = new HttpClient
        {
            BaseAddress = new Uri($"http://localhost:{GatewayHttpPort}")
        };
        client.DefaultRequestHeaders.Authorization =
            new AuthenticationHeaderValue("Bearer", token);

        return client;
    }
}

Gateway + Microservice Integration Tests

// Tests/Integration/StellaOps.Router.Integration.Tests/GatewayMicroserviceTests.cs
namespace StellaOps.Router.Integration.Tests;

public class GatewayMicroserviceTests : IntegrationTestBase
{
    [Fact]
    public async Task Request_FlowsThroughGatewayToMicroservice()
    {
        // Arrange
        using var client = CreateAuthenticatedClient();

        // Act
        var response = await client.GetAsync("/users/123");

        // Assert
        Assert.Equal(HttpStatusCode.OK, response.StatusCode);

        var user = await response.Content.ReadFromJsonAsync<TestUserDto>();
        Assert.NotNull(user);
        Assert.Equal("123", user.Id);
    }

    [Fact]
    public async Task Claims_ArePropagatedToMicroservice()
    {
        // Arrange
        var claims = new Dictionary<string, object>
        {
            ["sub"] = "claim-test-user",
            ["user:read"] = true,
            ["custom_claim"] = "custom_value",
            ["role"] = "admin"
        };

        using var client = CreateAuthenticatedClient(claims);

        // Act
        var response = await client.GetAsync("/users/me/claims");

        // Assert
        Assert.Equal(HttpStatusCode.OK, response.StatusCode);

        var receivedClaims = await response.Content
            .ReadFromJsonAsync<Dictionary<string, object>>();

        Assert.Equal("claim-test-user", receivedClaims!["sub"].ToString());
        Assert.Equal("custom_value", receivedClaims["custom_claim"].ToString());
        Assert.Equal("admin", receivedClaims["role"].ToString());
    }

    [Fact]
    public async Task LargePayload_HandledCorrectly()
    {
        // Arrange
        using var client = CreateAuthenticatedClient(new Dictionary<string, object>
        {
            ["sub"] = "test-user",
            ["user:write"] = true
        });

        var largeData = new string('x', 1_000_000); // 1MB payload

        // Act
        var response = await client.PostAsJsonAsync("/users/data", new { Data = largeData });

        // Assert
        Assert.Equal(HttpStatusCode.OK, response.StatusCode);

        var result = await response.Content.ReadFromJsonAsync<DataResponse>();
        Assert.Equal(1_000_000, result!.ReceivedLength);
    }

    [Fact]
    public async Task ConcurrentRequests_HandledCorrectly()
    {
        // Arrange
        using var client = CreateAuthenticatedClient();
        var requestCount = 100;

        // Act
        var tasks = Enumerable.Range(0, requestCount)
            .Select(i => client.GetAsync($"/users/{i}"));

        var responses = await Task.WhenAll(tasks);

        // Assert
        Assert.All(responses, r => Assert.Equal(HttpStatusCode.OK, r.StatusCode));
    }

    [Fact]
    public async Task MicroserviceTimeout_ReturnsGatewayTimeout()
    {
        // Arrange
        using var client = CreateAuthenticatedClient();

        // Act - endpoint that simulates slow response
        var response = await client.GetAsync("/users/slow?delay=35000"); // 35 seconds

        // Assert - should timeout at 30 seconds
        Assert.Equal(HttpStatusCode.GatewayTimeout, response.StatusCode);
    }

    [Fact]
    public async Task MicroserviceError_ReturnsBadGateway()
    {
        // Arrange
        using var client = CreateAuthenticatedClient();

        // Act - endpoint that throws exception
        var response = await client.GetAsync("/users/error");

        // Assert
        Assert.Equal(HttpStatusCode.BadGateway, response.StatusCode);
    }
}

Security Integration Tests

// Tests/Integration/StellaOps.Router.Integration.Tests/SecurityIntegrationTests.cs
namespace StellaOps.Router.Integration.Tests;

public class SecurityIntegrationTests : IntegrationTestBase
{
    [Fact]
    public async Task NoToken_ReturnsUnauthorized()
    {
        // Arrange - no auth header

        // Act
        var response = await HttpClient.GetAsync("/users/123");

        // Assert
        Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode);
    }

    [Fact]
    public async Task InvalidToken_ReturnsUnauthorized()
    {
        // Arrange
        HttpClient.DefaultRequestHeaders.Authorization =
            new AuthenticationHeaderValue("Bearer", "invalid.token.here");

        // Act
        var response = await HttpClient.GetAsync("/users/123");

        // Assert
        Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode);
    }

    [Fact]
    public async Task ExpiredToken_ReturnsUnauthorized()
    {
        // Arrange
        var expiredToken = TestTokenGenerator.GenerateExpired();
        HttpClient.DefaultRequestHeaders.Authorization =
            new AuthenticationHeaderValue("Bearer", expiredToken);

        // Act
        var response = await HttpClient.GetAsync("/users/123");

        // Assert
        Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode);
    }

    [Fact]
    public async Task MissingRequiredClaim_ReturnsForbidden()
    {
        // Arrange - token without required 'user:read' claim
        var token = TestTokenGenerator.Generate(new Dictionary<string, object>
        {
            ["sub"] = "test-user"
            // Missing user:read claim
        });

        HttpClient.DefaultRequestHeaders.Authorization =
            new AuthenticationHeaderValue("Bearer", token);

        // Act
        var response = await HttpClient.GetAsync("/users/123");

        // Assert
        Assert.Equal(HttpStatusCode.Forbidden, response.StatusCode);
    }

    [Fact]
    public async Task ValidTokenWithClaims_ReturnsSuccess()
    {
        // Arrange
        using var client = CreateAuthenticatedClient(new Dictionary<string, object>
        {
            ["sub"] = "test-user",
            ["user:read"] = true
        });

        // Act
        var response = await client.GetAsync("/users/123");

        // Assert
        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
    }

    [Fact]
    public async Task ClaimHydration_EnrichesTokenClaims()
    {
        // Arrange
        using var client = CreateAuthenticatedClient(new Dictionary<string, object>
        {
            ["sub"] = "hydration-test-user",
            ["user:read"] = true
        });

        // Act
        var response = await client.GetAsync("/users/me/enriched-claims");

        // Assert
        Assert.Equal(HttpStatusCode.OK, response.StatusCode);

        var claims = await response.Content.ReadFromJsonAsync<Dictionary<string, object>>();

        // Hydrated claims should be present
        Assert.True(claims!.ContainsKey("org_id"));
        Assert.True(claims.ContainsKey("permissions"));
        Assert.True(claims.ContainsKey("feature_flags"));
    }
}

Rate Limiting Integration Tests

// Tests/Integration/StellaOps.Router.Integration.Tests/RateLimitingIntegrationTests.cs
namespace StellaOps.Router.Integration.Tests;

public class RateLimitingIntegrationTests : IntegrationTestBase
{
    protected override void ConfigureGateway(WebApplicationBuilder builder)
    {
        builder.Services.Configure<RateLimitOptions>(options =>
        {
            options.DefaultLimits = new RateLimitConfiguration
            {
                RequestsPerMinute = 10,
                WindowSize = TimeSpan.FromSeconds(10)
            };
        });
    }

    [Fact]
    public async Task ExceedingRateLimit_ReturnsTooManyRequests()
    {
        // Arrange
        using var client = CreateAuthenticatedClient(new Dictionary<string, object>
        {
            ["sub"] = "rate-limit-test-user",
            ["order:access"] = true,
            ["user_id"] = "rl-user-1"
        });

        // Act - send requests beyond the limit
        var responses = new List<HttpResponseMessage>();
        for (int i = 0; i < 15; i++)
        {
            responses.Add(await client.GetAsync("/orders"));
        }

        // Assert - first 10 should succeed, rest should be rate limited
        var successCount = responses.Count(r => r.StatusCode == HttpStatusCode.OK);
        var rateLimitedCount = responses.Count(r => r.StatusCode == HttpStatusCode.TooManyRequests);

        Assert.Equal(10, successCount);
        Assert.Equal(5, rateLimitedCount);
    }

    [Fact]
    public async Task RateLimitHeaders_AreReturned()
    {
        // Arrange
        using var client = CreateAuthenticatedClient(new Dictionary<string, object>
        {
            ["sub"] = "header-test-user",
            ["order:access"] = true,
            ["user_id"] = "rl-user-2"
        });

        // Act
        var response = await client.GetAsync("/orders");

        // Assert
        Assert.True(response.Headers.Contains("X-RateLimit-Limit"));
        Assert.True(response.Headers.Contains("X-RateLimit-Remaining"));
        Assert.True(response.Headers.Contains("X-RateLimit-Reset"));

        var limit = int.Parse(response.Headers.GetValues("X-RateLimit-Limit").First());
        var remaining = int.Parse(response.Headers.GetValues("X-RateLimit-Remaining").First());

        Assert.Equal(10, limit);
        Assert.Equal(9, remaining);
    }

    [Fact]
    public async Task DifferentUsers_HaveIndependentLimits()
    {
        // Arrange
        using var client1 = CreateAuthenticatedClient(new Dictionary<string, object>
        {
            ["sub"] = "user-1",
            ["order:access"] = true,
            ["user_id"] = "independent-user-1"
        });

        using var client2 = CreateAuthenticatedClient(new Dictionary<string, object>
        {
            ["sub"] = "user-2",
            ["order:access"] = true,
            ["user_id"] = "independent-user-2"
        });

        // Act - exhaust limit for user 1
        for (int i = 0; i < 12; i++)
        {
            await client1.GetAsync("/orders");
        }

        // User 2 should still have their full limit
        var response = await client2.GetAsync("/orders");

        // Assert
        Assert.Equal(HttpStatusCode.OK, response.StatusCode);

        var remaining = int.Parse(response.Headers.GetValues("X-RateLimit-Remaining").First());
        Assert.Equal(9, remaining);
    }

    [Fact]
    public async Task RateLimit_ResetsAfterWindow()
    {
        // Arrange
        using var client = CreateAuthenticatedClient(new Dictionary<string, object>
        {
            ["sub"] = "reset-test-user",
            ["order:access"] = true,
            ["user_id"] = "rl-reset-user"
        });

        // Act - exhaust limit
        for (int i = 0; i < 12; i++)
        {
            await client.GetAsync("/orders");
        }

        // Wait for window to reset
        await Task.Delay(TimeSpan.FromSeconds(11));

        // Should be able to make requests again
        var response = await client.GetAsync("/orders");

        // Assert
        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
    }
}

Transport Integration Tests

// Tests/Integration/StellaOps.Router.Integration.Tests/TransportIntegrationTests.cs
namespace StellaOps.Router.Integration.Tests;

public class TransportIntegrationTests : IntegrationTestBase
{
    [Fact]
    public async Task TcpTransport_HandlesConnectionDrop()
    {
        // Arrange
        using var client = CreateAuthenticatedClient();

        // First request establishes connection
        await client.GetAsync("/users/123");

        // Simulate connection drop by restarting microservice
        await MicroserviceHost.StopAsync();
        await MicroserviceHost.StartAsync();
        await Task.Delay(1000); // Wait for reconnection

        // Act - should auto-reconnect
        var response = await client.GetAsync("/users/456");

        // Assert
        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
    }

    [Fact]
    public async Task Heartbeat_KeepsConnectionAlive()
    {
        // Arrange
        using var client = CreateAuthenticatedClient();

        // Establish connection
        await client.GetAsync("/users/123");

        // Wait longer than idle timeout but within heartbeat interval
        await Task.Delay(TimeSpan.FromSeconds(45));

        // Act - connection should still be alive
        var response = await client.GetAsync("/users/456");

        // Assert
        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
    }

    [Fact]
    public async Task ConnectionPooling_ReusesConnections()
    {
        // Arrange
        using var client = CreateAuthenticatedClient();
        var connectionCountBefore = GetActiveConnectionCount();

        // Act - make many requests
        var tasks = Enumerable.Range(0, 50)
            .Select(_ => client.GetAsync("/products/1"));

        await Task.WhenAll(tasks);

        var connectionCountAfter = GetActiveConnectionCount();

        // Assert - should use pooled connections, not 50 new ones
        Assert.True(connectionCountAfter - connectionCountBefore < 10);
    }

    private int GetActiveConnectionCount()
    {
        var metrics = GatewayHost.Services.GetRequiredService<ITransportMetrics>();
        return metrics.ActiveConnections;
    }
}

Performance Benchmarks

Benchmark Framework

// Tests/Benchmarks/StellaOps.Router.Benchmarks/RouterBenchmarks.cs
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Running;

namespace StellaOps.Router.Benchmarks;

[MemoryDiagnoser]
[ThreadingDiagnoser]
public class RouterBenchmarks
{
    private HttpClient _client = null!;
    private IHost _gatewayHost = null!;
    private IHost _microserviceHost = null!;
    private string _validToken = null!;

    [GlobalSetup]
    public async Task Setup()
    {
        _microserviceHost = await CreateMicroserviceHostAsync();
        await _microserviceHost.StartAsync();

        _gatewayHost = await CreateGatewayHostAsync();
        await _gatewayHost.StartAsync();

        _client = new HttpClient
        {
            BaseAddress = new Uri("http://localhost:5000")
        };

        _validToken = TestTokenGenerator.Generate(new Dictionary<string, object>
        {
            ["sub"] = "bench-user",
            ["user:read"] = true
        });

        _client.DefaultRequestHeaders.Authorization =
            new AuthenticationHeaderValue("Bearer", _validToken);

        // Warmup
        await _client.GetAsync("/users/1");
    }

    [GlobalCleanup]
    public async Task Cleanup()
    {
        _client.Dispose();
        await _gatewayHost.StopAsync();
        await _microserviceHost.StopAsync();
        _gatewayHost.Dispose();
        _microserviceHost.Dispose();
    }

    [Benchmark(Baseline = true)]
    public async Task SimpleGetRequest()
    {
        var response = await _client.GetAsync("/users/123");
        response.EnsureSuccessStatusCode();
    }

    [Benchmark]
    public async Task GetRequestWithQueryParams()
    {
        var response = await _client.GetAsync("/users?page=1&size=20&search=test");
        response.EnsureSuccessStatusCode();
    }

    [Benchmark]
    public async Task PostRequestWithSmallBody()
    {
        var content = new StringContent(
            """{"name": "Test User", "email": "test@example.com"}""",
            Encoding.UTF8,
            "application/json");

        var response = await _client.PostAsync("/users", content);
        response.EnsureSuccessStatusCode();
    }

    [Benchmark]
    public async Task PostRequestWithLargeBody()
    {
        var data = new string('x', 100_000);
        var content = new StringContent(
            $$"""{"data": "{{data}}"}""",
            Encoding.UTF8,
            "application/json");

        var response = await _client.PostAsync("/users/data", content);
        response.EnsureSuccessStatusCode();
    }

    [Benchmark]
    [Arguments(10)]
    [Arguments(50)]
    [Arguments(100)]
    public async Task ConcurrentRequests(int concurrency)
    {
        var tasks = Enumerable.Range(0, concurrency)
            .Select(_ => _client.GetAsync("/users/123"));

        var responses = await Task.WhenAll(tasks);

        foreach (var response in responses)
        {
            response.EnsureSuccessStatusCode();
        }
    }
}

[MemoryDiagnoser]
public class JwtValidationBenchmarks
{
    private IJwtValidator _validator = null!;
    private string _validToken = null!;
    private string _tokenWithManyClaims = null!;

    [GlobalSetup]
    public void Setup()
    {
        var options = Options.Create(new JwtValidationOptions
        {
            Issuer = "https://auth.example.com",
            Audience = "stella-router",
            SigningKey = TestKeys.SymmetricKey
        });

        _validator = new JwtValidator(options, new InMemoryKeyProvider(),
            NullLogger<JwtValidator>.Instance);

        _validToken = TestTokenGenerator.Generate(new Dictionary<string, object>
        {
            ["sub"] = "test-user"
        });

        _tokenWithManyClaims = TestTokenGenerator.Generate(
            Enumerable.Range(0, 50)
                .ToDictionary(i => $"claim_{i}", i => (object)$"value_{i}"));
    }

    [Benchmark(Baseline = true)]
    public async Task ValidateSimpleToken()
    {
        await _validator.ValidateAsync(_validToken);
    }

    [Benchmark]
    public async Task ValidateTokenWithManyClaims()
    {
        await _validator.ValidateAsync(_tokenWithManyClaims);
    }
}

[MemoryDiagnoser]
public class SerializationBenchmarks
{
    private readonly IPayloadSerializer _messagePackSerializer = new MessagePackPayloadSerializer();
    private readonly IPayloadSerializer _jsonSerializer = new JsonPayloadSerializer();
    private RequestPayload _smallPayload = null!;
    private RequestPayload _largePayload = null!;

    [GlobalSetup]
    public void Setup()
    {
        _smallPayload = new RequestPayload
        {
            Method = "GET",
            Path = "/users/123",
            Headers = new Dictionary<string, string>
            {
                ["Content-Type"] = "application/json",
                ["Authorization"] = "Bearer token"
            },
            Body = Array.Empty<byte>()
        };

        _largePayload = new RequestPayload
        {
            Method = "POST",
            Path = "/users",
            Headers = new Dictionary<string, string>
            {
                ["Content-Type"] = "application/json"
            },
            Body = new byte[100_000]
        };
    }

    [Benchmark]
    public byte[] MessagePack_SerializeSmall()
    {
        return _messagePackSerializer.Serialize(_smallPayload);
    }

    [Benchmark]
    public byte[] Json_SerializeSmall()
    {
        return _jsonSerializer.Serialize(_smallPayload);
    }

    [Benchmark]
    public byte[] MessagePack_SerializeLarge()
    {
        return _messagePackSerializer.Serialize(_largePayload);
    }

    [Benchmark]
    public byte[] Json_SerializeLarge()
    {
        return _jsonSerializer.Serialize(_largePayload);
    }
}

Performance Regression Detection

// Tests/Benchmarks/StellaOps.Router.Benchmarks/RegressionDetector.cs
namespace StellaOps.Router.Benchmarks;

public class RegressionDetector
{
    private readonly string _baselinePath;
    private readonly double _regressionThreshold;

    public RegressionDetector(string baselinePath, double regressionThreshold = 0.10)
    {
        _baselinePath = baselinePath;
        _regressionThreshold = regressionThreshold;
    }

    public RegressionReport Compare(BenchmarkReport current)
    {
        var baseline = LoadBaseline();
        var regressions = new List<BenchmarkRegression>();

        foreach (var currentResult in current.Results)
        {
            if (baseline.TryGetValue(currentResult.Name, out var baselineResult))
            {
                var percentChange = (currentResult.MeanNs - baselineResult.MeanNs)
                    / baselineResult.MeanNs;

                if (percentChange > _regressionThreshold)
                {
                    regressions.Add(new BenchmarkRegression
                    {
                        BenchmarkName = currentResult.Name,
                        BaselineMeanNs = baselineResult.MeanNs,
                        CurrentMeanNs = currentResult.MeanNs,
                        PercentChange = percentChange * 100
                    });
                }
            }
        }

        return new RegressionReport
        {
            Regressions = regressions,
            HasRegressions = regressions.Count > 0
        };
    }

    private Dictionary<string, BenchmarkResult> LoadBaseline()
    {
        if (!File.Exists(_baselinePath))
            return new Dictionary<string, BenchmarkResult>();

        var json = File.ReadAllText(_baselinePath);
        return JsonSerializer.Deserialize<Dictionary<string, BenchmarkResult>>(json)
            ?? new Dictionary<string, BenchmarkResult>();
    }

    public void SaveBaseline(BenchmarkReport report)
    {
        var baseline = report.Results.ToDictionary(r => r.Name);
        var json = JsonSerializer.Serialize(baseline, new JsonSerializerOptions
        {
            WriteIndented = true
        });
        File.WriteAllText(_baselinePath, json);
    }
}

public record BenchmarkResult(string Name, double MeanNs, double StdDevNs, long AllocatedBytes);
public record BenchmarkRegression
{
    public string BenchmarkName { get; init; } = "";
    public double BaselineMeanNs { get; init; }
    public double CurrentMeanNs { get; init; }
    public double PercentChange { get; init; }
}

public record RegressionReport
{
    public List<BenchmarkRegression> Regressions { get; init; } = new();
    public bool HasRegressions { get; init; }
}

Chaos Testing

Chaos Test Framework

// Tests/Chaos/StellaOps.Router.Chaos.Tests/ChaosTestBase.cs
namespace StellaOps.Router.Chaos.Tests;

public abstract class ChaosTestBase : IntegrationTestBase
{
    protected IChaosMonkey ChaosMonkey { get; private set; } = null!;

    public override async Task InitializeAsync()
    {
        await base.InitializeAsync();

        ChaosMonkey = GatewayHost.Services.GetRequiredService<IChaosMonkey>();
    }

    protected override void ConfigureGateway(WebApplicationBuilder builder)
    {
        builder.Services.AddStellaChaos(options =>
        {
            options.Enabled = true;
        });
    }
}

public interface IChaosMonkey
{
    void InjectLatency(string target, TimeSpan delay, double probability = 1.0);
    void InjectError(string target, Exception error, double probability = 1.0);
    void InjectConnectionDrop(string target, double probability = 1.0);
    void DisableTarget(string target);
    void EnableTarget(string target);
    void Reset();
}

public class ChaosMonkey : IChaosMonkey
{
    private readonly ConcurrentDictionary<string, ChaosRule> _rules = new();

    public void InjectLatency(string target, TimeSpan delay, double probability = 1.0)
    {
        _rules[target] = new ChaosRule
        {
            Type = ChaosType.Latency,
            Delay = delay,
            Probability = probability
        };
    }

    public void InjectError(string target, Exception error, double probability = 1.0)
    {
        _rules[target] = new ChaosRule
        {
            Type = ChaosType.Error,
            Error = error,
            Probability = probability
        };
    }

    public void InjectConnectionDrop(string target, double probability = 1.0)
    {
        _rules[target] = new ChaosRule
        {
            Type = ChaosType.ConnectionDrop,
            Probability = probability
        };
    }

    public void DisableTarget(string target)
    {
        _rules[target] = new ChaosRule
        {
            Type = ChaosType.Disabled,
            Probability = 1.0
        };
    }

    public void EnableTarget(string target)
    {
        _rules.TryRemove(target, out _);
    }

    public void Reset()
    {
        _rules.Clear();
    }

    public async Task ApplyAsync(string target, CancellationToken cancellationToken)
    {
        if (!_rules.TryGetValue(target, out var rule))
            return;

        if (Random.Shared.NextDouble() > rule.Probability)
            return;

        switch (rule.Type)
        {
            case ChaosType.Latency:
                await Task.Delay(rule.Delay, cancellationToken);
                break;
            case ChaosType.Error:
                throw rule.Error!;
            case ChaosType.ConnectionDrop:
                throw new IOException("Connection forcibly closed by chaos monkey");
            case ChaosType.Disabled:
                throw new ServiceUnavailableException($"Target {target} is disabled");
        }
    }
}

public enum ChaosType { Latency, Error, ConnectionDrop, Disabled }

public class ChaosRule
{
    public ChaosType Type { get; init; }
    public TimeSpan Delay { get; init; }
    public Exception? Error { get; init; }
    public double Probability { get; init; }
}

Chaos Test Scenarios

// Tests/Chaos/StellaOps.Router.Chaos.Tests/ResilienceChaosTests.cs
namespace StellaOps.Router.Chaos.Tests;

public class ResilienceChaosTests : ChaosTestBase
{
    [Fact]
    public async Task CircuitBreaker_OpensOnRepeatedFailures()
    {
        // Arrange
        using var client = CreateAuthenticatedClient();
        ChaosMonkey.InjectError("users-service", new Exception("Service unavailable"));

        // Act - trigger circuit breaker
        var responses = new List<HttpResponseMessage>();
        for (int i = 0; i < 10; i++)
        {
            responses.Add(await client.GetAsync("/users/123"));
        }

        // Assert - circuit should be open now
        var lastResponse = responses.Last();
        Assert.Equal(HttpStatusCode.ServiceUnavailable, lastResponse.StatusCode);

        // Verify circuit breaker is open by checking fast fail
        var sw = Stopwatch.StartNew();
        var fastFailResponse = await client.GetAsync("/users/456");
        sw.Stop();

        Assert.True(sw.ElapsedMilliseconds < 100); // Should fail fast
        Assert.Equal(HttpStatusCode.ServiceUnavailable, fastFailResponse.StatusCode);
    }

    [Fact]
    public async Task CircuitBreaker_HalfOpensAndRecovers()
    {
        // Arrange
        using var client = CreateAuthenticatedClient();
        ChaosMonkey.InjectError("users-service", new Exception("Service unavailable"));

        // Trip the circuit breaker
        for (int i = 0; i < 10; i++)
        {
            await client.GetAsync("/users/123");
        }

        // Wait for half-open state
        await Task.Delay(TimeSpan.FromSeconds(31));

        // Remove chaos
        ChaosMonkey.Reset();

        // Act - next request should test the circuit
        var response = await client.GetAsync("/users/789");

        // Assert - should succeed and close circuit
        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
    }

    [Fact]
    public async Task Retry_SucceedsAfterTransientFailure()
    {
        // Arrange
        using var client = CreateAuthenticatedClient();
        var failureCount = 0;

        // Inject 2 failures then succeed
        ChaosMonkey.InjectError("users-service",
            new TransientException("Temporary failure"),
            probability: 0.66); // ~66% chance of failure (2/3 requests fail)

        // Act
        var response = await client.GetAsync("/users/123");

        // Assert - should eventually succeed due to retry
        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
    }

    [Fact]
    public async Task HighLatency_TriggersTimeout()
    {
        // Arrange
        using var client = CreateAuthenticatedClient();
        ChaosMonkey.InjectLatency("users-service", TimeSpan.FromSeconds(35));

        // Act
        var sw = Stopwatch.StartNew();
        var response = await client.GetAsync("/users/123");
        sw.Stop();

        // Assert
        Assert.Equal(HttpStatusCode.GatewayTimeout, response.StatusCode);
        Assert.True(sw.ElapsedMilliseconds >= 30000 && sw.ElapsedMilliseconds < 35000);
    }

    [Fact]
    public async Task PartialOutage_RoutesToHealthyInstances()
    {
        // This test requires multiple microservice instances
        // Arrange
        ChaosMonkey.DisableTarget("users-service-1");
        // users-service-2 and users-service-3 remain healthy

        using var client = CreateAuthenticatedClient();

        // Act
        var tasks = Enumerable.Range(0, 30)
            .Select(_ => client.GetAsync("/users/123"));

        var responses = await Task.WhenAll(tasks);

        // Assert - all should succeed via healthy instances
        Assert.All(responses, r => Assert.Equal(HttpStatusCode.OK, r.StatusCode));
    }

    [Fact]
    public async Task GracefulDegradation_WhenAuthorityUnavailable()
    {
        // Arrange
        ChaosMonkey.DisableTarget("authority-service");

        // Use endpoint that allows degraded mode
        using var client = CreateAuthenticatedClient();

        // Act
        var response = await client.GetAsync("/products/123"); // No auth required

        // Assert - should succeed with default/limited functionality
        Assert.Equal(HttpStatusCode.OK, response.StatusCode);

        // But authenticated endpoint should fail gracefully
        var authResponse = await client.GetAsync("/users/123");
        Assert.Equal(HttpStatusCode.ServiceUnavailable, authResponse.StatusCode);
    }
}

CI/CD Pipeline Configuration

GitHub Actions Workflow

# .github/workflows/router-ci.yml
name: Router CI

on:
  push:
    branches: [main]
    paths:
      - 'src/Router/**'
      - 'tests/Router/**'
  pull_request:
    branches: [main]
    paths:
      - 'src/Router/**'
      - 'tests/Router/**'

env:
  DOTNET_VERSION: '10.0.x'

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Setup .NET
        uses: actions/setup-dotnet@v4
        with:
          dotnet-version: ${{ env.DOTNET_VERSION }}

      - name: Restore dependencies
        run: dotnet restore src/Router/StellaOps.Router.sln

      - name: Build
        run: dotnet build src/Router/StellaOps.Router.sln --configuration Release --no-restore

      - name: Upload build artifacts
        uses: actions/upload-artifact@v4
        with:
          name: router-build
          path: src/Router/**/bin/Release/

  unit-tests:
    needs: build
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Setup .NET
        uses: actions/setup-dotnet@v4
        with:
          dotnet-version: ${{ env.DOTNET_VERSION }}

      - name: Run unit tests
        run: |
          dotnet test src/Router/StellaOps.Router.sln \
            --configuration Release \
            --filter "Category!=Integration&Category!=Chaos&Category!=Benchmark" \
            --logger "trx;LogFileName=unit-tests.trx" \
            --collect:"XPlat Code Coverage"

      - name: Upload test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: unit-test-results
          path: '**/TestResults/**/*.trx'

      - name: Upload coverage
        uses: codecov/codecov-action@v4
        with:
          files: '**/coverage.cobertura.xml'
          flags: unit-tests

  integration-tests:
    needs: build
    runs-on: ubuntu-latest
    services:
      redis:
        image: redis:7
        ports:
          - 6379:6379
    steps:
      - uses: actions/checkout@v4

      - name: Setup .NET
        uses: actions/setup-dotnet@v4
        with:
          dotnet-version: ${{ env.DOTNET_VERSION }}

      - name: Run integration tests
        run: |
          dotnet test src/Router/StellaOps.Router.sln \
            --configuration Release \
            --filter "Category=Integration" \
            --logger "trx;LogFileName=integration-tests.trx" \
            --collect:"XPlat Code Coverage"
        env:
          REDIS_CONNECTION: localhost:6379

      - name: Upload test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: integration-test-results
          path: '**/TestResults/**/*.trx'

  chaos-tests:
    needs: integration-tests
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Setup .NET
        uses: actions/setup-dotnet@v4
        with:
          dotnet-version: ${{ env.DOTNET_VERSION }}

      - name: Run chaos tests
        run: |
          dotnet test tests/Router/StellaOps.Router.Chaos.Tests \
            --configuration Release \
            --logger "trx;LogFileName=chaos-tests.trx"

      - name: Upload test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: chaos-test-results
          path: '**/TestResults/**/*.trx'

  benchmarks:
    needs: build
    runs-on: ubuntu-latest
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
    steps:
      - uses: actions/checkout@v4

      - name: Setup .NET
        uses: actions/setup-dotnet@v4
        with:
          dotnet-version: ${{ env.DOTNET_VERSION }}

      - name: Download baseline
        uses: actions/cache@v4
        with:
          path: benchmarks/baseline.json
          key: benchmark-baseline-${{ github.sha }}
          restore-keys: |
            benchmark-baseline-

      - name: Run benchmarks
        run: |
          dotnet run --project tests/Benchmarks/StellaOps.Router.Benchmarks \
            --configuration Release \
            -- --exporters json

      - name: Check for regressions
        run: |
          dotnet run --project tests/Benchmarks/StellaOps.Router.Benchmarks.Analyzer \
            -- check \
            --baseline benchmarks/baseline.json \
            --current BenchmarkDotNet.Artifacts/results/*.json \
            --threshold 0.10

      - name: Update baseline
        if: success()
        run: |
          cp BenchmarkDotNet.Artifacts/results/*.json benchmarks/baseline.json

      - name: Upload benchmark results
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-results
          path: BenchmarkDotNet.Artifacts/

  security-scan:
    needs: build
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Run security scan
        uses: snyk/actions/dotnet@master
        with:
          args: --file=src/Router/StellaOps.Router.sln
        env:
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}

  publish:
    needs: [unit-tests, integration-tests, chaos-tests, security-scan]
    runs-on: ubuntu-latest
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
    steps:
      - uses: actions/checkout@v4

      - name: Setup .NET
        uses: actions/setup-dotnet@v4
        with:
          dotnet-version: ${{ env.DOTNET_VERSION }}

      - name: Pack NuGet packages
        run: |
          dotnet pack src/Router/StellaOps.Router.Core \
            --configuration Release \
            --output ./packages

          dotnet pack src/Router/StellaOps.Router.Gateway \
            --configuration Release \
            --output ./packages

          dotnet pack src/Router/StellaOps.Router.Microservice \
            --configuration Release \
            --output ./packages

      - name: Push to NuGet
        run: |
          dotnet nuget push ./packages/*.nupkg \
            --source https://api.nuget.org/v3/index.json \
            --api-key ${{ secrets.NUGET_API_KEY }}

Gitea Actions Workflow

# .gitea/workflows/router-ci.yml
name: Router CI

on:
  push:
    branches: [main]
    paths:
      - 'src/Router/**'
      - 'tests/Router/**'
  pull_request:
    branches: [main]

jobs:
  build-and-test:
    runs-on: ubuntu-latest
    container:
      image: mcr.microsoft.com/dotnet/sdk:10.0
    steps:
      - uses: actions/checkout@v4

      - name: Restore
        run: dotnet restore src/Router/StellaOps.Router.sln

      - name: Build
        run: dotnet build src/Router/StellaOps.Router.sln -c Release --no-restore

      - name: Test
        run: |
          dotnet test src/Router/StellaOps.Router.sln \
            -c Release \
            --no-build \
            --logger "trx" \
            --collect:"XPlat Code Coverage"

      - name: Publish coverage
        run: |
          dotnet tool install -g dotnet-reportgenerator-globaltool
          reportgenerator \
            -reports:**/coverage.cobertura.xml \
            -targetdir:coverage \
            -reporttypes:Html

      - name: Upload artifacts
        uses: actions/upload-artifact@v4
        with:
          name: coverage-report
          path: coverage/

Test Data Generation

// Tests/Common/StellaOps.Router.Testing/TestDataGenerators.cs
namespace StellaOps.Router.Testing;

public static class TestTokenGenerator
{
    private static readonly byte[] Key = Encoding.UTF8.GetBytes(
        "this-is-a-test-key-for-jwt-tokens-32-bytes!");

    public static string Generate(Dictionary<string, object>? claims = null)
    {
        var tokenHandler = new JwtSecurityTokenHandler();
        var tokenDescriptor = new SecurityTokenDescriptor
        {
            Issuer = "https://test.auth.local",
            Audience = "stella-router-test",
            Expires = DateTime.UtcNow.AddHours(1),
            SigningCredentials = new SigningCredentials(
                new SymmetricSecurityKey(Key),
                SecurityAlgorithms.HmacSha256Signature),
            Claims = claims ?? new Dictionary<string, object>()
        };

        var token = tokenHandler.CreateToken(tokenDescriptor);
        return tokenHandler.WriteToken(token);
    }

    public static string GenerateExpired()
    {
        var tokenHandler = new JwtSecurityTokenHandler();
        var tokenDescriptor = new SecurityTokenDescriptor
        {
            Issuer = "https://test.auth.local",
            Audience = "stella-router-test",
            Expires = DateTime.UtcNow.AddHours(-1), // Expired
            SigningCredentials = new SigningCredentials(
                new SymmetricSecurityKey(Key),
                SecurityAlgorithms.HmacSha256Signature)
        };

        var token = tokenHandler.CreateToken(tokenDescriptor);
        return tokenHandler.WriteToken(token);
    }
}

public class TestUserFactory
{
    private int _counter = 0;

    public TestUserDto Create(Action<TestUserDto>? configure = null)
    {
        var user = new TestUserDto
        {
            Id = Guid.NewGuid().ToString(),
            Name = $"Test User {Interlocked.Increment(ref _counter)}",
            Email = $"user{_counter}@test.local",
            CreatedAt = DateTime.UtcNow
        };

        configure?.Invoke(user);
        return user;
    }

    public IEnumerable<TestUserDto> CreateMany(int count)
    {
        return Enumerable.Range(0, count).Select(_ => Create());
    }
}

YAML Configuration

# config/test-config.yaml
testing:
  integration:
    gatewayPort: 15000
    microservicePort: 19000
    timeout: 30s
    retryAttempts: 3

  chaos:
    enabled: true
    defaultLatency: 100ms
    defaultFailureProbability: 0.1
    circuitBreakerThreshold: 5
    circuitBreakerTimeout: 30s

  benchmarks:
    warmupIterations: 3
    targetIterations: 10
    regressionThreshold: 0.10

  coverage:
    minimumCoverage: 80
    excludePatterns:
      - "**/Generated/**"
      - "**/Migrations/**"

Deliverables

Artifact	Path
Integration Test Base	tests/Router/StellaOps.Router.Integration.Tests/
Chaos Test Framework	tests/Router/StellaOps.Router.Chaos.Tests/
Benchmarks	tests/Router/StellaOps.Router.Benchmarks/
CI Workflows	.gitea/workflows/router-ci.yml
Test Utilities	tests/Router/StellaOps.Router.Testing/

Implementation Complete

Congratulations! You have completed all 29 steps of the Stella Router implementation plan. The router now includes:

Phase 1-2: Core Infrastructure

• Route configuration and matching
• Route table with concurrent updates
• Request pipeline and middleware
• JWT authentication with per-endpoint keys
• Claim hydration system
• Tiered rate limiting

Phase 3-4: Transport Layer

• Request/response serialization (MessagePack)
• Frame-based protocol
• InMemory, TCP, and TLS transports
• Connection pooling and management

Phase 5: Route Handlers

• Microservice handler
• GraphQL handler
• S3/Storage handler
• Reverse proxy handler

Phase 6: Microservice SDK

• Host builder with fluent API
• Endpoint discovery and registration
• Request/response context
• Dual exposure mode

Phase 7: Observability & Resilience

• Structured logging with correlation
• OpenTelemetry tracing
• Prometheus metrics
• Health checks
• Circuit breaker and retry policies
• Configuration hot-reload

Phase 8: Quality & Deployment

• Reference implementation and examples
• Migration tooling
• Agent guidelines
• Integration testing and CI/CD

← Back to Specs | Start Implementation →