# Step 18: Reverse Proxy Handler Implementation **Phase 4: Handler Plugins** **Estimated Complexity:** Medium **Dependencies:** Step 10 (Microservice Handler) --- ## Overview The Reverse Proxy handler forwards requests to external HTTP services without using the internal transport protocol. It's used for legacy services, third-party APIs, and services that can't be modified to use the Stella transport layer. --- ## Goals 1. Forward HTTP requests to configurable upstream servers 2. Support connection pooling and HTTP/2 multiplexing 3. Handle request/response transformation 4. Support health checks and circuit breaking 5. Maintain correlation IDs for tracing --- ## Core Architecture ``` ┌────────────────────────────────────────────────────────────────┐ │ Reverse Proxy Handler │ ├────────────────────────────────────────────────────────────────┤ │ │ │ Incoming Request │ │ │ │ │ ▼ │ │ ┌───────────────┐ ┌─────────────────────┐ │ │ │Path Rewriter │───►│ URL Transformation │ │ │ └───────┬───────┘ └─────────────────────┘ │ │ │ │ │ ▼ │ │ ┌───────────────┐ ┌─────────────────────┐ │ │ │ Header Filter │───►│ Add/Remove Headers │ │ │ └───────┬───────┘ └─────────────────────┘ │ │ │ │ │ ▼ │ │ ┌───────────────┐ ┌─────────────────────┐ │ │ │ Load Balancer │───►│ Round Robin/Weighted │ │ │ └───────┬───────┘ └─────────────────────┘ │ │ │ │ │ ▼ │ │ ┌───────────────────────────────────────────┐ │ │ │ HttpClient Pool │ │ │ │ (Connection pooling, HTTP/2, retries) │ │ │ └───────────────────────────────────────────┘ │ │ │ └────────────────────────────────────────────────────────────────┘ ``` --- ## Configuration ```csharp namespace StellaOps.Router.Handlers.ReverseProxy; public class ReverseProxyConfig { /// Upstream definitions by name. public Dictionary Upstreams { get; set; } = new(); /// Route-to-upstream mappings. public List Routes { get; set; } = new(); /// Default timeout for upstream requests. public TimeSpan DefaultTimeout { get; set; } = TimeSpan.FromSeconds(30); /// Whether to forward X-Forwarded-* headers. public bool AddForwardedHeaders { get; set; } = true; /// Whether to preserve host header. public bool PreserveHost { get; set; } = false; /// Connection pool settings. public ConnectionPoolConfig ConnectionPool { get; set; } = new(); } public class UpstreamConfig { /// Upstream server addresses. public List Servers { get; set; } = new(); /// Load balancing strategy. public LoadBalanceStrategy LoadBalance { get; set; } = LoadBalanceStrategy.RoundRobin; /// Health check configuration. public HealthCheckConfig? HealthCheck { get; set; } /// Circuit breaker configuration. public CircuitBreakerConfig? CircuitBreaker { get; set; } /// Retry configuration. public RetryConfig? Retry { get; set; } } public class UpstreamServer { public string Address { get; set; } = ""; public int Weight { get; set; } = 1; public bool Backup { get; set; } = false; } public class ProxyRoute { /// Path pattern to match. public string PathPattern { get; set; } = ""; /// Target upstream name. public string Upstream { get; set; } = ""; /// Path rewrite rule. public PathRewriteRule? Rewrite { get; set; } /// Header transformations. public HeaderTransformConfig? Headers { get; set; } /// Timeout override. public TimeSpan? Timeout { get; set; } /// Required claims for access. public List? RequiredClaims { get; set; } } public class PathRewriteRule { public string Pattern { get; set; } = ""; public string Replacement { get; set; } = ""; } public class HeaderTransformConfig { public Dictionary Add { get; set; } = new(); public List Remove { get; set; } = new(); public Dictionary Set { get; set; } = new(); public bool ForwardClaims { get; set; } = false; public string ClaimsHeaderPrefix { get; set; } = "X-Claim-"; } public class HealthCheckConfig { public string Path { get; set; } = "/health"; public TimeSpan Interval { get; set; } = TimeSpan.FromSeconds(10); public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(5); public int UnhealthyThreshold { get; set; } = 3; public int HealthyThreshold { get; set; } = 2; } public class CircuitBreakerConfig { public int FailureThreshold { get; set; } = 5; public TimeSpan SamplingDuration { get; set; } = TimeSpan.FromSeconds(30); public TimeSpan BreakDuration { get; set; } = TimeSpan.FromSeconds(30); public double FailureRatioThreshold { get; set; } = 0.5; } public class RetryConfig { public int MaxRetries { get; set; } = 3; public TimeSpan InitialDelay { get; set; } = TimeSpan.FromMilliseconds(100); public double BackoffMultiplier { get; set; } = 2.0; public List RetryableStatusCodes { get; set; } = new() { 502, 503, 504 }; } public class ConnectionPoolConfig { public int MaxConnectionsPerServer { get; set; } = 100; public TimeSpan ConnectionIdleTimeout { get; set; } = TimeSpan.FromMinutes(2); public bool EnableHttp2 { get; set; } = true; } public enum LoadBalanceStrategy { RoundRobin, Random, LeastConnections, WeightedRoundRobin, IPHash } ``` --- ## Reverse Proxy Handler Implementation ```csharp namespace StellaOps.Router.Handlers.ReverseProxy; public sealed class ReverseProxyHandler : IRouteHandler { public string HandlerType => "ReverseProxy"; public int Priority => 50; private readonly ReverseProxyConfig _config; private readonly IUpstreamManager _upstreamManager; private readonly IHttpClientFactory _httpClientFactory; private readonly ILogger _logger; public ReverseProxyHandler( IOptions config, IUpstreamManager upstreamManager, IHttpClientFactory httpClientFactory, ILogger logger) { _config = config.Value; _upstreamManager = upstreamManager; _httpClientFactory = httpClientFactory; _logger = logger; } public bool CanHandle(RouteMatchResult match) { if (match.Handler == "ReverseProxy") return true; return _config.Routes.Any(r => IsRouteMatch(match.Route.Path, r.PathPattern)); } public async Task HandleAsync( HttpContext context, RouteMatchResult match, IReadOnlyDictionary claims, CancellationToken cancellationToken) { // Find matching route var route = _config.Routes.FirstOrDefault(r => IsRouteMatch(context.Request.Path, r.PathPattern)); if (route == null) { return new RouteHandlerResult { Handled = false }; } // Check required claims if (route.RequiredClaims?.Any() == true) { if (!route.RequiredClaims.All(c => claims.ContainsKey(c))) { return new RouteHandlerResult { Handled = true, StatusCode = 403, Body = Encoding.UTF8.GetBytes("Forbidden") }; } } // Get upstream server var server = await _upstreamManager.GetServerAsync(route.Upstream, context, cancellationToken); if (server == null) { _logger.LogWarning("No healthy upstream for {Upstream}", route.Upstream); return new RouteHandlerResult { Handled = true, StatusCode = 503, Body = Encoding.UTF8.GetBytes("Service unavailable") }; } try { return await ForwardRequestAsync(context, route, server, claims, cancellationToken); } catch (Exception ex) { _logger.LogError(ex, "Proxy error for {Upstream}", route.Upstream); _upstreamManager.ReportFailure(route.Upstream, server.Address); return new RouteHandlerResult { Handled = true, StatusCode = 502, Body = Encoding.UTF8.GetBytes("Bad gateway") }; } } private bool IsRouteMatch(string path, string pattern) { if (pattern.EndsWith("*")) { return path.StartsWith(pattern.TrimEnd('*'), StringComparison.OrdinalIgnoreCase); } return string.Equals(path, pattern, StringComparison.OrdinalIgnoreCase); } private async Task ForwardRequestAsync( HttpContext context, ProxyRoute route, UpstreamServer server, IReadOnlyDictionary claims, CancellationToken cancellationToken) { var request = context.Request; // Build upstream URL var targetUri = BuildTargetUri(server.Address, request, route.Rewrite); // Create HTTP request var httpRequest = new HttpRequestMessage { Method = new HttpMethod(request.Method), RequestUri = targetUri }; // Copy headers CopyRequestHeaders(request, httpRequest, route.Headers, claims); // Add forwarded headers if (_config.AddForwardedHeaders) { AddForwardedHeaders(context, httpRequest); } // Copy body for non-GET/HEAD requests if (!HttpMethods.IsGet(request.Method) && !HttpMethods.IsHead(request.Method)) { httpRequest.Content = new StreamContent(request.Body); if (request.ContentType != null) { httpRequest.Content.Headers.ContentType = MediaTypeHeaderValue.Parse(request.ContentType); } } // Send request var client = _httpClientFactory.CreateClient("proxy"); var timeout = route.Timeout ?? _config.DefaultTimeout; using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(timeout); var response = await client.SendAsync(httpRequest, HttpCompletionOption.ResponseHeadersRead, cts.Token); // Copy response return await BuildResponseAsync(context, response, route.Headers, cancellationToken); } private Uri BuildTargetUri(string serverAddress, HttpRequest request, PathRewriteRule? rewrite) { var path = request.Path.Value ?? "/"; if (rewrite != null) { path = Regex.Replace(path, rewrite.Pattern, rewrite.Replacement); } var query = request.QueryString.Value ?? ""; var baseUri = new Uri(serverAddress.TrimEnd('/')); return new Uri(baseUri, path + query); } private void CopyRequestHeaders( HttpRequest source, HttpRequestMessage target, HeaderTransformConfig? transform, IReadOnlyDictionary claims) { // Skip hop-by-hop headers var skipHeaders = new HashSet(StringComparer.OrdinalIgnoreCase) { "Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", "TE", "Trailer", "Transfer-Encoding", "Upgrade", "Host" }; // Headers to remove if (transform?.Remove != null) { foreach (var header in transform.Remove) { skipHeaders.Add(header); } } foreach (var header in source.Headers) { if (skipHeaders.Contains(header.Key)) continue; target.Headers.TryAddWithoutValidation(header.Key, header.Value.ToArray()); } // Add configured headers if (transform?.Add != null) { foreach (var (key, value) in transform.Add) { target.Headers.TryAddWithoutValidation(key, value); } } // Set configured headers (overwrite) if (transform?.Set != null) { foreach (var (key, value) in transform.Set) { target.Headers.Remove(key); target.Headers.TryAddWithoutValidation(key, value); } } // Forward claims as headers if (transform?.ForwardClaims == true) { var prefix = transform.ClaimsHeaderPrefix ?? "X-Claim-"; foreach (var (key, value) in claims) { var headerName = prefix + key.Replace('/', '-').Replace(':', '-'); target.Headers.TryAddWithoutValidation(headerName, value); } } // Preserve or set Host if (_config.PreserveHost) { target.Headers.Host = source.Host.Value; } } private void AddForwardedHeaders(HttpContext context, HttpRequestMessage request) { var connection = context.Connection; var httpRequest = context.Request; // X-Forwarded-For var forwardedFor = httpRequest.Headers["X-Forwarded-For"].FirstOrDefault(); var clientIp = connection.RemoteIpAddress?.ToString(); if (!string.IsNullOrEmpty(clientIp)) { forwardedFor = string.IsNullOrEmpty(forwardedFor) ? clientIp : $"{forwardedFor}, {clientIp}"; } request.Headers.TryAddWithoutValidation("X-Forwarded-For", forwardedFor); // X-Forwarded-Proto request.Headers.TryAddWithoutValidation("X-Forwarded-Proto", httpRequest.Scheme); // X-Forwarded-Host request.Headers.TryAddWithoutValidation("X-Forwarded-Host", httpRequest.Host.Value); // X-Real-IP if (connection.RemoteIpAddress != null) { request.Headers.TryAddWithoutValidation("X-Real-IP", connection.RemoteIpAddress.ToString()); } // X-Request-ID (correlation) request.Headers.TryAddWithoutValidation("X-Request-ID", context.TraceIdentifier); } private async Task BuildResponseAsync( HttpContext context, HttpResponseMessage response, HeaderTransformConfig? transform, CancellationToken cancellationToken) { var httpResponse = context.Response; httpResponse.StatusCode = (int)response.StatusCode; // Copy response headers var skipHeaders = new HashSet(StringComparer.OrdinalIgnoreCase) { "Transfer-Encoding", "Connection" }; foreach (var header in response.Headers) { if (skipHeaders.Contains(header.Key)) continue; httpResponse.Headers[header.Key] = header.Value.ToArray(); } foreach (var header in response.Content.Headers) { if (skipHeaders.Contains(header.Key)) continue; httpResponse.Headers[header.Key] = header.Value.ToArray(); } // Stream response body await response.Content.CopyToAsync(httpResponse.Body, cancellationToken); return new RouteHandlerResult { Handled = true, StatusCode = (int)response.StatusCode }; } } ``` --- ## Upstream Manager ```csharp namespace StellaOps.Router.Handlers.ReverseProxy; public interface IUpstreamManager { Task GetServerAsync( string upstreamName, HttpContext context, CancellationToken cancellationToken); void ReportSuccess(string upstreamName, string serverAddress); void ReportFailure(string upstreamName, string serverAddress); } public sealed class UpstreamManager : IUpstreamManager, IHostedService { private readonly ReverseProxyConfig _config; private readonly ILogger _logger; private readonly ConcurrentDictionary _serverStates = new(); private readonly ConcurrentDictionary _roundRobinCounters = new(); private Timer? _healthCheckTimer; public UpstreamManager( IOptions config, ILogger logger) { _config = config.Value; _logger = logger; InitializeServerStates(); } private void InitializeServerStates() { foreach (var (name, upstream) in _config.Upstreams) { foreach (var server in upstream.Servers) { var key = $"{name}:{server.Address}"; _serverStates[key] = new ServerState { Address = server.Address, Weight = server.Weight, IsHealthy = true, IsBackup = server.Backup }; } } } public Task GetServerAsync( string upstreamName, HttpContext context, CancellationToken cancellationToken) { if (!_config.Upstreams.TryGetValue(upstreamName, out var upstream)) { return Task.FromResult(null); } var healthyServers = upstream.Servers .Where(s => IsServerHealthy(upstreamName, s.Address) && !s.Backup) .ToList(); // Fall back to backup servers if no primary available if (healthyServers.Count == 0) { healthyServers = upstream.Servers .Where(s => IsServerHealthy(upstreamName, s.Address) && s.Backup) .ToList(); } if (healthyServers.Count == 0) { return Task.FromResult(null); } var server = upstream.LoadBalance switch { LoadBalanceStrategy.RoundRobin => SelectRoundRobin(upstreamName, healthyServers), LoadBalanceStrategy.Random => SelectRandom(healthyServers), LoadBalanceStrategy.WeightedRoundRobin => SelectWeightedRoundRobin(upstreamName, healthyServers), LoadBalanceStrategy.LeastConnections => SelectLeastConnections(upstreamName, healthyServers), LoadBalanceStrategy.IPHash => SelectIPHash(context, healthyServers), _ => healthyServers[0] }; return Task.FromResult(server); } private bool IsServerHealthy(string upstreamName, string address) { var key = $"{upstreamName}:{address}"; return _serverStates.TryGetValue(key, out var state) && state.IsHealthy; } private UpstreamServer SelectRoundRobin(string upstreamName, List servers) { var counter = _roundRobinCounters.AddOrUpdate(upstreamName, 0, (_, c) => c + 1); return servers[counter % servers.Count]; } private UpstreamServer SelectRandom(List servers) { return servers[Random.Shared.Next(servers.Count)]; } private UpstreamServer SelectWeightedRoundRobin(string upstreamName, List servers) { var totalWeight = servers.Sum(s => s.Weight); var counter = _roundRobinCounters.AddOrUpdate(upstreamName, 0, (_, c) => c + 1); var position = counter % totalWeight; var cumulative = 0; foreach (var server in servers) { cumulative += server.Weight; if (position < cumulative) return server; } return servers[^1]; } private UpstreamServer SelectLeastConnections(string upstreamName, List servers) { return servers .OrderBy(s => { var key = $"{upstreamName}:{s.Address}"; return _serverStates.TryGetValue(key, out var state) ? state.ActiveConnections : 0; }) .First(); } private UpstreamServer SelectIPHash(HttpContext context, List servers) { var ip = context.Connection.RemoteIpAddress?.ToString() ?? "127.0.0.1"; var hash = ip.GetHashCode(); return servers[Math.Abs(hash) % servers.Count]; } public void ReportSuccess(string upstreamName, string serverAddress) { var key = $"{upstreamName}:{serverAddress}"; if (_serverStates.TryGetValue(key, out var state)) { state.ConsecutiveFailures = 0; state.ConsecutiveSuccesses++; // Check circuit breaker reset if (!state.IsHealthy && state.ConsecutiveSuccesses >= GetHealthyThreshold(upstreamName)) { state.IsHealthy = true; _logger.LogInformation("Server {Server} marked healthy", serverAddress); } } } public void ReportFailure(string upstreamName, string serverAddress) { var key = $"{upstreamName}:{serverAddress}"; if (_serverStates.TryGetValue(key, out var state)) { state.ConsecutiveSuccesses = 0; state.ConsecutiveFailures++; // Check circuit breaker trip if (state.IsHealthy && state.ConsecutiveFailures >= GetUnhealthyThreshold(upstreamName)) { state.IsHealthy = false; _logger.LogWarning("Server {Server} marked unhealthy after {Failures} failures", serverAddress, state.ConsecutiveFailures); } } } private int GetUnhealthyThreshold(string upstreamName) { return _config.Upstreams.TryGetValue(upstreamName, out var upstream) ? upstream.HealthCheck?.UnhealthyThreshold ?? 3 : 3; } private int GetHealthyThreshold(string upstreamName) { return _config.Upstreams.TryGetValue(upstreamName, out var upstream) ? upstream.HealthCheck?.HealthyThreshold ?? 2 : 2; } public Task StartAsync(CancellationToken cancellationToken) { _healthCheckTimer = new Timer(PerformHealthChecks, null, TimeSpan.Zero, TimeSpan.FromSeconds(10)); return Task.CompletedTask; } private async void PerformHealthChecks(object? state) { foreach (var (name, upstream) in _config.Upstreams) { if (upstream.HealthCheck == null) continue; foreach (var server in upstream.Servers) { await CheckServerHealthAsync(name, server, upstream.HealthCheck); } } } private async Task CheckServerHealthAsync( string upstreamName, UpstreamServer server, HealthCheckConfig config) { try { using var client = new HttpClient { Timeout = config.Timeout }; var uri = new Uri(new Uri(server.Address), config.Path); var response = await client.GetAsync(uri); if (response.IsSuccessStatusCode) { ReportSuccess(upstreamName, server.Address); } else { ReportFailure(upstreamName, server.Address); } } catch { ReportFailure(upstreamName, server.Address); } } public Task StopAsync(CancellationToken cancellationToken) { _healthCheckTimer?.Dispose(); return Task.CompletedTask; } } internal class ServerState { public string Address { get; set; } = ""; public int Weight { get; set; } = 1; public bool IsHealthy { get; set; } = true; public bool IsBackup { get; set; } public int ConsecutiveFailures { get; set; } public int ConsecutiveSuccesses { get; set; } public int ActiveConnections { get; set; } } ``` --- ## Service Registration ```csharp namespace StellaOps.Router.Handlers.ReverseProxy; public static class ReverseProxyExtensions { public static IServiceCollection AddReverseProxyHandler( this IServiceCollection services, IConfiguration configuration) { services.Configure( configuration.GetSection("ReverseProxy")); services.AddSingleton(); services.AddHostedService(sp => (UpstreamManager)sp.GetRequiredService()); services.AddHttpClient("proxy", client => { client.DefaultRequestVersion = HttpVersion.Version20; client.DefaultVersionPolicy = HttpVersionPolicy.RequestVersionOrLower; }) .ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler { PooledConnectionLifetime = TimeSpan.FromMinutes(5), MaxConnectionsPerServer = 100, EnableMultipleHttp2Connections = true }); services.AddSingleton(); return services; } } ``` --- ## YAML Configuration ```yaml ReverseProxy: DefaultTimeout: "00:00:30" AddForwardedHeaders: true PreserveHost: false ConnectionPool: MaxConnectionsPerServer: 100 ConnectionIdleTimeout: "00:02:00" EnableHttp2: true Upstreams: legacy-api: LoadBalance: RoundRobin Servers: - Address: "http://legacy-api-1:8080" Weight: 2 - Address: "http://legacy-api-2:8080" Weight: 1 - Address: "http://legacy-api-backup:8080" Backup: true HealthCheck: Path: "/health" Interval: "00:00:10" Timeout: "00:00:05" UnhealthyThreshold: 3 HealthyThreshold: 2 CircuitBreaker: FailureThreshold: 5 SamplingDuration: "00:00:30" BreakDuration: "00:00:30" Retry: MaxRetries: 3 InitialDelay: "00:00:00.100" BackoffMultiplier: 2.0 RetryableStatusCodes: [502, 503, 504] external-service: LoadBalance: LeastConnections Servers: - Address: "https://api.external-service.com" Routes: - PathPattern: "/legacy/*" Upstream: "legacy-api" Rewrite: Pattern: "^/legacy" Replacement: "/api/v1" Headers: Add: X-Proxy-Source: "stella-router" Remove: - "X-Internal-Token" ForwardClaims: true ClaimsHeaderPrefix: "X-User-" RequiredClaims: - "sub" - PathPattern: "/external/*" Upstream: "external-service" Timeout: "00:01:00" Headers: Set: Authorization: "Bearer ${EXTERNAL_API_KEY}" ``` --- ## Deliverables 1. `StellaOps.Router.Handlers.ReverseProxy/ReverseProxyHandler.cs` 2. `StellaOps.Router.Handlers.ReverseProxy/ReverseProxyConfig.cs` 3. `StellaOps.Router.Handlers.ReverseProxy/IUpstreamManager.cs` 4. `StellaOps.Router.Handlers.ReverseProxy/UpstreamManager.cs` 5. `StellaOps.Router.Handlers.ReverseProxy/ReverseProxyExtensions.cs` 6. Load balancing strategy tests 7. Health check tests 8. Circuit breaker tests 9. Header transformation tests --- ## Next Step Proceed to [Step 19: Additional Handler Plugins](19-Step.md) to implement static files and WebSocket handlers.