audit notes work completed, test fixes work (95% done), new sprints, new data sources setup and configuration
This commit is contained in:
master
@@ -0,0 +1,343 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// MirrorRateLimitingExtensions.cs
|
||||
// Sprint: SPRINT_20260114_SOURCES_SETUP
|
||||
// Task: 3.2 - Mirror Server Rate Limiting Setup
|
||||
// Description: Extension methods for integrating mirror rate limiting with Router
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Extensions;
|
||||
|
||||
/// <summary>
|
||||
/// Extension methods for configuring mirror server rate limiting using Router library.
|
||||
/// </summary>
|
||||
public static class MirrorRateLimitingExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Configuration section path for sources configuration.
|
||||
/// </summary>
|
||||
public const string SourcesConfigSection = "sources";
|
||||
|
||||
/// <summary>
|
||||
/// Configuration section path for mirror server.
|
||||
/// </summary>
|
||||
public const string MirrorServerConfigSection = "sources:mirrorServer";
|
||||
|
||||
/// <summary>
|
||||
/// Configuration section path for rate limits.
|
||||
/// </summary>
|
||||
public const string RateLimitsConfigSection = "sources:mirrorServer:rateLimits";
|
||||
|
||||
/// <summary>
|
||||
/// Adds mirror rate limiting services using Router library integration.
|
||||
/// </summary>
|
||||
/// <param name="services">Service collection.</param>
|
||||
/// <param name="configuration">Configuration instance.</param>
|
||||
/// <returns>Service collection for chaining.</returns>
|
||||
public static IServiceCollection AddMirrorRateLimiting(
|
||||
this IServiceCollection services,
|
||||
IConfiguration configuration)
|
||||
{
|
||||
var mirrorConfig = configuration
|
||||
.GetSection(RateLimitsConfigSection)
|
||||
.Get<MirrorRateLimitConfig>();
|
||||
|
||||
if (mirrorConfig is null || !mirrorConfig.IsEnabled)
|
||||
{
|
||||
return services;
|
||||
}
|
||||
|
||||
// Validate configuration
|
||||
mirrorConfig.Validate();
|
||||
|
||||
// Build Router-compatible configuration and register
|
||||
var routerConfigSection = BuildRouterConfiguration(mirrorConfig);
|
||||
|
||||
// Register Router rate limiting services
|
||||
// This maps our MirrorRateLimitConfig to Router's RateLimitConfig
|
||||
var routerConfig = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(routerConfigSection)
|
||||
.Build();
|
||||
|
||||
// Register the Router rate limiting services
|
||||
// Note: The actual Router library would have its own AddRouterRateLimiting extension
|
||||
// This is a bridge to that library
|
||||
services.Configure<MirrorRateLimitConfig>(
|
||||
configuration.GetSection(RateLimitsConfigSection));
|
||||
|
||||
// Register middleware options
|
||||
services.AddSingleton(mirrorConfig);
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds rate limiting middleware for mirror endpoints.
|
||||
/// </summary>
|
||||
/// <param name="app">Application builder.</param>
|
||||
/// <returns>Application builder for chaining.</returns>
|
||||
public static IApplicationBuilder UseMirrorRateLimiting(this IApplicationBuilder app)
|
||||
{
|
||||
var config = app.ApplicationServices.GetService<MirrorRateLimitConfig>();
|
||||
|
||||
if (config is null || !config.IsEnabled)
|
||||
{
|
||||
return app;
|
||||
}
|
||||
|
||||
// Apply rate limiting to mirror endpoints
|
||||
app.UseWhen(
|
||||
context => context.Request.Path.StartsWithSegments("/api/mirror"),
|
||||
branch =>
|
||||
{
|
||||
// The Router library middleware would be used here
|
||||
// branch.UseMiddleware<RateLimitMiddleware>();
|
||||
|
||||
// For now, use our custom middleware adapter
|
||||
branch.UseMiddleware<MirrorRateLimitMiddleware>();
|
||||
});
|
||||
|
||||
return app;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds Router-compatible configuration from MirrorRateLimitConfig.
|
||||
/// </summary>
|
||||
private static IEnumerable<KeyValuePair<string, string?>> BuildRouterConfiguration(
|
||||
MirrorRateLimitConfig mirrorConfig)
|
||||
{
|
||||
var config = new Dictionary<string, string?>();
|
||||
|
||||
// Map activation threshold
|
||||
config["rate_limiting:process_back_pressure_when_more_than_per_5min"] =
|
||||
mirrorConfig.ActivationThresholdPer5Min.ToString();
|
||||
|
||||
// Map instance-level config
|
||||
if (mirrorConfig.ForInstance is not null)
|
||||
{
|
||||
config["rate_limiting:for_instance:per_seconds"] =
|
||||
mirrorConfig.ForInstance.PerSeconds.ToString();
|
||||
config["rate_limiting:for_instance:max_requests"] =
|
||||
mirrorConfig.ForInstance.MaxRequests.ToString();
|
||||
|
||||
if (mirrorConfig.ForInstance.AllowBurstForSeconds.HasValue)
|
||||
{
|
||||
config["rate_limiting:for_instance:allow_burst_for_seconds"] =
|
||||
mirrorConfig.ForInstance.AllowBurstForSeconds.Value.ToString();
|
||||
}
|
||||
|
||||
if (mirrorConfig.ForInstance.AllowMaxBurstRequests.HasValue)
|
||||
{
|
||||
config["rate_limiting:for_instance:allow_max_burst_requests"] =
|
||||
mirrorConfig.ForInstance.AllowMaxBurstRequests.Value.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
// Map environment-level config
|
||||
if (mirrorConfig.ForEnvironment is not null)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(mirrorConfig.ForEnvironment.ValkeyConnection))
|
||||
{
|
||||
config["rate_limiting:for_environment:valkey_connection"] =
|
||||
mirrorConfig.ForEnvironment.ValkeyConnection;
|
||||
}
|
||||
|
||||
config["rate_limiting:for_environment:valkey_bucket"] =
|
||||
mirrorConfig.ForEnvironment.ValkeyBucket;
|
||||
config["rate_limiting:for_environment:per_seconds"] =
|
||||
mirrorConfig.ForEnvironment.PerSeconds.ToString();
|
||||
config["rate_limiting:for_environment:max_requests"] =
|
||||
mirrorConfig.ForEnvironment.MaxRequests.ToString();
|
||||
|
||||
if (mirrorConfig.ForEnvironment.AllowBurstForSeconds.HasValue)
|
||||
{
|
||||
config["rate_limiting:for_environment:allow_burst_for_seconds"] =
|
||||
mirrorConfig.ForEnvironment.AllowBurstForSeconds.Value.ToString();
|
||||
}
|
||||
|
||||
if (mirrorConfig.ForEnvironment.AllowMaxBurstRequests.HasValue)
|
||||
{
|
||||
config["rate_limiting:for_environment:allow_max_burst_requests"] =
|
||||
mirrorConfig.ForEnvironment.AllowMaxBurstRequests.Value.ToString();
|
||||
}
|
||||
|
||||
// Map route-specific limits
|
||||
foreach (var (routeName, routeConfig) in mirrorConfig.ForEnvironment.Routes)
|
||||
{
|
||||
var routePrefix = $"rate_limiting:for_environment:microservices:mirror:routes:{routeName}";
|
||||
|
||||
config[$"{routePrefix}:pattern"] = routeConfig.Pattern;
|
||||
config[$"{routePrefix}:match_type"] = routeConfig.MatchType.ToString();
|
||||
config[$"{routePrefix}:per_seconds"] = routeConfig.PerSeconds.ToString();
|
||||
config[$"{routePrefix}:max_requests"] = routeConfig.MaxRequests.ToString();
|
||||
|
||||
if (routeConfig.AllowBurstForSeconds.HasValue)
|
||||
{
|
||||
config[$"{routePrefix}:allow_burst_for_seconds"] =
|
||||
routeConfig.AllowBurstForSeconds.Value.ToString();
|
||||
}
|
||||
|
||||
if (routeConfig.AllowMaxBurstRequests.HasValue)
|
||||
{
|
||||
config[$"{routePrefix}:allow_max_burst_requests"] =
|
||||
routeConfig.AllowMaxBurstRequests.Value.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
// Map circuit breaker config
|
||||
if (mirrorConfig.ForEnvironment.CircuitBreaker is not null)
|
||||
{
|
||||
config["rate_limiting:for_environment:circuit_breaker:failure_threshold"] =
|
||||
mirrorConfig.ForEnvironment.CircuitBreaker.FailureThreshold.ToString();
|
||||
config["rate_limiting:for_environment:circuit_breaker:timeout_seconds"] =
|
||||
mirrorConfig.ForEnvironment.CircuitBreaker.TimeoutSeconds.ToString();
|
||||
config["rate_limiting:for_environment:circuit_breaker:half_open_timeout"] =
|
||||
mirrorConfig.ForEnvironment.CircuitBreaker.HalfOpenTimeoutSeconds.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
return config;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Middleware for applying rate limits to mirror endpoints.
|
||||
/// Bridges to Router library rate limiting.
|
||||
/// </summary>
|
||||
public class MirrorRateLimitMiddleware
|
||||
{
|
||||
private readonly RequestDelegate _next;
|
||||
private readonly MirrorRateLimitConfig _config;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
// Simple in-memory counters for instance-level limiting
|
||||
// Environment-level would use Valkey via Router library
|
||||
private readonly Dictionary<string, RateLimitCounter> _counters = new();
|
||||
private readonly object _lock = new();
|
||||
|
||||
public MirrorRateLimitMiddleware(
|
||||
RequestDelegate next,
|
||||
MirrorRateLimitConfig config,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_next = next;
|
||||
_config = config;
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public async Task InvokeAsync(HttpContext context)
|
||||
{
|
||||
if (!_config.IsEnabled)
|
||||
{
|
||||
await _next(context);
|
||||
return;
|
||||
}
|
||||
|
||||
var path = context.Request.Path.Value ?? "";
|
||||
var clientId = GetClientIdentifier(context);
|
||||
|
||||
// Check instance-level limits first
|
||||
if (_config.ForInstance is not null)
|
||||
{
|
||||
var (allowed, retryAfter) = CheckInstanceLimit(clientId, path);
|
||||
if (!allowed)
|
||||
{
|
||||
await WriteRateLimitResponse(context, retryAfter, "instance");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Environment-level checking would go through Router's ValkeyRateLimitStore
|
||||
// For now, we skip environment checks if no Valkey is configured
|
||||
if (_config.ForEnvironment is not null &&
|
||||
!string.IsNullOrWhiteSpace(_config.ForEnvironment.ValkeyConnection))
|
||||
{
|
||||
// TODO: Integrate with Router's EnvironmentRateLimiter via Valkey
|
||||
// var (allowed, retryAfter) = await CheckEnvironmentLimitAsync(clientId, path);
|
||||
// if (!allowed) { ... }
|
||||
}
|
||||
|
||||
// Add rate limit headers
|
||||
AddRateLimitHeaders(context, path);
|
||||
|
||||
await _next(context);
|
||||
}
|
||||
|
||||
private (bool Allowed, TimeSpan RetryAfter) CheckInstanceLimit(string clientId, string path)
|
||||
{
|
||||
if (_config.ForInstance is null)
|
||||
return (true, TimeSpan.Zero);
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var window = TimeSpan.FromSeconds(_config.ForInstance.PerSeconds);
|
||||
var key = $"{clientId}:{path}";
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
if (!_counters.TryGetValue(key, out var counter) ||
|
||||
now - counter.WindowStart >= window)
|
||||
{
|
||||
counter = new RateLimitCounter(now, 0);
|
||||
}
|
||||
|
||||
if (counter.Count >= _config.ForInstance.MaxRequests)
|
||||
{
|
||||
var windowEnd = counter.WindowStart + window;
|
||||
var retryAfter = windowEnd > now ? windowEnd - now : TimeSpan.Zero;
|
||||
return (false, retryAfter);
|
||||
}
|
||||
|
||||
_counters[key] = counter with { Count = counter.Count + 1 };
|
||||
return (true, TimeSpan.Zero);
|
||||
}
|
||||
}
|
||||
|
||||
private static string GetClientIdentifier(HttpContext context)
|
||||
{
|
||||
// Use client IP or authenticated user ID
|
||||
var forwardedFor = context.Request.Headers["X-Forwarded-For"].FirstOrDefault();
|
||||
if (!string.IsNullOrWhiteSpace(forwardedFor))
|
||||
{
|
||||
return forwardedFor.Split(',')[0].Trim();
|
||||
}
|
||||
|
||||
return context.Connection.RemoteIpAddress?.ToString() ?? "unknown";
|
||||
}
|
||||
|
||||
private void AddRateLimitHeaders(HttpContext context, string path)
|
||||
{
|
||||
if (_config.ForInstance is not null)
|
||||
{
|
||||
context.Response.Headers["X-RateLimit-Limit"] =
|
||||
_config.ForInstance.MaxRequests.ToString();
|
||||
context.Response.Headers["X-RateLimit-Window"] =
|
||||
_config.ForInstance.PerSeconds.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task WriteRateLimitResponse(
|
||||
HttpContext context,
|
||||
TimeSpan retryAfter,
|
||||
string scope)
|
||||
{
|
||||
context.Response.StatusCode = StatusCodes.Status429TooManyRequests;
|
||||
context.Response.Headers["Retry-After"] =
|
||||
((int)Math.Ceiling(retryAfter.TotalSeconds)).ToString();
|
||||
context.Response.Headers["X-RateLimit-Scope"] = scope;
|
||||
|
||||
context.Response.ContentType = "application/json";
|
||||
await context.Response.WriteAsJsonAsync(new
|
||||
{
|
||||
error = "rate_limit_exceeded",
|
||||
message = $"Mirror rate limit exceeded. Try again in {(int)retryAfter.TotalSeconds} seconds.",
|
||||
retryAfter = (int)retryAfter.TotalSeconds,
|
||||
scope
|
||||
});
|
||||
}
|
||||
|
||||
private sealed record RateLimitCounter(DateTimeOffset WindowStart, int Count);
|
||||
}
|
||||
Reference in New Issue
Block a user