using System.Diagnostics.Metrics; using System.Net; using System.Net.Http.Headers; using System.Net.Http.Json; using System.Text.Json; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.IdentityModel.JsonWebTokens; using Microsoft.IdentityModel.Protocols.OpenIdConnect; using Microsoft.IdentityModel.Tokens; using Polly.Utilities; using StellaOps.Auth.Client; using StellaOps.Auth.Abstractions; using StellaOps.Policy.Gateway.Clients; using StellaOps.Policy.Gateway.Contracts; using StellaOps.Policy.Gateway.Options; using StellaOps.Policy.Gateway.Services; using Xunit; using Xunit.Sdk; namespace StellaOps.Policy.Gateway.Tests; public sealed class GatewayActivationTests { [Fact] public async Task ActivateRevision_UsesServiceTokenFallback_And_RecordsMetrics() { await using var factory = new PolicyGatewayWebApplicationFactory(); var tokenClient = factory.Services.GetRequiredService(); tokenClient.Reset(); var recordingHandler = factory.Services.GetRequiredService(); recordingHandler.Reset(); using var listener = new MeterListener(); var activationMeasurements = new List<(long Value, string Outcome, string Source)>(); var latencyMeasurements = new List<(double Value, string Outcome, string Source)>(); listener.InstrumentPublished += (instrument, meterListener) => { if (instrument.Meter.Name != "StellaOps.Policy.Gateway") { return; } meterListener.EnableMeasurementEvents(instrument); }; listener.SetMeasurementEventCallback((instrument, value, tags, _) => { if (instrument.Name != "policy_gateway_activation_requests_total") { return; } activationMeasurements.Add((value, GetTag(tags, "outcome"), GetTag(tags, "source"))); }); listener.SetMeasurementEventCallback((instrument, value, tags, _) => { if (instrument.Name != "policy_gateway_activation_latency_ms") { return; } latencyMeasurements.Add((value, GetTag(tags, "outcome"), GetTag(tags, "source"))); }); listener.Start(); using var client = factory.CreateClient(); var response = await client.PostAsJsonAsync( "/api/policy/packs/example/revisions/5:activate", new ActivatePolicyRevisionRequest("rollout window start")); listener.Dispose(); var forwardedRequest = recordingHandler.LastRequest; var issuedTokens = tokenClient.RequestCount; var responseBody = await response.Content.ReadAsStringAsync(); if (!response.IsSuccessStatusCode) { throw new Xunit.Sdk.XunitException( $"Gateway response was {(int)response.StatusCode} {response.StatusCode}. " + $"Body: {responseBody}. IssuedTokens: {issuedTokens}. Forwarded: { (forwardedRequest is null ? "no" : "yes") }."); } Assert.Equal(1, tokenClient.RequestCount); Assert.NotNull(forwardedRequest); Assert.Equal(HttpMethod.Post, forwardedRequest!.Method); Assert.Equal("https://policy-engine.test/api/policy/packs/example/revisions/5:activate", forwardedRequest.RequestUri!.ToString()); Assert.Equal("Bearer", forwardedRequest.Headers.Authorization?.Scheme); Assert.Equal("service-token", forwardedRequest.Headers.Authorization?.Parameter); Assert.False(forwardedRequest.Headers.TryGetValues("DPoP", out _), "Expected no DPoP header when DPoP is disabled."); Assert.Contains(activationMeasurements, measurement => measurement.Value == 1 && measurement.Outcome == "activated" && measurement.Source == "service"); Assert.Contains(latencyMeasurements, measurement => measurement.Outcome == "activated" && measurement.Source == "service"); } [Fact] public async Task ActivateRevision_RecordsMetrics_WhenUpstreamReturnsUnauthorized() { await using var factory = new PolicyGatewayWebApplicationFactory(); var tokenClient = factory.Services.GetRequiredService(); tokenClient.Reset(); var recordingHandler = factory.Services.GetRequiredService(); recordingHandler.Reset(); recordingHandler.SetResponseFactory(_ => { var problem = new ProblemDetails { Title = "Unauthorized", Detail = "Caller token rejected.", Status = StatusCodes.Status401Unauthorized }; return new HttpResponseMessage(HttpStatusCode.Unauthorized) { Content = JsonContent.Create(problem) }; }); using var listener = new MeterListener(); var activationMeasurements = new List<(long Value, string Outcome, string Source)>(); var latencyMeasurements = new List<(double Value, string Outcome, string Source)>(); listener.InstrumentPublished += (instrument, meterListener) => { if (instrument.Meter.Name != "StellaOps.Policy.Gateway") { return; } meterListener.EnableMeasurementEvents(instrument); }; listener.SetMeasurementEventCallback((instrument, value, tags, _) => { if (instrument.Name != "policy_gateway_activation_requests_total") { return; } activationMeasurements.Add((value, GetTag(tags, "outcome"), GetTag(tags, "source"))); }); listener.SetMeasurementEventCallback((instrument, value, tags, _) => { if (instrument.Name != "policy_gateway_activation_latency_ms") { return; } latencyMeasurements.Add((value, GetTag(tags, "outcome"), GetTag(tags, "source"))); }); listener.Start(); using var client = factory.CreateClient(); var response = await client.PostAsJsonAsync( "/api/policy/packs/example/revisions/2:activate", new ActivatePolicyRevisionRequest("failure path")); listener.Dispose(); Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode); Assert.Equal(1, tokenClient.RequestCount); var forwardedRequest = recordingHandler.LastRequest; Assert.NotNull(forwardedRequest); Assert.Equal("service-token", forwardedRequest!.Headers.Authorization?.Parameter); Assert.Contains(activationMeasurements, measurement => measurement.Value == 1 && measurement.Outcome == "unauthorized" && measurement.Source == "service"); Assert.Contains(latencyMeasurements, measurement => measurement.Outcome == "unauthorized" && measurement.Source == "service"); } [Fact] public async Task ActivateRevision_RecordsMetrics_WhenUpstreamReturnsBadGateway() { await using var factory = new PolicyGatewayWebApplicationFactory(); var tokenClient = factory.Services.GetRequiredService(); tokenClient.Reset(); var recordingHandler = factory.Services.GetRequiredService(); recordingHandler.Reset(); recordingHandler.SetResponseFactory(_ => { var problem = new ProblemDetails { Title = "Upstream error", Detail = "Policy Engine returned 502.", Status = StatusCodes.Status502BadGateway }; return new HttpResponseMessage(HttpStatusCode.BadGateway) { Content = JsonContent.Create(problem) }; }); using var listener = new MeterListener(); var activationMeasurements = new List<(long Value, string Outcome, string Source)>(); var latencyMeasurements = new List<(double Value, string Outcome, string Source)>(); listener.InstrumentPublished += (instrument, meterListener) => { if (instrument.Meter.Name != "StellaOps.Policy.Gateway") { return; } meterListener.EnableMeasurementEvents(instrument); }; listener.SetMeasurementEventCallback((instrument, value, tags, _) => { if (instrument.Name != "policy_gateway_activation_requests_total") { return; } activationMeasurements.Add((value, GetTag(tags, "outcome"), GetTag(tags, "source"))); }); listener.SetMeasurementEventCallback((instrument, value, tags, _) => { if (instrument.Name != "policy_gateway_activation_latency_ms") { return; } latencyMeasurements.Add((value, GetTag(tags, "outcome"), GetTag(tags, "source"))); }); listener.Start(); using var client = factory.CreateClient(); var response = await client.PostAsJsonAsync( "/api/policy/packs/example/revisions/3:activate", new ActivatePolicyRevisionRequest("upstream failure")); listener.Dispose(); Assert.Equal(HttpStatusCode.BadGateway, response.StatusCode); Assert.Equal(1, tokenClient.RequestCount); var forwardedRequest = recordingHandler.LastRequest; Assert.NotNull(forwardedRequest); Assert.Equal("service-token", forwardedRequest!.Headers.Authorization?.Parameter); Assert.Contains(activationMeasurements, measurement => measurement.Value == 1 && measurement.Outcome == "error" && measurement.Source == "service"); Assert.Contains(latencyMeasurements, measurement => measurement.Outcome == "error" && measurement.Source == "service"); } [Fact] public async Task ActivateRevision_RetriesOnTooManyRequests() { await using var factory = new PolicyGatewayWebApplicationFactory(); var recordedDelays = new List(); var originalSleep = SystemClock.SleepAsync; SystemClock.SleepAsync = (delay, cancellationToken) => { recordedDelays.Add(delay); return Task.CompletedTask; }; var tokenClient = factory.Services.GetRequiredService(); tokenClient.Reset(); var recordingHandler = factory.Services.GetRequiredService(); recordingHandler.Reset(); recordingHandler.SetResponseSequence(new[] { CreateThrottleResponse(), CreateThrottleResponse(), RecordingPolicyEngineHandler.CreateSuccessResponse() }); using var client = factory.CreateClient(); try { var response = await client.PostAsJsonAsync( "/api/policy/packs/example/revisions/7:activate", new ActivatePolicyRevisionRequest("retry after throttle")); Assert.True(response.IsSuccessStatusCode, "Gateway should succeed after retrying throttled upstream responses."); Assert.Equal(1, tokenClient.RequestCount); Assert.Equal(3, recordingHandler.RequestCount); } finally { SystemClock.SleepAsync = originalSleep; } Assert.Equal(new[] { TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(4) }, recordedDelays); } private static HttpResponseMessage CreateThrottleResponse() { var problem = new ProblemDetails { Title = "Too many requests", Detail = "Slow down.", Status = StatusCodes.Status429TooManyRequests }; var response = new HttpResponseMessage((HttpStatusCode)StatusCodes.Status429TooManyRequests) { Content = JsonContent.Create(problem) }; response.Headers.RetryAfter = new RetryConditionHeaderValue(TimeSpan.FromMilliseconds(10)); return response; } private static string GetTag(ReadOnlySpan> tags, string key) { foreach (var tag in tags) { if (string.Equals(tag.Key, key, StringComparison.Ordinal)) { return tag.Value?.ToString() ?? string.Empty; } } return string.Empty; } private sealed class PolicyGatewayWebApplicationFactory : WebApplicationFactory { protected override void ConfigureWebHost(IWebHostBuilder builder) { builder.UseEnvironment("Development"); builder.ConfigureAppConfiguration((_, configurationBuilder) => { var settings = new Dictionary { ["PolicyGateway:Telemetry:MinimumLogLevel"] = "Warning", ["PolicyGateway:ResourceServer:Authority"] = "https://authority.test", ["PolicyGateway:ResourceServer:RequireHttpsMetadata"] = "false", ["PolicyGateway:ResourceServer:BypassNetworks:0"] = "127.0.0.1/32", ["PolicyGateway:ResourceServer:BypassNetworks:1"] = "::1/128", ["PolicyGateway:PolicyEngine:BaseAddress"] = "https://policy-engine.test/", ["PolicyGateway:PolicyEngine:ClientCredentials:Enabled"] = "true", ["PolicyGateway:PolicyEngine:ClientCredentials:ClientId"] = "policy-gateway", ["PolicyGateway:PolicyEngine:ClientCredentials:ClientSecret"] = "secret", ["PolicyGateway:PolicyEngine:ClientCredentials:Scopes:0"] = "policy:activate", ["PolicyGateway:PolicyEngine:Dpop:Enabled"] = "false" }; configurationBuilder.AddInMemoryCollection(settings); }); builder.ConfigureServices(services => { services.RemoveAll(); services.AddSingleton(); services.AddSingleton(sp => sp.GetRequiredService()); services.RemoveAll(); services.RemoveAll(); services.AddSingleton(); services.AddHttpClient() .ConfigureHttpClient(client => { client.BaseAddress = new Uri("https://policy-engine.test/"); }) .ConfigurePrimaryHttpMessageHandler(sp => sp.GetRequiredService()); services.AddSingleton(new RemoteIpStartupFilter()); services.PostConfigure(StellaOpsAuthenticationDefaults.AuthenticationScheme, options => { options.RequireHttpsMetadata = false; options.Configuration = new OpenIdConnectConfiguration { Issuer = "https://authority.test", TokenEndpoint = "https://authority.test/token" }; options.TokenValidationParameters = new TokenValidationParameters { ValidateIssuer = false, ValidateAudience = false, ValidateIssuerSigningKey = false, SignatureValidator = (token, parameters) => new JsonWebToken(token) }; options.BackchannelHttpHandler = new NoOpBackchannelHandler(); }); }); } } private sealed class RemoteIpStartupFilter : IStartupFilter { public Action Configure(Action next) { return app => { app.Use(async (context, innerNext) => { context.Connection.RemoteIpAddress ??= IPAddress.Loopback; await innerNext(); }); next(app); }; } } private sealed class RecordingPolicyEngineHandler : HttpMessageHandler { private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); public HttpRequestMessage? LastRequest { get; private set; } public int RequestCount { get; private set; } private Func? responseFactory; private Queue? responseQueue; protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { LastRequest = request; RequestCount++; if (responseQueue is { Count: > 0 }) { return Task.FromResult(responseQueue.Dequeue()); } var response = responseFactory is not null ? responseFactory(request) : CreateSuccessResponse(); return Task.FromResult(response); } public void Reset() { LastRequest = null; RequestCount = 0; responseFactory = null; responseQueue?.Clear(); responseQueue = null; } public void SetResponseFactory(Func? factory) { responseFactory = factory; } public void SetResponseSequence(IEnumerable responses) { responseQueue = new Queue(responses ?? Array.Empty()); } public static HttpResponseMessage CreateSuccessResponse() { var now = DateTimeOffset.UtcNow; var payload = new PolicyRevisionActivationDto( "activated", new PolicyRevisionDto( 5, "activated", false, now, now, Array.Empty())); return new HttpResponseMessage(HttpStatusCode.OK) { Content = JsonContent.Create(payload, options: SerializerOptions) }; } } private sealed class NoOpBackchannelHandler : HttpMessageHandler { protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)); } private sealed class StubTokenClient : IStellaOpsTokenClient { public int RequestCount { get; private set; } public void Reset() { RequestCount = 0; } public Task RequestClientCredentialsTokenAsync(string? scope = null, IReadOnlyDictionary? additionalParameters = null, CancellationToken cancellationToken = default) { RequestCount++; var expiresAt = DateTimeOffset.UtcNow.AddMinutes(5); return Task.FromResult(new StellaOpsTokenResult("service-token", "Bearer", expiresAt, Array.Empty())); } public Task RequestPasswordTokenAsync(string username, string password, string? scope = null, IReadOnlyDictionary? additionalParameters = null, CancellationToken cancellationToken = default) => throw new NotSupportedException(); public Task GetJsonWebKeySetAsync(CancellationToken cancellationToken = default) => throw new NotSupportedException(); public ValueTask GetCachedTokenAsync(string key, CancellationToken cancellationToken = default) => ValueTask.FromResult(null); public ValueTask CacheTokenAsync(string key, StellaOpsTokenCacheEntry entry, CancellationToken cancellationToken = default) => ValueTask.CompletedTask; public ValueTask ClearCachedTokenAsync(string key, CancellationToken cancellationToken = default) => ValueTask.CompletedTask; } }