using System.Collections.Generic; using System.Diagnostics.Metrics; using System.Net; using System.Net.Http.Headers; using System.Net.Http.Json; using System.Text.Json; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.IdentityModel.JsonWebTokens; using Microsoft.IdentityModel.Protocols.OpenIdConnect; using Microsoft.IdentityModel.Tokens; using Polly.Utilities; using StellaOps.Auth.Client; using StellaOps.Auth.Abstractions; using StellaOps.Policy.Gateway.Clients; using StellaOps.Policy.Gateway.Contracts; using StellaOps.Policy.Gateway.Options; using StellaOps.Policy.Gateway.Services; using Xunit; using Xunit.Sdk; namespace StellaOps.Policy.Gateway.Tests; public sealed class GatewayActivationTests { [Trait("Category", TestCategories.Unit)] [Fact] public async Task ActivateRevision_UsesServiceTokenFallback_And_RecordsMetrics() { await using var factory = new PolicyGatewayWebApplicationFactory(); var tokenClient = factory.Services.GetRequiredService(); tokenClient.Reset(); var recordingHandler = factory.Services.GetRequiredService(); recordingHandler.Reset(); using var listener = new MeterListener(); var activationMeasurements = new List<(long Value, string Outcome, string Source)>(); var latencyMeasurements = new List<(double Value, string Outcome, string Source)>(); listener.InstrumentPublished += (instrument, meterListener) => { if (instrument.Meter.Name != "StellaOps.Policy.Gateway") { return; } meterListener.EnableMeasurementEvents(instrument); }; listener.SetMeasurementEventCallback((instrument, value, tags, _) => { if (instrument.Name != "policy_gateway_activation_requests_total") { return; } activationMeasurements.Add((value, GetTag(tags, "outcome"), GetTag(tags, "source"))); }); listener.SetMeasurementEventCallback((instrument, value, tags, _) => { if (instrument.Name != "policy_gateway_activation_latency_ms") { return; } latencyMeasurements.Add((value, GetTag(tags, "outcome"), GetTag(tags, "source"))); }); listener.Start(); using var client = factory.CreateClient(); var response = await client.PostAsJsonAsync( "/api/policy/packs/example/revisions/5:activate", new ActivatePolicyRevisionRequest("rollout window start")); listener.Dispose(); var forwardedRequest = recordingHandler.LastRequest; var issuedTokens = tokenClient.RequestCount; var responseBody = await response.Content.ReadAsStringAsync(); if (!response.IsSuccessStatusCode) { throw new Xunit.Sdk.XunitException( $"Gateway response was {(int)response.StatusCode} {response.StatusCode}. " + $"Body: {responseBody}. IssuedTokens: {issuedTokens}. Forwarded: { (forwardedRequest is null ? "no" : "yes") }."); } Assert.Equal(1, tokenClient.RequestCount); Assert.NotNull(forwardedRequest); Assert.Equal(HttpMethod.Post, forwardedRequest!.Method); Assert.Equal("https://policy-engine.test/api/policy/packs/example/revisions/5:activate", forwardedRequest.RequestUri!.ToString()); Assert.Equal("Bearer", forwardedRequest.Headers.Authorization?.Scheme); Assert.Equal("service-token", forwardedRequest.Headers.Authorization?.Parameter); Assert.False(forwardedRequest.Headers.TryGetValues("DPoP", out _), "Expected no DPoP header when DPoP is disabled."); Assert.Contains(activationMeasurements, measurement => measurement.Value == 1 && measurement.Outcome == "activated" && measurement.Source == "service"); Assert.Contains(latencyMeasurements, measurement => measurement.Outcome == "activated" && measurement.Source == "service"); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task ActivateRevision_CompletesDualControlWorkflow() { await using var factory = new PolicyGatewayWebApplicationFactory(); var recordingHandler = factory.Services.GetRequiredService(); recordingHandler.Reset(); recordingHandler.SetResponseSequence(new[] { RecordingPolicyEngineHandler.CreatePendingSecondApprovalResponse(), RecordingPolicyEngineHandler.CreateDualControlSuccessResponse() }); using var client = factory.CreateClient(); var firstResponse = await client.PostAsJsonAsync( "/api/policy/packs/example/revisions/5:activate", new ActivatePolicyRevisionRequest("first approval")); Assert.Equal(HttpStatusCode.Accepted, firstResponse.StatusCode); var pendingPayload = await firstResponse.Content.ReadFromJsonAsync(); Assert.NotNull(pendingPayload); Assert.Equal("pending_second_approval", pendingPayload!.Status); Assert.True(pendingPayload.Revision.RequiresTwoPersonApproval); Assert.Single(pendingPayload.Revision.Approvals); var secondResponse = await client.PostAsJsonAsync( "/api/policy/packs/example/revisions/5:activate", new ActivatePolicyRevisionRequest("second approval")); Assert.Equal(HttpStatusCode.OK, secondResponse.StatusCode); var activatedPayload = await secondResponse.Content.ReadFromJsonAsync(); Assert.NotNull(activatedPayload); Assert.Equal("activated", activatedPayload!.Status); Assert.Equal(2, activatedPayload.Revision.Approvals.Count); Assert.True(activatedPayload.Revision.RequiresTwoPersonApproval); Assert.Equal(2, recordingHandler.RequestCount); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task ActivateRevision_RecordsMetrics_WhenUpstreamReturnsUnauthorized() { await using var factory = new PolicyGatewayWebApplicationFactory(); var tokenClient = factory.Services.GetRequiredService(); tokenClient.Reset(); var recordingHandler = factory.Services.GetRequiredService(); recordingHandler.Reset(); recordingHandler.SetResponseFactory(_ => { var problem = new ProblemDetails { Title = "Unauthorized", Detail = "Caller token rejected.", Status = StatusCodes.Status401Unauthorized }; return new HttpResponseMessage(HttpStatusCode.Unauthorized) { Content = JsonContent.Create(problem) }; }); using var listener = new MeterListener(); var activationMeasurements = new List<(long Value, string Outcome, string Source)>(); var latencyMeasurements = new List<(double Value, string Outcome, string Source)>(); listener.InstrumentPublished += (instrument, meterListener) => { if (instrument.Meter.Name != "StellaOps.Policy.Gateway") { return; } meterListener.EnableMeasurementEvents(instrument); }; listener.SetMeasurementEventCallback((instrument, value, tags, _) => { if (instrument.Name != "policy_gateway_activation_requests_total") { return; } activationMeasurements.Add((value, GetTag(tags, "outcome"), GetTag(tags, "source"))); }); listener.SetMeasurementEventCallback((instrument, value, tags, _) => { if (instrument.Name != "policy_gateway_activation_latency_ms") { return; } latencyMeasurements.Add((value, GetTag(tags, "outcome"), GetTag(tags, "source"))); }); listener.Start(); using var client = factory.CreateClient(); var response = await client.PostAsJsonAsync( "/api/policy/packs/example/revisions/2:activate", new ActivatePolicyRevisionRequest("failure path")); listener.Dispose(); Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode); Assert.Equal(1, tokenClient.RequestCount); var forwardedRequest = recordingHandler.LastRequest; Assert.NotNull(forwardedRequest); Assert.Equal("service-token", forwardedRequest!.Headers.Authorization?.Parameter); Assert.Contains(activationMeasurements, measurement => measurement.Value == 1 && measurement.Outcome == "unauthorized" && measurement.Source == "service"); Assert.Contains(latencyMeasurements, measurement => measurement.Outcome == "unauthorized" && measurement.Source == "service"); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task ActivateRevision_RecordsMetrics_WhenUpstreamReturnsBadGateway() { await using var factory = new PolicyGatewayWebApplicationFactory(); var tokenClient = factory.Services.GetRequiredService(); tokenClient.Reset(); var recordingHandler = factory.Services.GetRequiredService(); recordingHandler.Reset(); recordingHandler.SetResponseFactory(_ => { var problem = new ProblemDetails { Title = "Upstream error", Detail = "Policy Engine returned 502.", Status = StatusCodes.Status502BadGateway }; return new HttpResponseMessage(HttpStatusCode.BadGateway) { Content = JsonContent.Create(problem) }; }); using var listener = new MeterListener(); var activationMeasurements = new List<(long Value, string Outcome, string Source)>(); var latencyMeasurements = new List<(double Value, string Outcome, string Source)>(); listener.InstrumentPublished += (instrument, meterListener) => { if (instrument.Meter.Name != "StellaOps.Policy.Gateway") { return; } meterListener.EnableMeasurementEvents(instrument); }; listener.SetMeasurementEventCallback((instrument, value, tags, _) => { if (instrument.Name != "policy_gateway_activation_requests_total") { return; } activationMeasurements.Add((value, GetTag(tags, "outcome"), GetTag(tags, "source"))); }); listener.SetMeasurementEventCallback((instrument, value, tags, _) => { if (instrument.Name != "policy_gateway_activation_latency_ms") { return; } latencyMeasurements.Add((value, GetTag(tags, "outcome"), GetTag(tags, "source"))); }); listener.Start(); using var client = factory.CreateClient(); var response = await client.PostAsJsonAsync( "/api/policy/packs/example/revisions/3:activate", new ActivatePolicyRevisionRequest("upstream failure")); listener.Dispose(); Assert.Equal(HttpStatusCode.BadGateway, response.StatusCode); Assert.Equal(1, tokenClient.RequestCount); var forwardedRequest = recordingHandler.LastRequest; Assert.NotNull(forwardedRequest); Assert.Equal("service-token", forwardedRequest!.Headers.Authorization?.Parameter); Assert.Contains(activationMeasurements, measurement => measurement.Value == 1 && measurement.Outcome == "error" && measurement.Source == "service"); Assert.Contains(latencyMeasurements, measurement => measurement.Outcome == "error" && measurement.Source == "service"); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task ActivateRevision_RetriesOnTooManyRequests() { await using var factory = new PolicyGatewayWebApplicationFactory(); var recordedDelays = new List(); var originalSleep = SystemClock.SleepAsync; SystemClock.SleepAsync = (delay, cancellationToken) => { recordedDelays.Add(delay); return Task.CompletedTask; }; var tokenClient = factory.Services.GetRequiredService(); tokenClient.Reset(); var recordingHandler = factory.Services.GetRequiredService(); recordingHandler.Reset(); recordingHandler.SetResponseSequence(new[] { CreateThrottleResponse(), CreateThrottleResponse(), RecordingPolicyEngineHandler.CreateSuccessResponse() }); using var client = factory.CreateClient(); using StellaOps.TestKit; try { var response = await client.PostAsJsonAsync( "/api/policy/packs/example/revisions/7:activate", new ActivatePolicyRevisionRequest("retry after throttle")); Assert.True(response.IsSuccessStatusCode, "Gateway should succeed after retrying throttled upstream responses."); Assert.Equal(1, tokenClient.RequestCount); Assert.Equal(3, recordingHandler.RequestCount); } finally { SystemClock.SleepAsync = originalSleep; } Assert.Equal(new[] { TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(4) }, recordedDelays); } private static HttpResponseMessage CreateThrottleResponse() { var problem = new ProblemDetails { Title = "Too many requests", Detail = "Slow down.", Status = StatusCodes.Status429TooManyRequests }; var response = new HttpResponseMessage((HttpStatusCode)StatusCodes.Status429TooManyRequests) { Content = JsonContent.Create(problem) }; response.Headers.RetryAfter = new RetryConditionHeaderValue(TimeSpan.FromMilliseconds(10)); return response; } private static string GetTag(ReadOnlySpan> tags, string key) { foreach (var tag in tags) { if (string.Equals(tag.Key, key, StringComparison.Ordinal)) { return tag.Value?.ToString() ?? string.Empty; } } return string.Empty; } private sealed class PolicyGatewayWebApplicationFactory : WebApplicationFactory { protected override void ConfigureWebHost(IWebHostBuilder builder) { builder.UseEnvironment("Development"); builder.ConfigureAppConfiguration((_, configurationBuilder) => { var settings = new Dictionary { ["PolicyGateway:Telemetry:MinimumLogLevel"] = "Warning", ["PolicyGateway:ResourceServer:Authority"] = "https://authority.test", ["PolicyGateway:ResourceServer:RequireHttpsMetadata"] = "false", ["PolicyGateway:ResourceServer:BypassNetworks:0"] = "127.0.0.1/32", ["PolicyGateway:ResourceServer:BypassNetworks:1"] = "::1/128", ["PolicyGateway:PolicyEngine:BaseAddress"] = "https://policy-engine.test/", ["PolicyGateway:PolicyEngine:ClientCredentials:Enabled"] = "true", ["PolicyGateway:PolicyEngine:ClientCredentials:ClientId"] = "policy-gateway", ["PolicyGateway:PolicyEngine:ClientCredentials:ClientSecret"] = "secret", ["PolicyGateway:PolicyEngine:ClientCredentials:Scopes:0"] = "policy:activate", ["PolicyGateway:PolicyEngine:Dpop:Enabled"] = "false" }; configurationBuilder.AddInMemoryCollection(settings); }); builder.ConfigureServices(services => { services.RemoveAll(); services.AddSingleton(); services.AddSingleton(sp => sp.GetRequiredService()); services.RemoveAll(); services.RemoveAll(); services.AddSingleton(); services.AddHttpClient() .ConfigureHttpClient(client => { client.BaseAddress = new Uri("https://policy-engine.test/"); }) .ConfigurePrimaryHttpMessageHandler(sp => sp.GetRequiredService()); services.AddSingleton(new RemoteIpStartupFilter()); services.PostConfigure(StellaOpsAuthenticationDefaults.AuthenticationScheme, options => { options.RequireHttpsMetadata = false; options.Configuration = new OpenIdConnectConfiguration { Issuer = "https://authority.test", TokenEndpoint = "https://authority.test/token" }; options.TokenValidationParameters = new TokenValidationParameters { ValidateIssuer = false, ValidateAudience = false, ValidateIssuerSigningKey = false, SignatureValidator = (token, parameters) => new JsonWebToken(token) }; options.BackchannelHttpHandler = new NoOpBackchannelHandler(); }); }); } } private sealed class RemoteIpStartupFilter : IStartupFilter { public Action Configure(Action next) { return app => { app.Use(async (context, innerNext) => { context.Connection.RemoteIpAddress ??= IPAddress.Loopback; await innerNext(); }); next(app); }; } } private sealed class RecordingPolicyEngineHandler : HttpMessageHandler { private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); public HttpRequestMessage? LastRequest { get; private set; } public int RequestCount { get; private set; } private Func? responseFactory; private Queue? responseQueue; protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { LastRequest = request; RequestCount++; if (responseQueue is { Count: > 0 }) { return Task.FromResult(responseQueue.Dequeue()); } var response = responseFactory is not null ? responseFactory(request) : CreateSuccessResponse(); return Task.FromResult(response); } public void Reset() { LastRequest = null; RequestCount = 0; responseFactory = null; responseQueue?.Clear(); responseQueue = null; } public void SetResponseFactory(Func? factory) { responseFactory = factory; } public void SetResponseSequence(IEnumerable responses) { responseQueue = new Queue(responses ?? Array.Empty()); } public static HttpResponseMessage CreateSuccessResponse() => CreateActivationResponse( HttpStatusCode.OK, "activated", "activated", false, DateTimeOffset.UtcNow, Array.Empty()); public static HttpResponseMessage CreatePendingSecondApprovalResponse() { var firstApproval = new PolicyActivationApprovalDto( "alice@example.com", DateTimeOffset.UtcNow, "first approval"); return CreateActivationResponse( HttpStatusCode.Accepted, "pending_second_approval", "approved", true, activatedAt: null, approvals: new[] { firstApproval }); } public static HttpResponseMessage CreateDualControlSuccessResponse() { var approvals = new[] { new PolicyActivationApprovalDto("alice@example.com", DateTimeOffset.UtcNow.AddSeconds(-15), "first approval"), new PolicyActivationApprovalDto("bob@example.com", DateTimeOffset.UtcNow, "final approval") }; return CreateActivationResponse( HttpStatusCode.OK, "activated", "activated", true, DateTimeOffset.UtcNow, approvals); } private static HttpResponseMessage CreateActivationResponse( HttpStatusCode httpStatus, string activationStatus, string revisionStatus, bool requiresTwoPersonApproval, DateTimeOffset? activatedAt, IReadOnlyList? approvals) { var now = DateTimeOffset.UtcNow; var payload = new PolicyRevisionActivationDto( activationStatus, new PolicyRevisionDto( 5, revisionStatus, requiresTwoPersonApproval, now, activatedAt, approvals ?? Array.Empty())); return new HttpResponseMessage(httpStatus) { Content = JsonContent.Create(payload, options: SerializerOptions) }; } } private sealed class NoOpBackchannelHandler : HttpMessageHandler { protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)); } private sealed class StubTokenClient : IStellaOpsTokenClient { public int RequestCount { get; private set; } public void Reset() { RequestCount = 0; } public Task RequestClientCredentialsTokenAsync(string? scope = null, IReadOnlyDictionary? additionalParameters = null, CancellationToken cancellationToken = default) { RequestCount++; var expiresAt = DateTimeOffset.UtcNow.AddMinutes(5); return Task.FromResult(new StellaOpsTokenResult("service-token", "Bearer", expiresAt, Array.Empty())); } public Task RequestPasswordTokenAsync(string username, string password, string? scope = null, IReadOnlyDictionary? additionalParameters = null, CancellationToken cancellationToken = default) => throw new NotSupportedException(); public Task GetJsonWebKeySetAsync(CancellationToken cancellationToken = default) => throw new NotSupportedException(); public ValueTask GetCachedTokenAsync(string key, CancellationToken cancellationToken = default) => ValueTask.FromResult(null); public ValueTask CacheTokenAsync(string key, StellaOpsTokenCacheEntry entry, CancellationToken cancellationToken = default) => ValueTask.CompletedTask; public ValueTask ClearCachedTokenAsync(string key, CancellationToken cancellationToken = default) => ValueTask.CompletedTask; } }