f98cea3bcf
- Introduced AuthorityAdvisoryAiOptions and related classes for managing advisory AI configurations, including remote inference options and tenant-specific settings. - Added AuthorityApiLifecycleOptions to control API lifecycle settings, including legacy OAuth endpoint configurations. - Implemented validation and normalization methods for both advisory AI and API lifecycle options to ensure proper configuration. - Created AuthorityNotificationsOptions and its related classes for managing notification settings, including ack tokens, webhooks, and escalation options. - Developed IssuerDirectoryClient and related models for interacting with the issuer directory service, including caching mechanisms and HTTP client configurations. - Added support for dependency injection through ServiceCollectionExtensions for the Issuer Directory Client. - Updated project file to include necessary package references for the new Issuer Directory Client library.
549 lines
21 KiB
C#
549 lines
21 KiB
C#
using System.Diagnostics.Metrics;
|
|
using System.Net;
|
|
using System.Net.Http.Headers;
|
|
using System.Net.Http.Json;
|
|
using System.Text.Json;
|
|
using Microsoft.AspNetCore.Authentication.JwtBearer;
|
|
using Microsoft.AspNetCore.Builder;
|
|
using Microsoft.AspNetCore.Hosting;
|
|
using Microsoft.AspNetCore.Http;
|
|
using Microsoft.AspNetCore.Mvc;
|
|
using Microsoft.AspNetCore.Mvc.Testing;
|
|
using Microsoft.Extensions.Configuration;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.DependencyInjection.Extensions;
|
|
using Microsoft.IdentityModel.JsonWebTokens;
|
|
using Microsoft.IdentityModel.Protocols.OpenIdConnect;
|
|
using Microsoft.IdentityModel.Tokens;
|
|
using Polly.Utilities;
|
|
using StellaOps.Auth.Client;
|
|
using StellaOps.Auth.Abstractions;
|
|
using StellaOps.Policy.Gateway.Clients;
|
|
using StellaOps.Policy.Gateway.Contracts;
|
|
using StellaOps.Policy.Gateway.Options;
|
|
using StellaOps.Policy.Gateway.Services;
|
|
using Xunit;
|
|
using Xunit.Sdk;
|
|
|
|
namespace StellaOps.Policy.Gateway.Tests;
|
|
|
|
public sealed class GatewayActivationTests
|
|
{
|
|
[Fact]
|
|
public async Task ActivateRevision_UsesServiceTokenFallback_And_RecordsMetrics()
|
|
{
|
|
await using var factory = new PolicyGatewayWebApplicationFactory();
|
|
|
|
var tokenClient = factory.Services.GetRequiredService<StubTokenClient>();
|
|
tokenClient.Reset();
|
|
|
|
var recordingHandler = factory.Services.GetRequiredService<RecordingPolicyEngineHandler>();
|
|
recordingHandler.Reset();
|
|
|
|
using var listener = new MeterListener();
|
|
var activationMeasurements = new List<(long Value, string Outcome, string Source)>();
|
|
var latencyMeasurements = new List<(double Value, string Outcome, string Source)>();
|
|
|
|
listener.InstrumentPublished += (instrument, meterListener) =>
|
|
{
|
|
if (instrument.Meter.Name != "StellaOps.Policy.Gateway")
|
|
{
|
|
return;
|
|
}
|
|
|
|
meterListener.EnableMeasurementEvents(instrument);
|
|
};
|
|
|
|
listener.SetMeasurementEventCallback<long>((instrument, value, tags, _) =>
|
|
{
|
|
if (instrument.Name != "policy_gateway_activation_requests_total")
|
|
{
|
|
return;
|
|
}
|
|
|
|
activationMeasurements.Add((value, GetTag(tags, "outcome"), GetTag(tags, "source")));
|
|
});
|
|
|
|
listener.SetMeasurementEventCallback<double>((instrument, value, tags, _) =>
|
|
{
|
|
if (instrument.Name != "policy_gateway_activation_latency_ms")
|
|
{
|
|
return;
|
|
}
|
|
|
|
latencyMeasurements.Add((value, GetTag(tags, "outcome"), GetTag(tags, "source")));
|
|
});
|
|
|
|
listener.Start();
|
|
|
|
using var client = factory.CreateClient();
|
|
|
|
var response = await client.PostAsJsonAsync(
|
|
"/api/policy/packs/example/revisions/5:activate",
|
|
new ActivatePolicyRevisionRequest("rollout window start"));
|
|
|
|
listener.Dispose();
|
|
|
|
var forwardedRequest = recordingHandler.LastRequest;
|
|
var issuedTokens = tokenClient.RequestCount;
|
|
var responseBody = await response.Content.ReadAsStringAsync();
|
|
if (!response.IsSuccessStatusCode)
|
|
{
|
|
throw new Xunit.Sdk.XunitException(
|
|
$"Gateway response was {(int)response.StatusCode} {response.StatusCode}. " +
|
|
$"Body: {responseBody}. IssuedTokens: {issuedTokens}. Forwarded: { (forwardedRequest is null ? "no" : "yes") }.");
|
|
}
|
|
|
|
Assert.Equal(1, tokenClient.RequestCount);
|
|
|
|
Assert.NotNull(forwardedRequest);
|
|
Assert.Equal(HttpMethod.Post, forwardedRequest!.Method);
|
|
Assert.Equal("https://policy-engine.test/api/policy/packs/example/revisions/5:activate", forwardedRequest.RequestUri!.ToString());
|
|
Assert.Equal("Bearer", forwardedRequest.Headers.Authorization?.Scheme);
|
|
Assert.Equal("service-token", forwardedRequest.Headers.Authorization?.Parameter);
|
|
Assert.False(forwardedRequest.Headers.TryGetValues("DPoP", out _), "Expected no DPoP header when DPoP is disabled.");
|
|
|
|
Assert.Contains(activationMeasurements, measurement =>
|
|
measurement.Value == 1 &&
|
|
measurement.Outcome == "activated" &&
|
|
measurement.Source == "service");
|
|
|
|
Assert.Contains(latencyMeasurements, measurement =>
|
|
measurement.Outcome == "activated" &&
|
|
measurement.Source == "service");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ActivateRevision_RecordsMetrics_WhenUpstreamReturnsUnauthorized()
|
|
{
|
|
await using var factory = new PolicyGatewayWebApplicationFactory();
|
|
|
|
var tokenClient = factory.Services.GetRequiredService<StubTokenClient>();
|
|
tokenClient.Reset();
|
|
|
|
var recordingHandler = factory.Services.GetRequiredService<RecordingPolicyEngineHandler>();
|
|
recordingHandler.Reset();
|
|
recordingHandler.SetResponseFactory(_ =>
|
|
{
|
|
var problem = new ProblemDetails
|
|
{
|
|
Title = "Unauthorized",
|
|
Detail = "Caller token rejected.",
|
|
Status = StatusCodes.Status401Unauthorized
|
|
};
|
|
return new HttpResponseMessage(HttpStatusCode.Unauthorized)
|
|
{
|
|
Content = JsonContent.Create(problem)
|
|
};
|
|
});
|
|
|
|
using var listener = new MeterListener();
|
|
var activationMeasurements = new List<(long Value, string Outcome, string Source)>();
|
|
var latencyMeasurements = new List<(double Value, string Outcome, string Source)>();
|
|
|
|
listener.InstrumentPublished += (instrument, meterListener) =>
|
|
{
|
|
if (instrument.Meter.Name != "StellaOps.Policy.Gateway")
|
|
{
|
|
return;
|
|
}
|
|
|
|
meterListener.EnableMeasurementEvents(instrument);
|
|
};
|
|
|
|
listener.SetMeasurementEventCallback<long>((instrument, value, tags, _) =>
|
|
{
|
|
if (instrument.Name != "policy_gateway_activation_requests_total")
|
|
{
|
|
return;
|
|
}
|
|
|
|
activationMeasurements.Add((value, GetTag(tags, "outcome"), GetTag(tags, "source")));
|
|
});
|
|
|
|
listener.SetMeasurementEventCallback<double>((instrument, value, tags, _) =>
|
|
{
|
|
if (instrument.Name != "policy_gateway_activation_latency_ms")
|
|
{
|
|
return;
|
|
}
|
|
|
|
latencyMeasurements.Add((value, GetTag(tags, "outcome"), GetTag(tags, "source")));
|
|
});
|
|
|
|
listener.Start();
|
|
|
|
using var client = factory.CreateClient();
|
|
|
|
var response = await client.PostAsJsonAsync(
|
|
"/api/policy/packs/example/revisions/2:activate",
|
|
new ActivatePolicyRevisionRequest("failure path"));
|
|
|
|
listener.Dispose();
|
|
|
|
Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode);
|
|
|
|
Assert.Equal(1, tokenClient.RequestCount);
|
|
|
|
var forwardedRequest = recordingHandler.LastRequest;
|
|
Assert.NotNull(forwardedRequest);
|
|
Assert.Equal("service-token", forwardedRequest!.Headers.Authorization?.Parameter);
|
|
|
|
Assert.Contains(activationMeasurements, measurement =>
|
|
measurement.Value == 1 &&
|
|
measurement.Outcome == "unauthorized" &&
|
|
measurement.Source == "service");
|
|
|
|
Assert.Contains(latencyMeasurements, measurement =>
|
|
measurement.Outcome == "unauthorized" &&
|
|
measurement.Source == "service");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ActivateRevision_RecordsMetrics_WhenUpstreamReturnsBadGateway()
|
|
{
|
|
await using var factory = new PolicyGatewayWebApplicationFactory();
|
|
|
|
var tokenClient = factory.Services.GetRequiredService<StubTokenClient>();
|
|
tokenClient.Reset();
|
|
|
|
var recordingHandler = factory.Services.GetRequiredService<RecordingPolicyEngineHandler>();
|
|
recordingHandler.Reset();
|
|
recordingHandler.SetResponseFactory(_ =>
|
|
{
|
|
var problem = new ProblemDetails
|
|
{
|
|
Title = "Upstream error",
|
|
Detail = "Policy Engine returned 502.",
|
|
Status = StatusCodes.Status502BadGateway
|
|
};
|
|
return new HttpResponseMessage(HttpStatusCode.BadGateway)
|
|
{
|
|
Content = JsonContent.Create(problem)
|
|
};
|
|
});
|
|
|
|
using var listener = new MeterListener();
|
|
var activationMeasurements = new List<(long Value, string Outcome, string Source)>();
|
|
var latencyMeasurements = new List<(double Value, string Outcome, string Source)>();
|
|
|
|
listener.InstrumentPublished += (instrument, meterListener) =>
|
|
{
|
|
if (instrument.Meter.Name != "StellaOps.Policy.Gateway")
|
|
{
|
|
return;
|
|
}
|
|
|
|
meterListener.EnableMeasurementEvents(instrument);
|
|
};
|
|
|
|
listener.SetMeasurementEventCallback<long>((instrument, value, tags, _) =>
|
|
{
|
|
if (instrument.Name != "policy_gateway_activation_requests_total")
|
|
{
|
|
return;
|
|
}
|
|
|
|
activationMeasurements.Add((value, GetTag(tags, "outcome"), GetTag(tags, "source")));
|
|
});
|
|
|
|
listener.SetMeasurementEventCallback<double>((instrument, value, tags, _) =>
|
|
{
|
|
if (instrument.Name != "policy_gateway_activation_latency_ms")
|
|
{
|
|
return;
|
|
}
|
|
|
|
latencyMeasurements.Add((value, GetTag(tags, "outcome"), GetTag(tags, "source")));
|
|
});
|
|
|
|
listener.Start();
|
|
|
|
using var client = factory.CreateClient();
|
|
|
|
var response = await client.PostAsJsonAsync(
|
|
"/api/policy/packs/example/revisions/3:activate",
|
|
new ActivatePolicyRevisionRequest("upstream failure"));
|
|
|
|
listener.Dispose();
|
|
|
|
Assert.Equal(HttpStatusCode.BadGateway, response.StatusCode);
|
|
|
|
Assert.Equal(1, tokenClient.RequestCount);
|
|
|
|
var forwardedRequest = recordingHandler.LastRequest;
|
|
Assert.NotNull(forwardedRequest);
|
|
Assert.Equal("service-token", forwardedRequest!.Headers.Authorization?.Parameter);
|
|
|
|
Assert.Contains(activationMeasurements, measurement =>
|
|
measurement.Value == 1 &&
|
|
measurement.Outcome == "error" &&
|
|
measurement.Source == "service");
|
|
|
|
Assert.Contains(latencyMeasurements, measurement =>
|
|
measurement.Outcome == "error" &&
|
|
measurement.Source == "service");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ActivateRevision_RetriesOnTooManyRequests()
|
|
{
|
|
await using var factory = new PolicyGatewayWebApplicationFactory();
|
|
|
|
var recordedDelays = new List<TimeSpan>();
|
|
var originalSleep = SystemClock.SleepAsync;
|
|
SystemClock.SleepAsync = (delay, cancellationToken) =>
|
|
{
|
|
recordedDelays.Add(delay);
|
|
return Task.CompletedTask;
|
|
};
|
|
|
|
var tokenClient = factory.Services.GetRequiredService<StubTokenClient>();
|
|
tokenClient.Reset();
|
|
|
|
var recordingHandler = factory.Services.GetRequiredService<RecordingPolicyEngineHandler>();
|
|
recordingHandler.Reset();
|
|
recordingHandler.SetResponseSequence(new[]
|
|
{
|
|
CreateThrottleResponse(),
|
|
CreateThrottleResponse(),
|
|
RecordingPolicyEngineHandler.CreateSuccessResponse()
|
|
});
|
|
|
|
using var client = factory.CreateClient();
|
|
|
|
try
|
|
{
|
|
var response = await client.PostAsJsonAsync(
|
|
"/api/policy/packs/example/revisions/7:activate",
|
|
new ActivatePolicyRevisionRequest("retry after throttle"));
|
|
|
|
Assert.True(response.IsSuccessStatusCode, "Gateway should succeed after retrying throttled upstream responses.");
|
|
Assert.Equal(1, tokenClient.RequestCount);
|
|
Assert.Equal(3, recordingHandler.RequestCount);
|
|
}
|
|
finally
|
|
{
|
|
SystemClock.SleepAsync = originalSleep;
|
|
}
|
|
|
|
Assert.Equal(new[] { TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(4) }, recordedDelays);
|
|
}
|
|
|
|
private static HttpResponseMessage CreateThrottleResponse()
|
|
{
|
|
var problem = new ProblemDetails
|
|
{
|
|
Title = "Too many requests",
|
|
Detail = "Slow down.",
|
|
Status = StatusCodes.Status429TooManyRequests
|
|
};
|
|
|
|
var response = new HttpResponseMessage((HttpStatusCode)StatusCodes.Status429TooManyRequests)
|
|
{
|
|
Content = JsonContent.Create(problem)
|
|
};
|
|
response.Headers.RetryAfter = new RetryConditionHeaderValue(TimeSpan.FromMilliseconds(10));
|
|
return response;
|
|
}
|
|
|
|
private static string GetTag(ReadOnlySpan<KeyValuePair<string, object?>> tags, string key)
|
|
{
|
|
foreach (var tag in tags)
|
|
{
|
|
if (string.Equals(tag.Key, key, StringComparison.Ordinal))
|
|
{
|
|
return tag.Value?.ToString() ?? string.Empty;
|
|
}
|
|
}
|
|
|
|
return string.Empty;
|
|
}
|
|
|
|
private sealed class PolicyGatewayWebApplicationFactory : WebApplicationFactory<Program>
|
|
{
|
|
protected override void ConfigureWebHost(IWebHostBuilder builder)
|
|
{
|
|
builder.UseEnvironment("Development");
|
|
|
|
builder.ConfigureAppConfiguration((_, configurationBuilder) =>
|
|
{
|
|
var settings = new Dictionary<string, string?>
|
|
{
|
|
["PolicyGateway:Telemetry:MinimumLogLevel"] = "Warning",
|
|
["PolicyGateway:ResourceServer:Authority"] = "https://authority.test",
|
|
["PolicyGateway:ResourceServer:RequireHttpsMetadata"] = "false",
|
|
["PolicyGateway:ResourceServer:BypassNetworks:0"] = "127.0.0.1/32",
|
|
["PolicyGateway:ResourceServer:BypassNetworks:1"] = "::1/128",
|
|
["PolicyGateway:PolicyEngine:BaseAddress"] = "https://policy-engine.test/",
|
|
["PolicyGateway:PolicyEngine:ClientCredentials:Enabled"] = "true",
|
|
["PolicyGateway:PolicyEngine:ClientCredentials:ClientId"] = "policy-gateway",
|
|
["PolicyGateway:PolicyEngine:ClientCredentials:ClientSecret"] = "secret",
|
|
["PolicyGateway:PolicyEngine:ClientCredentials:Scopes:0"] = "policy:activate",
|
|
["PolicyGateway:PolicyEngine:Dpop:Enabled"] = "false"
|
|
};
|
|
|
|
configurationBuilder.AddInMemoryCollection(settings);
|
|
});
|
|
|
|
builder.ConfigureServices(services =>
|
|
{
|
|
services.RemoveAll<IStellaOpsTokenClient>();
|
|
services.AddSingleton<StubTokenClient>();
|
|
services.AddSingleton<IStellaOpsTokenClient>(sp => sp.GetRequiredService<StubTokenClient>());
|
|
|
|
services.RemoveAll<PolicyEngineClient>();
|
|
services.RemoveAll<IPolicyEngineClient>();
|
|
services.AddSingleton<RecordingPolicyEngineHandler>();
|
|
services.AddHttpClient<IPolicyEngineClient, PolicyEngineClient>()
|
|
.ConfigureHttpClient(client =>
|
|
{
|
|
client.BaseAddress = new Uri("https://policy-engine.test/");
|
|
})
|
|
.ConfigurePrimaryHttpMessageHandler(sp => sp.GetRequiredService<RecordingPolicyEngineHandler>());
|
|
|
|
services.AddSingleton<IStartupFilter>(new RemoteIpStartupFilter());
|
|
|
|
services.PostConfigure<JwtBearerOptions>(StellaOpsAuthenticationDefaults.AuthenticationScheme, options =>
|
|
{
|
|
options.RequireHttpsMetadata = false;
|
|
options.Configuration = new OpenIdConnectConfiguration
|
|
{
|
|
Issuer = "https://authority.test",
|
|
TokenEndpoint = "https://authority.test/token"
|
|
};
|
|
options.TokenValidationParameters = new TokenValidationParameters
|
|
{
|
|
ValidateIssuer = false,
|
|
ValidateAudience = false,
|
|
ValidateIssuerSigningKey = false,
|
|
SignatureValidator = (token, parameters) => new JsonWebToken(token)
|
|
};
|
|
options.BackchannelHttpHandler = new NoOpBackchannelHandler();
|
|
});
|
|
|
|
});
|
|
}
|
|
}
|
|
|
|
private sealed class RemoteIpStartupFilter : IStartupFilter
|
|
{
|
|
public Action<IApplicationBuilder> Configure(Action<IApplicationBuilder> next)
|
|
{
|
|
return app =>
|
|
{
|
|
app.Use(async (context, innerNext) =>
|
|
{
|
|
context.Connection.RemoteIpAddress ??= IPAddress.Loopback;
|
|
await innerNext();
|
|
});
|
|
|
|
next(app);
|
|
};
|
|
}
|
|
}
|
|
|
|
private sealed class RecordingPolicyEngineHandler : HttpMessageHandler
|
|
{
|
|
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
|
|
|
|
public HttpRequestMessage? LastRequest { get; private set; }
|
|
public int RequestCount { get; private set; }
|
|
private Func<HttpRequestMessage, HttpResponseMessage>? responseFactory;
|
|
private Queue<HttpResponseMessage>? responseQueue;
|
|
|
|
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
|
{
|
|
LastRequest = request;
|
|
RequestCount++;
|
|
|
|
if (responseQueue is { Count: > 0 })
|
|
{
|
|
return Task.FromResult(responseQueue.Dequeue());
|
|
}
|
|
|
|
var response = responseFactory is not null
|
|
? responseFactory(request)
|
|
: CreateSuccessResponse();
|
|
|
|
return Task.FromResult(response);
|
|
}
|
|
|
|
public void Reset()
|
|
{
|
|
LastRequest = null;
|
|
RequestCount = 0;
|
|
responseFactory = null;
|
|
responseQueue?.Clear();
|
|
responseQueue = null;
|
|
}
|
|
|
|
public void SetResponseFactory(Func<HttpRequestMessage, HttpResponseMessage>? factory)
|
|
{
|
|
responseFactory = factory;
|
|
}
|
|
|
|
public void SetResponseSequence(IEnumerable<HttpResponseMessage> responses)
|
|
{
|
|
responseQueue = new Queue<HttpResponseMessage>(responses ?? Array.Empty<HttpResponseMessage>());
|
|
}
|
|
|
|
public static HttpResponseMessage CreateSuccessResponse()
|
|
{
|
|
var now = DateTimeOffset.UtcNow;
|
|
var payload = new PolicyRevisionActivationDto(
|
|
"activated",
|
|
new PolicyRevisionDto(
|
|
5,
|
|
"activated",
|
|
false,
|
|
now,
|
|
now,
|
|
Array.Empty<PolicyActivationApprovalDto>()));
|
|
|
|
return new HttpResponseMessage(HttpStatusCode.OK)
|
|
{
|
|
Content = JsonContent.Create(payload, options: SerializerOptions)
|
|
};
|
|
}
|
|
}
|
|
|
|
private sealed class NoOpBackchannelHandler : HttpMessageHandler
|
|
{
|
|
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
|
=> Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK));
|
|
}
|
|
|
|
private sealed class StubTokenClient : IStellaOpsTokenClient
|
|
{
|
|
public int RequestCount { get; private set; }
|
|
|
|
public void Reset()
|
|
{
|
|
RequestCount = 0;
|
|
}
|
|
|
|
public Task<StellaOpsTokenResult> RequestClientCredentialsTokenAsync(string? scope = null, IReadOnlyDictionary<string, string>? additionalParameters = null, CancellationToken cancellationToken = default)
|
|
{
|
|
RequestCount++;
|
|
var expiresAt = DateTimeOffset.UtcNow.AddMinutes(5);
|
|
return Task.FromResult(new StellaOpsTokenResult("service-token", "Bearer", expiresAt, Array.Empty<string>()));
|
|
}
|
|
|
|
public Task<StellaOpsTokenResult> RequestPasswordTokenAsync(string username, string password, string? scope = null, IReadOnlyDictionary<string, string>? additionalParameters = null, CancellationToken cancellationToken = default)
|
|
=> throw new NotSupportedException();
|
|
|
|
public Task<JsonWebKeySet> GetJsonWebKeySetAsync(CancellationToken cancellationToken = default)
|
|
=> throw new NotSupportedException();
|
|
|
|
public ValueTask<StellaOpsTokenCacheEntry?> GetCachedTokenAsync(string key, CancellationToken cancellationToken = default)
|
|
=> ValueTask.FromResult<StellaOpsTokenCacheEntry?>(null);
|
|
|
|
public ValueTask CacheTokenAsync(string key, StellaOpsTokenCacheEntry entry, CancellationToken cancellationToken = default)
|
|
=> ValueTask.CompletedTask;
|
|
|
|
public ValueTask ClearCachedTokenAsync(string key, CancellationToken cancellationToken = default)
|
|
=> ValueTask.CompletedTask;
|
|
}
|
|
}
|