Add unit tests for SBOM ingestion and transformation
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
- Implement `SbomIngestServiceCollectionExtensionsTests` to verify the SBOM ingestion pipeline exports snapshots correctly. - Create `SbomIngestTransformerTests` to ensure the transformation produces expected nodes and edges, including deduplication of license nodes and normalization of timestamps. - Add `SbomSnapshotExporterTests` to test the export functionality for manifest, adjacency, nodes, and edges. - Introduce `VexOverlayTransformerTests` to validate the transformation of VEX nodes and edges. - Set up project file for the test project with necessary dependencies and configurations. - Include JSON fixture files for testing purposes.
This commit is contained in:
@@ -0,0 +1,363 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using StellaOps.Auth.Abstractions;
|
||||
using StellaOps.Scheduler.Models;
|
||||
using StellaOps.Scheduler.WebService.Auth;
|
||||
using StellaOps.Scheduler.WebService.PolicyRuns;
|
||||
|
||||
namespace StellaOps.Scheduler.WebService.PolicySimulations;
|
||||
|
||||
internal static class PolicySimulationEndpointExtensions
|
||||
{
|
||||
private const string Scope = StellaOpsScopes.PolicySimulate;
|
||||
|
||||
public static void MapPolicySimulationEndpoints(this IEndpointRouteBuilder builder)
|
||||
{
|
||||
var group = builder.MapGroup("/api/v1/scheduler/policies/simulations");
|
||||
|
||||
group.MapGet("/", ListSimulationsAsync);
|
||||
group.MapGet("/{simulationId}", GetSimulationAsync);
|
||||
group.MapGet("/{simulationId}/stream", StreamSimulationAsync);
|
||||
group.MapGet("/metrics", GetMetricsAsync);
|
||||
group.MapPost("/", CreateSimulationAsync);
|
||||
group.MapPost("/{simulationId}/cancel", CancelSimulationAsync);
|
||||
group.MapPost("/{simulationId}/retry", RetrySimulationAsync);
|
||||
}
|
||||
|
||||
private static async Task<IResult> ListSimulationsAsync(
|
||||
HttpContext httpContext,
|
||||
[FromServices] ITenantContextAccessor tenantAccessor,
|
||||
[FromServices] IScopeAuthorizer scopeAuthorizer,
|
||||
[FromServices] IPolicyRunService policyRunService,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
scopeAuthorizer.EnsureScope(httpContext, Scope);
|
||||
var tenant = tenantAccessor.GetTenant(httpContext);
|
||||
var options = PolicyRunQueryOptions
|
||||
.FromRequest(httpContext.Request)
|
||||
.ForceMode(PolicyRunMode.Simulate);
|
||||
|
||||
var simulations = await policyRunService
|
||||
.ListAsync(tenant.TenantId, options, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return Results.Ok(new PolicySimulationCollectionResponse(simulations));
|
||||
}
|
||||
catch (UnauthorizedAccessException ex)
|
||||
{
|
||||
return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status401Unauthorized);
|
||||
}
|
||||
catch (InvalidOperationException ex)
|
||||
{
|
||||
return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status403Forbidden);
|
||||
}
|
||||
catch (ValidationException ex)
|
||||
{
|
||||
return Results.BadRequest(new { error = ex.Message });
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<IResult> GetSimulationAsync(
|
||||
HttpContext httpContext,
|
||||
string simulationId,
|
||||
[FromServices] ITenantContextAccessor tenantAccessor,
|
||||
[FromServices] IScopeAuthorizer scopeAuthorizer,
|
||||
[FromServices] IPolicyRunService policyRunService,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
scopeAuthorizer.EnsureScope(httpContext, Scope);
|
||||
var tenant = tenantAccessor.GetTenant(httpContext);
|
||||
|
||||
var simulation = await policyRunService
|
||||
.GetAsync(tenant.TenantId, simulationId, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return simulation is null
|
||||
? Results.NotFound()
|
||||
: Results.Ok(new PolicySimulationResponse(simulation));
|
||||
}
|
||||
catch (UnauthorizedAccessException ex)
|
||||
{
|
||||
return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status401Unauthorized);
|
||||
}
|
||||
catch (InvalidOperationException ex)
|
||||
{
|
||||
return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status403Forbidden);
|
||||
}
|
||||
catch (ValidationException ex)
|
||||
{
|
||||
return Results.BadRequest(new { error = ex.Message });
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<IResult> GetMetricsAsync(
|
||||
HttpContext httpContext,
|
||||
[FromServices] ITenantContextAccessor tenantAccessor,
|
||||
[FromServices] IScopeAuthorizer scopeAuthorizer,
|
||||
[FromServices] IPolicySimulationMetricsProvider? metricsProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
scopeAuthorizer.EnsureScope(httpContext, Scope);
|
||||
var tenant = tenantAccessor.GetTenant(httpContext);
|
||||
|
||||
if (metricsProvider is null)
|
||||
{
|
||||
return Results.StatusCode(StatusCodes.Status501NotImplemented);
|
||||
}
|
||||
|
||||
var metrics = await metricsProvider
|
||||
.CaptureAsync(tenant.TenantId, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return Results.Ok(metrics);
|
||||
}
|
||||
catch (UnauthorizedAccessException ex)
|
||||
{
|
||||
return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status401Unauthorized);
|
||||
}
|
||||
catch (InvalidOperationException ex)
|
||||
{
|
||||
return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status403Forbidden);
|
||||
}
|
||||
catch (ValidationException ex)
|
||||
{
|
||||
return Results.BadRequest(new { error = ex.Message });
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<IResult> CreateSimulationAsync(
|
||||
HttpContext httpContext,
|
||||
PolicySimulationCreateRequest request,
|
||||
[FromServices] ITenantContextAccessor tenantAccessor,
|
||||
[FromServices] IScopeAuthorizer scopeAuthorizer,
|
||||
[FromServices] IPolicyRunService policyRunService,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
scopeAuthorizer.EnsureScope(httpContext, Scope);
|
||||
var tenant = tenantAccessor.GetTenant(httpContext);
|
||||
var actor = SchedulerEndpointHelpers.ResolveActorId(httpContext);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.PolicyId))
|
||||
{
|
||||
throw new ValidationException("policyId must be provided.");
|
||||
}
|
||||
|
||||
if (request.PolicyVersion is null || request.PolicyVersion <= 0)
|
||||
{
|
||||
throw new ValidationException("policyVersion must be provided and greater than zero.");
|
||||
}
|
||||
|
||||
var normalizedMetadata = NormalizeMetadata(request.Metadata);
|
||||
var inputs = request.Inputs ?? PolicyRunInputs.Empty;
|
||||
|
||||
var policyRequest = new PolicyRunRequest(
|
||||
tenant.TenantId,
|
||||
request.PolicyId,
|
||||
PolicyRunMode.Simulate,
|
||||
inputs,
|
||||
request.Priority,
|
||||
runId: null,
|
||||
policyVersion: request.PolicyVersion,
|
||||
requestedBy: actor,
|
||||
queuedAt: null,
|
||||
correlationId: request.CorrelationId,
|
||||
metadata: normalizedMetadata);
|
||||
|
||||
var status = await policyRunService
|
||||
.EnqueueAsync(tenant.TenantId, policyRequest, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return Results.Created(
|
||||
$"/api/v1/scheduler/policies/simulations/{status.RunId}",
|
||||
new PolicySimulationResponse(status));
|
||||
}
|
||||
catch (UnauthorizedAccessException ex)
|
||||
{
|
||||
return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status401Unauthorized);
|
||||
}
|
||||
catch (InvalidOperationException ex)
|
||||
{
|
||||
return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status403Forbidden);
|
||||
}
|
||||
catch (ValidationException ex)
|
||||
{
|
||||
return Results.BadRequest(new { error = ex.Message });
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<IResult> CancelSimulationAsync(
|
||||
HttpContext httpContext,
|
||||
string simulationId,
|
||||
PolicySimulationCancelRequest? request,
|
||||
[FromServices] ITenantContextAccessor tenantAccessor,
|
||||
[FromServices] IScopeAuthorizer scopeAuthorizer,
|
||||
[FromServices] IPolicyRunService policyRunService,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
scopeAuthorizer.EnsureScope(httpContext, Scope);
|
||||
var tenant = tenantAccessor.GetTenant(httpContext);
|
||||
var cancellation = await policyRunService
|
||||
.RequestCancellationAsync(tenant.TenantId, simulationId, request?.Reason, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return cancellation is null
|
||||
? Results.NotFound()
|
||||
: Results.Ok(new PolicySimulationResponse(cancellation));
|
||||
}
|
||||
catch (UnauthorizedAccessException ex)
|
||||
{
|
||||
return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status401Unauthorized);
|
||||
}
|
||||
catch (InvalidOperationException ex)
|
||||
{
|
||||
return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status403Forbidden);
|
||||
}
|
||||
catch (ValidationException ex)
|
||||
{
|
||||
return Results.BadRequest(new { error = ex.Message });
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<IResult> RetrySimulationAsync(
|
||||
HttpContext httpContext,
|
||||
string simulationId,
|
||||
[FromServices] ITenantContextAccessor tenantAccessor,
|
||||
[FromServices] IScopeAuthorizer scopeAuthorizer,
|
||||
[FromServices] IPolicyRunService policyRunService,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
scopeAuthorizer.EnsureScope(httpContext, Scope);
|
||||
var tenant = tenantAccessor.GetTenant(httpContext);
|
||||
var actor = SchedulerEndpointHelpers.ResolveActorId(httpContext);
|
||||
|
||||
var status = await policyRunService
|
||||
.RetryAsync(tenant.TenantId, simulationId, actor, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return Results.Created(
|
||||
$"/api/v1/scheduler/policies/simulations/{status.RunId}",
|
||||
new PolicySimulationResponse(status));
|
||||
}
|
||||
catch (KeyNotFoundException)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
catch (UnauthorizedAccessException ex)
|
||||
{
|
||||
return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status401Unauthorized);
|
||||
}
|
||||
catch (InvalidOperationException ex)
|
||||
{
|
||||
return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status409Conflict);
|
||||
}
|
||||
catch (ValidationException ex)
|
||||
{
|
||||
return Results.BadRequest(new { error = ex.Message });
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task StreamSimulationAsync(
|
||||
HttpContext httpContext,
|
||||
string simulationId,
|
||||
[FromServices] ITenantContextAccessor tenantAccessor,
|
||||
[FromServices] IScopeAuthorizer scopeAuthorizer,
|
||||
[FromServices] IPolicyRunService policyRunService,
|
||||
[FromServices] IPolicySimulationStreamCoordinator streamCoordinator,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
scopeAuthorizer.EnsureScope(httpContext, Scope);
|
||||
var tenant = tenantAccessor.GetTenant(httpContext);
|
||||
|
||||
var simulation = await policyRunService
|
||||
.GetAsync(tenant.TenantId, simulationId, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (simulation is null)
|
||||
{
|
||||
await Results.NotFound().ExecuteAsync(httpContext);
|
||||
return;
|
||||
}
|
||||
|
||||
await streamCoordinator
|
||||
.StreamAsync(httpContext, tenant.TenantId, simulation, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (UnauthorizedAccessException ex)
|
||||
{
|
||||
await Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status401Unauthorized)
|
||||
.ExecuteAsync(httpContext);
|
||||
}
|
||||
catch (InvalidOperationException ex)
|
||||
{
|
||||
await Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status403Forbidden)
|
||||
.ExecuteAsync(httpContext);
|
||||
}
|
||||
catch (ValidationException ex)
|
||||
{
|
||||
await Results.BadRequest(new { error = ex.Message }).ExecuteAsync(httpContext);
|
||||
}
|
||||
}
|
||||
|
||||
private static ImmutableSortedDictionary<string, string>? NormalizeMetadata(IReadOnlyDictionary<string, string>? metadata)
|
||||
{
|
||||
if (metadata is null || metadata.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var builder = ImmutableSortedDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
|
||||
foreach (var (key, value) in metadata)
|
||||
{
|
||||
var normalizedKey = key?.Trim();
|
||||
var normalizedValue = value?.Trim();
|
||||
if (string.IsNullOrEmpty(normalizedKey) || string.IsNullOrEmpty(normalizedValue))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var lowerKey = normalizedKey.ToLowerInvariant();
|
||||
if (!builder.ContainsKey(lowerKey))
|
||||
{
|
||||
builder[lowerKey] = normalizedValue;
|
||||
}
|
||||
}
|
||||
|
||||
return builder.Count == 0 ? null : builder.ToImmutable();
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record PolicySimulationCreateRequest(
|
||||
[property: JsonPropertyName("policyId")] string PolicyId,
|
||||
[property: JsonPropertyName("policyVersion")] int? PolicyVersion,
|
||||
[property: JsonPropertyName("priority")] PolicyRunPriority Priority = PolicyRunPriority.Normal,
|
||||
[property: JsonPropertyName("correlationId")] string? CorrelationId = null,
|
||||
[property: JsonPropertyName("metadata")] IReadOnlyDictionary<string, string>? Metadata = null,
|
||||
[property: JsonPropertyName("inputs")] PolicyRunInputs? Inputs = null);
|
||||
|
||||
internal sealed record PolicySimulationCancelRequest(
|
||||
[property: JsonPropertyName("reason")] string? Reason);
|
||||
|
||||
internal sealed record PolicySimulationCollectionResponse(
|
||||
[property: JsonPropertyName("simulations")] IReadOnlyList<PolicyRunStatus> Simulations);
|
||||
|
||||
internal sealed record PolicySimulationResponse(
|
||||
[property: JsonPropertyName("simulation")] PolicyRunStatus Simulation);
|
||||
@@ -0,0 +1,234 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.Metrics;
|
||||
using System.Linq;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Scheduler.Models;
|
||||
using StellaOps.Scheduler.Storage.Mongo.Repositories;
|
||||
|
||||
namespace StellaOps.Scheduler.WebService.PolicySimulations;
|
||||
|
||||
internal interface IPolicySimulationMetricsProvider
|
||||
{
|
||||
Task<PolicySimulationMetricsResponse> CaptureAsync(string tenantId, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
internal interface IPolicySimulationMetricsRecorder
|
||||
{
|
||||
void RecordLatency(PolicyRunStatus status, DateTimeOffset observedAt);
|
||||
}
|
||||
|
||||
internal sealed class PolicySimulationMetricsProvider : IPolicySimulationMetricsProvider, IPolicySimulationMetricsRecorder, IDisposable
|
||||
{
|
||||
private static readonly PolicyRunJobStatus[] QueueStatuses =
|
||||
{
|
||||
PolicyRunJobStatus.Pending,
|
||||
PolicyRunJobStatus.Dispatching,
|
||||
PolicyRunJobStatus.Submitted,
|
||||
};
|
||||
|
||||
private static readonly PolicyRunJobStatus[] TerminalStatuses =
|
||||
{
|
||||
PolicyRunJobStatus.Completed,
|
||||
PolicyRunJobStatus.Failed,
|
||||
PolicyRunJobStatus.Cancelled,
|
||||
};
|
||||
|
||||
private readonly IPolicyRunJobRepository _repository;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly Meter _meter;
|
||||
private readonly ObservableGauge<long> _queueGauge;
|
||||
private readonly Histogram<double> _latencyHistogram;
|
||||
private readonly object _snapshotLock = new();
|
||||
private IReadOnlyDictionary<string, long> _latestQueueSnapshot = new Dictionary<string, long>(StringComparer.Ordinal);
|
||||
private bool _disposed;
|
||||
|
||||
public PolicySimulationMetricsProvider(IPolicyRunJobRepository repository, TimeProvider? timeProvider = null)
|
||||
{
|
||||
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_meter = new Meter("StellaOps.Scheduler.WebService.PolicySimulations");
|
||||
_queueGauge = _meter.CreateObservableGauge<long>(
|
||||
"policy_simulation_queue_depth",
|
||||
ObserveQueueDepth,
|
||||
unit: "runs",
|
||||
description: "Queued policy simulation jobs grouped by status.");
|
||||
_latencyHistogram = _meter.CreateHistogram<double>(
|
||||
"policy_simulation_latency",
|
||||
unit: "s",
|
||||
description: "End-to-end policy simulation latency (seconds).");
|
||||
}
|
||||
|
||||
public async Task<PolicySimulationMetricsResponse> CaptureAsync(string tenantId, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
if (string.IsNullOrWhiteSpace(tenantId))
|
||||
{
|
||||
throw new ArgumentException("Tenant id must be provided.", nameof(tenantId));
|
||||
}
|
||||
|
||||
var queueCounts = new Dictionary<string, long>(StringComparer.OrdinalIgnoreCase);
|
||||
long totalQueueDepth = 0;
|
||||
|
||||
foreach (var status in QueueStatuses)
|
||||
{
|
||||
var count = await _repository.CountAsync(
|
||||
tenantId,
|
||||
PolicyRunMode.Simulate,
|
||||
new[] { status },
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
queueCounts[status.ToString().ToLowerInvariant()] = count;
|
||||
totalQueueDepth += count;
|
||||
}
|
||||
|
||||
lock (_snapshotLock)
|
||||
{
|
||||
_latestQueueSnapshot = queueCounts;
|
||||
}
|
||||
|
||||
var sampleSize = 200;
|
||||
var recentJobs = await _repository.ListAsync(
|
||||
tenantId,
|
||||
policyId: null,
|
||||
mode: PolicyRunMode.Simulate,
|
||||
statuses: TerminalStatuses,
|
||||
queuedAfter: null,
|
||||
limit: sampleSize,
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var durations = recentJobs
|
||||
.Select(job => CalculateLatencySeconds(job, _timeProvider.GetUtcNow()))
|
||||
.Where(duration => duration >= 0)
|
||||
.OrderBy(duration => duration)
|
||||
.ToArray();
|
||||
|
||||
var latencyMetrics = new PolicySimulationLatencyMetrics(
|
||||
durations.Length,
|
||||
Percentile(durations, 0.50),
|
||||
Percentile(durations, 0.90),
|
||||
Percentile(durations, 0.95),
|
||||
Percentile(durations, 0.99),
|
||||
Average(durations));
|
||||
|
||||
return new PolicySimulationMetricsResponse(
|
||||
new PolicySimulationQueueDepth(totalQueueDepth, queueCounts),
|
||||
latencyMetrics);
|
||||
}
|
||||
|
||||
public void RecordLatency(PolicyRunStatus status, DateTimeOffset observedAt)
|
||||
{
|
||||
if (status is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(status));
|
||||
}
|
||||
|
||||
var latencySeconds = CalculateLatencySeconds(status, observedAt);
|
||||
if (latencySeconds >= 0)
|
||||
{
|
||||
_latencyHistogram.Record(latencySeconds);
|
||||
}
|
||||
}
|
||||
|
||||
private IEnumerable<Measurement<long>> ObserveQueueDepth()
|
||||
{
|
||||
IReadOnlyDictionary<string, long> snapshot;
|
||||
lock (_snapshotLock)
|
||||
{
|
||||
snapshot = _latestQueueSnapshot;
|
||||
}
|
||||
|
||||
foreach (var pair in snapshot)
|
||||
{
|
||||
yield return new Measurement<long>(
|
||||
pair.Value,
|
||||
new KeyValuePair<string, object?>("status", pair.Key));
|
||||
}
|
||||
}
|
||||
|
||||
private static double CalculateLatencySeconds(PolicyRunJob job, DateTimeOffset now)
|
||||
{
|
||||
var started = job.QueuedAt ?? job.CreatedAt;
|
||||
var finished = job.CompletedAt ?? job.CancelledAt ?? job.UpdatedAt;
|
||||
if (started == default)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
var duration = (finished - started).TotalSeconds;
|
||||
return duration < 0 ? 0 : duration;
|
||||
}
|
||||
|
||||
private static double CalculateLatencySeconds(PolicyRunStatus status, DateTimeOffset now)
|
||||
{
|
||||
var started = status.QueuedAt;
|
||||
var finished = status.FinishedAt ?? now;
|
||||
if (started == default)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
var duration = (finished - started).TotalSeconds;
|
||||
return duration < 0 ? 0 : duration;
|
||||
}
|
||||
|
||||
private static double? Percentile(IReadOnlyList<double> values, double percentile)
|
||||
{
|
||||
if (values.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var position = percentile * (values.Count - 1);
|
||||
var lowerIndex = (int)Math.Floor(position);
|
||||
var upperIndex = (int)Math.Ceiling(position);
|
||||
|
||||
if (lowerIndex == upperIndex)
|
||||
{
|
||||
return Math.Round(values[lowerIndex], 4);
|
||||
}
|
||||
|
||||
var fraction = position - lowerIndex;
|
||||
var interpolated = values[lowerIndex] + (values[upperIndex] - values[lowerIndex]) * fraction;
|
||||
return Math.Round(interpolated, 4);
|
||||
}
|
||||
|
||||
private static double? Average(IReadOnlyList<double> values)
|
||||
{
|
||||
if (values.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var sum = values.Sum();
|
||||
return Math.Round(sum / values.Count, 4);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_meter.Dispose();
|
||||
_disposed = true;
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record PolicySimulationMetricsResponse(
|
||||
[property: JsonPropertyName("policy_simulation_queue_depth")] PolicySimulationQueueDepth QueueDepth,
|
||||
[property: JsonPropertyName("policy_simulation_latency")] PolicySimulationLatencyMetrics Latency);
|
||||
|
||||
internal sealed record PolicySimulationQueueDepth(
|
||||
[property: JsonPropertyName("total")] long Total,
|
||||
[property: JsonPropertyName("by_status")] IReadOnlyDictionary<string, long> ByStatus);
|
||||
|
||||
internal sealed record PolicySimulationLatencyMetrics(
|
||||
[property: JsonPropertyName("samples")] int Samples,
|
||||
[property: JsonPropertyName("p50_seconds")] double? P50,
|
||||
[property: JsonPropertyName("p90_seconds")] double? P90,
|
||||
[property: JsonPropertyName("p95_seconds")] double? P95,
|
||||
[property: JsonPropertyName("p99_seconds")] double? P99,
|
||||
[property: JsonPropertyName("mean_seconds")] double? Mean);
|
||||
@@ -0,0 +1,198 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scheduler.Models;
|
||||
using StellaOps.Scheduler.WebService.PolicyRuns;
|
||||
using StellaOps.Scheduler.WebService.Runs;
|
||||
|
||||
namespace StellaOps.Scheduler.WebService.PolicySimulations;
|
||||
|
||||
internal interface IPolicySimulationStreamCoordinator
|
||||
{
|
||||
Task StreamAsync(HttpContext context, string tenantId, PolicyRunStatus initialStatus, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class PolicySimulationStreamCoordinator : IPolicySimulationStreamCoordinator
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
private readonly IPolicyRunService _policyRunService;
|
||||
private readonly IQueueLagSummaryProvider _queueLagProvider;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly RunStreamOptions _options;
|
||||
private readonly IPolicySimulationMetricsRecorder? _metricsRecorder;
|
||||
private readonly ILogger<PolicySimulationStreamCoordinator> _logger;
|
||||
|
||||
public PolicySimulationStreamCoordinator(
|
||||
IPolicyRunService policyRunService,
|
||||
IQueueLagSummaryProvider queueLagProvider,
|
||||
IOptions<RunStreamOptions> options,
|
||||
TimeProvider? timeProvider,
|
||||
ILogger<PolicySimulationStreamCoordinator> logger,
|
||||
IPolicySimulationMetricsRecorder? metricsRecorder = null)
|
||||
{
|
||||
_policyRunService = policyRunService ?? throw new ArgumentNullException(nameof(policyRunService));
|
||||
_queueLagProvider = queueLagProvider ?? throw new ArgumentNullException(nameof(queueLagProvider));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value.Validate();
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_metricsRecorder = metricsRecorder;
|
||||
}
|
||||
|
||||
public async Task StreamAsync(HttpContext context, string tenantId, PolicyRunStatus initialStatus, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
ArgumentNullException.ThrowIfNull(initialStatus);
|
||||
|
||||
ConfigureSseHeaders(context.Response);
|
||||
await SseWriter.WriteRetryAsync(context.Response, _options.ReconnectDelay, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var last = initialStatus;
|
||||
await SseWriter.WriteEventAsync(context.Response, "initial", PolicySimulationPayload.From(last), SerializerOptions, cancellationToken).ConfigureAwait(false);
|
||||
await SseWriter.WriteEventAsync(context.Response, "queueLag", _queueLagProvider.Capture(), SerializerOptions, cancellationToken).ConfigureAwait(false);
|
||||
await SseWriter.WriteEventAsync(context.Response, "heartbeat", HeartbeatPayload.Create(_timeProvider.GetUtcNow()), SerializerOptions, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (IsTerminal(last.Status))
|
||||
{
|
||||
_metricsRecorder?.RecordLatency(last, _timeProvider.GetUtcNow());
|
||||
await SseWriter.WriteEventAsync(context.Response, "completed", PolicySimulationPayload.From(last), SerializerOptions, cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
using var pollTimer = new PeriodicTimer(_options.PollInterval);
|
||||
using var queueTimer = new PeriodicTimer(_options.QueueLagInterval);
|
||||
using var heartbeatTimer = new PeriodicTimer(_options.HeartbeatInterval);
|
||||
|
||||
try
|
||||
{
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
var pollTask = pollTimer.WaitForNextTickAsync(cancellationToken).AsTask();
|
||||
var queueTask = queueTimer.WaitForNextTickAsync(cancellationToken).AsTask();
|
||||
var heartbeatTask = heartbeatTimer.WaitForNextTickAsync(cancellationToken).AsTask();
|
||||
|
||||
var completed = await Task.WhenAny(pollTask, queueTask, heartbeatTask).ConfigureAwait(false);
|
||||
|
||||
if (completed == pollTask && await pollTask.ConfigureAwait(false))
|
||||
{
|
||||
var current = await _policyRunService
|
||||
.GetAsync(tenantId, last.RunId, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (current is null)
|
||||
{
|
||||
_logger.LogWarning("Policy simulation {RunId} disappeared while streaming.", last.RunId);
|
||||
await SseWriter.WriteEventAsync(
|
||||
context.Response,
|
||||
"notFound",
|
||||
new PolicySimulationNotFoundPayload(last.RunId),
|
||||
SerializerOptions,
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
break;
|
||||
}
|
||||
|
||||
if (HasMeaningfulChange(last, current))
|
||||
{
|
||||
await SseWriter.WriteEventAsync(context.Response, "status", PolicySimulationPayload.From(current), SerializerOptions, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
last = current;
|
||||
|
||||
if (IsTerminal(last.Status))
|
||||
{
|
||||
_metricsRecorder?.RecordLatency(last, _timeProvider.GetUtcNow());
|
||||
await SseWriter.WriteEventAsync(context.Response, "completed", PolicySimulationPayload.From(last), SerializerOptions, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
break;
|
||||
}
|
||||
}
|
||||
else if (completed == queueTask && await queueTask.ConfigureAwait(false))
|
||||
{
|
||||
var summary = _queueLagProvider.Capture();
|
||||
await SseWriter.WriteEventAsync(context.Response, "queueLag", summary, SerializerOptions, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
else if (completed == heartbeatTask && await heartbeatTask.ConfigureAwait(false))
|
||||
{
|
||||
await SseWriter.WriteEventAsync(context.Response, "heartbeat", HeartbeatPayload.Create(_timeProvider.GetUtcNow()), SerializerOptions, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
_logger.LogDebug("Policy simulation stream cancelled for run {RunId}.", last.RunId);
|
||||
}
|
||||
}
|
||||
|
||||
private static void ConfigureSseHeaders(HttpResponse response)
|
||||
{
|
||||
response.StatusCode = StatusCodes.Status200OK;
|
||||
response.Headers.CacheControl = "no-store";
|
||||
response.Headers["X-Accel-Buffering"] = "no";
|
||||
response.Headers["Connection"] = "keep-alive";
|
||||
response.ContentType = "text/event-stream";
|
||||
}
|
||||
|
||||
private static bool HasMeaningfulChange(PolicyRunStatus previous, PolicyRunStatus current)
|
||||
{
|
||||
if (!EqualityComparer<PolicyRunExecutionStatus>.Default.Equals(previous.Status, current.Status))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!Nullable.Equals(previous.StartedAt, current.StartedAt) || !Nullable.Equals(previous.FinishedAt, current.FinishedAt))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (previous.Attempts != current.Attempts)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!string.Equals(previous.Error, current.Error, StringComparison.Ordinal) ||
|
||||
!string.Equals(previous.ErrorCode, current.ErrorCode, StringComparison.Ordinal) ||
|
||||
!string.Equals(previous.DeterminismHash, current.DeterminismHash, StringComparison.Ordinal))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (previous.CancellationRequested != current.CancellationRequested ||
|
||||
!Nullable.Equals(previous.CancellationRequestedAt, current.CancellationRequestedAt) ||
|
||||
!string.Equals(previous.CancellationReason, current.CancellationReason, StringComparison.Ordinal))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!EqualityComparer<PolicyRunStats>.Default.Equals(previous.Stats, current.Stats))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static bool IsTerminal(PolicyRunExecutionStatus status)
|
||||
=> status is PolicyRunExecutionStatus.Succeeded or PolicyRunExecutionStatus.Failed or PolicyRunExecutionStatus.Cancelled;
|
||||
|
||||
private sealed record PolicySimulationPayload(
|
||||
[property: JsonPropertyName("simulation")] PolicyRunStatus Simulation)
|
||||
{
|
||||
public static PolicySimulationPayload From(PolicyRunStatus status) => new(status);
|
||||
}
|
||||
|
||||
private sealed record PolicySimulationNotFoundPayload(
|
||||
[property: JsonPropertyName("runId")] string RunId);
|
||||
|
||||
private sealed record HeartbeatPayload(
|
||||
[property: JsonPropertyName("ts")] DateTimeOffset Timestamp)
|
||||
{
|
||||
public static HeartbeatPayload Create(DateTimeOffset timestamp) => new(timestamp);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user