using System; using System.Collections.Generic; using System.Collections.Immutable; using System.IO; using System.Linq; using System.Net.Http; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using StellaOps.Scheduler.Models; using StellaOps.Scheduler.Queue; using StellaOps.Scheduler.Persistence.Postgres.Repositories; using StellaOps.TestKit; namespace StellaOps.Scheduler.WebService.Tests; public sealed class RunEndpointTests : IClassFixture { private readonly SchedulerWebApplicationFactory _factory; public RunEndpointTests(SchedulerWebApplicationFactory factory) { _factory = factory; } [Trait("Category", TestCategories.Unit)] [Fact] public async Task CreateListCancelRun() { using var client = _factory.CreateClient(); client.DefaultRequestHeaders.Add("X-Tenant-Id", "tenant-runs"); client.DefaultRequestHeaders.Add("X-Scopes", "scheduler.schedules.write scheduler.schedules.read scheduler.runs.write scheduler.runs.read scheduler.runs.preview scheduler.runs.manage"); var scheduleResponse = await client.PostAsJsonAsync("/api/v1/scheduler/schedules", new { name = "RunSchedule", cronExpression = "0 3 * * *", timezone = "UTC", mode = "analysis-only", selection = new { scope = "all-images" } }); scheduleResponse.EnsureSuccessStatusCode(); var scheduleJson = await scheduleResponse.Content.ReadFromJsonAsync(); var scheduleId = scheduleJson.GetProperty("schedule").GetProperty("id").GetString(); Assert.False(string.IsNullOrEmpty(scheduleId)); var createRun = await client.PostAsJsonAsync("/api/v1/scheduler/runs", new { scheduleId, trigger = "manual" }); createRun.EnsureSuccessStatusCode(); Assert.Equal(System.Net.HttpStatusCode.Created, createRun.StatusCode); var runJson = await createRun.Content.ReadFromJsonAsync(); var runId = runJson.GetProperty("run").GetProperty("id").GetString(); Assert.False(string.IsNullOrEmpty(runId)); Assert.Equal("planning", runJson.GetProperty("run").GetProperty("state").GetString()); var listResponse = await client.GetAsync("/api/v1/scheduler/runs"); listResponse.EnsureSuccessStatusCode(); var listJson = await listResponse.Content.ReadFromJsonAsync(); Assert.True(listJson.GetProperty("runs").EnumerateArray().Any()); var cancelResponse = await client.PostAsync($"/api/v1/scheduler/runs/{runId}/cancel", null); cancelResponse.EnsureSuccessStatusCode(); var cancelled = await cancelResponse.Content.ReadFromJsonAsync(); Assert.Equal("cancelled", cancelled.GetProperty("run").GetProperty("state").GetString()); var getResponse = await client.GetAsync($"/api/v1/scheduler/runs/{runId}"); getResponse.EnsureSuccessStatusCode(); var runDetail = await getResponse.Content.ReadFromJsonAsync(); Assert.Equal("cancelled", runDetail.GetProperty("run").GetProperty("state").GetString()); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task PreviewImpactForSchedule() { using var client = _factory.CreateClient(); client.DefaultRequestHeaders.Add("X-Tenant-Id", "tenant-preview"); client.DefaultRequestHeaders.Add("X-Scopes", "scheduler.schedules.write scheduler.schedules.read scheduler.runs.write scheduler.runs.read scheduler.runs.preview scheduler.runs.manage"); var scheduleResponse = await client.PostAsJsonAsync("/api/v1/scheduler/schedules", new { name = "PreviewSchedule", cronExpression = "0 5 * * *", timezone = "UTC", mode = "analysis-only", selection = new { scope = "all-images" } }); scheduleResponse.EnsureSuccessStatusCode(); var scheduleJson = await scheduleResponse.Content.ReadFromJsonAsync(); var scheduleId = scheduleJson.GetProperty("schedule").GetProperty("id").GetString(); Assert.False(string.IsNullOrEmpty(scheduleId)); var previewResponse = await client.PostAsJsonAsync("/api/v1/scheduler/runs/preview", new { scheduleId, usageOnly = true, sampleSize = 3 }); previewResponse.EnsureSuccessStatusCode(); var preview = await previewResponse.Content.ReadFromJsonAsync(); Assert.True(preview.GetProperty("total").GetInt32() >= 0); Assert.True(preview.GetProperty("sample").GetArrayLength() <= 3); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task RetryRunCreatesNewRun() { using var client = _factory.CreateClient(); client.DefaultRequestHeaders.Add("X-Tenant-Id", "tenant-retry"); client.DefaultRequestHeaders.Add("X-Scopes", "scheduler.schedules.write scheduler.schedules.read scheduler.runs.write scheduler.runs.read scheduler.runs.preview scheduler.runs.manage"); var scheduleId = await CreateScheduleAsync(client, "RetrySchedule"); var createRun = await client.PostAsJsonAsync("/api/v1/scheduler/runs", new { scheduleId, trigger = "manual" }); createRun.EnsureSuccessStatusCode(); var runJson = await createRun.Content.ReadFromJsonAsync(); var runId = runJson.GetProperty("run").GetProperty("id").GetString(); Assert.False(string.IsNullOrEmpty(runId)); var cancelResponse = await client.PostAsync($"/api/v1/scheduler/runs/{runId}/cancel", null); cancelResponse.EnsureSuccessStatusCode(); var retryResponse = await client.PostAsync($"/api/v1/scheduler/runs/{runId}/retry", content: null); retryResponse.EnsureSuccessStatusCode(); Assert.Equal(System.Net.HttpStatusCode.Created, retryResponse.StatusCode); var retryJson = await retryResponse.Content.ReadFromJsonAsync(); var retryRun = retryJson.GetProperty("run"); Assert.Equal("planning", retryRun.GetProperty("state").GetString()); Assert.Equal(runId, retryRun.GetProperty("retryOf").GetString()); Assert.Equal("manual", retryRun.GetProperty("trigger").GetString()); Assert.Contains("retry-of:", retryRun.GetProperty("reason").GetProperty("manualReason").GetString()); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task GetRunDeltasReturnsMetadata() { using var client = _factory.CreateClient(); client.DefaultRequestHeaders.Add("X-Tenant-Id", "tenant-deltas"); client.DefaultRequestHeaders.Add("X-Scopes", "scheduler.schedules.write scheduler.schedules.read scheduler.runs.write scheduler.runs.read scheduler.runs.preview scheduler.runs.manage"); var scheduleId = await CreateScheduleAsync(client, "DeltaSchedule"); var runResponse = await client.PostAsJsonAsync("/api/v1/scheduler/runs", new { scheduleId, trigger = "manual" }); runResponse.EnsureSuccessStatusCode(); var runJson = await runResponse.Content.ReadFromJsonAsync(); var runId = runJson.GetProperty("run").GetProperty("id").GetString()!; using (var scope = _factory.Services.CreateScope()) { var repository = scope.ServiceProvider.GetRequiredService(); var existing = await repository.GetAsync("tenant-deltas", runId); Assert.NotNull(existing); var deltas = ImmutableArray.Create(new DeltaSummary( "sha256:" + new string('a', 64), newFindings: 2, newCriticals: 1, newHigh: 1, newMedium: 0, newLow: 0)); var updated = new Run( existing!.Id, existing.TenantId, existing.Trigger, existing.State, existing.Stats, existing.CreatedAt, existing.Reason, existing.ScheduleId, existing.StartedAt, existing.FinishedAt, existing.Error, deltas, existing.RetryOf, existing.SchemaVersion); await repository.UpdateAsync(updated); } var deltasResponse = await client.GetAsync($"/api/v1/scheduler/runs/{runId}/deltas"); deltasResponse.EnsureSuccessStatusCode(); var deltasJson = await deltasResponse.Content.ReadFromJsonAsync(); Assert.Equal(1, deltasJson.GetProperty("deltas").GetArrayLength()); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task QueueLagSummaryReturnsDepth() { SchedulerQueueMetrics.RecordDepth("redis", "scheduler-runner", 7); try { using var client = _factory.CreateClient(); client.DefaultRequestHeaders.Add("X-Tenant-Id", "tenant-queue"); client.DefaultRequestHeaders.Add("X-Scopes", "scheduler.runs.read scheduler.runs.manage"); var queueResponse = await client.GetAsync("/api/v1/scheduler/runs/queue/lag"); queueResponse.EnsureSuccessStatusCode(); var summary = await queueResponse.Content.ReadFromJsonAsync(); Assert.True(summary.GetProperty("totalDepth").GetInt64() >= 7); Assert.True(summary.GetProperty("queues").EnumerateArray().Any()); } finally { SchedulerQueueMetrics.RemoveDepth("redis", "scheduler-runner"); } } [Trait("Category", TestCategories.Unit)] [Fact] public async Task StreamRunEmitsInitialEvent() { using var client = _factory.CreateClient(); client.DefaultRequestHeaders.Add("X-Tenant-Id", "tenant-stream"); client.DefaultRequestHeaders.Add("X-Scopes", "scheduler.schedules.write scheduler.schedules.read scheduler.runs.write scheduler.runs.read scheduler.runs.preview scheduler.runs.manage"); var scheduleId = await CreateScheduleAsync(client, "StreamSchedule"); var runResponse = await client.PostAsJsonAsync("/api/v1/scheduler/runs", new { scheduleId, trigger = "manual" }); runResponse.EnsureSuccessStatusCode(); var runJson = await runResponse.Content.ReadFromJsonAsync(); var runId = runJson.GetProperty("run").GetProperty("id").GetString(); Assert.False(string.IsNullOrEmpty(runId)); using var request = new HttpRequestMessage(HttpMethod.Get, $"/api/v1/scheduler/runs/{runId}/stream"); request.Headers.Accept.ParseAdd("text/event-stream"); using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); response.EnsureSuccessStatusCode(); await using var stream = await response.Content.ReadAsStreamAsync(); using var reader = new StreamReader(stream); using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); var seenRetry = false; var seenInitial = false; var seenQueueLag = false; var seenHeartbeat = false; while (!cts.IsCancellationRequested && !(seenRetry && seenInitial && seenQueueLag && seenHeartbeat)) { string? line; try { line = await reader.ReadLineAsync(cts.Token); } catch (OperationCanceledException) { break; } if (line is null) { break; } if (line.Length == 0) { continue; } if (line.StartsWith("retry:", StringComparison.Ordinal)) { seenRetry = true; } else if (line.StartsWith("event: initial", StringComparison.Ordinal)) { seenInitial = true; } else if (line.StartsWith("event: queueLag", StringComparison.Ordinal)) { seenQueueLag = true; } else if (line.StartsWith("event: heartbeat", StringComparison.Ordinal)) { seenHeartbeat = true; } } Assert.True(seenRetry, "Retry directive was not observed."); Assert.True(seenInitial, "Initial snapshot was not observed."); Assert.True(seenQueueLag, "Queue lag event was not observed."); Assert.True(seenHeartbeat, "Heartbeat event was not observed."); } private static async Task CreateScheduleAsync(HttpClient client, string name) { var scheduleResponse = await client.PostAsJsonAsync("/api/v1/scheduler/schedules", new { name, cronExpression = "0 1 * * *", timezone = "UTC", mode = "analysis-only", selection = new { scope = "all-images" } }); scheduleResponse.EnsureSuccessStatusCode(); var scheduleJson = await scheduleResponse.Content.ReadFromJsonAsync(); var scheduleId = scheduleJson.GetProperty("schedule").GetProperty("id").GetString(); Assert.False(string.IsNullOrEmpty(scheduleId)); return scheduleId!; } }