git.stella-ops.org/src/JobEngine/StellaOps.Scheduler.__Tests/StellaOps.Scheduler.WebService.Tests/RunEndpointTests.cs

using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Queue;
using StellaOps.Scheduler.Persistence.Postgres.Repositories;


using StellaOps.TestKit;
namespace StellaOps.Scheduler.WebService.Tests;

public sealed class RunEndpointTests : IClassFixture<SchedulerWebApplicationFactory>
{
    private readonly SchedulerWebApplicationFactory _factory;

    public RunEndpointTests(SchedulerWebApplicationFactory factory)
    {
        _factory = factory;
    }

    [Trait("Category", TestCategories.Unit)]
        [Fact]
    public async Task CreateListCancelRun()
    {
        using var client = _factory.CreateClient();
        client.DefaultRequestHeaders.Add("X-Tenant-Id", "tenant-runs");
        client.DefaultRequestHeaders.Add("X-Scopes", "scheduler.schedules.write scheduler.schedules.read scheduler.runs.write scheduler.runs.read scheduler.runs.preview scheduler.runs.manage");

        var scheduleResponse = await client.PostAsJsonAsync("/api/v1/scheduler/schedules", new
        {
            name = "RunSchedule",
            cronExpression = "0 3 * * *",
            timezone = "UTC",
            mode = "analysis-only",
            selection = new
            {
                scope = "all-images"
            }
        });

        scheduleResponse.EnsureSuccessStatusCode();
        var scheduleJson = await scheduleResponse.Content.ReadFromJsonAsync<JsonElement>();
        var scheduleId = scheduleJson.GetProperty("schedule").GetProperty("id").GetString();
        Assert.False(string.IsNullOrEmpty(scheduleId));

        var createRun = await client.PostAsJsonAsync("/api/v1/scheduler/runs", new
        {
            scheduleId,
            trigger = "manual"
        });

        createRun.EnsureSuccessStatusCode();
        Assert.Equal(System.Net.HttpStatusCode.Created, createRun.StatusCode);
        var runJson = await createRun.Content.ReadFromJsonAsync<JsonElement>();
        var runId = runJson.GetProperty("run").GetProperty("id").GetString();
        Assert.False(string.IsNullOrEmpty(runId));
        Assert.Equal("planning", runJson.GetProperty("run").GetProperty("state").GetString());

        var listResponse = await client.GetAsync("/api/v1/scheduler/runs");
        listResponse.EnsureSuccessStatusCode();
        var listJson = await listResponse.Content.ReadFromJsonAsync<JsonElement>();
        Assert.True(listJson.GetProperty("runs").EnumerateArray().Any());

        var cancelResponse = await client.PostAsync($"/api/v1/scheduler/runs/{runId}/cancel", null);
        cancelResponse.EnsureSuccessStatusCode();
        var cancelled = await cancelResponse.Content.ReadFromJsonAsync<JsonElement>();
        Assert.Equal("cancelled", cancelled.GetProperty("run").GetProperty("state").GetString());

        var getResponse = await client.GetAsync($"/api/v1/scheduler/runs/{runId}");
        getResponse.EnsureSuccessStatusCode();
        var runDetail = await getResponse.Content.ReadFromJsonAsync<JsonElement>();
        Assert.Equal("cancelled", runDetail.GetProperty("run").GetProperty("state").GetString());
    }

    [Trait("Category", TestCategories.Unit)]
        [Fact]
    public async Task PreviewImpactForSchedule()
    {
        using var client = _factory.CreateClient();
        client.DefaultRequestHeaders.Add("X-Tenant-Id", "tenant-preview");
        client.DefaultRequestHeaders.Add("X-Scopes", "scheduler.schedules.write scheduler.schedules.read scheduler.runs.write scheduler.runs.read scheduler.runs.preview scheduler.runs.manage");

        var scheduleResponse = await client.PostAsJsonAsync("/api/v1/scheduler/schedules", new
        {
            name = "PreviewSchedule",
            cronExpression = "0 5 * * *",
            timezone = "UTC",
            mode = "analysis-only",
            selection = new
            {
                scope = "all-images"
            }
        });

        scheduleResponse.EnsureSuccessStatusCode();
        var scheduleJson = await scheduleResponse.Content.ReadFromJsonAsync<JsonElement>();
        var scheduleId = scheduleJson.GetProperty("schedule").GetProperty("id").GetString();
        Assert.False(string.IsNullOrEmpty(scheduleId));

        var previewResponse = await client.PostAsJsonAsync("/api/v1/scheduler/runs/preview", new
        {
            scheduleId,
            usageOnly = true,
            sampleSize = 3
        });

        previewResponse.EnsureSuccessStatusCode();
        var preview = await previewResponse.Content.ReadFromJsonAsync<JsonElement>();
        Assert.True(preview.GetProperty("total").GetInt32() >= 0);
        Assert.True(preview.GetProperty("sample").GetArrayLength() <= 3);
    }

    [Trait("Category", TestCategories.Unit)]
        [Fact]
    public async Task RetryRunCreatesNewRun()
    {
        using var client = _factory.CreateClient();
        client.DefaultRequestHeaders.Add("X-Tenant-Id", "tenant-retry");
        client.DefaultRequestHeaders.Add("X-Scopes", "scheduler.schedules.write scheduler.schedules.read scheduler.runs.write scheduler.runs.read scheduler.runs.preview scheduler.runs.manage");

        var scheduleId = await CreateScheduleAsync(client, "RetrySchedule");

        var createRun = await client.PostAsJsonAsync("/api/v1/scheduler/runs", new
        {
            scheduleId,
            trigger = "manual"
        });

        createRun.EnsureSuccessStatusCode();
        var runJson = await createRun.Content.ReadFromJsonAsync<JsonElement>();
        var runId = runJson.GetProperty("run").GetProperty("id").GetString();
        Assert.False(string.IsNullOrEmpty(runId));

        var cancelResponse = await client.PostAsync($"/api/v1/scheduler/runs/{runId}/cancel", null);
        cancelResponse.EnsureSuccessStatusCode();

        var retryResponse = await client.PostAsync($"/api/v1/scheduler/runs/{runId}/retry", content: null);
        retryResponse.EnsureSuccessStatusCode();
        Assert.Equal(System.Net.HttpStatusCode.Created, retryResponse.StatusCode);

        var retryJson = await retryResponse.Content.ReadFromJsonAsync<JsonElement>();
        var retryRun = retryJson.GetProperty("run");
        Assert.Equal("planning", retryRun.GetProperty("state").GetString());
        Assert.Equal(runId, retryRun.GetProperty("retryOf").GetString());
        Assert.Equal("manual", retryRun.GetProperty("trigger").GetString());
        Assert.Contains("retry-of:", retryRun.GetProperty("reason").GetProperty("manualReason").GetString());
    }

    [Trait("Category", TestCategories.Unit)]
        [Fact]
    public async Task GetRunDeltasReturnsMetadata()
    {
        using var client = _factory.CreateClient();
        client.DefaultRequestHeaders.Add("X-Tenant-Id", "tenant-deltas");
        client.DefaultRequestHeaders.Add("X-Scopes", "scheduler.schedules.write scheduler.schedules.read scheduler.runs.write scheduler.runs.read scheduler.runs.preview scheduler.runs.manage");

        var scheduleId = await CreateScheduleAsync(client, "DeltaSchedule");

        var runResponse = await client.PostAsJsonAsync("/api/v1/scheduler/runs", new
        {
            scheduleId,
            trigger = "manual"
        });

        runResponse.EnsureSuccessStatusCode();
        var runJson = await runResponse.Content.ReadFromJsonAsync<JsonElement>();
        var runId = runJson.GetProperty("run").GetProperty("id").GetString()!;

        using (var scope = _factory.Services.CreateScope())
        {
            var repository = scope.ServiceProvider.GetRequiredService<IRunRepository>();
            var existing = await repository.GetAsync("tenant-deltas", runId);
            Assert.NotNull(existing);

            var deltas = ImmutableArray.Create(new DeltaSummary(
                "sha256:" + new string('a', 64),
                newFindings: 2,
                newCriticals: 1,
                newHigh: 1,
                newMedium: 0,
                newLow: 0));

            var updated = new Run(
                existing!.Id,
                existing.TenantId,
                existing.Trigger,
                existing.State,
                existing.Stats,
                existing.CreatedAt,
                existing.Reason,
                existing.ScheduleId,
                existing.StartedAt,
                existing.FinishedAt,
                existing.Error,
                deltas,
                existing.RetryOf,
                existing.SchemaVersion);

            await repository.UpdateAsync(updated);
        }

        var deltasResponse = await client.GetAsync($"/api/v1/scheduler/runs/{runId}/deltas");
        deltasResponse.EnsureSuccessStatusCode();

        var deltasJson = await deltasResponse.Content.ReadFromJsonAsync<JsonElement>();
        Assert.Equal(1, deltasJson.GetProperty("deltas").GetArrayLength());
    }

    [Trait("Category", TestCategories.Unit)]
        [Fact]
    public async Task QueueLagSummaryReturnsDepth()
    {
        SchedulerQueueMetrics.RecordDepth("redis", "scheduler-runner", 7);

        try
        {
            using var client = _factory.CreateClient();
            client.DefaultRequestHeaders.Add("X-Tenant-Id", "tenant-queue");
            client.DefaultRequestHeaders.Add("X-Scopes", "scheduler.runs.read scheduler.runs.manage");

            var queueResponse = await client.GetAsync("/api/v1/scheduler/runs/queue/lag");
            queueResponse.EnsureSuccessStatusCode();

            var summary = await queueResponse.Content.ReadFromJsonAsync<JsonElement>();
            Assert.True(summary.GetProperty("totalDepth").GetInt64() >= 7);
            Assert.True(summary.GetProperty("queues").EnumerateArray().Any());
        }
        finally
        {
            SchedulerQueueMetrics.RemoveDepth("redis", "scheduler-runner");
        }
    }

    [Trait("Category", TestCategories.Unit)]
        [Fact]
    public async Task StreamRunEmitsInitialEvent()
    {
        using var client = _factory.CreateClient();
        client.DefaultRequestHeaders.Add("X-Tenant-Id", "tenant-stream");
        client.DefaultRequestHeaders.Add("X-Scopes", "scheduler.schedules.write scheduler.schedules.read scheduler.runs.write scheduler.runs.read scheduler.runs.preview scheduler.runs.manage");

        var scheduleId = await CreateScheduleAsync(client, "StreamSchedule");

        var runResponse = await client.PostAsJsonAsync("/api/v1/scheduler/runs", new
        {
            scheduleId,
            trigger = "manual"
        });

        runResponse.EnsureSuccessStatusCode();
        var runJson = await runResponse.Content.ReadFromJsonAsync<JsonElement>();
        var runId = runJson.GetProperty("run").GetProperty("id").GetString();
        Assert.False(string.IsNullOrEmpty(runId));

        using var request = new HttpRequestMessage(HttpMethod.Get, $"/api/v1/scheduler/runs/{runId}/stream");
        request.Headers.Accept.ParseAdd("text/event-stream");

        using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
        response.EnsureSuccessStatusCode();

        await using var stream = await response.Content.ReadAsStreamAsync();
        using var reader = new StreamReader(stream);

        using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2));
        var seenRetry = false;
        var seenInitial = false;
        var seenQueueLag = false;
        var seenHeartbeat = false;

        while (!cts.IsCancellationRequested && !(seenRetry && seenInitial && seenQueueLag && seenHeartbeat))
        {
            string? line;
            try
            {
                line = await reader.ReadLineAsync(cts.Token);
            }
            catch (OperationCanceledException)
            {
                break;
            }

            if (line is null)
            {
                break;
            }

            if (line.Length == 0)
            {
                continue;
            }

            if (line.StartsWith("retry:", StringComparison.Ordinal))
            {
                seenRetry = true;
            }
            else if (line.StartsWith("event: initial", StringComparison.Ordinal))
            {
                seenInitial = true;
            }
            else if (line.StartsWith("event: queueLag", StringComparison.Ordinal))
            {
                seenQueueLag = true;
            }
            else if (line.StartsWith("event: heartbeat", StringComparison.Ordinal))
            {
                seenHeartbeat = true;
            }
        }

        Assert.True(seenRetry, "Retry directive was not observed.");
        Assert.True(seenInitial, "Initial snapshot was not observed.");
        Assert.True(seenQueueLag, "Queue lag event was not observed.");
        Assert.True(seenHeartbeat, "Heartbeat event was not observed.");
    }

    private static async Task<string> CreateScheduleAsync(HttpClient client, string name)
    {
        var scheduleResponse = await client.PostAsJsonAsync("/api/v1/scheduler/schedules", new
        {
            name,
            cronExpression = "0 1 * * *",
            timezone = "UTC",
            mode = "analysis-only",
            selection = new { scope = "all-images" }
        });

        scheduleResponse.EnsureSuccessStatusCode();
        var scheduleJson = await scheduleResponse.Content.ReadFromJsonAsync<JsonElement>();
        var scheduleId = scheduleJson.GetProperty("schedule").GetProperty("id").GetString();
        Assert.False(string.IsNullOrEmpty(scheduleId));
        return scheduleId!;
    }
}