feat: Add Scanner CI runner and related artifacts

- Implemented `run-scanner-ci.sh` to build and run tests for the Scanner solution with a warmed NuGet cache. - Created `excititor-vex-traces.json` dashboard for monitoring Excititor VEX observations. - Added Docker Compose configuration for the OTLP span sink in `docker-compose.spansink.yml`. - Configured OpenTelemetry collector in `otel-spansink.yaml` to receive and process traces. - Developed `run-spansink.sh` script to run the OTLP span sink for Excititor traces. - Introduced `FileSystemRiskBundleObjectStore` for storing risk bundle artifacts in the filesystem. - Built `RiskBundleBuilder` for creating risk bundles with associated metadata and providers. - Established `RiskBundleJob` to execute the risk bundle creation and storage process. - Defined models for risk bundle inputs, entries, and manifests in `RiskBundleModels.cs`. - Implemented signing functionality for risk bundle manifests with `HmacRiskBundleManifestSigner`. - Created unit tests for `RiskBundleBuilder`, `RiskBundleJob`, and signing functionality to ensure correctness. - Added filesystem artifact reader tests to validate manifest parsing and artifact listing. - Included test manifests for egress scenarios in the task runner tests. - Developed timeline query service tests to verify tenant and event ID handling.
2025-11-30 19:12:35 +02:00
parent 17d45a6d30
commit 71e9a56cfd
92 changed files with 2596 additions and 387 deletions
--- a/src/Orchestrator/StellaOps.Orchestrator/AGENTS.md
+++ b/src/Orchestrator/StellaOps.Orchestrator/AGENTS.md
@@ -27,3 +27,4 @@ Build and operate the Source & Job Orchestrator control plane described in Epic
 - 3. Keep changes deterministic (stable ordering, timestamps, hashes) and align with offline/air-gap expectations.
 - 4. Coordinate doc updates, tests, and cross-guild communication whenever contracts or workflows change.
 - 5. Revert to `TODO` if you pause the task without shipping changes; leave notes in commit/PR descriptions for context.
+- 6. **Contract guardrails:** Pack-run scheduling now requires `projectId` plus tenant headers; reject/422 if absent. Keep OpenAPI examples and worker/CLI samples aligned. Preserve idempotency semantics (`Idempotency-Key`) and deterministic pagination/stream ordering in all APIs.
--- a/src/Orchestrator/StellaOps.Orchestrator/StellaOps.Orchestrator.Tests/PackRun/PackRunStreamCoordinatorTests.cs
+++ b/src/Orchestrator/StellaOps.Orchestrator/StellaOps.Orchestrator.Tests/PackRun/PackRunStreamCoordinatorTests.cs
@@ -1,3 +1,4 @@
+using System.Net.WebSockets;
 using System.Text;
 using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.Logging.Abstractions;
@@ -72,6 +73,63 @@ public sealed class PackRunStreamCoordinatorTests
        Assert.Contains("event: completed", payload);
    }

+    [Fact]
+    public async Task StreamWebSocketAsync_TerminalRun_SendsInitialAndCompleted()
+    {
+        var now = DateTimeOffset.UtcNow;
+        var packRun = new PackRunDomain(
+            PackRunId: Guid.NewGuid(),
+            TenantId: "tenantA",
+            ProjectId: null,
+            PackId: "pack.demo",
+            PackVersion: "1.0.0",
+            Status: PackRunStatus.Succeeded,
+            Priority: 0,
+            Attempt: 1,
+            MaxAttempts: 3,
+            Parameters: "{}",
+            ParametersDigest: new string('a', 64),
+            IdempotencyKey: "idem-1",
+            CorrelationId: null,
+            LeaseId: null,
+            TaskRunnerId: "runner-1",
+            LeaseUntil: null,
+            CreatedAt: now.AddMinutes(-2),
+            ScheduledAt: now.AddMinutes(-2),
+            LeasedAt: now.AddMinutes(-1),
+            StartedAt: now.AddMinutes(-1),
+            CompletedAt: now,
+            NotBefore: null,
+            Reason: null,
+            ExitCode: 0,
+            DurationMs: 120_000,
+            CreatedBy: "tester",
+            Metadata: null);
+
+        var logRepo = new StubPackRunLogRepository((2, 5));
+        var streamOptions = Options.Create(new StreamOptions
+        {
+            PollInterval = TimeSpan.FromMilliseconds(150),
+            HeartbeatInterval = TimeSpan.FromMilliseconds(150),
+            MaxStreamDuration = TimeSpan.FromMinutes(1)
+        });
+        var coordinator = new PackRunStreamCoordinator(
+            new StubPackRunRepository(packRun),
+            logRepo,
+            streamOptions,
+            TimeProvider.System,
+            NullLogger<PackRunStreamCoordinator>.Instance);
+
+        var socket = new FakeWebSocket();
+
+        await coordinator.StreamWebSocketAsync(socket, packRun.TenantId, packRun, CancellationToken.None);
+
+        var messages = socket.SentMessages;
+        Assert.Contains(messages, m => m.Contains("\"type\":\"initial\""));
+        Assert.Contains(messages, m => m.Contains("\"type\":\"completed\""));
+        Assert.All(messages, m => Assert.Contains(packRun.PackRunId.ToString(), m));
+    }
+
    private sealed class StubPackRunRepository : IPackRunRepository
    {
        private readonly PackRunDomain _packRun;
@@ -117,4 +175,45 @@ public sealed class PackRunStreamCoordinatorTests
            => Task.FromResult(new PackRunLogBatch(packRunId, tenantId, afterSequence, new List<PackRunLog>()));
        public Task<long> DeleteLogsAsync(string tenantId, Guid packRunId, CancellationToken cancellationToken) => Task.FromResult(0L);
    }
+
+    private sealed class FakeWebSocket : WebSocket
+    {
+        private WebSocketState _state = WebSocketState.Open;
+        public List<string> SentMessages { get; } = new();
+
+        public override WebSocketCloseStatus? CloseStatus { get; }
+        public override string? CloseStatusDescription { get; }
+        public override string? SubProtocol { get; }
+        public override WebSocketState State => _state;
+
+        public override void Abort() => _state = WebSocketState.Aborted;
+
+        public override Task CloseAsync(WebSocketCloseStatus closeStatus, string? statusDescription, CancellationToken cancellationToken)
+        {
+            _state = WebSocketState.Closed;
+            return Task.CompletedTask;
+        }
+
+        public override Task CloseOutputAsync(WebSocketCloseStatus closeStatus, string? statusDescription, CancellationToken cancellationToken)
+        {
+            _state = WebSocketState.CloseSent;
+            return Task.CompletedTask;
+        }
+
+        public override void Dispose()
+        {
+            _state = WebSocketState.Closed;
+            base.Dispose();
+        }
+
+        public override Task<WebSocketReceiveResult> ReceiveAsync(ArraySegment<byte> buffer, CancellationToken cancellationToken)
+            => Task.FromResult(new WebSocketReceiveResult(0, WebSocketMessageType.Close, true));
+
+        public override Task SendAsync(ArraySegment<byte> buffer, WebSocketMessageType messageType, bool endOfMessage, CancellationToken cancellationToken)
+        {
+            var message = Encoding.UTF8.GetString(buffer.Array!, buffer.Offset, buffer.Count);
+            SentMessages.Add(message);
+            return Task.CompletedTask;
+        }
+    }
 }
--- a/src/Orchestrator/StellaOps.Orchestrator/StellaOps.Orchestrator.WebService/Endpoints/StreamEndpoints.cs
+++ b/src/Orchestrator/StellaOps.Orchestrator/StellaOps.Orchestrator.WebService/Endpoints/StreamEndpoints.cs
@@ -30,6 +30,10 @@ public static class StreamEndpoints
            .WithName("Orchestrator_StreamPackRun")
            .WithDescription("Stream real-time pack run log and status updates via SSE");

+        group.MapGet("pack-runs/{packRunId:guid}/ws", StreamPackRunWebSocket)
+            .WithName("Orchestrator_StreamPackRunWebSocket")
+            .WithDescription("Stream real-time pack run log and status updates via WebSocket");
+
        return group;
    }

@@ -138,4 +142,32 @@ public static class StreamEndpoints
            }
        }
    }
+
+    private static async Task StreamPackRunWebSocket(
+        HttpContext context,
+        [FromRoute] Guid packRunId,
+        [FromServices] TenantResolver tenantResolver,
+        [FromServices] IPackRunRepository packRunRepository,
+        [FromServices] IPackRunStreamCoordinator streamCoordinator,
+        CancellationToken cancellationToken)
+    {
+        if (!context.WebSockets.IsWebSocketRequest)
+        {
+            context.Response.StatusCode = StatusCodes.Status400BadRequest;
+            await context.Response.WriteAsJsonAsync(new { error = "Expected WebSocket request" }, cancellationToken).ConfigureAwait(false);
+            return;
+        }
+
+        var tenantId = tenantResolver.Resolve(context);
+        var packRun = await packRunRepository.GetByIdAsync(tenantId, packRunId, cancellationToken).ConfigureAwait(false);
+        if (packRun is null)
+        {
+            context.Response.StatusCode = StatusCodes.Status404NotFound;
+            await context.Response.WriteAsJsonAsync(new { error = "Pack run not found" }, cancellationToken).ConfigureAwait(false);
+            return;
+        }
+
+        using var socket = await context.WebSockets.AcceptWebSocketAsync().ConfigureAwait(false);
+        await streamCoordinator.StreamWebSocketAsync(socket, tenantId, packRun, cancellationToken).ConfigureAwait(false);
+    }
 }
--- a/src/Orchestrator/StellaOps.Orchestrator/StellaOps.Orchestrator.WebService/Program.cs
+++ b/src/Orchestrator/StellaOps.Orchestrator/StellaOps.Orchestrator.WebService/Program.cs
@@ -35,6 +35,9 @@ if (app.Environment.IsDevelopment())
    app.MapOpenApi();
 }

+// Enable WebSocket support for streaming endpoints
+app.UseWebSockets();
+
 // OpenAPI discovery endpoints (available in all environments)
 app.MapOpenApiEndpoints();

--- a/src/Orchestrator/StellaOps.Orchestrator/StellaOps.Orchestrator.WebService/Streaming/PackRunStreamCoordinator.cs
+++ b/src/Orchestrator/StellaOps.Orchestrator/StellaOps.Orchestrator.WebService/Streaming/PackRunStreamCoordinator.cs
@@ -1,3 +1,5 @@
+using System.Net.WebSockets;
+using System.Text;
 using System.Text.Json;
 using Microsoft.Extensions.Options;
 using StellaOps.Orchestrator.Core.Domain;
@@ -8,6 +10,7 @@ namespace StellaOps.Orchestrator.WebService.Streaming;
 public interface IPackRunStreamCoordinator
 {
    Task StreamAsync(HttpContext context, string tenantId, PackRun packRun, CancellationToken cancellationToken);
+    Task StreamWebSocketAsync(WebSocket socket, string tenantId, PackRun packRun, CancellationToken cancellationToken);
 }

 /// <summary>
@@ -117,6 +120,82 @@ public sealed class PackRunStreamCoordinator : IPackRunStreamCoordinator
        }
    }

+    public async Task StreamWebSocketAsync(WebSocket socket, string tenantId, PackRun packRun, CancellationToken cancellationToken)
+    {
+        if (socket is null) throw new ArgumentNullException(nameof(socket));
+
+        var (logCount, latestSeq) = await _logRepository.GetLogStatsAsync(tenantId, packRun.PackRunId, cancellationToken).ConfigureAwait(false);
+        await SendAsync(socket, "initial", PackRunSnapshotPayload.From(packRun, logCount, latestSeq), cancellationToken).ConfigureAwait(false);
+        await SendAsync(socket, "heartbeat", HeartbeatPayload.Create(_timeProvider.GetUtcNow()), cancellationToken).ConfigureAwait(false);
+
+        if (IsTerminal(packRun.Status))
+        {
+            await SendCompletedAsync(socket, packRun, logCount, latestSeq, cancellationToken).ConfigureAwait(false);
+            return;
+        }
+
+        var last = packRun;
+        var lastSeq = latestSeq;
+        var start = _timeProvider.GetUtcNow();
+        using var poll = new PeriodicTimer(_options.PollInterval);
+        using var heartbeat = new PeriodicTimer(_options.HeartbeatInterval);
+
+        try
+        {
+            while (!cancellationToken.IsCancellationRequested && socket.State == WebSocketState.Open)
+            {
+                if (_timeProvider.GetUtcNow() - start > _options.MaxStreamDuration)
+                {
+                    await SendAsync(socket, "timeout", new { packRunId = last.PackRunId, reason = "Max stream duration reached" }, cancellationToken).ConfigureAwait(false);
+                    break;
+                }
+
+                var pollTask = poll.WaitForNextTickAsync(cancellationToken).AsTask();
+                var hbTask = heartbeat.WaitForNextTickAsync(cancellationToken).AsTask();
+                var completed = await Task.WhenAny(pollTask, hbTask).ConfigureAwait(false);
+
+                if (completed == hbTask && await hbTask.ConfigureAwait(false))
+                {
+                    await SendAsync(socket, "heartbeat", HeartbeatPayload.Create(_timeProvider.GetUtcNow()), cancellationToken).ConfigureAwait(false);
+                    continue;
+                }
+
+                if (completed == pollTask && await pollTask.ConfigureAwait(false))
+                {
+                    var current = await _packRunRepository.GetByIdAsync(tenantId, last.PackRunId, cancellationToken).ConfigureAwait(false);
+                    if (current is null)
+                    {
+                        await SendAsync(socket, "notFound", new NotFoundPayload(last.PackRunId.ToString(), "pack-run"), cancellationToken).ConfigureAwait(false);
+                        break;
+                    }
+
+                    var batch = await _logRepository.GetLogsAsync(tenantId, current.PackRunId, lastSeq, DefaultBatchSize, cancellationToken).ConfigureAwait(false);
+                    if (batch.Logs.Count > 0)
+                    {
+                        lastSeq = batch.Logs[^1].Sequence;
+                        await SendAsync(socket, "logs", batch.Logs.Select(PackRunLogPayload.FromDomain), cancellationToken).ConfigureAwait(false);
+                    }
+
+                    if (HasStatusChanged(last, current))
+                    {
+                        await SendAsync(socket, "statusChanged", PackRunSnapshotPayload.From(current, batch.Logs.Count, lastSeq), cancellationToken).ConfigureAwait(false);
+                        last = current;
+
+                        if (IsTerminal(current.Status))
+                        {
+                            await SendCompletedAsync(socket, current, batch.Logs.Count, lastSeq, cancellationToken).ConfigureAwait(false);
+                            break;
+                        }
+                    }
+                }
+            }
+        }
+        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+        {
+            _logger.LogDebug("Pack run websocket stream cancelled for {PackRunId}.", packRun.PackRunId);
+        }
+    }
+
    private static bool HasStatusChanged(PackRun previous, PackRun current)
    {
        return previous.Status != current.Status || previous.Attempt != current.Attempt || previous.LeaseId != current.LeaseId;
@@ -139,8 +218,32 @@ public sealed class PackRunStreamCoordinator : IPackRunStreamCoordinator
        await SseWriter.WriteEventAsync(response, "completed", payload, SerializerOptions, cancellationToken).ConfigureAwait(false);
    }

+    private async Task SendCompletedAsync(WebSocket socket, PackRun packRun, long logCount, long latestSequence, CancellationToken cancellationToken)
+    {
+        var durationSeconds = packRun.CompletedAt.HasValue && packRun.StartedAt.HasValue
+            ? (packRun.CompletedAt.Value - packRun.StartedAt.Value).TotalSeconds
+            : packRun.CompletedAt.HasValue ? (packRun.CompletedAt.Value - packRun.CreatedAt).TotalSeconds : 0;
+
+        var payload = new PackRunCompletedPayload(
+            PackRunId: packRun.PackRunId,
+            Status: packRun.Status.ToString().ToLowerInvariant(),
+            CompletedAt: packRun.CompletedAt ?? _timeProvider.GetUtcNow(),
+            DurationSeconds: durationSeconds,
+            LogCount: logCount,
+            LatestSequence: latestSequence);
+
+        await SendAsync(socket, "completed", payload, cancellationToken).ConfigureAwait(false);
+    }
+
    private static bool IsTerminal(PackRunStatus status) =>
        status is PackRunStatus.Succeeded or PackRunStatus.Failed or PackRunStatus.Canceled or PackRunStatus.TimedOut;
+
+    private static async Task SendAsync(WebSocket socket, string type, object payload, CancellationToken cancellationToken)
+    {
+        var json = JsonSerializer.Serialize(new { type, data = payload }, SerializerOptions);
+        var buffer = Encoding.UTF8.GetBytes(json);
+        await socket.SendAsync(buffer, WebSocketMessageType.Text, true, cancellationToken).ConfigureAwait(false);
+    }
 }

 internal sealed record PackRunSnapshotPayload(