up

src/StellaOps.Scanner.Worker/Processing/AnalyzerStageExecutor.cs

@@ -0,0 +1,20 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace StellaOps.Scanner.Worker.Processing;
+
+public sealed class AnalyzerStageExecutor : IScanStageExecutor
+{
+    private readonly IScanAnalyzerDispatcher _dispatcher;
+
+    public AnalyzerStageExecutor(IScanAnalyzerDispatcher dispatcher)
+    {
+        _dispatcher = dispatcher ?? throw new ArgumentNullException(nameof(dispatcher));
+    }
+
+    public string StageName => ScanStageNames.ExecuteAnalyzers;
+
+    public ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken)
+        => _dispatcher.ExecuteAsync(context, cancellationToken);
+}

src/StellaOps.Scanner.Worker/Processing/IDelayScheduler.cs

@@ -0,0 +1,10 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace StellaOps.Scanner.Worker.Processing;
+
+public interface IDelayScheduler
+{
+    Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken);
+}

src/StellaOps.Scanner.Worker/Processing/IScanAnalyzerDispatcher.cs

@@ -0,0 +1,15 @@
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace StellaOps.Scanner.Worker.Processing;
+
+public interface IScanAnalyzerDispatcher
+{
+    ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken);
+}
+
+public sealed class NullScanAnalyzerDispatcher : IScanAnalyzerDispatcher
+{
+    public ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken)
+        => ValueTask.CompletedTask;
+}

src/StellaOps.Scanner.Worker/Processing/IScanJobLease.cs

@@ -0,0 +1,31 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace StellaOps.Scanner.Worker.Processing;
+
+public interface IScanJobLease : IAsyncDisposable
+{
+    string JobId { get; }
+
+    string ScanId { get; }
+
+    int Attempt { get; }
+
+    DateTimeOffset EnqueuedAtUtc { get; }
+
+    DateTimeOffset LeasedAtUtc { get; }
+
+    TimeSpan LeaseDuration { get; }
+
+    IReadOnlyDictionary<string, string> Metadata { get; }
+
+    ValueTask RenewAsync(CancellationToken cancellationToken);
+
+    ValueTask CompleteAsync(CancellationToken cancellationToken);
+
+    ValueTask AbandonAsync(string reason, CancellationToken cancellationToken);
+
+    ValueTask PoisonAsync(string reason, CancellationToken cancellationToken);
+}

src/StellaOps.Scanner.Worker/Processing/IScanJobSource.cs

@@ -0,0 +1,9 @@
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace StellaOps.Scanner.Worker.Processing;
+
+public interface IScanJobSource
+{
+    Task<IScanJobLease?> TryAcquireAsync(CancellationToken cancellationToken);
+}

src/StellaOps.Scanner.Worker/Processing/IScanStageExecutor.cs

@@ -0,0 +1,11 @@
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace StellaOps.Scanner.Worker.Processing;
+
+public interface IScanStageExecutor
+{
+    string StageName { get; }
+
+    ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken);
+}

src/StellaOps.Scanner.Worker/Processing/LeaseHeartbeatService.cs

@@ -0,0 +1,148 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using StellaOps.Scanner.Worker.Options;
+
+namespace StellaOps.Scanner.Worker.Processing;
+
+public sealed class LeaseHeartbeatService
+{
+    private readonly TimeProvider _timeProvider;
+    private readonly IOptionsMonitor<ScannerWorkerOptions> _options;
+    private readonly IDelayScheduler _delayScheduler;
+    private readonly ILogger<LeaseHeartbeatService> _logger;
+
+    public LeaseHeartbeatService(TimeProvider timeProvider, IDelayScheduler delayScheduler, IOptionsMonitor<ScannerWorkerOptions> options, ILogger<LeaseHeartbeatService> logger)
+    {
+        _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
+        _delayScheduler = delayScheduler ?? throw new ArgumentNullException(nameof(delayScheduler));
+        _options = options ?? throw new ArgumentNullException(nameof(options));
+        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+    }
+
+    public async Task RunAsync(IScanJobLease lease, CancellationToken cancellationToken)
+    {
+        ArgumentNullException.ThrowIfNull(lease);
+
+        var options = _options.CurrentValue;
+        var interval = ComputeInterval(options, lease);
+
+        while (!cancellationToken.IsCancellationRequested)
+        {
+            options = _options.CurrentValue;
+            var delay = ApplyJitter(interval, options.Queue.MaxHeartbeatJitterMilliseconds);
+            try
+            {
+                await _delayScheduler.DelayAsync(delay, cancellationToken).ConfigureAwait(false);
+            }
+            catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+            {
+                break;
+            }
+
+            if (cancellationToken.IsCancellationRequested)
+            {
+                break;
+            }
+
+            if (await TryRenewAsync(options, lease, cancellationToken).ConfigureAwait(false))
+            {
+                continue;
+            }
+
+            _logger.LogError(
+                "Job {JobId} (scan {ScanId}) lease renewal exhausted retries; cancelling processing.",
+                lease.JobId,
+                lease.ScanId);
+            throw new InvalidOperationException("Lease renewal retries exhausted.");
+        }
+    }
+
+    private static TimeSpan ComputeInterval(ScannerWorkerOptions options, IScanJobLease lease)
+    {
+        var divisor = options.Queue.HeartbeatSafetyFactor <= 0 ? 3.0 : options.Queue.HeartbeatSafetyFactor;
+        var recommended = TimeSpan.FromTicks((long)(lease.LeaseDuration.Ticks / Math.Max(2.0, divisor)));
+        if (recommended < options.Queue.MinHeartbeatInterval)
+        {
+            recommended = options.Queue.MinHeartbeatInterval;
+        }
+        else if (recommended > options.Queue.MaxHeartbeatInterval)
+        {
+            recommended = options.Queue.MaxHeartbeatInterval;
+        }
+
+        return recommended;
+    }
+
+    private static TimeSpan ApplyJitter(TimeSpan duration, int maxJitterMilliseconds)
+    {
+        if (maxJitterMilliseconds <= 0)
+        {
+            return duration;
+        }
+
+        var offset = Random.Shared.NextDouble() * maxJitterMilliseconds;
+        return duration + TimeSpan.FromMilliseconds(offset);
+    }
+
+    private async Task<bool> TryRenewAsync(ScannerWorkerOptions options, IScanJobLease lease, CancellationToken cancellationToken)
+    {
+        try
+        {
+            await lease.RenewAsync(cancellationToken).ConfigureAwait(false);
+            return true;
+        }
+        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+        {
+            return false;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(
+                ex,
+                "Job {JobId} (scan {ScanId}) heartbeat failed; retrying.",
+                lease.JobId,
+                lease.ScanId);
+        }
+
+        foreach (var delay in options.Queue.NormalizedHeartbeatRetryDelays)
+        {
+            if (cancellationToken.IsCancellationRequested)
+            {
+                return false;
+            }
+
+            try
+            {
+                await _delayScheduler.DelayAsync(delay, cancellationToken).ConfigureAwait(false);
+            }
+            catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+            {
+                return false;
+            }
+
+            try
+            {
+                await lease.RenewAsync(cancellationToken).ConfigureAwait(false);
+                return true;
+            }
+            catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+            {
+                return false;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning(
+                    ex,
+                    "Job {JobId} (scan {ScanId}) heartbeat retry failed; will retry after {Delay}.",
+                    lease.JobId,
+                    lease.ScanId,
+                    delay);
+            }
+        }
+
+        return false;
+    }
+}

src/StellaOps.Scanner.Worker/Processing/NoOpStageExecutor.cs

@@ -0,0 +1,18 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace StellaOps.Scanner.Worker.Processing;
+
+public sealed class NoOpStageExecutor : IScanStageExecutor
+{
+    public NoOpStageExecutor(string stageName)
+    {
+        StageName = stageName ?? throw new ArgumentNullException(nameof(stageName));
+    }
+
+    public string StageName { get; }
+
+    public ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken)
+        => ValueTask.CompletedTask;
+}

src/StellaOps.Scanner.Worker/Processing/NullScanJobSource.cs

@@ -0,0 +1,26 @@
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+
+namespace StellaOps.Scanner.Worker.Processing;
+
+public sealed class NullScanJobSource : IScanJobSource
+{
+    private readonly ILogger<NullScanJobSource> _logger;
+    private int _logged;
+
+    public NullScanJobSource(ILogger<NullScanJobSource> logger)
+    {
+        _logger = logger;
+    }
+
+    public Task<IScanJobLease?> TryAcquireAsync(CancellationToken cancellationToken)
+    {
+        if (Interlocked.Exchange(ref _logged, 1) == 0)
+        {
+            _logger.LogWarning("No queue provider registered. Scanner worker will idle until a queue adapter is configured.");
+        }
+
+        return Task.FromResult<IScanJobLease?>(null);
+    }
+}

src/StellaOps.Scanner.Worker/Processing/PollDelayStrategy.cs

@@ -0,0 +1,49 @@
+using System;
+
+using StellaOps.Scanner.Worker.Options;
+
+namespace StellaOps.Scanner.Worker.Processing;
+
+public sealed class PollDelayStrategy
+{
+    private readonly ScannerWorkerOptions.PollingOptions _options;
+    private TimeSpan _currentDelay;
+
+    public PollDelayStrategy(ScannerWorkerOptions.PollingOptions options)
+    {
+        _options = options ?? throw new ArgumentNullException(nameof(options));
+    }
+
+    public TimeSpan NextDelay()
+    {
+        if (_currentDelay == TimeSpan.Zero)
+        {
+            _currentDelay = _options.InitialDelay;
+            return ApplyJitter(_currentDelay);
+        }
+
+        var doubled = _currentDelay + _currentDelay;
+        _currentDelay = doubled < _options.MaxDelay ? doubled : _options.MaxDelay;
+        return ApplyJitter(_currentDelay);
+    }
+
+    public void Reset() => _currentDelay = TimeSpan.Zero;
+
+    private TimeSpan ApplyJitter(TimeSpan duration)
+    {
+        if (_options.JitterRatio <= 0)
+        {
+            return duration;
+        }
+
+        var maxOffset = duration.TotalMilliseconds * _options.JitterRatio;
+        if (maxOffset <= 0)
+        {
+            return duration;
+        }
+
+        var offset = (Random.Shared.NextDouble() * 2.0 - 1.0) * maxOffset;
+        var adjustedMs = Math.Max(0, duration.TotalMilliseconds + offset);
+        return TimeSpan.FromMilliseconds(adjustedMs);
+    }
+}

src/StellaOps.Scanner.Worker/Processing/ScanJobContext.cs

@@ -0,0 +1,27 @@
+using System;
+using System.Threading;
+
+namespace StellaOps.Scanner.Worker.Processing;
+
+public sealed class ScanJobContext
+{
+    public ScanJobContext(IScanJobLease lease, TimeProvider timeProvider, DateTimeOffset startUtc, CancellationToken cancellationToken)
+    {
+        Lease = lease ?? throw new ArgumentNullException(nameof(lease));
+        TimeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
+        StartUtc = startUtc;
+        CancellationToken = cancellationToken;
+    }
+
+    public IScanJobLease Lease { get; }
+
+    public TimeProvider TimeProvider { get; }
+
+    public DateTimeOffset StartUtc { get; }
+
+    public CancellationToken CancellationToken { get; }
+
+    public string JobId => Lease.JobId;
+
+    public string ScanId => Lease.ScanId;
+}

src/StellaOps.Scanner.Worker/Processing/ScanJobProcessor.cs

@@ -0,0 +1,65 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+
+namespace StellaOps.Scanner.Worker.Processing;
+
+public sealed class ScanJobProcessor
+{
+    private readonly IReadOnlyDictionary<string, IScanStageExecutor> _executors;
+    private readonly ScanProgressReporter _progressReporter;
+    private readonly ILogger<ScanJobProcessor> _logger;
+
+    public ScanJobProcessor(IEnumerable<IScanStageExecutor> executors, ScanProgressReporter progressReporter, ILogger<ScanJobProcessor> logger)
+    {
+        _progressReporter = progressReporter ?? throw new ArgumentNullException(nameof(progressReporter));
+        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+
+        var map = new Dictionary<string, IScanStageExecutor>(StringComparer.OrdinalIgnoreCase);
+        foreach (var executor in executors ?? Array.Empty<IScanStageExecutor>())
+        {
+            if (executor is null || string.IsNullOrWhiteSpace(executor.StageName))
+            {
+                continue;
+            }
+
+            map[executor.StageName] = executor;
+        }
+
+        foreach (var stage in ScanStageNames.Ordered)
+        {
+            if (map.ContainsKey(stage))
+            {
+                continue;
+            }
+
+            map[stage] = new NoOpStageExecutor(stage);
+            _logger.LogDebug("No executor registered for stage {Stage}; using no-op placeholder.", stage);
+        }
+
+        _executors = map;
+    }
+
+    public async ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken)
+    {
+        ArgumentNullException.ThrowIfNull(context);
+
+        foreach (var stage in ScanStageNames.Ordered)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            if (!_executors.TryGetValue(stage, out var executor))
+            {
+                continue;
+            }
+
+            await _progressReporter.ExecuteStageAsync(
+                context,
+                stage,
+                executor.ExecuteAsync,
+                cancellationToken).ConfigureAwait(false);
+        }
+    }
+}

src/StellaOps.Scanner.Worker/Processing/ScanProgressReporter.cs

@@ -0,0 +1,86 @@
+using System;
+using System.Diagnostics;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using StellaOps.Scanner.Worker.Diagnostics;
+
+namespace StellaOps.Scanner.Worker.Processing;
+
+public sealed partial class ScanProgressReporter
+{
+    private readonly ScannerWorkerMetrics _metrics;
+    private readonly ILogger<ScanProgressReporter> _logger;
+
+    public ScanProgressReporter(ScannerWorkerMetrics metrics, ILogger<ScanProgressReporter> logger)
+    {
+        _metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
+        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+    }
+
+    public async ValueTask ExecuteStageAsync(
+        ScanJobContext context,
+        string stageName,
+        Func<ScanJobContext, CancellationToken, ValueTask> stageWork,
+        CancellationToken cancellationToken)
+    {
+        ArgumentNullException.ThrowIfNull(context);
+        ArgumentException.ThrowIfNullOrWhiteSpace(stageName);
+        ArgumentNullException.ThrowIfNull(stageWork);
+
+        StageStarting(_logger, context.JobId, context.ScanId, stageName, context.Lease.Attempt);
+
+        var start = context.TimeProvider.GetUtcNow();
+        using var activity = ScannerWorkerInstrumentation.ActivitySource.StartActivity(
+            $"scanner.worker.{stageName}",
+            ActivityKind.Internal);
+
+        activity?.SetTag("scanner.worker.job_id", context.JobId);
+        activity?.SetTag("scanner.worker.scan_id", context.ScanId);
+        activity?.SetTag("scanner.worker.stage", stageName);
+
+        try
+        {
+            await stageWork(context, cancellationToken).ConfigureAwait(false);
+            var duration = context.TimeProvider.GetUtcNow() - start;
+            _metrics.RecordStageDuration(context, stageName, duration);
+            StageCompleted(_logger, context.JobId, context.ScanId, stageName, duration.TotalMilliseconds);
+        }
+        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+        {
+            StageCancelled(_logger, context.JobId, context.ScanId, stageName);
+            throw;
+        }
+        catch (Exception ex)
+        {
+            var duration = context.TimeProvider.GetUtcNow() - start;
+            _metrics.RecordStageDuration(context, stageName, duration);
+            StageFailed(_logger, context.JobId, context.ScanId, stageName, ex);
+            throw;
+        }
+    }
+
+    [LoggerMessage(
+        EventId = 1000,
+        Level = LogLevel.Information,
+        Message = "Job {JobId} (scan {ScanId}) entering stage {Stage} (attempt {Attempt}).")]
+    private static partial void StageStarting(ILogger logger, string jobId, string scanId, string stage, int attempt);
+
+    [LoggerMessage(
+        EventId = 1001,
+        Level = LogLevel.Information,
+        Message = "Job {JobId} (scan {ScanId}) finished stage {Stage} in {ElapsedMs:F0} ms.")]
+    private static partial void StageCompleted(ILogger logger, string jobId, string scanId, string stage, double elapsedMs);
+
+    [LoggerMessage(
+        EventId = 1002,
+        Level = LogLevel.Warning,
+        Message = "Job {JobId} (scan {ScanId}) stage {Stage} cancelled by request.")]
+    private static partial void StageCancelled(ILogger logger, string jobId, string scanId, string stage);
+
+    [LoggerMessage(
+        EventId = 1003,
+        Level = LogLevel.Error,
+        Message = "Job {JobId} (scan {ScanId}) stage {Stage} failed.")]
+    private static partial void StageFailed(ILogger logger, string jobId, string scanId, string stage, Exception exception);
+}

src/StellaOps.Scanner.Worker/Processing/ScanStageNames.cs

@@ -0,0 +1,23 @@
+using System.Collections.Generic;
+
+namespace StellaOps.Scanner.Worker.Processing;
+
+public static class ScanStageNames
+{
+    public const string ResolveImage = "resolve-image";
+    public const string PullLayers = "pull-layers";
+    public const string BuildFilesystem = "build-filesystem";
+    public const string ExecuteAnalyzers = "execute-analyzers";
+    public const string ComposeArtifacts = "compose-artifacts";
+    public const string EmitReports = "emit-reports";
+
+    public static readonly IReadOnlyList<string> Ordered = new[]
+    {
+        ResolveImage,
+        PullLayers,
+        BuildFilesystem,
+        ExecuteAnalyzers,
+        ComposeArtifacts,
+        EmitReports,
+    };
+}

src/StellaOps.Scanner.Worker/Processing/SystemDelayScheduler.cs

@@ -0,0 +1,18 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace StellaOps.Scanner.Worker.Processing;
+
+public sealed class SystemDelayScheduler : IDelayScheduler
+{
+    public Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken)
+    {
+        if (delay <= TimeSpan.Zero)
+        {
+            return Task.CompletedTask;
+        }
+
+        return Task.Delay(delay, cancellationToken);
+    }
+}