up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled

This commit is contained in:
StellaOps Bot
2025-12-13 00:20:26 +02:00
parent e1f1bef4c1
commit 564df71bfb
2376 changed files with 334389 additions and 328032 deletions

View File

@@ -1,9 +1,9 @@
using System;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Worker.Processing;
using System;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Worker.Processing;
public sealed class AnalyzerStageExecutor : IScanStageExecutor
{
private readonly IScanAnalyzerDispatcher _dispatcher;

View File

@@ -13,6 +13,7 @@ using StellaOps.Scanner.Analyzers.Lang.Internal;
using StellaOps.Scanner.Analyzers.Lang.Plugin;
using StellaOps.Scanner.Analyzers.OS;
using StellaOps.Scanner.Analyzers.OS.Abstractions;
using StellaOps.Scanner.Analyzers.OS.Internal;
using StellaOps.Scanner.Analyzers.OS.Mapping;
using StellaOps.Scanner.Analyzers.OS.Plugin;
using StellaOps.Scanner.Core.Contracts;
@@ -126,6 +127,9 @@ internal sealed class CompositeScanAnalyzerDispatcher : IScanAnalyzerDispatcher
await validatorRunner.EnsureAsync(validationContext, cancellationToken).ConfigureAwait(false);
var cache = services.GetRequiredService<ISurfaceCache>();
var cacheAdapter = new OsAnalyzerSurfaceCache(cache, surfaceEnvironment.Settings.Tenant);
foreach (var analyzer in analyzers)
{
cancellationToken.ThrowIfCancellationRequested();
@@ -135,7 +139,46 @@ internal sealed class CompositeScanAnalyzerDispatcher : IScanAnalyzerDispatcher
try
{
var result = await analyzer.AnalyzeAsync(analyzerContext, cancellationToken).ConfigureAwait(false);
string? fingerprint = null;
try
{
fingerprint = OsRootfsFingerprint.TryCompute(analyzer.AnalyzerId, rootfsPath, cancellationToken);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException)
{
_logger.LogDebug(
ex,
"Failed to compute rootfs fingerprint for OS analyzer {AnalyzerId} (job {JobId}); bypassing cache.",
analyzer.AnalyzerId,
context.JobId);
}
OSPackageAnalyzerResult result;
if (fingerprint is not null)
{
var cacheEntry = await cacheAdapter.GetOrCreateEntryAsync(
_logger,
analyzer.AnalyzerId,
fingerprint,
token => analyzer.AnalyzeAsync(analyzerContext, token),
cancellationToken)
.ConfigureAwait(false);
result = cacheEntry.Result;
if (cacheEntry.IsHit)
{
_metrics.RecordOsCacheHit(context, analyzer.AnalyzerId);
}
else
{
_metrics.RecordOsCacheMiss(context, analyzer.AnalyzerId);
}
}
else
{
result = await analyzer.AnalyzeAsync(analyzerContext, cancellationToken).ConfigureAwait(false);
}
results.Add(result);
}
catch (Exception ex)

View File

@@ -3,7 +3,7 @@ using StellaOps.Scanner.Worker.Determinism;
namespace StellaOps.Scanner.Worker.Processing;
internal sealed class DeterministicRandomService
public sealed class DeterministicRandomService
{
private readonly IDeterministicRandomProvider _provider;

View File

@@ -1,10 +1,10 @@
using System;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Worker.Processing;
public interface IDelayScheduler
{
Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken);
}
using System;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Worker.Processing;
public interface IDelayScheduler
{
Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken);
}

View File

@@ -1,9 +1,9 @@
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Worker.Processing;
public interface IEntryTraceExecutionService
{
ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken);
}
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Worker.Processing;
public interface IEntryTraceExecutionService
{
ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken);
}

View File

@@ -1,15 +1,15 @@
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Worker.Processing;
public interface IScanAnalyzerDispatcher
{
ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken);
}
public sealed class NullScanAnalyzerDispatcher : IScanAnalyzerDispatcher
{
public ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken)
=> ValueTask.CompletedTask;
}
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Worker.Processing;
public interface IScanAnalyzerDispatcher
{
ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken);
}
public sealed class NullScanAnalyzerDispatcher : IScanAnalyzerDispatcher
{
public ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken)
=> ValueTask.CompletedTask;
}

View File

@@ -1,31 +1,31 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Worker.Processing;
public interface IScanJobLease : IAsyncDisposable
{
string JobId { get; }
string ScanId { get; }
int Attempt { get; }
DateTimeOffset EnqueuedAtUtc { get; }
DateTimeOffset LeasedAtUtc { get; }
TimeSpan LeaseDuration { get; }
IReadOnlyDictionary<string, string> Metadata { get; }
ValueTask RenewAsync(CancellationToken cancellationToken);
ValueTask CompleteAsync(CancellationToken cancellationToken);
ValueTask AbandonAsync(string reason, CancellationToken cancellationToken);
ValueTask PoisonAsync(string reason, CancellationToken cancellationToken);
}
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Worker.Processing;
public interface IScanJobLease : IAsyncDisposable
{
string JobId { get; }
string ScanId { get; }
int Attempt { get; }
DateTimeOffset EnqueuedAtUtc { get; }
DateTimeOffset LeasedAtUtc { get; }
TimeSpan LeaseDuration { get; }
IReadOnlyDictionary<string, string> Metadata { get; }
ValueTask RenewAsync(CancellationToken cancellationToken);
ValueTask CompleteAsync(CancellationToken cancellationToken);
ValueTask AbandonAsync(string reason, CancellationToken cancellationToken);
ValueTask PoisonAsync(string reason, CancellationToken cancellationToken);
}

View File

@@ -1,9 +1,9 @@
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Worker.Processing;
public interface IScanJobSource
{
Task<IScanJobLease?> TryAcquireAsync(CancellationToken cancellationToken);
}
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Worker.Processing;
public interface IScanJobSource
{
Task<IScanJobLease?> TryAcquireAsync(CancellationToken cancellationToken);
}

View File

@@ -1,11 +1,11 @@
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Worker.Processing;
public interface IScanStageExecutor
{
string StageName { get; }
ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken);
}
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Worker.Processing;
public interface IScanStageExecutor
{
string StageName { get; }
ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken);
}

View File

@@ -1,14 +1,15 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Scanner.Worker.Options;
namespace StellaOps.Scanner.Worker.Processing;
public sealed class LeaseHeartbeatService
{
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Scanner.Worker.Determinism;
using StellaOps.Scanner.Worker.Options;
namespace StellaOps.Scanner.Worker.Processing;
public sealed class LeaseHeartbeatService
{
private readonly TimeProvider _timeProvider;
private readonly IOptionsMonitor<ScannerWorkerOptions> _options;
private readonly IDelayScheduler _delayScheduler;
@@ -28,7 +29,7 @@ public sealed class LeaseHeartbeatService
_randomProvider = randomProvider ?? throw new ArgumentNullException(nameof(randomProvider));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task RunAsync(IScanJobLease lease, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(lease);
@@ -45,27 +46,27 @@ public sealed class LeaseHeartbeatService
await _delayScheduler.DelayAsync(delay, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
break;
}
if (cancellationToken.IsCancellationRequested)
{
break;
}
if (await TryRenewAsync(options, lease, cancellationToken).ConfigureAwait(false))
{
continue;
}
_logger.LogError(
"Job {JobId} (scan {ScanId}) lease renewal exhausted retries; cancelling processing.",
lease.JobId,
lease.ScanId);
throw new InvalidOperationException("Lease renewal retries exhausted.");
}
}
{
break;
}
if (cancellationToken.IsCancellationRequested)
{
break;
}
if (await TryRenewAsync(options, lease, cancellationToken).ConfigureAwait(false))
{
continue;
}
_logger.LogError(
"Job {JobId} (scan {ScanId}) lease renewal exhausted retries; cancelling processing.",
lease.JobId,
lease.ScanId);
throw new InvalidOperationException("Lease renewal retries exhausted.");
}
}
private static TimeSpan ComputeInterval(ScannerWorkerOptions options, IScanJobLease lease)
{
@@ -77,9 +78,9 @@ public sealed class LeaseHeartbeatService
recommended = options.Queue.MinHeartbeatInterval;
}
else if (recommended > options.Queue.MaxHeartbeatInterval)
{
recommended = options.Queue.MaxHeartbeatInterval;
}
{
recommended = options.Queue.MaxHeartbeatInterval;
}
return recommended;
}
@@ -108,55 +109,55 @@ public sealed class LeaseHeartbeatService
await lease.RenewAsync(cancellationToken).ConfigureAwait(false);
return true;
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
return false;
}
catch (Exception ex)
{
_logger.LogWarning(
ex,
"Job {JobId} (scan {ScanId}) heartbeat failed; retrying.",
lease.JobId,
lease.ScanId);
}
foreach (var delay in options.Queue.NormalizedHeartbeatRetryDelays)
{
if (cancellationToken.IsCancellationRequested)
{
return false;
}
try
{
await _delayScheduler.DelayAsync(delay, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
return false;
}
try
{
await lease.RenewAsync(cancellationToken).ConfigureAwait(false);
return true;
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
return false;
}
catch (Exception ex)
{
_logger.LogWarning(
ex,
"Job {JobId} (scan {ScanId}) heartbeat retry failed; will retry after {Delay}.",
lease.JobId,
lease.ScanId,
delay);
}
}
return false;
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
return false;
}
catch (Exception ex)
{
_logger.LogWarning(
ex,
"Job {JobId} (scan {ScanId}) heartbeat failed; retrying.",
lease.JobId,
lease.ScanId);
}
foreach (var delay in options.Queue.NormalizedHeartbeatRetryDelays)
{
if (cancellationToken.IsCancellationRequested)
{
return false;
}
try
{
await _delayScheduler.DelayAsync(delay, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
return false;
}
try
{
await lease.RenewAsync(cancellationToken).ConfigureAwait(false);
return true;
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
return false;
}
catch (Exception ex)
{
_logger.LogWarning(
ex,
"Job {JobId} (scan {ScanId}) heartbeat retry failed; will retry after {Delay}.",
lease.JobId,
lease.ScanId,
delay);
}
}
return false;
}
}

View File

@@ -1,18 +1,18 @@
using System;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Worker.Processing;
public sealed class NoOpStageExecutor : IScanStageExecutor
{
public NoOpStageExecutor(string stageName)
{
StageName = stageName ?? throw new ArgumentNullException(nameof(stageName));
}
public string StageName { get; }
public ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken)
=> ValueTask.CompletedTask;
}
using System;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Worker.Processing;
public sealed class NoOpStageExecutor : IScanStageExecutor
{
public NoOpStageExecutor(string stageName)
{
StageName = stageName ?? throw new ArgumentNullException(nameof(stageName));
}
public string StageName { get; }
public ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken)
=> ValueTask.CompletedTask;
}

View File

@@ -1,26 +1,26 @@
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
namespace StellaOps.Scanner.Worker.Processing;
public sealed class NullScanJobSource : IScanJobSource
{
private readonly ILogger<NullScanJobSource> _logger;
private int _logged;
public NullScanJobSource(ILogger<NullScanJobSource> logger)
{
_logger = logger;
}
public Task<IScanJobLease?> TryAcquireAsync(CancellationToken cancellationToken)
{
if (Interlocked.Exchange(ref _logged, 1) == 0)
{
_logger.LogWarning("No queue provider registered. Scanner worker will idle until a queue adapter is configured.");
}
return Task.FromResult<IScanJobLease?>(null);
}
}
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
namespace StellaOps.Scanner.Worker.Processing;
public sealed class NullScanJobSource : IScanJobSource
{
private readonly ILogger<NullScanJobSource> _logger;
private int _logged;
public NullScanJobSource(ILogger<NullScanJobSource> logger)
{
_logger = logger;
}
public Task<IScanJobLease?> TryAcquireAsync(CancellationToken cancellationToken)
{
if (Interlocked.Exchange(ref _logged, 1) == 0)
{
_logger.LogWarning("No queue provider registered. Scanner worker will idle until a queue adapter is configured.");
}
return Task.FromResult<IScanJobLease?>(null);
}
}

View File

@@ -1,11 +1,11 @@
using System;
using StellaOps.Scanner.Worker.Options;
namespace StellaOps.Scanner.Worker.Processing;
public sealed class PollDelayStrategy
{
using System;
using StellaOps.Scanner.Worker.Options;
namespace StellaOps.Scanner.Worker.Processing;
public sealed class PollDelayStrategy
{
private readonly ScannerWorkerOptions.PollingOptions _options;
private readonly DeterministicRandomService _randomService;
private TimeSpan _currentDelay;
@@ -15,35 +15,35 @@ public sealed class PollDelayStrategy
_options = options ?? throw new ArgumentNullException(nameof(options));
_randomService = randomService ?? throw new ArgumentNullException(nameof(randomService));
}
public TimeSpan NextDelay()
{
if (_currentDelay == TimeSpan.Zero)
{
_currentDelay = _options.InitialDelay;
return ApplyJitter(_currentDelay);
}
var doubled = _currentDelay + _currentDelay;
_currentDelay = doubled < _options.MaxDelay ? doubled : _options.MaxDelay;
return ApplyJitter(_currentDelay);
}
public void Reset() => _currentDelay = TimeSpan.Zero;
private TimeSpan ApplyJitter(TimeSpan duration)
{
if (_options.JitterRatio <= 0)
{
return duration;
}
var maxOffset = duration.TotalMilliseconds * _options.JitterRatio;
if (maxOffset <= 0)
{
return duration;
}
public TimeSpan NextDelay()
{
if (_currentDelay == TimeSpan.Zero)
{
_currentDelay = _options.InitialDelay;
return ApplyJitter(_currentDelay);
}
var doubled = _currentDelay + _currentDelay;
_currentDelay = doubled < _options.MaxDelay ? doubled : _options.MaxDelay;
return ApplyJitter(_currentDelay);
}
public void Reset() => _currentDelay = TimeSpan.Zero;
private TimeSpan ApplyJitter(TimeSpan duration)
{
if (_options.JitterRatio <= 0)
{
return duration;
}
var maxOffset = duration.TotalMilliseconds * _options.JitterRatio;
if (maxOffset <= 0)
{
return duration;
}
var rng = _randomService.Create();
var offset = (rng.NextDouble() * 2.0 - 1.0) * maxOffset;
var adjustedMs = Math.Max(0, duration.TotalMilliseconds + offset);

View File

@@ -14,7 +14,7 @@ namespace StellaOps.Scanner.Worker.Processing.Replay;
/// Fetches a sealed replay bundle from the configured object store, verifies its SHA-256 hash,
/// and returns a local file path for downstream analyzers.
/// </summary>
internal sealed class ReplayBundleFetcher
public sealed class ReplayBundleFetcher
{
private readonly IArtifactObjectStore _objectStore;
private readonly ICryptoHash _cryptoHash;

View File

@@ -1,11 +1,11 @@
using System;
using System.Threading;
using StellaOps.Scanner.Core.Contracts;
namespace StellaOps.Scanner.Worker.Processing;
public sealed class ScanJobContext
{
namespace StellaOps.Scanner.Worker.Processing;
public sealed class ScanJobContext
{
public ScanJobContext(IScanJobLease lease, TimeProvider timeProvider, DateTimeOffset startUtc, CancellationToken cancellationToken)
{
Lease = lease ?? throw new ArgumentNullException(nameof(lease));
@@ -14,13 +14,13 @@ public sealed class ScanJobContext
CancellationToken = cancellationToken;
Analysis = new ScanAnalysisStore();
}
public IScanJobLease Lease { get; }
public TimeProvider TimeProvider { get; }
public DateTimeOffset StartUtc { get; }
public IScanJobLease Lease { get; }
public TimeProvider TimeProvider { get; }
public DateTimeOffset StartUtc { get; }
public CancellationToken CancellationToken { get; }
public string JobId => Lease.JobId;

View File

@@ -1,12 +1,13 @@
using System;
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.Reachability;
namespace StellaOps.Scanner.Worker.Processing;
namespace StellaOps.Scanner.Worker.Processing;
public sealed class ScanJobProcessor
{
private readonly IReadOnlyDictionary<string, IScanStageExecutor> _executors;
@@ -26,36 +27,36 @@ public sealed class ScanJobProcessor
_reachabilityPublisher = reachabilityPublisher ?? throw new ArgumentNullException(nameof(reachabilityPublisher));
_replayBundleFetcher = replayBundleFetcher ?? throw new ArgumentNullException(nameof(replayBundleFetcher));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
var map = new Dictionary<string, IScanStageExecutor>(StringComparer.OrdinalIgnoreCase);
foreach (var executor in executors ?? Array.Empty<IScanStageExecutor>())
{
if (executor is null || string.IsNullOrWhiteSpace(executor.StageName))
{
continue;
}
map[executor.StageName] = executor;
}
foreach (var stage in ScanStageNames.Ordered)
{
if (map.ContainsKey(stage))
{
continue;
}
map[stage] = new NoOpStageExecutor(stage);
_logger.LogDebug("No executor registered for stage {Stage}; using no-op placeholder.", stage);
}
_executors = map;
}
var map = new Dictionary<string, IScanStageExecutor>(StringComparer.OrdinalIgnoreCase);
foreach (var executor in executors ?? Array.Empty<IScanStageExecutor>())
{
if (executor is null || string.IsNullOrWhiteSpace(executor.StageName))
{
continue;
}
map[executor.StageName] = executor;
}
foreach (var stage in ScanStageNames.Ordered)
{
if (map.ContainsKey(stage))
{
continue;
}
map[stage] = new NoOpStageExecutor(stage);
_logger.LogDebug("No executor registered for stage {Stage}; using no-op placeholder.", stage);
}
_executors = map;
}
public async ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
await EnsureReplayBundleFetchedAsync(context, cancellationToken).ConfigureAwait(false);
await EnsureReplayBundleFetchedAsync(context, cancellationToken).ConfigureAwait(false);
foreach (var stage in ScanStageNames.Ordered)
{

View File

@@ -1,86 +1,86 @@
using System;
using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Scanner.Worker.Diagnostics;
namespace StellaOps.Scanner.Worker.Processing;
public sealed partial class ScanProgressReporter
{
private readonly ScannerWorkerMetrics _metrics;
private readonly ILogger<ScanProgressReporter> _logger;
public ScanProgressReporter(ScannerWorkerMetrics metrics, ILogger<ScanProgressReporter> logger)
{
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async ValueTask ExecuteStageAsync(
ScanJobContext context,
string stageName,
Func<ScanJobContext, CancellationToken, ValueTask> stageWork,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
ArgumentException.ThrowIfNullOrWhiteSpace(stageName);
ArgumentNullException.ThrowIfNull(stageWork);
StageStarting(_logger, context.JobId, context.ScanId, stageName, context.Lease.Attempt);
var start = context.TimeProvider.GetUtcNow();
using var activity = ScannerWorkerInstrumentation.ActivitySource.StartActivity(
$"scanner.worker.{stageName}",
ActivityKind.Internal);
activity?.SetTag("scanner.worker.job_id", context.JobId);
activity?.SetTag("scanner.worker.scan_id", context.ScanId);
activity?.SetTag("scanner.worker.stage", stageName);
try
{
await stageWork(context, cancellationToken).ConfigureAwait(false);
var duration = context.TimeProvider.GetUtcNow() - start;
_metrics.RecordStageDuration(context, stageName, duration);
StageCompleted(_logger, context.JobId, context.ScanId, stageName, duration.TotalMilliseconds);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
StageCancelled(_logger, context.JobId, context.ScanId, stageName);
throw;
}
catch (Exception ex)
{
var duration = context.TimeProvider.GetUtcNow() - start;
_metrics.RecordStageDuration(context, stageName, duration);
StageFailed(_logger, context.JobId, context.ScanId, stageName, ex);
throw;
}
}
[LoggerMessage(
EventId = 1000,
Level = LogLevel.Information,
Message = "Job {JobId} (scan {ScanId}) entering stage {Stage} (attempt {Attempt}).")]
private static partial void StageStarting(ILogger logger, string jobId, string scanId, string stage, int attempt);
[LoggerMessage(
EventId = 1001,
Level = LogLevel.Information,
Message = "Job {JobId} (scan {ScanId}) finished stage {Stage} in {ElapsedMs:F0} ms.")]
private static partial void StageCompleted(ILogger logger, string jobId, string scanId, string stage, double elapsedMs);
[LoggerMessage(
EventId = 1002,
Level = LogLevel.Warning,
Message = "Job {JobId} (scan {ScanId}) stage {Stage} cancelled by request.")]
private static partial void StageCancelled(ILogger logger, string jobId, string scanId, string stage);
[LoggerMessage(
EventId = 1003,
Level = LogLevel.Error,
Message = "Job {JobId} (scan {ScanId}) stage {Stage} failed.")]
private static partial void StageFailed(ILogger logger, string jobId, string scanId, string stage, Exception exception);
}
using System;
using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Scanner.Worker.Diagnostics;
namespace StellaOps.Scanner.Worker.Processing;
public sealed partial class ScanProgressReporter
{
private readonly ScannerWorkerMetrics _metrics;
private readonly ILogger<ScanProgressReporter> _logger;
public ScanProgressReporter(ScannerWorkerMetrics metrics, ILogger<ScanProgressReporter> logger)
{
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async ValueTask ExecuteStageAsync(
ScanJobContext context,
string stageName,
Func<ScanJobContext, CancellationToken, ValueTask> stageWork,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
ArgumentException.ThrowIfNullOrWhiteSpace(stageName);
ArgumentNullException.ThrowIfNull(stageWork);
StageStarting(_logger, context.JobId, context.ScanId, stageName, context.Lease.Attempt);
var start = context.TimeProvider.GetUtcNow();
using var activity = ScannerWorkerInstrumentation.ActivitySource.StartActivity(
$"scanner.worker.{stageName}",
ActivityKind.Internal);
activity?.SetTag("scanner.worker.job_id", context.JobId);
activity?.SetTag("scanner.worker.scan_id", context.ScanId);
activity?.SetTag("scanner.worker.stage", stageName);
try
{
await stageWork(context, cancellationToken).ConfigureAwait(false);
var duration = context.TimeProvider.GetUtcNow() - start;
_metrics.RecordStageDuration(context, stageName, duration);
StageCompleted(_logger, context.JobId, context.ScanId, stageName, duration.TotalMilliseconds);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
StageCancelled(_logger, context.JobId, context.ScanId, stageName);
throw;
}
catch (Exception ex)
{
var duration = context.TimeProvider.GetUtcNow() - start;
_metrics.RecordStageDuration(context, stageName, duration);
StageFailed(_logger, context.JobId, context.ScanId, stageName, ex);
throw;
}
}
[LoggerMessage(
EventId = 1000,
Level = LogLevel.Information,
Message = "Job {JobId} (scan {ScanId}) entering stage {Stage} (attempt {Attempt}).")]
private static partial void StageStarting(ILogger logger, string jobId, string scanId, string stage, int attempt);
[LoggerMessage(
EventId = 1001,
Level = LogLevel.Information,
Message = "Job {JobId} (scan {ScanId}) finished stage {Stage} in {ElapsedMs:F0} ms.")]
private static partial void StageCompleted(ILogger logger, string jobId, string scanId, string stage, double elapsedMs);
[LoggerMessage(
EventId = 1002,
Level = LogLevel.Warning,
Message = "Job {JobId} (scan {ScanId}) stage {Stage} cancelled by request.")]
private static partial void StageCancelled(ILogger logger, string jobId, string scanId, string stage);
[LoggerMessage(
EventId = 1003,
Level = LogLevel.Error,
Message = "Job {JobId} (scan {ScanId}) stage {Stage} failed.")]
private static partial void StageFailed(ILogger logger, string jobId, string scanId, string stage, Exception exception);
}

View File

@@ -1,7 +1,7 @@
using System.Collections.Generic;
namespace StellaOps.Scanner.Worker.Processing;
using System.Collections.Generic;
namespace StellaOps.Scanner.Worker.Processing;
public static class ScanStageNames
{
public const string IngestReplay = "ingest-replay";

View File

@@ -129,30 +129,14 @@ internal sealed class HmacDsseEnvelopeSigner : IDsseEnvelopeSigner
try
{
var tenant = environment?.Settings.Tenant ?? "default";
var request = new SurfaceSecretRequest(tenant, component: "scanner-worker", secretType: "attestation", name: "dsse-signing");
var handle = provider.TryGetSecret(request, CancellationToken.None);
if (handle is null)
{
return null;
}
if (handle.Secret.TryGetProperty("privateKeyPem", out var privateKeyPem) && privateKeyPem.ValueKind == JsonValueKind.String)
{
var pem = privateKeyPem.GetString();
if (!string.IsNullOrWhiteSpace(pem))
{
return Encoding.UTF8.GetBytes(pem);
}
}
if (handle.Secret.TryGetProperty("token", out var token) && token.ValueKind == JsonValueKind.String)
{
var value = token.GetString();
if (!string.IsNullOrWhiteSpace(value))
{
return Encoding.UTF8.GetBytes(value);
}
}
var request = new SurfaceSecretRequest(
Tenant: tenant,
Component: "scanner-worker",
SecretType: "attestation",
Name: "dsse-signing");
using var handle = provider.GetAsync(request, CancellationToken.None).GetAwaiter().GetResult();
var bytes = handle.AsBytes();
return bytes.IsEmpty ? null : bytes.Span.ToArray();
}
catch
{

View File

@@ -85,11 +85,6 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
await PersistRubyPackagesAsync(context, cancellationToken).ConfigureAwait(false);
await PersistBunPackagesAsync(context, cancellationToken).ConfigureAwait(false);
var determinismPayloads = BuildDeterminismPayloads(context, payloads, out var merkleRoot);
if (determinismPayloads is not null && determinismPayloads.Count > 0)
{
payloads.AddRange(determinismPayloads);
}
if (payloads.Count == 0)
{
_metrics.RecordSurfaceManifestSkipped(context);
@@ -97,6 +92,12 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
return;
}
var determinismPayloads = BuildDeterminismPayloads(context, payloads, out var merkleRoot);
if (determinismPayloads is not null && determinismPayloads.Count > 0)
{
payloads.AddRange(determinismPayloads);
}
var tenant = _surfaceEnvironment.Settings?.Tenant ?? string.Empty;
var stopwatch = Stopwatch.StartNew();
@@ -249,12 +250,6 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
}));
}
var determinismPayload = BuildDeterminismPayload(context, payloads);
if (determinismPayload is not null)
{
payloads.Add(determinismPayload);
}
return payloads;
}
@@ -326,7 +321,7 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
}));
// Attach DSSE envelope for layer fragments when present.
foreach (var fragmentPayload in payloadList.Where(p => p.Kind == "layer.fragments"))
foreach (var fragmentPayload in payloadList.Where(p => p.Kind == "layer.fragments").ToArray())
{
var dsse = _dsseSigner.SignAsync(
payloadType: fragmentPayload.MediaType,
@@ -362,10 +357,9 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
return payloadList.Skip(payloads.Count()).ToList();
}
private static (Dictionary<string, string> Hashes, byte[] RecipeBytes, string RecipeSha256) BuildCompositionRecipe(IEnumerable<SurfaceManifestPayload> payloads)
private (Dictionary<string, string> Hashes, byte[] RecipeBytes, string RecipeSha256) BuildCompositionRecipe(IEnumerable<SurfaceManifestPayload> payloads)
{
var map = new SortedDictionary<string, string>(StringComparer.Ordinal);
using var sha = SHA256.Create();
foreach (var payload in payloads.OrderBy(p => p.Kind, StringComparer.Ordinal))
{
@@ -381,8 +375,7 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
var recipeJson = JsonSerializer.Serialize(recipe, JsonOptions);
var recipeBytes = Encoding.UTF8.GetBytes(recipeJson);
var rootHash = sha.ComputeHash(recipeBytes);
var merkleRoot = Convert.ToHexString(rootHash).ToLowerInvariant();
var merkleRoot = _hash.ComputeHashHex(recipeBytes, HashAlgorithms.Sha256);
return (new Dictionary<string, string>(map, StringComparer.OrdinalIgnoreCase), recipeBytes, merkleRoot);
}
@@ -459,10 +452,10 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
if (bestPlan is not null)
{
metadata["best_terminal"] = bestPlan.Value.TerminalPath;
metadata["best_confidence"] = bestPlan.Value.Confidence.ToString("F4", CultureInfoInvariant);
metadata["best_user"] = bestPlan.Value.User;
metadata["best_workdir"] = bestPlan.Value.WorkingDirectory;
metadata["best_terminal"] = bestPlan.TerminalPath;
metadata["best_confidence"] = bestPlan.Confidence.ToString("F4", CultureInfoInvariant);
metadata["best_user"] = bestPlan.User;
metadata["best_workdir"] = bestPlan.WorkingDirectory;
}
return metadata;

View File

@@ -1,18 +1,18 @@
using System;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Worker.Processing;
public sealed class SystemDelayScheduler : IDelayScheduler
{
public Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken)
{
if (delay <= TimeSpan.Zero)
{
return Task.CompletedTask;
}
return Task.Delay(delay, cancellationToken);
}
}
using System;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Worker.Processing;
public sealed class SystemDelayScheduler : IDelayScheduler
{
public Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken)
{
if (delay <= TimeSpan.Zero)
{
return Task.CompletedTask;
}
return Task.Delay(delay, cancellationToken);
}
}