up
Some checks failed
api-governance / spectral-lint (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
oas-ci / oas-validate (push) Has been cancelled
SDK Publish & Sign / sdk-publish (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Policy Simulation / policy-simulate (push) Has been cancelled
devportal-offline / build-offline (push) Has been cancelled

This commit is contained in:
StellaOps Bot
2025-11-26 20:23:28 +02:00
parent 4831c7fcb0
commit d63af51f84
139 changed files with 8010 additions and 2795 deletions

View File

@@ -0,0 +1,25 @@
using System;
namespace StellaOps.Scanner.Worker.Determinism;
/// <summary>
/// Captures determinism-related toggles for the worker runtime.
/// </summary>
public sealed class DeterminismContext
{
public DeterminismContext(bool fixedClock, DateTimeOffset fixedInstantUtc, int? rngSeed, bool filterLogs)
{
FixedClock = fixedClock;
FixedInstantUtc = fixedInstantUtc.ToUniversalTime();
RngSeed = rngSeed;
FilterLogs = filterLogs;
}
public bool FixedClock { get; }
public DateTimeOffset FixedInstantUtc { get; }
public int? RngSeed { get; }
public bool FilterLogs { get; }
}

View File

@@ -0,0 +1,26 @@
using System;
namespace StellaOps.Scanner.Worker.Determinism;
public interface IDeterministicRandomProvider
{
Random Create();
}
/// <summary>
/// Provides seeded <see cref="Random"/> instances when a seed is configured, otherwise defaults to a thread-safe system random.
/// </summary>
public sealed class DeterministicRandomProvider : IDeterministicRandomProvider
{
private readonly int? _seed;
public DeterministicRandomProvider(int? seed)
{
_seed = seed;
}
public Random Create()
{
return _seed.HasValue ? new Random(_seed.Value) : Random.Shared;
}
}

View File

@@ -0,0 +1,18 @@
using System;
namespace StellaOps.Scanner.Worker.Determinism;
/// <summary>
/// Time provider that always returns a fixed instant, used to enforce deterministic timestamps.
/// </summary>
public sealed class DeterministicTimeProvider : TimeProvider
{
private readonly DateTimeOffset _fixedUtc;
public DeterministicTimeProvider(DateTimeOffset fixedUtc)
{
_fixedUtc = fixedUtc.ToUniversalTime();
}
public override DateTimeOffset GetUtcNow() => _fixedUtc;
}

View File

@@ -27,6 +27,8 @@ public sealed class ScannerWorkerOptions
public AnalyzerOptions Analyzers { get; } = new();
public StellaOpsCryptoOptions Crypto { get; } = new();
public DeterminismOptions Determinism { get; } = new();
public sealed class QueueOptions
{
@@ -177,4 +179,27 @@ public sealed class ScannerWorkerOptions
public string EntryTraceProcRootMetadataKey { get; set; } = ScanMetadataKeys.RuntimeProcRoot;
}
public sealed class DeterminismOptions
{
/// <summary>
/// If true, the worker uses a fixed clock to ensure deterministic timestamps.
/// </summary>
public bool FixedClock { get; set; }
/// <summary>
/// Fixed UTC timestamp to emit when FixedClock is enabled. Defaults to Unix epoch.
/// </summary>
public DateTimeOffset FixedInstantUtc { get; set; } = DateTimeOffset.UnixEpoch;
/// <summary>
/// Optional seed for RNG-based components when determinism is required.
/// </summary>
public int? RngSeed { get; set; }
/// <summary>
/// If true, trims noisy log fields (duration, PIDs) to stable placeholders.
/// </summary>
public bool FilterLogs { get; set; }
}
}

View File

@@ -0,0 +1,16 @@
using System;
using StellaOps.Scanner.Worker.Determinism;
namespace StellaOps.Scanner.Worker.Processing;
internal sealed class DeterministicRandomService
{
private readonly IDeterministicRandomProvider _provider;
public DeterministicRandomService(IDeterministicRandomProvider provider)
{
_provider = provider ?? throw new ArgumentNullException(nameof(provider));
}
public Random Create() => _provider.Create();
}

View File

@@ -5,9 +5,8 @@ using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Scanner.Core.Entropy;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.Worker.Utilities;
using StellaOps.Scanner.Core.Entropy;
namespace StellaOps.Scanner.Worker.Processing.Entropy;
@@ -26,7 +25,7 @@ public sealed class EntropyStageExecutor : IScanStageExecutor
_reportBuilder = new EntropyReportBuilder();
}
public string StageName => ScanStageNames.EmitReports;
public string StageName => ScanStageNames.Entropy;
public async ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken)
{
@@ -68,7 +67,7 @@ public sealed class EntropyStageExecutor : IScanStageExecutor
return;
}
var layerDigest = context.Lease.LayerDigest ?? string.Empty;
var layerDigest = ResolveLayerDigest(context.Lease.Metadata);
var layerSize = files.Sum(f => f.SizeBytes);
var imageOpaqueBytes = reports.Sum(r => r.OpaqueBytes);
var imageTotalBytes = files.Sum(f => f.SizeBytes);
@@ -81,7 +80,7 @@ public sealed class EntropyStageExecutor : IScanStageExecutor
imageTotalBytes);
var entropyReport = new EntropyReport(
ImageDigest: context.Lease.ImageDigest ?? string.Empty,
ImageDigest: ResolveImageDigest(context.Lease.Metadata),
LayerDigest: layerDigest,
Files: reports,
ImageOpaqueRatio: imageRatio);
@@ -138,4 +137,49 @@ public sealed class EntropyStageExecutor : IScanStageExecutor
await stream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
return buffer.ToArray();
}
private static string ResolveLayerDigest(IReadOnlyDictionary<string, string> metadata)
{
if (metadata is null)
{
return string.Empty;
}
if (metadata.TryGetValue("layerDigest", out var digest) && !string.IsNullOrWhiteSpace(digest))
{
return digest.Trim();
}
if (metadata.TryGetValue("layer.digest", out digest) && !string.IsNullOrWhiteSpace(digest))
{
return digest.Trim();
}
return string.Empty;
}
private static string ResolveImageDigest(IReadOnlyDictionary<string, string> metadata)
{
if (metadata is null)
{
return string.Empty;
}
if (metadata.TryGetValue("image.digest", out var digest) && !string.IsNullOrWhiteSpace(digest))
{
return digest.Trim();
}
if (metadata.TryGetValue("imageDigest", out digest) && !string.IsNullOrWhiteSpace(digest))
{
return digest.Trim();
}
if (metadata.TryGetValue("scanner.image.digest", out digest) && !string.IsNullOrWhiteSpace(digest))
{
return digest.Trim();
}
return string.Empty;
}
}

View File

@@ -9,18 +9,25 @@ namespace StellaOps.Scanner.Worker.Processing;
public sealed class LeaseHeartbeatService
{
private readonly TimeProvider _timeProvider;
private readonly IOptionsMonitor<ScannerWorkerOptions> _options;
private readonly IDelayScheduler _delayScheduler;
private readonly ILogger<LeaseHeartbeatService> _logger;
public LeaseHeartbeatService(TimeProvider timeProvider, IDelayScheduler delayScheduler, IOptionsMonitor<ScannerWorkerOptions> options, ILogger<LeaseHeartbeatService> logger)
{
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
_delayScheduler = delayScheduler ?? throw new ArgumentNullException(nameof(delayScheduler));
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
private readonly TimeProvider _timeProvider;
private readonly IOptionsMonitor<ScannerWorkerOptions> _options;
private readonly IDelayScheduler _delayScheduler;
private readonly IDeterministicRandomProvider _randomProvider;
private readonly ILogger<LeaseHeartbeatService> _logger;
public LeaseHeartbeatService(
TimeProvider timeProvider,
IDelayScheduler delayScheduler,
IOptionsMonitor<ScannerWorkerOptions> options,
IDeterministicRandomProvider randomProvider,
ILogger<LeaseHeartbeatService> logger)
{
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
_delayScheduler = delayScheduler ?? throw new ArgumentNullException(nameof(delayScheduler));
_options = options ?? throw new ArgumentNullException(nameof(options));
_randomProvider = randomProvider ?? throw new ArgumentNullException(nameof(randomProvider));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task RunAsync(IScanJobLease lease, CancellationToken cancellationToken)
{
@@ -32,7 +39,7 @@ public sealed class LeaseHeartbeatService
{
var options = _options.CurrentValue;
var interval = ComputeInterval(options, lease);
var delay = ApplyJitter(interval, options.Queue);
var delay = ApplyJitter(interval, options.Queue, _randomProvider);
try
{
await _delayScheduler.DelayAsync(delay, cancellationToken).ConfigureAwait(false);
@@ -77,14 +84,14 @@ public sealed class LeaseHeartbeatService
return recommended;
}
private static TimeSpan ApplyJitter(TimeSpan duration, ScannerWorkerOptions.QueueOptions queueOptions)
private static TimeSpan ApplyJitter(TimeSpan duration, ScannerWorkerOptions.QueueOptions queueOptions, IDeterministicRandomProvider randomProvider)
{
if (queueOptions.MaxHeartbeatJitterMilliseconds <= 0)
{
return duration;
}
var offsetMs = Random.Shared.NextDouble() * queueOptions.MaxHeartbeatJitterMilliseconds;
var offsetMs = randomProvider.Create().NextDouble() * queueOptions.MaxHeartbeatJitterMilliseconds;
var adjusted = duration - TimeSpan.FromMilliseconds(offsetMs);
if (adjusted < queueOptions.MinHeartbeatInterval)
{
@@ -97,10 +104,10 @@ public sealed class LeaseHeartbeatService
private async Task<bool> TryRenewAsync(ScannerWorkerOptions options, IScanJobLease lease, CancellationToken cancellationToken)
{
try
{
await lease.RenewAsync(cancellationToken).ConfigureAwait(false);
return true;
}
{
await lease.RenewAsync(cancellationToken).ConfigureAwait(false);
return true;
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
return false;

View File

@@ -6,13 +6,15 @@ namespace StellaOps.Scanner.Worker.Processing;
public sealed class PollDelayStrategy
{
private readonly ScannerWorkerOptions.PollingOptions _options;
private TimeSpan _currentDelay;
public PollDelayStrategy(ScannerWorkerOptions.PollingOptions options)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
}
private readonly ScannerWorkerOptions.PollingOptions _options;
private readonly DeterministicRandomService _randomService;
private TimeSpan _currentDelay;
public PollDelayStrategy(ScannerWorkerOptions.PollingOptions options, DeterministicRandomService randomService)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
_randomService = randomService ?? throw new ArgumentNullException(nameof(randomService));
}
public TimeSpan NextDelay()
{
@@ -42,8 +44,9 @@ public sealed class PollDelayStrategy
return duration;
}
var offset = (Random.Shared.NextDouble() * 2.0 - 1.0) * maxOffset;
var adjustedMs = Math.Max(0, duration.TotalMilliseconds + offset);
return TimeSpan.FromMilliseconds(adjustedMs);
}
}
var rng = _randomService.Create();
var offset = (rng.NextDouble() * 2.0 - 1.0) * maxOffset;
var adjustedMs = Math.Max(0, duration.TotalMilliseconds + offset);
return TimeSpan.FromMilliseconds(adjustedMs);
}
}

View File

@@ -22,6 +22,8 @@ using StellaOps.Scanner.Worker.Diagnostics;
using StellaOps.Scanner.Worker.Hosting;
using StellaOps.Scanner.Worker.Options;
using StellaOps.Scanner.Worker.Processing;
using StellaOps.Scanner.Worker.Processing.Entropy;
using StellaOps.Scanner.Worker.Determinism;
using StellaOps.Scanner.Worker.Processing.Surface;
using StellaOps.Scanner.Storage.Extensions;
using StellaOps.Scanner.Storage;
@@ -34,7 +36,23 @@ builder.Services.AddOptions<ScannerWorkerOptions>()
.ValidateOnStart();
builder.Services.AddSingleton<IValidateOptions<ScannerWorkerOptions>, ScannerWorkerOptionsValidator>();
builder.Services.AddSingleton(TimeProvider.System);
var workerOptions = builder.Configuration.GetSection(ScannerWorkerOptions.SectionName).Get<ScannerWorkerOptions>() ?? new ScannerWorkerOptions();
if (workerOptions.Determinism.FixedClock)
{
builder.Services.AddSingleton<TimeProvider>(_ => new DeterministicTimeProvider(workerOptions.Determinism.FixedInstantUtc));
}
else
{
builder.Services.AddSingleton(TimeProvider.System);
}
builder.Services.AddSingleton(new DeterminismContext(
workerOptions.Determinism.FixedClock,
workerOptions.Determinism.FixedInstantUtc,
workerOptions.Determinism.RngSeed,
workerOptions.Determinism.FilterLogs));
builder.Services.AddSingleton<IDeterministicRandomProvider>(_ => new DeterministicRandomProvider(workerOptions.Determinism.RngSeed));
builder.Services.AddScannerCache(builder.Configuration);
builder.Services.AddSurfaceEnvironment(options =>
{
@@ -85,12 +103,11 @@ builder.Services.AddSingleton<IScanStageExecutor, RegistrySecretStageExecutor>()
builder.Services.AddSingleton<IScanStageExecutor, AnalyzerStageExecutor>();
builder.Services.AddSingleton<IScanStageExecutor, Reachability.ReachabilityBuildStageExecutor>();
builder.Services.AddSingleton<IScanStageExecutor, Reachability.ReachabilityPublishStageExecutor>();
builder.Services.AddSingleton<IScanStageExecutor, Entropy.EntropyStageExecutor>();
builder.Services.AddSingleton<IScanStageExecutor, EntropyStageExecutor>();
builder.Services.AddSingleton<ScannerWorkerHostedService>();
builder.Services.AddHostedService(sp => sp.GetRequiredService<ScannerWorkerHostedService>());
var workerOptions = builder.Configuration.GetSection(ScannerWorkerOptions.SectionName).Get<ScannerWorkerOptions>() ?? new ScannerWorkerOptions();
builder.Services.AddStellaOpsCrypto(workerOptions.Crypto);
builder.Services.Configure<HostOptions>(options =>