up
Some checks failed
api-governance / spectral-lint (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
oas-ci / oas-validate (push) Has been cancelled
SDK Publish & Sign / sdk-publish (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Policy Simulation / policy-simulate (push) Has been cancelled
devportal-offline / build-offline (push) Has been cancelled
Some checks failed
api-governance / spectral-lint (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
oas-ci / oas-validate (push) Has been cancelled
SDK Publish & Sign / sdk-publish (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Policy Simulation / policy-simulate (push) Has been cancelled
devportal-offline / build-offline (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,25 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Determinism;
|
||||
|
||||
/// <summary>
|
||||
/// Captures determinism-related toggles for the worker runtime.
|
||||
/// </summary>
|
||||
public sealed class DeterminismContext
|
||||
{
|
||||
public DeterminismContext(bool fixedClock, DateTimeOffset fixedInstantUtc, int? rngSeed, bool filterLogs)
|
||||
{
|
||||
FixedClock = fixedClock;
|
||||
FixedInstantUtc = fixedInstantUtc.ToUniversalTime();
|
||||
RngSeed = rngSeed;
|
||||
FilterLogs = filterLogs;
|
||||
}
|
||||
|
||||
public bool FixedClock { get; }
|
||||
|
||||
public DateTimeOffset FixedInstantUtc { get; }
|
||||
|
||||
public int? RngSeed { get; }
|
||||
|
||||
public bool FilterLogs { get; }
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Determinism;
|
||||
|
||||
public interface IDeterministicRandomProvider
|
||||
{
|
||||
Random Create();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Provides seeded <see cref="Random"/> instances when a seed is configured, otherwise defaults to a thread-safe system random.
|
||||
/// </summary>
|
||||
public sealed class DeterministicRandomProvider : IDeterministicRandomProvider
|
||||
{
|
||||
private readonly int? _seed;
|
||||
|
||||
public DeterministicRandomProvider(int? seed)
|
||||
{
|
||||
_seed = seed;
|
||||
}
|
||||
|
||||
public Random Create()
|
||||
{
|
||||
return _seed.HasValue ? new Random(_seed.Value) : Random.Shared;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Determinism;
|
||||
|
||||
/// <summary>
|
||||
/// Time provider that always returns a fixed instant, used to enforce deterministic timestamps.
|
||||
/// </summary>
|
||||
public sealed class DeterministicTimeProvider : TimeProvider
|
||||
{
|
||||
private readonly DateTimeOffset _fixedUtc;
|
||||
|
||||
public DeterministicTimeProvider(DateTimeOffset fixedUtc)
|
||||
{
|
||||
_fixedUtc = fixedUtc.ToUniversalTime();
|
||||
}
|
||||
|
||||
public override DateTimeOffset GetUtcNow() => _fixedUtc;
|
||||
}
|
||||
@@ -27,6 +27,8 @@ public sealed class ScannerWorkerOptions
|
||||
public AnalyzerOptions Analyzers { get; } = new();
|
||||
|
||||
public StellaOpsCryptoOptions Crypto { get; } = new();
|
||||
|
||||
public DeterminismOptions Determinism { get; } = new();
|
||||
|
||||
public sealed class QueueOptions
|
||||
{
|
||||
@@ -177,4 +179,27 @@ public sealed class ScannerWorkerOptions
|
||||
|
||||
public string EntryTraceProcRootMetadataKey { get; set; } = ScanMetadataKeys.RuntimeProcRoot;
|
||||
}
|
||||
|
||||
public sealed class DeterminismOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// If true, the worker uses a fixed clock to ensure deterministic timestamps.
|
||||
/// </summary>
|
||||
public bool FixedClock { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Fixed UTC timestamp to emit when FixedClock is enabled. Defaults to Unix epoch.
|
||||
/// </summary>
|
||||
public DateTimeOffset FixedInstantUtc { get; set; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
/// <summary>
|
||||
/// Optional seed for RNG-based components when determinism is required.
|
||||
/// </summary>
|
||||
public int? RngSeed { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// If true, trims noisy log fields (duration, PIDs) to stable placeholders.
|
||||
/// </summary>
|
||||
public bool FilterLogs { get; set; }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
using System;
|
||||
using StellaOps.Scanner.Worker.Determinism;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Processing;
|
||||
|
||||
internal sealed class DeterministicRandomService
|
||||
{
|
||||
private readonly IDeterministicRandomProvider _provider;
|
||||
|
||||
public DeterministicRandomService(IDeterministicRandomProvider provider)
|
||||
{
|
||||
_provider = provider ?? throw new ArgumentNullException(nameof(provider));
|
||||
}
|
||||
|
||||
public Random Create() => _provider.Create();
|
||||
}
|
||||
@@ -5,9 +5,8 @@ using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Scanner.Core.Entropy;
|
||||
using StellaOps.Scanner.Core.Contracts;
|
||||
using StellaOps.Scanner.Worker.Utilities;
|
||||
using StellaOps.Scanner.Core.Entropy;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Processing.Entropy;
|
||||
|
||||
@@ -26,7 +25,7 @@ public sealed class EntropyStageExecutor : IScanStageExecutor
|
||||
_reportBuilder = new EntropyReportBuilder();
|
||||
}
|
||||
|
||||
public string StageName => ScanStageNames.EmitReports;
|
||||
public string StageName => ScanStageNames.Entropy;
|
||||
|
||||
public async ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -68,7 +67,7 @@ public sealed class EntropyStageExecutor : IScanStageExecutor
|
||||
return;
|
||||
}
|
||||
|
||||
var layerDigest = context.Lease.LayerDigest ?? string.Empty;
|
||||
var layerDigest = ResolveLayerDigest(context.Lease.Metadata);
|
||||
var layerSize = files.Sum(f => f.SizeBytes);
|
||||
var imageOpaqueBytes = reports.Sum(r => r.OpaqueBytes);
|
||||
var imageTotalBytes = files.Sum(f => f.SizeBytes);
|
||||
@@ -81,7 +80,7 @@ public sealed class EntropyStageExecutor : IScanStageExecutor
|
||||
imageTotalBytes);
|
||||
|
||||
var entropyReport = new EntropyReport(
|
||||
ImageDigest: context.Lease.ImageDigest ?? string.Empty,
|
||||
ImageDigest: ResolveImageDigest(context.Lease.Metadata),
|
||||
LayerDigest: layerDigest,
|
||||
Files: reports,
|
||||
ImageOpaqueRatio: imageRatio);
|
||||
@@ -138,4 +137,49 @@ public sealed class EntropyStageExecutor : IScanStageExecutor
|
||||
await stream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
|
||||
return buffer.ToArray();
|
||||
}
|
||||
|
||||
private static string ResolveLayerDigest(IReadOnlyDictionary<string, string> metadata)
|
||||
{
|
||||
if (metadata is null)
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
if (metadata.TryGetValue("layerDigest", out var digest) && !string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
return digest.Trim();
|
||||
}
|
||||
|
||||
if (metadata.TryGetValue("layer.digest", out digest) && !string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
return digest.Trim();
|
||||
}
|
||||
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
private static string ResolveImageDigest(IReadOnlyDictionary<string, string> metadata)
|
||||
{
|
||||
if (metadata is null)
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
if (metadata.TryGetValue("image.digest", out var digest) && !string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
return digest.Trim();
|
||||
}
|
||||
|
||||
if (metadata.TryGetValue("imageDigest", out digest) && !string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
return digest.Trim();
|
||||
}
|
||||
|
||||
if (metadata.TryGetValue("scanner.image.digest", out digest) && !string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
return digest.Trim();
|
||||
}
|
||||
|
||||
return string.Empty;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,18 +9,25 @@ namespace StellaOps.Scanner.Worker.Processing;
|
||||
|
||||
public sealed class LeaseHeartbeatService
|
||||
{
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly IOptionsMonitor<ScannerWorkerOptions> _options;
|
||||
private readonly IDelayScheduler _delayScheduler;
|
||||
private readonly ILogger<LeaseHeartbeatService> _logger;
|
||||
|
||||
public LeaseHeartbeatService(TimeProvider timeProvider, IDelayScheduler delayScheduler, IOptionsMonitor<ScannerWorkerOptions> options, ILogger<LeaseHeartbeatService> logger)
|
||||
{
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
_delayScheduler = delayScheduler ?? throw new ArgumentNullException(nameof(delayScheduler));
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly IOptionsMonitor<ScannerWorkerOptions> _options;
|
||||
private readonly IDelayScheduler _delayScheduler;
|
||||
private readonly IDeterministicRandomProvider _randomProvider;
|
||||
private readonly ILogger<LeaseHeartbeatService> _logger;
|
||||
|
||||
public LeaseHeartbeatService(
|
||||
TimeProvider timeProvider,
|
||||
IDelayScheduler delayScheduler,
|
||||
IOptionsMonitor<ScannerWorkerOptions> options,
|
||||
IDeterministicRandomProvider randomProvider,
|
||||
ILogger<LeaseHeartbeatService> logger)
|
||||
{
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
_delayScheduler = delayScheduler ?? throw new ArgumentNullException(nameof(delayScheduler));
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_randomProvider = randomProvider ?? throw new ArgumentNullException(nameof(randomProvider));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task RunAsync(IScanJobLease lease, CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -32,7 +39,7 @@ public sealed class LeaseHeartbeatService
|
||||
{
|
||||
var options = _options.CurrentValue;
|
||||
var interval = ComputeInterval(options, lease);
|
||||
var delay = ApplyJitter(interval, options.Queue);
|
||||
var delay = ApplyJitter(interval, options.Queue, _randomProvider);
|
||||
try
|
||||
{
|
||||
await _delayScheduler.DelayAsync(delay, cancellationToken).ConfigureAwait(false);
|
||||
@@ -77,14 +84,14 @@ public sealed class LeaseHeartbeatService
|
||||
return recommended;
|
||||
}
|
||||
|
||||
private static TimeSpan ApplyJitter(TimeSpan duration, ScannerWorkerOptions.QueueOptions queueOptions)
|
||||
private static TimeSpan ApplyJitter(TimeSpan duration, ScannerWorkerOptions.QueueOptions queueOptions, IDeterministicRandomProvider randomProvider)
|
||||
{
|
||||
if (queueOptions.MaxHeartbeatJitterMilliseconds <= 0)
|
||||
{
|
||||
return duration;
|
||||
}
|
||||
|
||||
var offsetMs = Random.Shared.NextDouble() * queueOptions.MaxHeartbeatJitterMilliseconds;
|
||||
var offsetMs = randomProvider.Create().NextDouble() * queueOptions.MaxHeartbeatJitterMilliseconds;
|
||||
var adjusted = duration - TimeSpan.FromMilliseconds(offsetMs);
|
||||
if (adjusted < queueOptions.MinHeartbeatInterval)
|
||||
{
|
||||
@@ -97,10 +104,10 @@ public sealed class LeaseHeartbeatService
|
||||
private async Task<bool> TryRenewAsync(ScannerWorkerOptions options, IScanJobLease lease, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await lease.RenewAsync(cancellationToken).ConfigureAwait(false);
|
||||
return true;
|
||||
}
|
||||
{
|
||||
await lease.RenewAsync(cancellationToken).ConfigureAwait(false);
|
||||
return true;
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
return false;
|
||||
|
||||
@@ -6,13 +6,15 @@ namespace StellaOps.Scanner.Worker.Processing;
|
||||
|
||||
public sealed class PollDelayStrategy
|
||||
{
|
||||
private readonly ScannerWorkerOptions.PollingOptions _options;
|
||||
private TimeSpan _currentDelay;
|
||||
|
||||
public PollDelayStrategy(ScannerWorkerOptions.PollingOptions options)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
}
|
||||
private readonly ScannerWorkerOptions.PollingOptions _options;
|
||||
private readonly DeterministicRandomService _randomService;
|
||||
private TimeSpan _currentDelay;
|
||||
|
||||
public PollDelayStrategy(ScannerWorkerOptions.PollingOptions options, DeterministicRandomService randomService)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_randomService = randomService ?? throw new ArgumentNullException(nameof(randomService));
|
||||
}
|
||||
|
||||
public TimeSpan NextDelay()
|
||||
{
|
||||
@@ -42,8 +44,9 @@ public sealed class PollDelayStrategy
|
||||
return duration;
|
||||
}
|
||||
|
||||
var offset = (Random.Shared.NextDouble() * 2.0 - 1.0) * maxOffset;
|
||||
var adjustedMs = Math.Max(0, duration.TotalMilliseconds + offset);
|
||||
return TimeSpan.FromMilliseconds(adjustedMs);
|
||||
}
|
||||
}
|
||||
var rng = _randomService.Create();
|
||||
var offset = (rng.NextDouble() * 2.0 - 1.0) * maxOffset;
|
||||
var adjustedMs = Math.Max(0, duration.TotalMilliseconds + offset);
|
||||
return TimeSpan.FromMilliseconds(adjustedMs);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,6 +22,8 @@ using StellaOps.Scanner.Worker.Diagnostics;
|
||||
using StellaOps.Scanner.Worker.Hosting;
|
||||
using StellaOps.Scanner.Worker.Options;
|
||||
using StellaOps.Scanner.Worker.Processing;
|
||||
using StellaOps.Scanner.Worker.Processing.Entropy;
|
||||
using StellaOps.Scanner.Worker.Determinism;
|
||||
using StellaOps.Scanner.Worker.Processing.Surface;
|
||||
using StellaOps.Scanner.Storage.Extensions;
|
||||
using StellaOps.Scanner.Storage;
|
||||
@@ -34,7 +36,23 @@ builder.Services.AddOptions<ScannerWorkerOptions>()
|
||||
.ValidateOnStart();
|
||||
|
||||
builder.Services.AddSingleton<IValidateOptions<ScannerWorkerOptions>, ScannerWorkerOptionsValidator>();
|
||||
builder.Services.AddSingleton(TimeProvider.System);
|
||||
var workerOptions = builder.Configuration.GetSection(ScannerWorkerOptions.SectionName).Get<ScannerWorkerOptions>() ?? new ScannerWorkerOptions();
|
||||
|
||||
if (workerOptions.Determinism.FixedClock)
|
||||
{
|
||||
builder.Services.AddSingleton<TimeProvider>(_ => new DeterministicTimeProvider(workerOptions.Determinism.FixedInstantUtc));
|
||||
}
|
||||
else
|
||||
{
|
||||
builder.Services.AddSingleton(TimeProvider.System);
|
||||
}
|
||||
|
||||
builder.Services.AddSingleton(new DeterminismContext(
|
||||
workerOptions.Determinism.FixedClock,
|
||||
workerOptions.Determinism.FixedInstantUtc,
|
||||
workerOptions.Determinism.RngSeed,
|
||||
workerOptions.Determinism.FilterLogs));
|
||||
builder.Services.AddSingleton<IDeterministicRandomProvider>(_ => new DeterministicRandomProvider(workerOptions.Determinism.RngSeed));
|
||||
builder.Services.AddScannerCache(builder.Configuration);
|
||||
builder.Services.AddSurfaceEnvironment(options =>
|
||||
{
|
||||
@@ -85,12 +103,11 @@ builder.Services.AddSingleton<IScanStageExecutor, RegistrySecretStageExecutor>()
|
||||
builder.Services.AddSingleton<IScanStageExecutor, AnalyzerStageExecutor>();
|
||||
builder.Services.AddSingleton<IScanStageExecutor, Reachability.ReachabilityBuildStageExecutor>();
|
||||
builder.Services.AddSingleton<IScanStageExecutor, Reachability.ReachabilityPublishStageExecutor>();
|
||||
builder.Services.AddSingleton<IScanStageExecutor, Entropy.EntropyStageExecutor>();
|
||||
builder.Services.AddSingleton<IScanStageExecutor, EntropyStageExecutor>();
|
||||
|
||||
builder.Services.AddSingleton<ScannerWorkerHostedService>();
|
||||
builder.Services.AddHostedService(sp => sp.GetRequiredService<ScannerWorkerHostedService>());
|
||||
|
||||
var workerOptions = builder.Configuration.GetSection(ScannerWorkerOptions.SectionName).Get<ScannerWorkerOptions>() ?? new ScannerWorkerOptions();
|
||||
builder.Services.AddStellaOpsCrypto(workerOptions.Crypto);
|
||||
|
||||
builder.Services.Configure<HostOptions>(options =>
|
||||
|
||||
Reference in New Issue
Block a user