up
Some checks failed
api-governance / spectral-lint (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
oas-ci / oas-validate (push) Has been cancelled
SDK Publish & Sign / sdk-publish (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Policy Simulation / policy-simulate (push) Has been cancelled
devportal-offline / build-offline (push) Has been cancelled
Some checks failed
api-governance / spectral-lint (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
oas-ci / oas-validate (push) Has been cancelled
SDK Publish & Sign / sdk-publish (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Policy Simulation / policy-simulate (push) Has been cancelled
devportal-offline / build-offline (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,14 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Scanner.WebService.Contracts;
|
||||
|
||||
public sealed record EntropyLayerRequest(
|
||||
[property: JsonPropertyName("layerDigest")] string LayerDigest,
|
||||
[property: JsonPropertyName("opaqueRatio")] double OpaqueRatio,
|
||||
[property: JsonPropertyName("opaqueBytes")] long OpaqueBytes,
|
||||
[property: JsonPropertyName("totalBytes")] long TotalBytes);
|
||||
|
||||
public sealed record EntropyIngestRequest(
|
||||
[property: JsonPropertyName("imageOpaqueRatio")] double ImageOpaqueRatio,
|
||||
[property: JsonPropertyName("layers")] IReadOnlyList<EntropyLayerRequest> Layers);
|
||||
@@ -7,6 +7,7 @@ public sealed record ScanStatusResponse(
|
||||
DateTimeOffset CreatedAt,
|
||||
DateTimeOffset UpdatedAt,
|
||||
string? FailureReason,
|
||||
EntropyStatusDto? Entropy,
|
||||
SurfacePointersDto? Surface,
|
||||
ReplayStatusDto? Replay);
|
||||
|
||||
@@ -14,6 +15,16 @@ public sealed record ScanStatusTarget(
|
||||
string? Reference,
|
||||
string? Digest);
|
||||
|
||||
public sealed record EntropyStatusDto(
|
||||
double ImageOpaqueRatio,
|
||||
IReadOnlyList<EntropyLayerStatusDto> Layers);
|
||||
|
||||
public sealed record EntropyLayerStatusDto(
|
||||
string LayerDigest,
|
||||
double OpaqueRatio,
|
||||
long OpaqueBytes,
|
||||
long TotalBytes);
|
||||
|
||||
public sealed record ReplayStatusDto(
|
||||
string ManifestHash,
|
||||
IReadOnlyList<ReplayBundleStatusDto> Bundles);
|
||||
|
||||
@@ -7,6 +7,7 @@ public sealed record ScanSnapshot(
|
||||
DateTimeOffset CreatedAt,
|
||||
DateTimeOffset UpdatedAt,
|
||||
string? FailureReason,
|
||||
EntropySnapshot? Entropy,
|
||||
ReplayArtifacts? Replay);
|
||||
|
||||
public sealed record ReplayArtifacts(
|
||||
@@ -18,3 +19,13 @@ public sealed record ReplayBundleSummary(
|
||||
string Digest,
|
||||
string CasUri,
|
||||
long SizeBytes);
|
||||
|
||||
public sealed record EntropySnapshot(
|
||||
double ImageOpaqueRatio,
|
||||
IReadOnlyList<EntropyLayerSnapshot> Layers);
|
||||
|
||||
public sealed record EntropyLayerSnapshot(
|
||||
string LayerDigest,
|
||||
double OpaqueRatio,
|
||||
long OpaqueBytes,
|
||||
long TotalBytes);
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.IO.Pipelines;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text.Json;
|
||||
@@ -45,6 +46,13 @@ internal static class ScanEndpoints
|
||||
.Produces(StatusCodes.Status404NotFound)
|
||||
.RequireAuthorization(ScannerPolicies.ScansRead);
|
||||
|
||||
scans.MapPost("/{scanId}/entropy", HandleAttachEntropyAsync)
|
||||
.WithName("scanner.scans.entropy")
|
||||
.Produces(StatusCodes.Status202Accepted)
|
||||
.Produces(StatusCodes.Status400BadRequest)
|
||||
.Produces(StatusCodes.Status404NotFound)
|
||||
.RequireAuthorization(ScannerPolicies.ScansWrite);
|
||||
|
||||
scans.MapGet("/{scanId}/events", HandleProgressStreamAsync)
|
||||
.WithName("scanner.scans.events")
|
||||
.Produces(StatusCodes.Status200OK)
|
||||
@@ -203,12 +211,79 @@ internal static class ScanEndpoints
|
||||
CreatedAt: snapshot.CreatedAt,
|
||||
UpdatedAt: snapshot.UpdatedAt,
|
||||
FailureReason: snapshot.FailureReason,
|
||||
Entropy: snapshot.Entropy is null
|
||||
? null
|
||||
: new EntropyStatusDto(
|
||||
snapshot.Entropy.ImageOpaqueRatio,
|
||||
snapshot.Entropy.Layers
|
||||
.Select(l => new EntropyLayerStatusDto(l.LayerDigest, l.OpaqueRatio, l.OpaqueBytes, l.TotalBytes))
|
||||
.ToArray()),
|
||||
Surface: surfacePointers,
|
||||
Replay: snapshot.Replay is null ? null : MapReplay(snapshot.Replay));
|
||||
|
||||
return Json(response, StatusCodes.Status200OK);
|
||||
}
|
||||
|
||||
private static async Task<IResult> HandleAttachEntropyAsync(
|
||||
string scanId,
|
||||
EntropyIngestRequest request,
|
||||
IScanCoordinator coordinator,
|
||||
HttpContext context,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(coordinator);
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
if (!ScanId.TryParse(scanId, out var parsed))
|
||||
{
|
||||
return ProblemResultFactory.Create(
|
||||
context,
|
||||
ProblemTypes.Validation,
|
||||
"Invalid scan identifier",
|
||||
StatusCodes.Status400BadRequest,
|
||||
detail: "Scan identifier is required.");
|
||||
}
|
||||
|
||||
if (request.Layers is null || request.Layers.Count == 0)
|
||||
{
|
||||
return ProblemResultFactory.Create(
|
||||
context,
|
||||
ProblemTypes.Validation,
|
||||
"Entropy layers are required",
|
||||
StatusCodes.Status400BadRequest);
|
||||
}
|
||||
|
||||
var layers = request.Layers
|
||||
.Where(l => !string.IsNullOrWhiteSpace(l.LayerDigest))
|
||||
.Select(l => new EntropyLayerSnapshot(
|
||||
l.LayerDigest.Trim(),
|
||||
l.OpaqueRatio,
|
||||
l.OpaqueBytes,
|
||||
l.TotalBytes))
|
||||
.ToArray();
|
||||
|
||||
if (layers.Length == 0)
|
||||
{
|
||||
return ProblemResultFactory.Create(
|
||||
context,
|
||||
ProblemTypes.Validation,
|
||||
"Entropy layers are required",
|
||||
StatusCodes.Status400BadRequest);
|
||||
}
|
||||
|
||||
var snapshot = new EntropySnapshot(
|
||||
request.ImageOpaqueRatio,
|
||||
layers);
|
||||
|
||||
var attached = await coordinator.AttachEntropyAsync(parsed, snapshot, cancellationToken).ConfigureAwait(false);
|
||||
if (!attached)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
return Results.Accepted();
|
||||
}
|
||||
|
||||
private static async Task<IResult> HandleProgressStreamAsync(
|
||||
string scanId,
|
||||
string? format,
|
||||
|
||||
@@ -11,4 +11,6 @@ public interface IScanCoordinator
|
||||
ValueTask<ScanSnapshot?> TryFindByTargetAsync(string? reference, string? digest, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask<bool> AttachReplayAsync(ScanId scanId, ReplayArtifacts replay, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask<bool> AttachEntropyAsync(ScanId scanId, EntropySnapshot entropy, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
@@ -45,15 +45,16 @@ public sealed class InMemoryScanCoordinator : IScanCoordinator
|
||||
scanId,
|
||||
normalizedTarget,
|
||||
ScanStatus.Pending,
|
||||
now,
|
||||
now,
|
||||
now,
|
||||
null,
|
||||
null,
|
||||
null)),
|
||||
(_, existing) =>
|
||||
{
|
||||
if (submission.Force)
|
||||
{
|
||||
var snapshot = existing.Snapshot with
|
||||
(_, existing) =>
|
||||
{
|
||||
if (submission.Force)
|
||||
{
|
||||
var snapshot = existing.Snapshot with
|
||||
{
|
||||
Status = ScanStatus.Pending,
|
||||
UpdatedAt = now,
|
||||
@@ -134,6 +135,30 @@ public sealed class InMemoryScanCoordinator : IScanCoordinator
|
||||
return ValueTask.FromResult(true);
|
||||
}
|
||||
|
||||
public ValueTask<bool> AttachEntropyAsync(ScanId scanId, EntropySnapshot entropy, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(entropy);
|
||||
|
||||
if (!scans.TryGetValue(scanId.Value, out var existing))
|
||||
{
|
||||
return ValueTask.FromResult(false);
|
||||
}
|
||||
|
||||
var updated = existing.Snapshot with
|
||||
{
|
||||
Entropy = entropy,
|
||||
UpdatedAt = timeProvider.GetUtcNow()
|
||||
};
|
||||
|
||||
scans[scanId.Value] = new ScanEntry(updated);
|
||||
progressPublisher.Publish(scanId, updated.Status.ToString(), "entropy-attached", new Dictionary<string, object?>
|
||||
{
|
||||
["entropy.imageOpaqueRatio"] = entropy.ImageOpaqueRatio,
|
||||
["entropy.layers"] = entropy.Layers.Count
|
||||
});
|
||||
return ValueTask.FromResult(true);
|
||||
}
|
||||
|
||||
private void IndexTarget(string scanId, ScanTarget target)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(target.Digest))
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Determinism;
|
||||
|
||||
/// <summary>
|
||||
/// Captures determinism-related toggles for the worker runtime.
|
||||
/// </summary>
|
||||
public sealed class DeterminismContext
|
||||
{
|
||||
public DeterminismContext(bool fixedClock, DateTimeOffset fixedInstantUtc, int? rngSeed, bool filterLogs)
|
||||
{
|
||||
FixedClock = fixedClock;
|
||||
FixedInstantUtc = fixedInstantUtc.ToUniversalTime();
|
||||
RngSeed = rngSeed;
|
||||
FilterLogs = filterLogs;
|
||||
}
|
||||
|
||||
public bool FixedClock { get; }
|
||||
|
||||
public DateTimeOffset FixedInstantUtc { get; }
|
||||
|
||||
public int? RngSeed { get; }
|
||||
|
||||
public bool FilterLogs { get; }
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Determinism;
|
||||
|
||||
public interface IDeterministicRandomProvider
|
||||
{
|
||||
Random Create();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Provides seeded <see cref="Random"/> instances when a seed is configured, otherwise defaults to a thread-safe system random.
|
||||
/// </summary>
|
||||
public sealed class DeterministicRandomProvider : IDeterministicRandomProvider
|
||||
{
|
||||
private readonly int? _seed;
|
||||
|
||||
public DeterministicRandomProvider(int? seed)
|
||||
{
|
||||
_seed = seed;
|
||||
}
|
||||
|
||||
public Random Create()
|
||||
{
|
||||
return _seed.HasValue ? new Random(_seed.Value) : Random.Shared;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Determinism;
|
||||
|
||||
/// <summary>
|
||||
/// Time provider that always returns a fixed instant, used to enforce deterministic timestamps.
|
||||
/// </summary>
|
||||
public sealed class DeterministicTimeProvider : TimeProvider
|
||||
{
|
||||
private readonly DateTimeOffset _fixedUtc;
|
||||
|
||||
public DeterministicTimeProvider(DateTimeOffset fixedUtc)
|
||||
{
|
||||
_fixedUtc = fixedUtc.ToUniversalTime();
|
||||
}
|
||||
|
||||
public override DateTimeOffset GetUtcNow() => _fixedUtc;
|
||||
}
|
||||
@@ -27,6 +27,8 @@ public sealed class ScannerWorkerOptions
|
||||
public AnalyzerOptions Analyzers { get; } = new();
|
||||
|
||||
public StellaOpsCryptoOptions Crypto { get; } = new();
|
||||
|
||||
public DeterminismOptions Determinism { get; } = new();
|
||||
|
||||
public sealed class QueueOptions
|
||||
{
|
||||
@@ -177,4 +179,27 @@ public sealed class ScannerWorkerOptions
|
||||
|
||||
public string EntryTraceProcRootMetadataKey { get; set; } = ScanMetadataKeys.RuntimeProcRoot;
|
||||
}
|
||||
|
||||
public sealed class DeterminismOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// If true, the worker uses a fixed clock to ensure deterministic timestamps.
|
||||
/// </summary>
|
||||
public bool FixedClock { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Fixed UTC timestamp to emit when FixedClock is enabled. Defaults to Unix epoch.
|
||||
/// </summary>
|
||||
public DateTimeOffset FixedInstantUtc { get; set; } = DateTimeOffset.UnixEpoch;
|
||||
|
||||
/// <summary>
|
||||
/// Optional seed for RNG-based components when determinism is required.
|
||||
/// </summary>
|
||||
public int? RngSeed { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// If true, trims noisy log fields (duration, PIDs) to stable placeholders.
|
||||
/// </summary>
|
||||
public bool FilterLogs { get; set; }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
using System;
|
||||
using StellaOps.Scanner.Worker.Determinism;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Processing;
|
||||
|
||||
internal sealed class DeterministicRandomService
|
||||
{
|
||||
private readonly IDeterministicRandomProvider _provider;
|
||||
|
||||
public DeterministicRandomService(IDeterministicRandomProvider provider)
|
||||
{
|
||||
_provider = provider ?? throw new ArgumentNullException(nameof(provider));
|
||||
}
|
||||
|
||||
public Random Create() => _provider.Create();
|
||||
}
|
||||
@@ -5,9 +5,8 @@ using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Scanner.Core.Entropy;
|
||||
using StellaOps.Scanner.Core.Contracts;
|
||||
using StellaOps.Scanner.Worker.Utilities;
|
||||
using StellaOps.Scanner.Core.Entropy;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Processing.Entropy;
|
||||
|
||||
@@ -26,7 +25,7 @@ public sealed class EntropyStageExecutor : IScanStageExecutor
|
||||
_reportBuilder = new EntropyReportBuilder();
|
||||
}
|
||||
|
||||
public string StageName => ScanStageNames.EmitReports;
|
||||
public string StageName => ScanStageNames.Entropy;
|
||||
|
||||
public async ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -68,7 +67,7 @@ public sealed class EntropyStageExecutor : IScanStageExecutor
|
||||
return;
|
||||
}
|
||||
|
||||
var layerDigest = context.Lease.LayerDigest ?? string.Empty;
|
||||
var layerDigest = ResolveLayerDigest(context.Lease.Metadata);
|
||||
var layerSize = files.Sum(f => f.SizeBytes);
|
||||
var imageOpaqueBytes = reports.Sum(r => r.OpaqueBytes);
|
||||
var imageTotalBytes = files.Sum(f => f.SizeBytes);
|
||||
@@ -81,7 +80,7 @@ public sealed class EntropyStageExecutor : IScanStageExecutor
|
||||
imageTotalBytes);
|
||||
|
||||
var entropyReport = new EntropyReport(
|
||||
ImageDigest: context.Lease.ImageDigest ?? string.Empty,
|
||||
ImageDigest: ResolveImageDigest(context.Lease.Metadata),
|
||||
LayerDigest: layerDigest,
|
||||
Files: reports,
|
||||
ImageOpaqueRatio: imageRatio);
|
||||
@@ -138,4 +137,49 @@ public sealed class EntropyStageExecutor : IScanStageExecutor
|
||||
await stream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
|
||||
return buffer.ToArray();
|
||||
}
|
||||
|
||||
private static string ResolveLayerDigest(IReadOnlyDictionary<string, string> metadata)
|
||||
{
|
||||
if (metadata is null)
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
if (metadata.TryGetValue("layerDigest", out var digest) && !string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
return digest.Trim();
|
||||
}
|
||||
|
||||
if (metadata.TryGetValue("layer.digest", out digest) && !string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
return digest.Trim();
|
||||
}
|
||||
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
private static string ResolveImageDigest(IReadOnlyDictionary<string, string> metadata)
|
||||
{
|
||||
if (metadata is null)
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
if (metadata.TryGetValue("image.digest", out var digest) && !string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
return digest.Trim();
|
||||
}
|
||||
|
||||
if (metadata.TryGetValue("imageDigest", out digest) && !string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
return digest.Trim();
|
||||
}
|
||||
|
||||
if (metadata.TryGetValue("scanner.image.digest", out digest) && !string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
return digest.Trim();
|
||||
}
|
||||
|
||||
return string.Empty;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,18 +9,25 @@ namespace StellaOps.Scanner.Worker.Processing;
|
||||
|
||||
public sealed class LeaseHeartbeatService
|
||||
{
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly IOptionsMonitor<ScannerWorkerOptions> _options;
|
||||
private readonly IDelayScheduler _delayScheduler;
|
||||
private readonly ILogger<LeaseHeartbeatService> _logger;
|
||||
|
||||
public LeaseHeartbeatService(TimeProvider timeProvider, IDelayScheduler delayScheduler, IOptionsMonitor<ScannerWorkerOptions> options, ILogger<LeaseHeartbeatService> logger)
|
||||
{
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
_delayScheduler = delayScheduler ?? throw new ArgumentNullException(nameof(delayScheduler));
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly IOptionsMonitor<ScannerWorkerOptions> _options;
|
||||
private readonly IDelayScheduler _delayScheduler;
|
||||
private readonly IDeterministicRandomProvider _randomProvider;
|
||||
private readonly ILogger<LeaseHeartbeatService> _logger;
|
||||
|
||||
public LeaseHeartbeatService(
|
||||
TimeProvider timeProvider,
|
||||
IDelayScheduler delayScheduler,
|
||||
IOptionsMonitor<ScannerWorkerOptions> options,
|
||||
IDeterministicRandomProvider randomProvider,
|
||||
ILogger<LeaseHeartbeatService> logger)
|
||||
{
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
_delayScheduler = delayScheduler ?? throw new ArgumentNullException(nameof(delayScheduler));
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_randomProvider = randomProvider ?? throw new ArgumentNullException(nameof(randomProvider));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task RunAsync(IScanJobLease lease, CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -32,7 +39,7 @@ public sealed class LeaseHeartbeatService
|
||||
{
|
||||
var options = _options.CurrentValue;
|
||||
var interval = ComputeInterval(options, lease);
|
||||
var delay = ApplyJitter(interval, options.Queue);
|
||||
var delay = ApplyJitter(interval, options.Queue, _randomProvider);
|
||||
try
|
||||
{
|
||||
await _delayScheduler.DelayAsync(delay, cancellationToken).ConfigureAwait(false);
|
||||
@@ -77,14 +84,14 @@ public sealed class LeaseHeartbeatService
|
||||
return recommended;
|
||||
}
|
||||
|
||||
private static TimeSpan ApplyJitter(TimeSpan duration, ScannerWorkerOptions.QueueOptions queueOptions)
|
||||
private static TimeSpan ApplyJitter(TimeSpan duration, ScannerWorkerOptions.QueueOptions queueOptions, IDeterministicRandomProvider randomProvider)
|
||||
{
|
||||
if (queueOptions.MaxHeartbeatJitterMilliseconds <= 0)
|
||||
{
|
||||
return duration;
|
||||
}
|
||||
|
||||
var offsetMs = Random.Shared.NextDouble() * queueOptions.MaxHeartbeatJitterMilliseconds;
|
||||
var offsetMs = randomProvider.Create().NextDouble() * queueOptions.MaxHeartbeatJitterMilliseconds;
|
||||
var adjusted = duration - TimeSpan.FromMilliseconds(offsetMs);
|
||||
if (adjusted < queueOptions.MinHeartbeatInterval)
|
||||
{
|
||||
@@ -97,10 +104,10 @@ public sealed class LeaseHeartbeatService
|
||||
private async Task<bool> TryRenewAsync(ScannerWorkerOptions options, IScanJobLease lease, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await lease.RenewAsync(cancellationToken).ConfigureAwait(false);
|
||||
return true;
|
||||
}
|
||||
{
|
||||
await lease.RenewAsync(cancellationToken).ConfigureAwait(false);
|
||||
return true;
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
return false;
|
||||
|
||||
@@ -6,13 +6,15 @@ namespace StellaOps.Scanner.Worker.Processing;
|
||||
|
||||
public sealed class PollDelayStrategy
|
||||
{
|
||||
private readonly ScannerWorkerOptions.PollingOptions _options;
|
||||
private TimeSpan _currentDelay;
|
||||
|
||||
public PollDelayStrategy(ScannerWorkerOptions.PollingOptions options)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
}
|
||||
private readonly ScannerWorkerOptions.PollingOptions _options;
|
||||
private readonly DeterministicRandomService _randomService;
|
||||
private TimeSpan _currentDelay;
|
||||
|
||||
public PollDelayStrategy(ScannerWorkerOptions.PollingOptions options, DeterministicRandomService randomService)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_randomService = randomService ?? throw new ArgumentNullException(nameof(randomService));
|
||||
}
|
||||
|
||||
public TimeSpan NextDelay()
|
||||
{
|
||||
@@ -42,8 +44,9 @@ public sealed class PollDelayStrategy
|
||||
return duration;
|
||||
}
|
||||
|
||||
var offset = (Random.Shared.NextDouble() * 2.0 - 1.0) * maxOffset;
|
||||
var adjustedMs = Math.Max(0, duration.TotalMilliseconds + offset);
|
||||
return TimeSpan.FromMilliseconds(adjustedMs);
|
||||
}
|
||||
}
|
||||
var rng = _randomService.Create();
|
||||
var offset = (rng.NextDouble() * 2.0 - 1.0) * maxOffset;
|
||||
var adjustedMs = Math.Max(0, duration.TotalMilliseconds + offset);
|
||||
return TimeSpan.FromMilliseconds(adjustedMs);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,6 +22,8 @@ using StellaOps.Scanner.Worker.Diagnostics;
|
||||
using StellaOps.Scanner.Worker.Hosting;
|
||||
using StellaOps.Scanner.Worker.Options;
|
||||
using StellaOps.Scanner.Worker.Processing;
|
||||
using StellaOps.Scanner.Worker.Processing.Entropy;
|
||||
using StellaOps.Scanner.Worker.Determinism;
|
||||
using StellaOps.Scanner.Worker.Processing.Surface;
|
||||
using StellaOps.Scanner.Storage.Extensions;
|
||||
using StellaOps.Scanner.Storage;
|
||||
@@ -34,7 +36,23 @@ builder.Services.AddOptions<ScannerWorkerOptions>()
|
||||
.ValidateOnStart();
|
||||
|
||||
builder.Services.AddSingleton<IValidateOptions<ScannerWorkerOptions>, ScannerWorkerOptionsValidator>();
|
||||
builder.Services.AddSingleton(TimeProvider.System);
|
||||
var workerOptions = builder.Configuration.GetSection(ScannerWorkerOptions.SectionName).Get<ScannerWorkerOptions>() ?? new ScannerWorkerOptions();
|
||||
|
||||
if (workerOptions.Determinism.FixedClock)
|
||||
{
|
||||
builder.Services.AddSingleton<TimeProvider>(_ => new DeterministicTimeProvider(workerOptions.Determinism.FixedInstantUtc));
|
||||
}
|
||||
else
|
||||
{
|
||||
builder.Services.AddSingleton(TimeProvider.System);
|
||||
}
|
||||
|
||||
builder.Services.AddSingleton(new DeterminismContext(
|
||||
workerOptions.Determinism.FixedClock,
|
||||
workerOptions.Determinism.FixedInstantUtc,
|
||||
workerOptions.Determinism.RngSeed,
|
||||
workerOptions.Determinism.FilterLogs));
|
||||
builder.Services.AddSingleton<IDeterministicRandomProvider>(_ => new DeterministicRandomProvider(workerOptions.Determinism.RngSeed));
|
||||
builder.Services.AddScannerCache(builder.Configuration);
|
||||
builder.Services.AddSurfaceEnvironment(options =>
|
||||
{
|
||||
@@ -85,12 +103,11 @@ builder.Services.AddSingleton<IScanStageExecutor, RegistrySecretStageExecutor>()
|
||||
builder.Services.AddSingleton<IScanStageExecutor, AnalyzerStageExecutor>();
|
||||
builder.Services.AddSingleton<IScanStageExecutor, Reachability.ReachabilityBuildStageExecutor>();
|
||||
builder.Services.AddSingleton<IScanStageExecutor, Reachability.ReachabilityPublishStageExecutor>();
|
||||
builder.Services.AddSingleton<IScanStageExecutor, Entropy.EntropyStageExecutor>();
|
||||
builder.Services.AddSingleton<IScanStageExecutor, EntropyStageExecutor>();
|
||||
|
||||
builder.Services.AddSingleton<ScannerWorkerHostedService>();
|
||||
builder.Services.AddHostedService(sp => sp.GetRequiredService<ScannerWorkerHostedService>());
|
||||
|
||||
var workerOptions = builder.Configuration.GetSection(ScannerWorkerOptions.SectionName).Get<ScannerWorkerOptions>() ?? new ScannerWorkerOptions();
|
||||
builder.Services.AddStellaOpsCrypto(workerOptions.Crypto);
|
||||
|
||||
builder.Services.Configure<HostOptions>(options =>
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StellaOps.Scanner.Core.Contracts;
|
||||
|
||||
/// <summary>
|
||||
/// Minimal representation of a file discovered during analyzer stages.
|
||||
/// </summary>
|
||||
public sealed record ScanFileEntry(
|
||||
string Path,
|
||||
long SizeBytes,
|
||||
string Kind,
|
||||
IReadOnlyDictionary<string, string>? Metadata = null);
|
||||
@@ -45,7 +45,7 @@ public sealed class EntropyReportBuilder
|
||||
.ToList();
|
||||
|
||||
var opaqueBytes = windows
|
||||
.Where(w => w.Entropy >= _opaqueThreshold)
|
||||
.Where(w => w.EntropyBits >= _opaqueThreshold)
|
||||
.Sum(w => (long)w.Length);
|
||||
|
||||
var size = data.Length;
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
using System;
|
||||
using System.Net;
|
||||
using System.Net.Http.Json;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Scanner.WebService.Contracts;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.WebService.Tests;
|
||||
|
||||
public sealed partial class ScansEndpointsTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task EntropyEndpoint_AttachesSnapshot_AndSurfacesInStatus()
|
||||
{
|
||||
using var secrets = new TestSurfaceSecretsScope();
|
||||
using var factory = new ScannerApplicationFactory(cfg =>
|
||||
{
|
||||
cfg["scanner:authority:enabled"] = "false";
|
||||
cfg["scanner:authority:allowAnonymousFallback"] = "true";
|
||||
});
|
||||
|
||||
using var client = factory.CreateClient();
|
||||
|
||||
var submitResponse = await client.PostAsJsonAsync("/api/v1/scans", new
|
||||
{
|
||||
image = new { digest = "sha256:image-demo" }
|
||||
});
|
||||
submitResponse.EnsureSuccessStatusCode();
|
||||
|
||||
var submit = await submitResponse.Content.ReadFromJsonAsync<ScanSubmitResponse>();
|
||||
Assert.NotNull(submit);
|
||||
|
||||
var entropyPayload = new EntropyIngestRequest(
|
||||
ImageOpaqueRatio: 0.42,
|
||||
Layers: new[]
|
||||
{
|
||||
new EntropyLayerRequest("sha256:layer-demo", 0.35, 3500, 10_000)
|
||||
});
|
||||
|
||||
var attachResponse = await client.PostAsJsonAsync($"/api/v1/scans/{submit!.ScanId}/entropy", entropyPayload);
|
||||
Assert.Equal(HttpStatusCode.Accepted, attachResponse.StatusCode);
|
||||
|
||||
var status = await client.GetFromJsonAsync<ScanStatusResponse>($"/api/v1/scans/{submit.ScanId}");
|
||||
Assert.NotNull(status);
|
||||
Assert.NotNull(status!.Entropy);
|
||||
Assert.Equal(0.42, status.Entropy!.ImageOpaqueRatio, 3);
|
||||
Assert.Single(status.Entropy!.Layers);
|
||||
var layer = status.Entropy!.Layers[0];
|
||||
Assert.Equal("sha256:layer-demo", layer.LayerDigest);
|
||||
Assert.Equal(0.35, layer.OpaqueRatio, 3);
|
||||
Assert.Equal(3500, layer.OpaqueBytes);
|
||||
Assert.Equal(10_000, layer.TotalBytes);
|
||||
}
|
||||
}
|
||||
@@ -167,6 +167,12 @@ public sealed partial class ScansEndpointsTests
|
||||
|
||||
public ValueTask<ScanSnapshot?> TryFindByTargetAsync(string? reference, string? digest, CancellationToken cancellationToken)
|
||||
=> _inner.TryFindByTargetAsync(reference, digest, cancellationToken);
|
||||
|
||||
public ValueTask<bool> AttachReplayAsync(ScanId scanId, ReplayArtifacts replay, CancellationToken cancellationToken)
|
||||
=> _inner.AttachReplayAsync(scanId, replay, cancellationToken);
|
||||
|
||||
public ValueTask<bool> AttachEntropyAsync(ScanId scanId, EntropySnapshot entropy, CancellationToken cancellationToken)
|
||||
=> _inner.AttachEntropyAsync(scanId, entropy, cancellationToken);
|
||||
}
|
||||
|
||||
private sealed class StubEntryTraceResultStore : IEntryTraceResultStore
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
using System;
|
||||
using StellaOps.Scanner.Worker.Determinism;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Tests.Determinism;
|
||||
|
||||
public class DeterministicTimeProviderTests
|
||||
{
|
||||
[Fact]
|
||||
public void GetUtcNow_ReturnsFixedInstant()
|
||||
{
|
||||
var fixedInstant = new DateTimeOffset(2024, 01, 01, 12, 0, 0, TimeSpan.Zero);
|
||||
var provider = new DeterministicTimeProvider(fixedInstant);
|
||||
|
||||
Assert.Equal(fixedInstant, provider.GetUtcNow());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeterministicRandomProvider_ReturnsStableSequence_WhenSeeded()
|
||||
{
|
||||
var provider = new DeterministicRandomProvider(1234);
|
||||
var rng1 = provider.Create();
|
||||
var rng2 = provider.Create();
|
||||
|
||||
var seq1 = new[] { rng1.Next(), rng1.Next(), rng1.Next() };
|
||||
var seq2 = new[] { rng2.Next(), rng2.Next(), rng2.Next() };
|
||||
|
||||
Assert.Equal(seq1, seq2);
|
||||
}
|
||||
}
|
||||
@@ -26,7 +26,7 @@ public class EntropyStageExecutorTests
|
||||
|
||||
var fileEntries = new List<ScanFileEntry>
|
||||
{
|
||||
new ScanFileEntry(tmp, sizeBytes: bytes.LongLength, kind: "blob", metadata: new Dictionary<string, string>())
|
||||
new ScanFileEntry(tmp, SizeBytes: bytes.LongLength, Kind: "blob", Metadata: new Dictionary<string, string>())
|
||||
};
|
||||
|
||||
var lease = new StubLease("job-1", "scan-1", imageDigest: "sha256:test", layerDigest: "sha256:layer");
|
||||
@@ -55,13 +55,25 @@ public class EntropyStageExecutorTests
|
||||
{
|
||||
JobId = jobId;
|
||||
ScanId = scanId;
|
||||
ImageDigest = imageDigest;
|
||||
LayerDigest = layerDigest;
|
||||
Metadata = new Dictionary<string, string>
|
||||
{
|
||||
["image.digest"] = imageDigest,
|
||||
["layerDigest"] = layerDigest
|
||||
};
|
||||
}
|
||||
|
||||
public string JobId { get; }
|
||||
public string ScanId { get; }
|
||||
public string? ImageDigest { get; }
|
||||
public string? LayerDigest { get; }
|
||||
public int Attempt => 1;
|
||||
public DateTimeOffset EnqueuedAtUtc => DateTimeOffset.UtcNow;
|
||||
public DateTimeOffset LeasedAtUtc => DateTimeOffset.UtcNow;
|
||||
public TimeSpan LeaseDuration => TimeSpan.FromMinutes(5);
|
||||
public IReadOnlyDictionary<string, string> Metadata { get; }
|
||||
|
||||
public ValueTask RenewAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask;
|
||||
public ValueTask CompleteAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask;
|
||||
public ValueTask AbandonAsync(string reason, CancellationToken cancellationToken) => ValueTask.CompletedTask;
|
||||
public ValueTask PoisonAsync(string reason, CancellationToken cancellationToken) => ValueTask.CompletedTask;
|
||||
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Analyzers.Lang;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Ruby;
|
||||
using StellaOps.Scanner.Core.Contracts;
|
||||
using StellaOps.Scanner.Core.Entropy;
|
||||
using StellaOps.Scanner.EntryTrace;
|
||||
using StellaOps.Scanner.Surface.Env;
|
||||
using StellaOps.Scanner.Surface.FS;
|
||||
@@ -120,6 +121,62 @@ public sealed class SurfaceManifestStageExecutorTests
|
||||
Assert.Contains(payloadMetrics, m => Equals("layer.fragments", m["surface.kind"]));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_IncludesEntropyPayloads_WhenPresent()
|
||||
{
|
||||
var metrics = new ScannerWorkerMetrics();
|
||||
var publisher = new TestSurfaceManifestPublisher("tenant-a");
|
||||
var cache = new RecordingSurfaceCache();
|
||||
var environment = new TestSurfaceEnvironment("tenant-a");
|
||||
var hash = CreateCryptoHash();
|
||||
|
||||
var executor = new SurfaceManifestStageExecutor(
|
||||
publisher,
|
||||
cache,
|
||||
environment,
|
||||
metrics,
|
||||
NullLogger<SurfaceManifestStageExecutor>.Instance,
|
||||
hash,
|
||||
new NullRubyPackageInventoryStore());
|
||||
|
||||
var context = CreateContext();
|
||||
|
||||
var entropyReport = new EntropyReport(
|
||||
ImageDigest: "sha256:image",
|
||||
LayerDigest: "sha256:layer",
|
||||
Files: new[]
|
||||
{
|
||||
new EntropyFileReport(
|
||||
Path: "/bin/app",
|
||||
Size: 1024 * 32,
|
||||
OpaqueBytes: 1024 * 8,
|
||||
OpaqueRatio: 0.25,
|
||||
Flags: Array.Empty<string>(),
|
||||
Windows: Array.Empty<EntropyFileWindow>())
|
||||
},
|
||||
ImageOpaqueRatio: 0.2);
|
||||
|
||||
var entropySummary = new EntropyLayerSummary(
|
||||
LayerDigest: "sha256:layer",
|
||||
OpaqueBytes: 1024 * 8,
|
||||
TotalBytes: 1024 * 32,
|
||||
OpaqueRatio: 0.25,
|
||||
Indicators: Array.Empty<string>());
|
||||
|
||||
context.Analysis.Set(ScanAnalysisKeys.EntropyReport, entropyReport);
|
||||
context.Analysis.Set(ScanAnalysisKeys.EntropyLayerSummary, entropySummary);
|
||||
|
||||
await executor.ExecuteAsync(context, CancellationToken.None);
|
||||
|
||||
Assert.Equal(1, publisher.PublishCalls);
|
||||
Assert.NotNull(publisher.LastRequest);
|
||||
Assert.Contains(publisher.LastRequest!.Payloads, p => p.Kind == "entropy.report");
|
||||
Assert.Contains(publisher.LastRequest!.Payloads, p => p.Kind == "entropy.layer-summary");
|
||||
|
||||
// Two payloads + manifest persisted to cache.
|
||||
Assert.Equal(3, cache.Entries.Count);
|
||||
}
|
||||
|
||||
private static ScanJobContext CreateContext()
|
||||
{
|
||||
var lease = new FakeJobLease();
|
||||
|
||||
Reference in New Issue
Block a user