up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled

This commit is contained in:
StellaOps Bot
2025-12-13 00:20:26 +02:00
parent e1f1bef4c1
commit 564df71bfb
2376 changed files with 334389 additions and 328032 deletions

View File

@@ -1,5 +1,5 @@
using System;
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Collections.ObjectModel;
using System.Diagnostics.Metrics;
@@ -13,6 +13,7 @@ using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Scanner.Analyzers.Lang;
using StellaOps.Scanner.Analyzers.Lang.Plugin;
using StellaOps.Scanner.Analyzers.OS;
using StellaOps.Scanner.Analyzers.OS.Abstractions;
using StellaOps.Scanner.Analyzers.OS.Plugin;
using StellaOps.Scanner.Core.Contracts;
@@ -24,12 +25,12 @@ using StellaOps.Scanner.Worker.Diagnostics;
using StellaOps.Scanner.Worker.Processing;
using StellaOps.Scanner.Worker.Tests.TestInfrastructure;
using Xunit;
using WorkerOptions = StellaOps.Scanner.Worker.Options.ScannerWorkerOptions;
namespace StellaOps.Scanner.Worker.Tests;
public sealed class CompositeScanAnalyzerDispatcherTests
{
using WorkerOptions = StellaOps.Scanner.Worker.Options.ScannerWorkerOptions;
namespace StellaOps.Scanner.Worker.Tests;
public sealed class CompositeScanAnalyzerDispatcherTests
{
[Fact]
public async Task ExecuteAsync_RunsLanguageAnalyzers_StoresResults()
{
@@ -67,6 +68,7 @@ public sealed class CompositeScanAnalyzerDispatcherTests
serviceCollection.AddSingleton(TimeProvider.System);
serviceCollection.AddSurfaceEnvironment(options => options.ComponentName = "Scanner.Worker");
serviceCollection.AddSurfaceValidation();
serviceCollection.AddSingleton<ISurfaceValidatorRunner, NoopSurfaceValidatorRunner>();
serviceCollection.AddSurfaceFileCache(options => options.RootDirectory = cacheRoot.Path);
serviceCollection.AddSurfaceSecrets();
@@ -145,42 +147,203 @@ public sealed class CompositeScanAnalyzerDispatcherTests
services?.Dispose();
}
}
private sealed class FakeOsCatalog : IOSAnalyzerPluginCatalog
{
public IReadOnlyCollection<IOSAnalyzerPlugin> Plugins => Array.Empty<IOSAnalyzerPlugin>();
public void LoadFromDirectory(string directory, bool seal = true)
{
}
public IReadOnlyList<IOSPackageAnalyzer> CreateAnalyzers(IServiceProvider services) => Array.Empty<IOSPackageAnalyzer>();
}
private sealed class FakeLanguageCatalog : ILanguageAnalyzerPluginCatalog
{
private readonly IReadOnlyList<ILanguageAnalyzer> _analyzers;
public FakeLanguageCatalog(params ILanguageAnalyzer[] analyzers)
{
_analyzers = analyzers ?? Array.Empty<ILanguageAnalyzer>();
}
public IReadOnlyCollection<ILanguageAnalyzerPlugin> Plugins => Array.Empty<ILanguageAnalyzerPlugin>();
public void LoadFromDirectory(string directory, bool seal = true)
{
}
public IReadOnlyList<ILanguageAnalyzer> CreateAnalyzers(IServiceProvider services) => _analyzers;
}
private sealed class FakeLanguageAnalyzer : ILanguageAnalyzer
{
public string Id => "lang.fake";
public string DisplayName => "Fake Language Analyzer";
[Fact]
public async Task ExecuteAsync_RunsOsAnalyzers_UsesSurfaceCache()
{
using var rootfs = new TempDirectory();
using var cacheRoot = new TempDirectory();
Environment.SetEnvironmentVariable("SCANNER_SURFACE_FS_ENDPOINT", "https://surface.test");
Environment.SetEnvironmentVariable("SCANNER_SURFACE_FS_BUCKET", "unit-test-bucket");
Environment.SetEnvironmentVariable("SCANNER_SURFACE_CACHE_ROOT", cacheRoot.Path);
Environment.SetEnvironmentVariable("SCANNER_SURFACE_SECRETS_PROVIDER", "inline");
Environment.SetEnvironmentVariable("SCANNER_SURFACE_SECRETS_TENANT", "testtenant");
Environment.SetEnvironmentVariable(
"SURFACE_SECRET_TESTTENANT_SCANNERWORKEROSANALYZERS_REGISTRY_DEFAULT",
Convert.ToBase64String(Encoding.UTF8.GetBytes("token-placeholder")));
var dpkgStatusPath = Path.Combine(rootfs.Path, "var", "lib", "dpkg", "status");
Directory.CreateDirectory(Path.GetDirectoryName(dpkgStatusPath)!);
await File.WriteAllTextAsync(dpkgStatusPath, "Package: demo\nStatus: install ok installed\n", CancellationToken.None);
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
{
{ ScanMetadataKeys.RootFilesystemPath, rootfs.Path },
{ ScanMetadataKeys.WorkspacePath, rootfs.Path },
};
var analyzer = new FakeOsAnalyzer();
var osCatalog = new FakeOsCatalog(analyzer);
var languageCatalog = new FakeLanguageCatalog();
long hits = 0;
long misses = 0;
MeterListener? meterListener = null;
ServiceProvider? services = null;
try
{
var serviceCollection = new ServiceCollection();
serviceCollection.AddSingleton<IConfiguration>(new ConfigurationBuilder().Build());
serviceCollection.AddLogging(builder => builder.SetMinimumLevel(LogLevel.Debug));
serviceCollection.AddSingleton(TimeProvider.System);
serviceCollection.AddSurfaceEnvironment(options => options.ComponentName = "Scanner.Worker");
serviceCollection.AddSurfaceValidation();
serviceCollection.AddSingleton<ISurfaceValidatorRunner, NoopSurfaceValidatorRunner>();
serviceCollection.AddSurfaceFileCache(options => options.RootDirectory = cacheRoot.Path);
serviceCollection.AddSurfaceSecrets();
var metrics = new ScannerWorkerMetrics();
serviceCollection.AddSingleton(metrics);
meterListener = new MeterListener();
meterListener.InstrumentPublished = (instrument, listener) =>
{
if (instrument.Meter.Name == ScannerWorkerInstrumentation.MeterName &&
(instrument.Name == "scanner_worker_os_cache_hits_total" || instrument.Name == "scanner_worker_os_cache_misses_total"))
{
listener.EnableMeasurementEvents(instrument);
}
};
meterListener.SetMeasurementEventCallback<long>((instrument, measurement, tags, state) =>
{
if (instrument.Name == "scanner_worker_os_cache_hits_total")
{
Interlocked.Add(ref hits, measurement);
}
else if (instrument.Name == "scanner_worker_os_cache_misses_total")
{
Interlocked.Add(ref misses, measurement);
}
});
meterListener.Start();
services = serviceCollection.BuildServiceProvider();
var scopeFactory = services.GetRequiredService<IServiceScopeFactory>();
var loggerFactory = services.GetRequiredService<ILoggerFactory>();
var options = Microsoft.Extensions.Options.Options.Create(new WorkerOptions());
var dispatcher = new CompositeScanAnalyzerDispatcher(
scopeFactory,
osCatalog,
languageCatalog,
options,
loggerFactory.CreateLogger<CompositeScanAnalyzerDispatcher>(),
metrics,
new TestCryptoHash());
var lease = new TestJobLease(metadata);
var context = new ScanJobContext(lease, TimeProvider.System, TimeProvider.System.GetUtcNow(), CancellationToken.None);
await dispatcher.ExecuteAsync(context, CancellationToken.None);
var leaseSecond = new TestJobLease(metadata);
var contextSecond = new ScanJobContext(leaseSecond, TimeProvider.System, TimeProvider.System.GetUtcNow(), CancellationToken.None);
await dispatcher.ExecuteAsync(contextSecond, CancellationToken.None);
Assert.Equal(1, analyzer.InvocationCount);
Assert.True(context.Analysis.TryGet<Dictionary<string, OSPackageAnalyzerResult>>(ScanAnalysisKeys.OsPackageAnalyzers, out var results));
Assert.Single(results);
Assert.True(context.Analysis.TryGet<ImmutableArray<LayerComponentFragment>>(ScanAnalysisKeys.OsComponentFragments, out var fragments));
Assert.False(fragments.IsDefaultOrEmpty);
Assert.Equal(1, hits);
Assert.Equal(1, misses);
}
finally
{
Environment.SetEnvironmentVariable("SCANNER_SURFACE_FS_ENDPOINT", null);
Environment.SetEnvironmentVariable("SCANNER_SURFACE_FS_BUCKET", null);
Environment.SetEnvironmentVariable("SCANNER_SURFACE_CACHE_ROOT", null);
Environment.SetEnvironmentVariable("SCANNER_SURFACE_SECRETS_PROVIDER", null);
Environment.SetEnvironmentVariable("SCANNER_SURFACE_SECRETS_TENANT", null);
Environment.SetEnvironmentVariable("SURFACE_SECRET_TESTTENANT_SCANNERWORKEROSANALYZERS_REGISTRY_DEFAULT", null);
meterListener?.Dispose();
services?.Dispose();
}
}
private sealed class FakeOsCatalog : IOSAnalyzerPluginCatalog
{
private readonly IReadOnlyList<IOSPackageAnalyzer> _analyzers;
public FakeOsCatalog(params IOSPackageAnalyzer[] analyzers)
{
_analyzers = analyzers ?? Array.Empty<IOSPackageAnalyzer>();
}
public IReadOnlyCollection<IOSAnalyzerPlugin> Plugins => Array.Empty<IOSAnalyzerPlugin>();
public void LoadFromDirectory(string directory, bool seal = true)
{
}
public IReadOnlyList<IOSPackageAnalyzer> CreateAnalyzers(IServiceProvider services) => _analyzers;
}
private sealed class FakeLanguageCatalog : ILanguageAnalyzerPluginCatalog
{
private readonly IReadOnlyList<ILanguageAnalyzer> _analyzers;
public FakeLanguageCatalog(params ILanguageAnalyzer[] analyzers)
{
_analyzers = analyzers ?? Array.Empty<ILanguageAnalyzer>();
}
public IReadOnlyCollection<ILanguageAnalyzerPlugin> Plugins => Array.Empty<ILanguageAnalyzerPlugin>();
public void LoadFromDirectory(string directory, bool seal = true)
{
}
public IReadOnlyList<ILanguageAnalyzer> CreateAnalyzers(IServiceProvider services) => _analyzers;
}
private sealed class NoopSurfaceValidatorRunner : ISurfaceValidatorRunner
{
public ValueTask<SurfaceValidationResult> RunAllAsync(SurfaceValidationContext context, CancellationToken cancellationToken = default)
=> ValueTask.FromResult(SurfaceValidationResult.Success());
public ValueTask EnsureAsync(SurfaceValidationContext context, CancellationToken cancellationToken = default)
=> ValueTask.CompletedTask;
}
private sealed class FakeOsAnalyzer : IOSPackageAnalyzer
{
public string AnalyzerId => "dpkg";
public int InvocationCount { get; private set; }
private int _invocationCount;
public ValueTask<OSPackageAnalyzerResult> AnalyzeAsync(OSPackageAnalyzerContext context, CancellationToken cancellationToken)
{
Interlocked.Increment(ref _invocationCount);
InvocationCount = _invocationCount;
var package = new OSPackageRecord(
analyzerId: AnalyzerId,
packageUrl: "pkg:deb/debian/demo@1.0?arch=amd64",
name: "demo",
version: "1.0",
architecture: "amd64",
evidenceSource: PackageEvidenceSource.DpkgStatus,
files: new[] { new OSPackageFileEvidence("/usr/bin/demo") });
var telemetry = new OSAnalyzerTelemetry(TimeSpan.Zero, 1, 1);
return ValueTask.FromResult(new OSPackageAnalyzerResult(AnalyzerId, new[] { package }, telemetry));
}
}
private sealed class FakeLanguageAnalyzer : ILanguageAnalyzer
{
public string Id => "lang.fake";
public string DisplayName => "Fake Language Analyzer";
public int InvocationCount { get; private set; }
public ValueTask AnalyzeAsync(LanguageAnalyzerContext context, LanguageComponentWriter writer, CancellationToken cancellationToken)
@@ -198,69 +361,69 @@ public sealed class CompositeScanAnalyzerDispatcherTests
private int _invocationCount;
}
private sealed class TestJobLease : IScanJobLease
{
private readonly Dictionary<string, string> _metadata;
public TestJobLease(Dictionary<string, string> metadata)
{
_metadata = metadata;
JobId = Guid.NewGuid().ToString("n");
ScanId = $"scan-{Guid.NewGuid():n}";
Attempt = 1;
EnqueuedAtUtc = DateTimeOffset.UtcNow.AddSeconds(-1);
LeasedAtUtc = DateTimeOffset.UtcNow;
LeaseDuration = TimeSpan.FromMinutes(5);
}
public string JobId { get; }
public string ScanId { get; }
public int Attempt { get; }
public DateTimeOffset EnqueuedAtUtc { get; }
public DateTimeOffset LeasedAtUtc { get; }
public TimeSpan LeaseDuration { get; }
public IReadOnlyDictionary<string, string> Metadata => _metadata;
public ValueTask RenewAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask;
public ValueTask CompleteAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask;
public ValueTask AbandonAsync(string reason, CancellationToken cancellationToken) => ValueTask.CompletedTask;
public ValueTask PoisonAsync(string reason, CancellationToken cancellationToken) => ValueTask.CompletedTask;
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
}
private sealed class TempDirectory : IDisposable
{
public TempDirectory()
{
Path = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"stellaops-test-{Guid.NewGuid():n}");
Directory.CreateDirectory(Path);
}
public string Path { get; }
public void Dispose()
{
try
{
if (Directory.Exists(Path))
{
Directory.Delete(Path, recursive: true);
}
}
catch
{
}
}
}
}
private sealed class TestJobLease : IScanJobLease
{
private readonly Dictionary<string, string> _metadata;
public TestJobLease(Dictionary<string, string> metadata)
{
_metadata = metadata;
JobId = Guid.NewGuid().ToString("n");
ScanId = $"scan-{Guid.NewGuid():n}";
Attempt = 1;
EnqueuedAtUtc = DateTimeOffset.UtcNow.AddSeconds(-1);
LeasedAtUtc = DateTimeOffset.UtcNow;
LeaseDuration = TimeSpan.FromMinutes(5);
}
public string JobId { get; }
public string ScanId { get; }
public int Attempt { get; }
public DateTimeOffset EnqueuedAtUtc { get; }
public DateTimeOffset LeasedAtUtc { get; }
public TimeSpan LeaseDuration { get; }
public IReadOnlyDictionary<string, string> Metadata => _metadata;
public ValueTask RenewAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask;
public ValueTask CompleteAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask;
public ValueTask AbandonAsync(string reason, CancellationToken cancellationToken) => ValueTask.CompletedTask;
public ValueTask PoisonAsync(string reason, CancellationToken cancellationToken) => ValueTask.CompletedTask;
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
}
private sealed class TempDirectory : IDisposable
{
public TempDirectory()
{
Path = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"stellaops-test-{Guid.NewGuid():n}");
Directory.CreateDirectory(Path);
}
public string Path { get; }
public void Dispose()
{
try
{
if (Directory.Exists(Path))
{
Directory.Delete(Path, recursive: true);
}
}
catch
{
}
}
}
}

View File

@@ -6,6 +6,7 @@ using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using StellaOps.Cryptography;
using StellaOps.Scanner.Worker.Options;
using StellaOps.Scanner.Worker.Processing.Surface;
using Xunit;
@@ -24,12 +25,16 @@ public sealed class HmacDsseEnvelopeSignerTests
signing.KeyId = "scanner-hmac";
});
var signer = new HmacDsseEnvelopeSigner(options, NullLogger<HmacDsseEnvelopeSigner>.Instance, new ServiceCollection().BuildServiceProvider());
var signer = new HmacDsseEnvelopeSigner(
options,
DefaultCryptoHmac.CreateForTests(),
NullLogger<HmacDsseEnvelopeSigner>.Instance,
new ServiceCollection().BuildServiceProvider());
var payload = Encoding.UTF8.GetBytes("{\"hello\":\"world\"}");
var envelope = await signer.SignAsync("application/json", payload, "test.kind", "root", view: null, CancellationToken.None);
var json = JsonDocument.Parse(envelope.Content.Span);
var json = JsonDocument.Parse(envelope.Content);
var sig = json.RootElement.GetProperty("signatures")[0].GetProperty("sig").GetString();
var expectedSig = ComputeExpectedSignature("application/json", payload, "a2V5LXNlY3JldA==");
@@ -49,11 +54,15 @@ public sealed class HmacDsseEnvelopeSignerTests
signing.AllowDeterministicFallback = true;
});
var signer = new HmacDsseEnvelopeSigner(options, NullLogger<HmacDsseEnvelopeSigner>.Instance, new ServiceCollection().BuildServiceProvider());
var signer = new HmacDsseEnvelopeSigner(
options,
DefaultCryptoHmac.CreateForTests(),
NullLogger<HmacDsseEnvelopeSigner>.Instance,
new ServiceCollection().BuildServiceProvider());
var payload = Encoding.UTF8.GetBytes("abc");
var envelope = await signer.SignAsync("text/plain", payload, "kind", "root", view: null, CancellationToken.None);
var json = JsonDocument.Parse(envelope.Content.Span);
var json = JsonDocument.Parse(envelope.Content);
var sig = json.RootElement.GetProperty("signatures")[0].GetProperty("sig").GetString();
// Deterministic signer encodes sha256 hex of payload as signature.

View File

@@ -1,121 +1,128 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using StellaOps.Scanner.Worker.Options;
using StellaOps.Scanner.Worker.Processing;
using Xunit;
namespace StellaOps.Scanner.Worker.Tests;
public sealed class LeaseHeartbeatServiceTests
{
[Fact]
public async Task RunAsync_RespectsSafetyFactorBudget()
{
var options = new ScannerWorkerOptions
{
MaxConcurrentJobs = 1,
};
options.Queue.HeartbeatSafetyFactor = 3.0;
options.Queue.MinHeartbeatInterval = TimeSpan.FromSeconds(5);
options.Queue.MaxHeartbeatInterval = TimeSpan.FromSeconds(60);
options.Queue.SetHeartbeatRetryDelays(Array.Empty<TimeSpan>());
options.Queue.MaxHeartbeatJitterMilliseconds = 750;
var optionsMonitor = new StaticOptionsMonitor<ScannerWorkerOptions>(options);
using var cts = new CancellationTokenSource();
var scheduler = new RecordingDelayScheduler(cts);
var lease = new TestJobLease(TimeSpan.FromSeconds(90));
var service = new LeaseHeartbeatService(TimeProvider.System, scheduler, optionsMonitor, NullLogger<LeaseHeartbeatService>.Instance);
await service.RunAsync(lease, cts.Token);
var delay = Assert.Single(scheduler.Delays);
var expectedMax = TimeSpan.FromTicks((long)(lease.LeaseDuration.Ticks / Math.Max(3.0, options.Queue.HeartbeatSafetyFactor)));
Assert.True(delay <= expectedMax, $"Heartbeat delay {delay} should stay within safety factor budget {expectedMax}.");
Assert.True(delay >= options.Queue.MinHeartbeatInterval, $"Heartbeat delay {delay} should respect minimum interval {options.Queue.MinHeartbeatInterval}.");
}
private sealed class RecordingDelayScheduler : IDelayScheduler
{
private readonly CancellationTokenSource _cts;
public RecordingDelayScheduler(CancellationTokenSource cts)
{
_cts = cts ?? throw new ArgumentNullException(nameof(cts));
}
public List<TimeSpan> Delays { get; } = new();
public Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken)
{
Delays.Add(delay);
_cts.Cancel();
return Task.CompletedTask;
}
}
private sealed class TestJobLease : IScanJobLease
{
public TestJobLease(TimeSpan leaseDuration)
{
LeaseDuration = leaseDuration;
EnqueuedAtUtc = DateTimeOffset.UtcNow - leaseDuration;
LeasedAtUtc = DateTimeOffset.UtcNow;
}
public string JobId { get; } = Guid.NewGuid().ToString("n");
public string ScanId { get; } = $"scan-{Guid.NewGuid():n}";
public int Attempt { get; } = 1;
public DateTimeOffset EnqueuedAtUtc { get; }
public DateTimeOffset LeasedAtUtc { get; }
public TimeSpan LeaseDuration { get; }
public IReadOnlyDictionary<string, string> Metadata { get; } = new Dictionary<string, string>();
public ValueTask RenewAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask;
public ValueTask CompleteAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask;
public ValueTask AbandonAsync(string reason, CancellationToken cancellationToken) => ValueTask.CompletedTask;
public ValueTask PoisonAsync(string reason, CancellationToken cancellationToken) => ValueTask.CompletedTask;
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
}
private sealed class StaticOptionsMonitor<TOptions> : IOptionsMonitor<TOptions>
where TOptions : class
{
private readonly TOptions _value;
public StaticOptionsMonitor(TOptions value)
{
_value = value ?? throw new ArgumentNullException(nameof(value));
}
public TOptions CurrentValue => _value;
public TOptions Get(string? name) => _value;
public IDisposable OnChange(Action<TOptions, string?> listener) => NullDisposable.Instance;
private sealed class NullDisposable : IDisposable
{
public static readonly NullDisposable Instance = new();
public void Dispose()
{
}
}
}
}
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using StellaOps.Scanner.Worker.Options;
using StellaOps.Scanner.Worker.Determinism;
using StellaOps.Scanner.Worker.Processing;
using Xunit;
namespace StellaOps.Scanner.Worker.Tests;
public sealed class LeaseHeartbeatServiceTests
{
[Fact]
public async Task RunAsync_RespectsSafetyFactorBudget()
{
var options = new ScannerWorkerOptions
{
MaxConcurrentJobs = 1,
};
options.Queue.HeartbeatSafetyFactor = 3.0;
options.Queue.MinHeartbeatInterval = TimeSpan.FromSeconds(5);
options.Queue.MaxHeartbeatInterval = TimeSpan.FromSeconds(60);
options.Queue.SetHeartbeatRetryDelays(Array.Empty<TimeSpan>());
options.Queue.MaxHeartbeatJitterMilliseconds = 750;
var optionsMonitor = new StaticOptionsMonitor<ScannerWorkerOptions>(options);
using var cts = new CancellationTokenSource();
var scheduler = new RecordingDelayScheduler(cts);
var lease = new TestJobLease(TimeSpan.FromSeconds(90));
var randomProvider = new DeterministicRandomProvider(seed: 1337);
var service = new LeaseHeartbeatService(
TimeProvider.System,
scheduler,
optionsMonitor,
randomProvider,
NullLogger<LeaseHeartbeatService>.Instance);
await service.RunAsync(lease, cts.Token);
var delay = Assert.Single(scheduler.Delays);
var expectedMax = TimeSpan.FromTicks((long)(lease.LeaseDuration.Ticks / Math.Max(3.0, options.Queue.HeartbeatSafetyFactor)));
Assert.True(delay <= expectedMax, $"Heartbeat delay {delay} should stay within safety factor budget {expectedMax}.");
Assert.True(delay >= options.Queue.MinHeartbeatInterval, $"Heartbeat delay {delay} should respect minimum interval {options.Queue.MinHeartbeatInterval}.");
}
private sealed class RecordingDelayScheduler : IDelayScheduler
{
private readonly CancellationTokenSource _cts;
public RecordingDelayScheduler(CancellationTokenSource cts)
{
_cts = cts ?? throw new ArgumentNullException(nameof(cts));
}
public List<TimeSpan> Delays { get; } = new();
public Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken)
{
Delays.Add(delay);
_cts.Cancel();
return Task.CompletedTask;
}
}
private sealed class TestJobLease : IScanJobLease
{
public TestJobLease(TimeSpan leaseDuration)
{
LeaseDuration = leaseDuration;
EnqueuedAtUtc = DateTimeOffset.UtcNow - leaseDuration;
LeasedAtUtc = DateTimeOffset.UtcNow;
}
public string JobId { get; } = Guid.NewGuid().ToString("n");
public string ScanId { get; } = $"scan-{Guid.NewGuid():n}";
public int Attempt { get; } = 1;
public DateTimeOffset EnqueuedAtUtc { get; }
public DateTimeOffset LeasedAtUtc { get; }
public TimeSpan LeaseDuration { get; }
public IReadOnlyDictionary<string, string> Metadata { get; } = new Dictionary<string, string>();
public ValueTask RenewAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask;
public ValueTask CompleteAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask;
public ValueTask AbandonAsync(string reason, CancellationToken cancellationToken) => ValueTask.CompletedTask;
public ValueTask PoisonAsync(string reason, CancellationToken cancellationToken) => ValueTask.CompletedTask;
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
}
private sealed class StaticOptionsMonitor<TOptions> : IOptionsMonitor<TOptions>
where TOptions : class
{
private readonly TOptions _value;
public StaticOptionsMonitor(TOptions value)
{
_value = value ?? throw new ArgumentNullException(nameof(value));
}
public TOptions CurrentValue => _value;
public TOptions Get(string? name) => _value;
public IDisposable OnChange(Action<TOptions, string?> listener) => NullDisposable.Instance;
private sealed class NullDisposable : IDisposable
{
public static readonly NullDisposable Instance = new();
public void Dispose()
{
}
}
}
}

View File

@@ -1,245 +1,245 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Scanner.Queue;
using StellaOps.Scanner.Worker.Diagnostics;
using StellaOps.Scanner.Worker.Hosting;
using StellaOps.Scanner.Worker.Options;
using StellaOps.Scanner.Worker.Processing;
using StellaOps.Scanner.Worker.Tests.TestInfrastructure;
using Xunit;
namespace StellaOps.Scanner.Worker.Tests;
public sealed class RedisWorkerSmokeTests
{
[Fact]
public async Task Worker_CompletesJob_ViaRedisQueue()
{
var flag = Environment.GetEnvironmentVariable("STELLAOPS_REDIS_SMOKE");
if (string.IsNullOrWhiteSpace(flag))
{
return;
}
var redisConnection = Environment.GetEnvironmentVariable("STELLAOPS_REDIS_CONNECTION") ?? "localhost:6379";
var streamName = $"scanner:jobs:{Guid.NewGuid():n}";
var consumerGroup = $"worker-smoke-{Guid.NewGuid():n}";
var configuration = BuildQueueConfiguration(redisConnection, streamName, consumerGroup);
var queueOptions = new ScannerQueueOptions();
configuration.GetSection("scanner:queue").Bind(queueOptions);
var workerOptions = new ScannerWorkerOptions
{
MaxConcurrentJobs = 1,
};
workerOptions.Queue.HeartbeatSafetyFactor = 3.0;
workerOptions.Queue.MinHeartbeatInterval = TimeSpan.FromSeconds(2);
workerOptions.Queue.MaxHeartbeatInterval = TimeSpan.FromSeconds(8);
workerOptions.Queue.SetHeartbeatRetryDelays(new[]
{
TimeSpan.FromMilliseconds(200),
TimeSpan.FromMilliseconds(500),
TimeSpan.FromSeconds(1),
});
var services = new ServiceCollection();
services.AddLogging(builder =>
{
builder.SetMinimumLevel(LogLevel.Debug);
builder.AddConsole();
});
services.AddSingleton(TimeProvider.System);
services.AddScannerQueue(configuration, "scanner:queue");
services.AddSingleton<IScanJobSource, QueueBackedScanJobSource>();
services.AddSingleton<QueueBackedScanJobSourceDependencies>();
services.AddSingleton(queueOptions);
services.AddSingleton<IScanAnalyzerDispatcher, SmokeAnalyzerDispatcher>();
services.AddSingleton<IScanStageExecutor, AnalyzerStageExecutor>();
services.AddSingleton<ScannerWorkerMetrics>();
services.AddSingleton<ScanProgressReporter>();
services.AddSingleton<ScanJobProcessor>();
services.AddSingleton<LeaseHeartbeatService>();
services.AddSingleton<IDelayScheduler, SystemDelayScheduler>();
services.AddSingleton<IOptionsMonitor<ScannerWorkerOptions>>(new StaticOptionsMonitor<ScannerWorkerOptions>(workerOptions));
services.AddSingleton<ScannerWorkerHostedService>();
using var provider = services.BuildServiceProvider();
var queue = provider.GetRequiredService<IScanQueue>();
var jobId = $"job-{Guid.NewGuid():n}";
var scanId = $"scan-{Guid.NewGuid():n}";
await queue.EnqueueAsync(new ScanQueueMessage(jobId, Encoding.UTF8.GetBytes("smoke"))
{
Attributes = new Dictionary<string, string>(StringComparer.Ordinal)
{
["scanId"] = scanId,
["queue"] = "redis",
}
});
var hostedService = provider.GetRequiredService<ScannerWorkerHostedService>();
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
await hostedService.StartAsync(cts.Token);
var smokeObserver = provider.GetRequiredService<QueueBackedScanJobSourceDependencies>();
await smokeObserver.JobCompleted.Task.WaitAsync(TimeSpan.FromSeconds(20));
await hostedService.StopAsync(CancellationToken.None);
}
private static IConfiguration BuildQueueConfiguration(string connection, string stream, string consumerGroup)
{
return new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary<string, string?>
{
["scanner:queue:kind"] = "redis",
["scanner:queue:defaultLeaseDuration"] = "00:00:30",
["scanner:queue:redis:connectionString"] = connection,
["scanner:queue:redis:streamName"] = stream,
["scanner:queue:redis:consumerGroup"] = consumerGroup,
["scanner:queue:redis:idempotencyKeyPrefix"] = $"{stream}:idemp:",
["scanner:queue:redis:initializationTimeout"] = "00:00:10",
})
.Build();
}
private sealed class SmokeAnalyzerDispatcher : IScanAnalyzerDispatcher
{
public ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken)
{
return ValueTask.CompletedTask;
}
}
private sealed class QueueBackedScanJobSourceDependencies
{
public QueueBackedScanJobSourceDependencies()
{
JobCompleted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
}
public TaskCompletionSource JobCompleted { get; }
}
private sealed class QueueBackedScanJobSource : IScanJobSource
{
private readonly IScanQueue _queue;
private readonly ScannerQueueOptions _queueOptions;
private readonly QueueBackedScanJobSourceDependencies _deps;
private readonly TimeProvider _timeProvider;
private readonly string _consumerName = $"worker-smoke-{Guid.NewGuid():n}";
public QueueBackedScanJobSource(
IScanQueue queue,
ScannerQueueOptions queueOptions,
QueueBackedScanJobSourceDependencies deps,
TimeProvider timeProvider)
{
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
_queueOptions = queueOptions ?? throw new ArgumentNullException(nameof(queueOptions));
_deps = deps ?? throw new ArgumentNullException(nameof(deps));
_timeProvider = timeProvider ?? TimeProvider.System;
}
public async Task<IScanJobLease?> TryAcquireAsync(CancellationToken cancellationToken)
{
var request = new QueueLeaseRequest(_consumerName, 1, _queueOptions.DefaultLeaseDuration);
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Scanner.Queue;
using StellaOps.Scanner.Worker.Diagnostics;
using StellaOps.Scanner.Worker.Hosting;
using StellaOps.Scanner.Worker.Options;
using StellaOps.Scanner.Worker.Processing;
using StellaOps.Scanner.Worker.Tests.TestInfrastructure;
using Xunit;
namespace StellaOps.Scanner.Worker.Tests;
public sealed class RedisWorkerSmokeTests
{
[Fact]
public async Task Worker_CompletesJob_ViaRedisQueue()
{
var flag = Environment.GetEnvironmentVariable("STELLAOPS_REDIS_SMOKE");
if (string.IsNullOrWhiteSpace(flag))
{
return;
}
var redisConnection = Environment.GetEnvironmentVariable("STELLAOPS_REDIS_CONNECTION") ?? "localhost:6379";
var streamName = $"scanner:jobs:{Guid.NewGuid():n}";
var consumerGroup = $"worker-smoke-{Guid.NewGuid():n}";
var configuration = BuildQueueConfiguration(redisConnection, streamName, consumerGroup);
var queueOptions = new ScannerQueueOptions();
configuration.GetSection("scanner:queue").Bind(queueOptions);
var workerOptions = new ScannerWorkerOptions
{
MaxConcurrentJobs = 1,
};
workerOptions.Queue.HeartbeatSafetyFactor = 3.0;
workerOptions.Queue.MinHeartbeatInterval = TimeSpan.FromSeconds(2);
workerOptions.Queue.MaxHeartbeatInterval = TimeSpan.FromSeconds(8);
workerOptions.Queue.SetHeartbeatRetryDelays(new[]
{
TimeSpan.FromMilliseconds(200),
TimeSpan.FromMilliseconds(500),
TimeSpan.FromSeconds(1),
});
var services = new ServiceCollection();
services.AddLogging(builder =>
{
builder.SetMinimumLevel(LogLevel.Debug);
builder.AddConsole();
});
services.AddSingleton(TimeProvider.System);
services.AddScannerQueue(configuration, "scanner:queue");
services.AddSingleton<IScanJobSource, QueueBackedScanJobSource>();
services.AddSingleton<QueueBackedScanJobSourceDependencies>();
services.AddSingleton(queueOptions);
services.AddSingleton<IScanAnalyzerDispatcher, SmokeAnalyzerDispatcher>();
services.AddSingleton<IScanStageExecutor, AnalyzerStageExecutor>();
services.AddSingleton<ScannerWorkerMetrics>();
services.AddSingleton<ScanProgressReporter>();
services.AddSingleton<ScanJobProcessor>();
services.AddSingleton<LeaseHeartbeatService>();
services.AddSingleton<IDelayScheduler, SystemDelayScheduler>();
services.AddSingleton<IOptionsMonitor<ScannerWorkerOptions>>(new StaticOptionsMonitor<ScannerWorkerOptions>(workerOptions));
services.AddSingleton<ScannerWorkerHostedService>();
using var provider = services.BuildServiceProvider();
var queue = provider.GetRequiredService<IScanQueue>();
var jobId = $"job-{Guid.NewGuid():n}";
var scanId = $"scan-{Guid.NewGuid():n}";
await queue.EnqueueAsync(new ScanQueueMessage(jobId, Encoding.UTF8.GetBytes("smoke"))
{
Attributes = new Dictionary<string, string>(StringComparer.Ordinal)
{
["scanId"] = scanId,
["queue"] = "redis",
}
});
var hostedService = provider.GetRequiredService<ScannerWorkerHostedService>();
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
await hostedService.StartAsync(cts.Token);
var smokeObserver = provider.GetRequiredService<QueueBackedScanJobSourceDependencies>();
await smokeObserver.JobCompleted.Task.WaitAsync(TimeSpan.FromSeconds(20));
await hostedService.StopAsync(CancellationToken.None);
}
private static IConfiguration BuildQueueConfiguration(string connection, string stream, string consumerGroup)
{
return new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary<string, string?>
{
["scanner:queue:kind"] = "redis",
["scanner:queue:defaultLeaseDuration"] = "00:00:30",
["scanner:queue:redis:connectionString"] = connection,
["scanner:queue:redis:streamName"] = stream,
["scanner:queue:redis:consumerGroup"] = consumerGroup,
["scanner:queue:redis:idempotencyKeyPrefix"] = $"{stream}:idemp:",
["scanner:queue:redis:initializationTimeout"] = "00:00:10",
})
.Build();
}
private sealed class SmokeAnalyzerDispatcher : IScanAnalyzerDispatcher
{
public ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken)
{
return ValueTask.CompletedTask;
}
}
private sealed class QueueBackedScanJobSourceDependencies
{
public QueueBackedScanJobSourceDependencies()
{
JobCompleted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
}
public TaskCompletionSource JobCompleted { get; }
}
private sealed class QueueBackedScanJobSource : IScanJobSource
{
private readonly IScanQueue _queue;
private readonly ScannerQueueOptions _queueOptions;
private readonly QueueBackedScanJobSourceDependencies _deps;
private readonly TimeProvider _timeProvider;
private readonly string _consumerName = $"worker-smoke-{Guid.NewGuid():n}";
public QueueBackedScanJobSource(
IScanQueue queue,
ScannerQueueOptions queueOptions,
QueueBackedScanJobSourceDependencies deps,
TimeProvider timeProvider)
{
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
_queueOptions = queueOptions ?? throw new ArgumentNullException(nameof(queueOptions));
_deps = deps ?? throw new ArgumentNullException(nameof(deps));
_timeProvider = timeProvider ?? TimeProvider.System;
}
public async Task<IScanJobLease?> TryAcquireAsync(CancellationToken cancellationToken)
{
var request = new QueueLeaseRequest(_consumerName, 1, _queueOptions.DefaultLeaseDuration);
var leases = await _queue.LeaseAsync(request, cancellationToken);
if (leases.Count == 0)
{
return null;
}
return new QueueBackedScanJobLease(
leases[0],
_queueOptions,
_deps,
_timeProvider.GetUtcNow());
}
}
private sealed class QueueBackedScanJobLease : IScanJobLease
{
private readonly IScanQueueLease _lease;
private readonly ScannerQueueOptions _options;
private readonly QueueBackedScanJobSourceDependencies _deps;
private readonly DateTimeOffset _leasedAt;
private readonly IReadOnlyDictionary<string, string> _metadata;
public QueueBackedScanJobLease(
IScanQueueLease lease,
ScannerQueueOptions options,
QueueBackedScanJobSourceDependencies deps,
DateTimeOffset leasedAt)
{
_lease = lease ?? throw new ArgumentNullException(nameof(lease));
_options = options ?? throw new ArgumentNullException(nameof(options));
_deps = deps ?? throw new ArgumentNullException(nameof(deps));
_leasedAt = leasedAt;
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
{
["queue"] = _options.Kind.ToString(),
["queue.consumer"] = lease.Consumer,
};
if (!string.IsNullOrWhiteSpace(lease.IdempotencyKey))
{
metadata["job.idempotency"] = lease.IdempotencyKey;
}
foreach (var attribute in lease.Attributes)
{
metadata[attribute.Key] = attribute.Value;
}
_metadata = metadata;
}
public string JobId => _lease.JobId;
public string ScanId => _metadata.TryGetValue("scanId", out var scanId) ? scanId : _lease.JobId;
public int Attempt => _lease.Attempt;
public DateTimeOffset EnqueuedAtUtc => _lease.EnqueuedAt;
public DateTimeOffset LeasedAtUtc => _leasedAt;
public TimeSpan LeaseDuration => _lease.LeaseExpiresAt - _leasedAt;
public IReadOnlyDictionary<string, string> Metadata => _metadata;
public async ValueTask RenewAsync(CancellationToken cancellationToken)
{
if (leases.Count == 0)
{
return null;
}
return new QueueBackedScanJobLease(
leases[0],
_queueOptions,
_deps,
_timeProvider.GetUtcNow());
}
}
private sealed class QueueBackedScanJobLease : IScanJobLease
{
private readonly IScanQueueLease _lease;
private readonly ScannerQueueOptions _options;
private readonly QueueBackedScanJobSourceDependencies _deps;
private readonly DateTimeOffset _leasedAt;
private readonly IReadOnlyDictionary<string, string> _metadata;
public QueueBackedScanJobLease(
IScanQueueLease lease,
ScannerQueueOptions options,
QueueBackedScanJobSourceDependencies deps,
DateTimeOffset leasedAt)
{
_lease = lease ?? throw new ArgumentNullException(nameof(lease));
_options = options ?? throw new ArgumentNullException(nameof(options));
_deps = deps ?? throw new ArgumentNullException(nameof(deps));
_leasedAt = leasedAt;
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
{
["queue"] = _options.Kind.ToString(),
["queue.consumer"] = lease.Consumer,
};
if (!string.IsNullOrWhiteSpace(lease.IdempotencyKey))
{
metadata["job.idempotency"] = lease.IdempotencyKey;
}
foreach (var attribute in lease.Attributes)
{
metadata[attribute.Key] = attribute.Value;
}
_metadata = metadata;
}
public string JobId => _lease.JobId;
public string ScanId => _metadata.TryGetValue("scanId", out var scanId) ? scanId : _lease.JobId;
public int Attempt => _lease.Attempt;
public DateTimeOffset EnqueuedAtUtc => _lease.EnqueuedAt;
public DateTimeOffset LeasedAtUtc => _leasedAt;
public TimeSpan LeaseDuration => _lease.LeaseExpiresAt - _leasedAt;
public IReadOnlyDictionary<string, string> Metadata => _metadata;
public async ValueTask RenewAsync(CancellationToken cancellationToken)
{
await _lease.RenewAsync(_options.DefaultLeaseDuration, cancellationToken);
}
public async ValueTask CompleteAsync(CancellationToken cancellationToken)
{
}
public async ValueTask CompleteAsync(CancellationToken cancellationToken)
{
await _lease.AcknowledgeAsync(cancellationToken);
_deps.JobCompleted.TrySetResult();
}
public async ValueTask AbandonAsync(string reason, CancellationToken cancellationToken)
{
_deps.JobCompleted.TrySetResult();
}
public async ValueTask AbandonAsync(string reason, CancellationToken cancellationToken)
{
await _lease.ReleaseAsync(QueueReleaseDisposition.Retry, cancellationToken);
}
public async ValueTask PoisonAsync(string reason, CancellationToken cancellationToken)
{
}
public async ValueTask PoisonAsync(string reason, CancellationToken cancellationToken)
{
await _lease.DeadLetterAsync(reason, cancellationToken);
}
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
}
}
}
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
}
}

View File

@@ -3,6 +3,7 @@ using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging.Abstractions;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.Worker.Processing;
using StellaOps.Scanner.Worker.Processing.Replay;
using Xunit;
@@ -14,27 +15,27 @@ public sealed class ReplaySealedBundleStageExecutorTests
public async Task ExecuteAsync_SetsMetadata_WhenUriAndHashProvided()
{
var executor = new ReplaySealedBundleStageExecutor(NullLogger<ReplaySealedBundleStageExecutor>.Instance);
var context = TestContexts.Create();
context.Lease.Metadata["replay.bundle.uri"] = "cas://replay/input.tar.zst";
context.Lease.Metadata["replay.bundle.sha256"] = "abc123";
context.Lease.Metadata["determinism.policy"] = "rev-1";
context.Lease.Metadata["determinism.feed"] = "feed-2";
var context = TestContexts.Create(out var metadata);
metadata["replay.bundle.uri"] = "cas://replay/input.tar.zst";
metadata["replay.bundle.sha256"] = "abc123";
metadata["determinism.policy"] = "rev-1";
metadata["determinism.feed"] = "feed-2";
await executor.ExecuteAsync(context, CancellationToken.None);
Assert.True(context.Analysis.TryGet<ReplaySealedBundleMetadata>(ScanAnalysisKeys.ReplaySealedBundleMetadata, out var metadata));
Assert.Equal("abc123", metadata.ManifestHash);
Assert.Equal("cas://replay/input.tar.zst", metadata.BundleUri);
Assert.Equal("rev-1", metadata.PolicySnapshotId);
Assert.Equal("feed-2", metadata.FeedSnapshotId);
Assert.True(context.Analysis.TryGet<ReplaySealedBundleMetadata>(ScanAnalysisKeys.ReplaySealedBundleMetadata, out var sealedMetadata));
Assert.Equal("abc123", sealedMetadata.ManifestHash);
Assert.Equal("cas://replay/input.tar.zst", sealedMetadata.BundleUri);
Assert.Equal("rev-1", sealedMetadata.PolicySnapshotId);
Assert.Equal("feed-2", sealedMetadata.FeedSnapshotId);
}
[Fact]
public async Task ExecuteAsync_Skips_WhenHashMissing()
{
var executor = new ReplaySealedBundleStageExecutor(NullLogger<ReplaySealedBundleStageExecutor>.Instance);
var context = TestContexts.Create();
context.Lease.Metadata["replay.bundle.uri"] = "cas://replay/input.tar.zst";
var context = TestContexts.Create(out var metadata);
metadata["replay.bundle.uri"] = "cas://replay/input.tar.zst";
await executor.ExecuteAsync(context, CancellationToken.None);
@@ -44,9 +45,10 @@ public sealed class ReplaySealedBundleStageExecutorTests
internal static class TestContexts
{
public static ScanJobContext Create()
public static ScanJobContext Create(out Dictionary<string, string> metadata)
{
var lease = new TestScanJobLease();
metadata = lease.MutableMetadata;
return new ScanJobContext(lease, TimeProvider.System, TimeProvider.System.GetUtcNow(), CancellationToken.None);
}

View File

@@ -1,26 +1,26 @@
using System;
using System.Linq;
using StellaOps.Scanner.Worker.Options;
using Xunit;
namespace StellaOps.Scanner.Worker.Tests;
public sealed class ScannerWorkerOptionsValidatorTests
{
[Fact]
public void Validate_Fails_WhenHeartbeatSafetyFactorBelowThree()
{
var options = new ScannerWorkerOptions();
options.Queue.HeartbeatSafetyFactor = 2.5;
var validator = new ScannerWorkerOptionsValidator();
var result = validator.Validate(string.Empty, options);
Assert.True(result.Failed, "Validation should fail when HeartbeatSafetyFactor < 3.");
Assert.Contains(result.Failures, failure => failure.Contains("HeartbeatSafetyFactor", StringComparison.OrdinalIgnoreCase));
}
[Fact]
using System;
using System.Linq;
using StellaOps.Scanner.Worker.Options;
using Xunit;
namespace StellaOps.Scanner.Worker.Tests;
public sealed class ScannerWorkerOptionsValidatorTests
{
[Fact]
public void Validate_Fails_WhenHeartbeatSafetyFactorBelowThree()
{
var options = new ScannerWorkerOptions();
options.Queue.HeartbeatSafetyFactor = 2.5;
var validator = new ScannerWorkerOptionsValidator();
var result = validator.Validate(string.Empty, options);
Assert.True(result.Failed, "Validation should fail when HeartbeatSafetyFactor < 3.");
Assert.Contains(result.Failures, failure => failure.Contains("HeartbeatSafetyFactor", StringComparison.OrdinalIgnoreCase));
}
[Fact]
public void Validate_Succeeds_WhenHeartbeatSafetyFactorAtLeastThree()
{
var options = new ScannerWorkerOptions();

View File

@@ -43,14 +43,17 @@ public sealed class SurfaceManifestStageExecutorTests
listener.Start();
var hash = CreateCryptoHash();
var manifestWriter = new TestSurfaceManifestWriter();
var executor = new SurfaceManifestStageExecutor(
publisher,
manifestWriter,
cache,
environment,
metrics,
NullLogger<SurfaceManifestStageExecutor>.Instance,
hash,
new NullRubyPackageInventoryStore(),
new NullBunPackageInventoryStore(),
new DeterminismContext(true, DateTimeOffset.Parse("2024-01-01T00:00:00Z"), 1337, true, 1),
new DeterministicDsseEnvelopeSigner());
@@ -82,14 +85,17 @@ public sealed class SurfaceManifestStageExecutorTests
listener.Start();
var hash = CreateCryptoHash();
var manifestWriter = new TestSurfaceManifestWriter();
var executor = new SurfaceManifestStageExecutor(
publisher,
manifestWriter,
cache,
environment,
metrics,
NullLogger<SurfaceManifestStageExecutor>.Instance,
hash,
new NullRubyPackageInventoryStore(),
new NullBunPackageInventoryStore(),
new DeterminismContext(false, DateTimeOffset.UnixEpoch, null, false, null),
new DeterministicDsseEnvelopeSigner());
@@ -125,10 +131,14 @@ public sealed class SurfaceManifestStageExecutorTests
var payloadMetrics = listener.Measurements
.Where(m => m.InstrumentName == "scanner_worker_surface_payload_persisted_total")
.ToArray();
Assert.Equal(3, payloadMetrics.Length);
Assert.Equal(7, payloadMetrics.Length);
Assert.Contains(payloadMetrics, m => Equals("entrytrace.graph", m["surface.kind"]));
Assert.Contains(payloadMetrics, m => Equals("entrytrace.ndjson", m["surface.kind"]));
Assert.Contains(payloadMetrics, m => Equals("layer.fragments", m["surface.kind"]));
Assert.Contains(payloadMetrics, m => Equals("composition.recipe", m["surface.kind"]));
Assert.Contains(payloadMetrics, m => Equals("composition.recipe.dsse", m["surface.kind"]));
Assert.Contains(payloadMetrics, m => Equals("layer.fragments.dsse", m["surface.kind"]));
Assert.Contains(payloadMetrics, m => Equals("determinism.json", m["surface.kind"]));
}
[Fact]
@@ -148,22 +158,28 @@ public sealed class SurfaceManifestStageExecutorTests
var executor = new SurfaceManifestStageExecutor(
publisher,
new TestSurfaceManifestWriter(),
cache,
environment,
metrics,
NullLogger<SurfaceManifestStageExecutor>.Instance,
hash,
new NullRubyPackageInventoryStore(),
determinism);
new NullBunPackageInventoryStore(),
determinism,
new DeterministicDsseEnvelopeSigner());
var context = CreateContext();
context.Lease.Metadata["determinism.feed"] = "feed-001";
context.Lease.Metadata["determinism.policy"] = "rev-77";
var context = CreateContext(new Dictionary<string, string>
{
["determinism.feed"] = "feed-001",
["determinism.policy"] = "rev-77"
});
PopulateAnalysis(context);
await executor.ExecuteAsync(context, CancellationToken.None);
var determinismPayload = publisher.LastRequest!.Payloads.Single(p => p.Kind == "determinism.json");
var json = JsonDocument.Parse(determinismPayload.Content.Span);
var json = JsonDocument.Parse(determinismPayload.Content);
Assert.True(json.RootElement.GetProperty("fixedClock").GetBoolean());
Assert.Equal(42, json.RootElement.GetProperty("rngSeed").GetInt32());
@@ -188,13 +204,16 @@ public sealed class SurfaceManifestStageExecutorTests
var executor = new SurfaceManifestStageExecutor(
publisher,
new TestSurfaceManifestWriter(),
cache,
environment,
metrics,
NullLogger<SurfaceManifestStageExecutor>.Instance,
hash,
new NullRubyPackageInventoryStore(),
new DeterminismContext(false, DateTimeOffset.UnixEpoch, null, false, null));
new NullBunPackageInventoryStore(),
new DeterminismContext(false, DateTimeOffset.UnixEpoch, null, false, null),
new DeterministicDsseEnvelopeSigner());
var context = CreateContext();
@@ -230,8 +249,8 @@ public sealed class SurfaceManifestStageExecutorTests
Assert.Contains(publisher.LastRequest!.Payloads, p => p.Kind == "entropy.report");
Assert.Contains(publisher.LastRequest!.Payloads, p => p.Kind == "entropy.layer-summary");
// Two payloads + manifest persisted to cache.
Assert.Equal(3, cache.Entries.Count);
// Two payloads + determinism + manifest persisted to cache.
Assert.Equal(6, cache.Entries.Count);
}
private static ScanJobContext CreateContext(Dictionary<string, string>? metadata = null)
@@ -310,12 +329,16 @@ public sealed class SurfaceManifestStageExecutorTests
var executor = new SurfaceManifestStageExecutor(
publisher,
new TestSurfaceManifestWriter(),
cache,
environment,
metrics,
NullLogger<SurfaceManifestStageExecutor>.Instance,
hash,
packageStore);
packageStore,
new NullBunPackageInventoryStore(),
new DeterminismContext(false, DateTimeOffset.UnixEpoch, null, false, null),
new DeterministicDsseEnvelopeSigner());
var context = CreateContext();
PopulateAnalysis(context);
@@ -340,12 +363,16 @@ public sealed class SurfaceManifestStageExecutorTests
var executor = new SurfaceManifestStageExecutor(
publisher,
new TestSurfaceManifestWriter(),
cache,
environment,
metrics,
NullLogger<SurfaceManifestStageExecutor>.Instance,
hash,
packageStore);
packageStore,
new NullBunPackageInventoryStore(),
new DeterminismContext(false, DateTimeOffset.UnixEpoch, null, false, null),
new DeterministicDsseEnvelopeSigner());
var context = CreateContext();
PopulateAnalysis(context);
@@ -407,15 +434,19 @@ public sealed class SurfaceManifestStageExecutorTests
var cache = new RecordingSurfaceCache();
var environment = new TestSurfaceEnvironment("tenant-a");
var hash = CreateCryptoHash();
var manifestWriter = new TestSurfaceManifestWriter();
var executor = new SurfaceManifestStageExecutor(
publisher,
manifestWriter,
cache,
environment,
metrics,
NullLogger<SurfaceManifestStageExecutor>.Instance,
hash,
new NullRubyPackageInventoryStore(),
new DeterminismContext(false, DateTimeOffset.UnixEpoch, null, false, null));
new NullBunPackageInventoryStore(),
new DeterminismContext(false, DateTimeOffset.UnixEpoch, null, false, null),
new DeterministicDsseEnvelopeSigner());
var context = CreateContext();
var observationBytes = Encoding.UTF8.GetBytes("{\"entrypoints\":[\"mod.ts\"]}");
@@ -461,13 +492,16 @@ public sealed class SurfaceManifestStageExecutorTests
var executor = new SurfaceManifestStageExecutor(
publisher,
new TestSurfaceManifestWriter(),
cache,
environment,
metrics,
NullLogger<SurfaceManifestStageExecutor>.Instance,
hash,
new NullRubyPackageInventoryStore(),
determinism);
new NullBunPackageInventoryStore(),
determinism,
new DeterministicDsseEnvelopeSigner());
var leaseMetadata = new Dictionary<string, string>
{
@@ -553,6 +587,46 @@ public sealed class SurfaceManifestStageExecutorTests
}
}
private sealed class TestSurfaceManifestWriter : ISurfaceManifestWriter
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
WriteIndented = false,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
public int PublishCalls { get; private set; }
public SurfaceManifestDocument? LastDocument { get; private set; }
public Task<SurfaceManifestPublishResult> PublishAsync(
SurfaceManifestDocument document,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(document);
PublishCalls++;
LastDocument = document;
var json = JsonSerializer.SerializeToUtf8Bytes(document, SerializerOptions);
var digest = ComputeDigest(json);
return Task.FromResult(new SurfaceManifestPublishResult(
ManifestDigest: digest,
ManifestUri: $"cas://test/surface.manifests/{digest}",
ArtifactId: $"surface-manifest::{digest}",
Document: document,
DeterminismMerkleRoot: document.DeterminismMerkleRoot));
}
private static string ComputeDigest(ReadOnlySpan<byte> content)
{
Span<byte> hash = stackalloc byte[32];
SHA256.HashData(content, hash);
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
}
}
private sealed class TestSurfaceManifestPublisher : ISurfaceManifestPublisher
{
private readonly string _tenant;
@@ -664,30 +738,7 @@ public sealed class SurfaceManifestStageExecutorTests
}
private static ICryptoHash CreateCryptoHash()
=> new DefaultCryptoHash(new StaticOptionsMonitor<CryptoHashOptions>(new CryptoHashOptions()), NullLogger<DefaultCryptoHash>.Instance);
private sealed class StaticOptionsMonitor<T> : IOptionsMonitor<T>
{
public StaticOptionsMonitor(T value)
{
CurrentValue = value;
}
public T CurrentValue { get; }
public T Get(string? name) => CurrentValue;
public IDisposable OnChange(Action<T, string?> listener) => Disposable.Instance;
private sealed class Disposable : IDisposable
{
public static readonly Disposable Instance = new();
public void Dispose()
{
}
}
}
=> DefaultCryptoHash.CreateForTests();
private sealed class RecordingRubyPackageStore : IRubyPackageInventoryStore
{

View File

@@ -1,29 +1,29 @@
using System;
using Microsoft.Extensions.Options;
namespace StellaOps.Scanner.Worker.Tests.TestInfrastructure;
public sealed class StaticOptionsMonitor<TOptions> : IOptionsMonitor<TOptions>
where TOptions : class
{
private readonly TOptions _value;
public StaticOptionsMonitor(TOptions value)
{
_value = value ?? throw new ArgumentNullException(nameof(value));
}
public TOptions CurrentValue => _value;
public TOptions Get(string? name) => _value;
public IDisposable OnChange(Action<TOptions, string?> listener) => NullDisposable.Instance;
private sealed class NullDisposable : IDisposable
{
public static readonly NullDisposable Instance = new();
public void Dispose()
{
}
}
}
using System;
using Microsoft.Extensions.Options;
namespace StellaOps.Scanner.Worker.Tests.TestInfrastructure;
public sealed class StaticOptionsMonitor<TOptions> : IOptionsMonitor<TOptions>
where TOptions : class
{
private readonly TOptions _value;
public StaticOptionsMonitor(TOptions value)
{
_value = value ?? throw new ArgumentNullException(nameof(value));
}
public TOptions CurrentValue => _value;
public TOptions Get(string? name) => _value;
public IDisposable OnChange(Action<TOptions, string?> listener) => NullDisposable.Instance;
private sealed class NullDisposable : IDisposable
{
public static readonly NullDisposable Instance = new();
public void Dispose()
{
}
}
}

View File

@@ -35,6 +35,30 @@ internal sealed class TestCryptoHash : ICryptoHash
return Convert.ToHexString(bytes).ToLowerInvariant();
}
public byte[] ComputeHashForPurpose(ReadOnlySpan<byte> data, string purpose)
=> ComputeHash(data, GetAlgorithmForPurpose(purpose));
public string ComputeHashHexForPurpose(ReadOnlySpan<byte> data, string purpose)
=> ComputeHashHex(data, GetAlgorithmForPurpose(purpose));
public string ComputeHashBase64ForPurpose(ReadOnlySpan<byte> data, string purpose)
=> ComputeHashBase64(data, GetAlgorithmForPurpose(purpose));
public ValueTask<byte[]> ComputeHashForPurposeAsync(Stream stream, string purpose, CancellationToken cancellationToken = default)
=> ComputeHashAsync(stream, GetAlgorithmForPurpose(purpose), cancellationToken);
public ValueTask<string> ComputeHashHexForPurposeAsync(Stream stream, string purpose, CancellationToken cancellationToken = default)
=> ComputeHashHexAsync(stream, GetAlgorithmForPurpose(purpose), cancellationToken);
public string GetAlgorithmForPurpose(string purpose)
=> HashAlgorithms.Sha256;
public string GetHashPrefix(string purpose)
=> "sha256:";
public string ComputePrefixedHashForPurpose(ReadOnlySpan<byte> data, string purpose)
=> $"{GetHashPrefix(purpose)}{ComputeHashHexForPurpose(data, purpose)}";
private static HashAlgorithm CreateAlgorithm(string? algorithmId)
{
return algorithmId?.ToUpperInvariant() switch

View File

@@ -1,69 +1,85 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Microsoft.Extensions.Time.Testing;
using StellaOps.Cryptography;
using StellaOps.Scanner.Reachability;
using StellaOps.Scanner.Storage;
using StellaOps.Scanner.Storage.ObjectStore;
using StellaOps.Scanner.Worker.Diagnostics;
using StellaOps.Scanner.Worker.Determinism;
using StellaOps.Scanner.Worker.Hosting;
using StellaOps.Scanner.Worker.Options;
using StellaOps.Scanner.Worker.Processing;
using StellaOps.Scanner.Worker.Processing.Replay;
using StellaOps.Scanner.Worker.Tests.TestInfrastructure;
using Xunit;
namespace StellaOps.Scanner.Worker.Tests;
public sealed class WorkerBasicScanScenarioTests
{
[Fact]
public async Task DelayAsync_CompletesAfterTimeAdvance()
{
var scheduler = new ControlledDelayScheduler();
var delayTask = scheduler.DelayAsync(TimeSpan.FromSeconds(5), CancellationToken.None);
scheduler.AdvanceBy(TimeSpan.FromSeconds(5));
await delayTask.WaitAsync(TimeSpan.FromSeconds(1));
}
[Fact]
public async Task Worker_CompletesJob_RecordsTelemetry_And_Heartbeats()
{
var fakeTime = new FakeTimeProvider();
fakeTime.SetUtcNow(DateTimeOffset.UtcNow);
var options = new ScannerWorkerOptions
{
MaxConcurrentJobs = 1,
};
options.Telemetry.EnableTelemetry = false;
options.Telemetry.EnableMetrics = true;
var optionsMonitor = new StaticOptionsMonitor<ScannerWorkerOptions>(options);
var testLoggerProvider = new TestLoggerProvider();
var lease = new TestJobLease(fakeTime);
var jobSource = new TestJobSource(lease);
var scheduler = new ControlledDelayScheduler();
var analyzer = new TestAnalyzerDispatcher(scheduler);
namespace StellaOps.Scanner.Worker.Tests;
public sealed class WorkerBasicScanScenarioTests
{
[Fact]
public async Task DelayAsync_CompletesAfterTimeAdvance()
{
var scheduler = new ControlledDelayScheduler();
var delayTask = scheduler.DelayAsync(TimeSpan.FromSeconds(5), CancellationToken.None);
scheduler.AdvanceBy(TimeSpan.FromSeconds(5));
await delayTask.WaitAsync(TimeSpan.FromSeconds(1));
}
[Fact]
public async Task Worker_CompletesJob_RecordsTelemetry_And_Heartbeats()
{
var fakeTime = new FakeTimeProvider();
fakeTime.SetUtcNow(DateTimeOffset.UtcNow);
var options = new ScannerWorkerOptions
{
MaxConcurrentJobs = 1,
};
options.Telemetry.EnableTelemetry = false;
options.Telemetry.EnableMetrics = true;
var optionsMonitor = new StaticOptionsMonitor<ScannerWorkerOptions>(options);
var testLoggerProvider = new TestLoggerProvider();
var lease = new TestJobLease(fakeTime);
var jobSource = new TestJobSource(lease);
var scheduler = new ControlledDelayScheduler();
var analyzer = new TestAnalyzerDispatcher(scheduler);
using var listener = new WorkerMeterListener();
listener.Start();
using var services = new ServiceCollection()
.AddLogging(builder =>
{
builder.ClearProviders();
builder.AddProvider(testLoggerProvider);
builder.SetMinimumLevel(LogLevel.Debug);
})
.AddSingleton(fakeTime)
.AddSingleton<TimeProvider>(fakeTime)
.AddSingleton<IOptionsMonitor<ScannerWorkerOptions>>(optionsMonitor)
.AddSingleton<ScannerWorkerMetrics>()
.AddSingleton<ScanProgressReporter>()
using var services = new ServiceCollection()
.AddLogging(builder =>
{
builder.ClearProviders();
builder.AddProvider(testLoggerProvider);
builder.SetMinimumLevel(LogLevel.Debug);
})
.AddSingleton(fakeTime)
.AddSingleton<TimeProvider>(fakeTime)
.AddSingleton<IOptionsMonitor<ScannerWorkerOptions>>(optionsMonitor)
.AddSingleton<ScannerWorkerMetrics>()
.AddSingleton<ScanProgressReporter>()
.AddSingleton<ScanJobProcessor>()
.AddSingleton<IDeterministicRandomProvider>(new DeterministicRandomProvider(seed: 1337))
.AddSingleton<DeterministicRandomService>()
.AddSingleton<IReachabilityUnionPublisherService, NullReachabilityUnionPublisherService>()
.AddSingleton<ReplayBundleFetcher>(_ => new ReplayBundleFetcher(
new NullArtifactObjectStore(),
DefaultCryptoHash.CreateForTests(),
new ScannerStorageOptions(),
NullLogger<ReplayBundleFetcher>.Instance))
.AddSingleton<LeaseHeartbeatService>()
.AddSingleton<IDelayScheduler>(scheduler)
.AddSingleton<IScanJobSource>(_ => jobSource)
@@ -72,147 +88,155 @@ public sealed class WorkerBasicScanScenarioTests
.AddSingleton<IScanStageExecutor, AnalyzerStageExecutor>()
.AddSingleton<ScannerWorkerHostedService>()
.BuildServiceProvider();
var worker = services.GetRequiredService<ScannerWorkerHostedService>();
await worker.StartAsync(CancellationToken.None);
await jobSource.LeaseIssued.Task.WaitAsync(TimeSpan.FromSeconds(5));
await Task.Yield();
var spin = 0;
while (!lease.Completed.Task.IsCompleted && spin++ < 24)
{
fakeTime.Advance(TimeSpan.FromSeconds(15));
scheduler.AdvanceBy(TimeSpan.FromSeconds(15));
await Task.Delay(1);
}
try
{
await lease.Completed.Task.WaitAsync(TimeSpan.FromSeconds(30));
}
catch (TimeoutException ex)
{
var stageLogs = string.Join(Environment.NewLine, testLoggerProvider
.GetEntriesForCategory(typeof(ScanProgressReporter).FullName!)
.Select(entry => entry.ToFormattedString()));
throw new TimeoutException($"Worker did not complete within timeout. Logs:{Environment.NewLine}{stageLogs}", ex);
}
await worker.StopAsync(CancellationToken.None);
Assert.True(lease.Completed.Task.IsCompletedSuccessfully, "Job should complete successfully.");
var worker = services.GetRequiredService<ScannerWorkerHostedService>();
await worker.StartAsync(CancellationToken.None);
await jobSource.LeaseIssued.Task.WaitAsync(TimeSpan.FromSeconds(5));
await Task.Yield();
var spin = 0;
while (!lease.Completed.Task.IsCompleted && spin++ < 24)
{
fakeTime.Advance(TimeSpan.FromSeconds(15));
scheduler.AdvanceBy(TimeSpan.FromSeconds(15));
await Task.Delay(1);
}
try
{
await lease.Completed.Task.WaitAsync(TimeSpan.FromSeconds(30));
}
catch (TimeoutException ex)
{
var stageLogs = string.Join(Environment.NewLine, testLoggerProvider
.GetEntriesForCategory(typeof(ScanProgressReporter).FullName!)
.Select(entry => entry.ToFormattedString()));
throw new TimeoutException($"Worker did not complete within timeout. Logs:{Environment.NewLine}{stageLogs}", ex);
}
await worker.StopAsync(CancellationToken.None);
Assert.True(lease.Completed.Task.IsCompletedSuccessfully, "Job should complete successfully.");
Assert.Single(analyzer.Executions);
var stageOrder = testLoggerProvider
.GetEntriesForCategory(typeof(ScanProgressReporter).FullName!)
.Where(entry => entry.EventId.Id == 1000)
.Select(entry => entry.GetScopeProperty<string>("Stage"))
.Where(stage => stage is not null)
.Cast<string>()
.ToArray();
Assert.Equal(ScanStageNames.Ordered, stageOrder);
var queueLatency = listener.Measurements.Where(m => m.InstrumentName == "scanner_worker_queue_latency_ms").ToArray();
Assert.Single(queueLatency);
Assert.True(queueLatency[0].Value > 0, "Queue latency should be positive.");
var jobDuration = listener.Measurements.Where(m => m.InstrumentName == "scanner_worker_job_duration_ms").ToArray();
Assert.Single(jobDuration);
var queueLatency = listener.Measurements.Where(m => m.InstrumentName == "scanner_worker_queue_latency_ms").ToArray();
Assert.Single(queueLatency);
Assert.True(queueLatency[0].Value > 0, "Queue latency should be positive.");
var jobDuration = listener.Measurements.Where(m => m.InstrumentName == "scanner_worker_job_duration_ms").ToArray();
Assert.Single(jobDuration);
var jobDurationMs = jobDuration[0].Value;
Assert.True(jobDurationMs > 0, "Job duration should be positive.");
var stageDurations = listener.Measurements.Where(m => m.InstrumentName == "scanner_worker_stage_duration_ms").ToArray();
Assert.Contains(stageDurations, m => m.Tags.TryGetValue("stage", out var stage) && Equals(stage, ScanStageNames.ExecuteAnalyzers));
}
private sealed class TestJobSource : IScanJobSource
{
private readonly TestJobLease _lease;
private int _delivered;
public TestJobSource(TestJobLease lease)
{
_lease = lease;
}
public TaskCompletionSource LeaseIssued { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public Task<IScanJobLease?> TryAcquireAsync(CancellationToken cancellationToken)
{
if (Interlocked.Exchange(ref _delivered, 1) == 0)
{
LeaseIssued.TrySetResult();
return Task.FromResult<IScanJobLease?>(_lease);
}
return Task.FromResult<IScanJobLease?>(null);
}
}
private sealed class TestJobLease : IScanJobLease
{
private readonly FakeTimeProvider _timeProvider;
private readonly Dictionary<string, string> _metadata = new()
{
{ "queue", "tests" },
{ "job.kind", "basic" },
};
public TestJobLease(FakeTimeProvider timeProvider)
{
_timeProvider = timeProvider;
EnqueuedAtUtc = _timeProvider.GetUtcNow() - TimeSpan.FromSeconds(5);
LeasedAtUtc = _timeProvider.GetUtcNow();
}
public string JobId { get; } = Guid.NewGuid().ToString("n");
public string ScanId { get; } = $"scan-{Guid.NewGuid():n}";
public int Attempt { get; } = 1;
public DateTimeOffset EnqueuedAtUtc { get; }
public DateTimeOffset LeasedAtUtc { get; }
public TimeSpan LeaseDuration { get; } = TimeSpan.FromSeconds(90);
public IReadOnlyDictionary<string, string> Metadata => _metadata;
public TaskCompletionSource Completed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public int RenewalCount => _renewalCount;
public ValueTask RenewAsync(CancellationToken cancellationToken)
{
Interlocked.Increment(ref _renewalCount);
return ValueTask.CompletedTask;
}
public ValueTask CompleteAsync(CancellationToken cancellationToken)
{
Completed.TrySetResult();
return ValueTask.CompletedTask;
}
public ValueTask AbandonAsync(string reason, CancellationToken cancellationToken)
{
Completed.TrySetException(new InvalidOperationException($"Abandoned: {reason}"));
return ValueTask.CompletedTask;
}
public ValueTask PoisonAsync(string reason, CancellationToken cancellationToken)
{
Completed.TrySetException(new InvalidOperationException($"Poisoned: {reason}"));
return ValueTask.CompletedTask;
}
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
var stageDurations = listener.Measurements.Where(m => m.InstrumentName == "scanner_worker_stage_duration_ms").ToArray();
Assert.Contains(stageDurations, m => m.Tags.TryGetValue("stage", out var stage) && Equals(stage, ScanStageNames.ExecuteAnalyzers));
}
private sealed class NullReachabilityUnionPublisherService : IReachabilityUnionPublisherService
{
public Task<ReachabilityUnionPublishResult> PublishAsync(ReachabilityUnionGraph graph, string analysisId, CancellationToken cancellationToken = default)
=> Task.FromResult(new ReachabilityUnionPublishResult("none", "none", 0));
}
private sealed class NullArtifactObjectStore : IArtifactObjectStore
{
public Task PutAsync(ArtifactObjectDescriptor descriptor, Stream content, CancellationToken cancellationToken)
=> Task.CompletedTask;
public Task<Stream?> GetAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken)
=> Task.FromResult<Stream?>(null);
public Task DeleteAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken)
=> Task.CompletedTask;
}
private sealed class TestJobSource : IScanJobSource
{
private readonly TestJobLease _lease;
private int _delivered;
public TestJobSource(TestJobLease lease)
{
_lease = lease;
}
public TaskCompletionSource LeaseIssued { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public Task<IScanJobLease?> TryAcquireAsync(CancellationToken cancellationToken)
{
if (Interlocked.Exchange(ref _delivered, 1) == 0)
{
LeaseIssued.TrySetResult();
return Task.FromResult<IScanJobLease?>(_lease);
}
return Task.FromResult<IScanJobLease?>(null);
}
}
private sealed class TestJobLease : IScanJobLease
{
private readonly FakeTimeProvider _timeProvider;
private readonly Dictionary<string, string> _metadata = new()
{
{ "queue", "tests" },
{ "job.kind", "basic" },
};
public TestJobLease(FakeTimeProvider timeProvider)
{
_timeProvider = timeProvider;
EnqueuedAtUtc = _timeProvider.GetUtcNow() - TimeSpan.FromSeconds(5);
LeasedAtUtc = _timeProvider.GetUtcNow();
}
public string JobId { get; } = Guid.NewGuid().ToString("n");
public string ScanId { get; } = $"scan-{Guid.NewGuid():n}";
public int Attempt { get; } = 1;
public DateTimeOffset EnqueuedAtUtc { get; }
public DateTimeOffset LeasedAtUtc { get; }
public TimeSpan LeaseDuration { get; } = TimeSpan.FromSeconds(90);
public IReadOnlyDictionary<string, string> Metadata => _metadata;
public TaskCompletionSource Completed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public int RenewalCount => _renewalCount;
public ValueTask RenewAsync(CancellationToken cancellationToken)
{
Interlocked.Increment(ref _renewalCount);
return ValueTask.CompletedTask;
}
public ValueTask CompleteAsync(CancellationToken cancellationToken)
{
Completed.TrySetResult();
return ValueTask.CompletedTask;
}
public ValueTask AbandonAsync(string reason, CancellationToken cancellationToken)
{
Completed.TrySetException(new InvalidOperationException($"Abandoned: {reason}"));
return ValueTask.CompletedTask;
}
public ValueTask PoisonAsync(string reason, CancellationToken cancellationToken)
{
Completed.TrySetException(new InvalidOperationException($"Poisoned: {reason}"));
return ValueTask.CompletedTask;
}
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
private int _renewalCount;
}
@@ -227,191 +251,191 @@ public sealed class WorkerBasicScanScenarioTests
private readonly IDelayScheduler _scheduler;
public TestAnalyzerDispatcher(IDelayScheduler scheduler)
{
_scheduler = scheduler;
}
public List<string> Executions { get; } = new();
public async ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken)
{
Executions.Add(context.JobId);
await _scheduler.DelayAsync(TimeSpan.FromSeconds(45), cancellationToken);
}
}
private sealed class ControlledDelayScheduler : IDelayScheduler
{
private readonly object _lock = new();
private readonly SortedDictionary<double, List<ScheduledDelay>> _scheduled = new();
private double _currentMilliseconds;
public Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken)
{
if (delay <= TimeSpan.Zero)
{
return Task.CompletedTask;
}
var tcs = new TaskCompletionSource<object?>(TaskCreationOptions.RunContinuationsAsynchronously);
var scheduled = new ScheduledDelay(tcs, cancellationToken);
lock (_lock)
{
var due = _currentMilliseconds + delay.TotalMilliseconds;
if (!_scheduled.TryGetValue(due, out var list))
{
list = new List<ScheduledDelay>();
_scheduled.Add(due, list);
}
list.Add(scheduled);
}
return scheduled.Task;
}
public void AdvanceBy(TimeSpan delta)
{
lock (_lock)
{
_currentMilliseconds += delta.TotalMilliseconds;
var dueKeys = _scheduled.Keys.Where(key => key <= _currentMilliseconds).ToList();
foreach (var due in dueKeys)
{
foreach (var scheduled in _scheduled[due])
{
scheduled.Complete();
}
_scheduled.Remove(due);
}
}
}
private sealed class ScheduledDelay
{
private readonly TaskCompletionSource<object?> _tcs;
private readonly CancellationTokenRegistration _registration;
public ScheduledDelay(TaskCompletionSource<object?> tcs, CancellationToken cancellationToken)
{
_tcs = tcs;
if (cancellationToken.CanBeCanceled)
{
_registration = cancellationToken.Register(state =>
{
var source = (TaskCompletionSource<object?>)state!;
source.TrySetCanceled(cancellationToken);
}, tcs);
}
}
public Task Task => _tcs.Task;
public void Complete()
{
_registration.Dispose();
_tcs.TrySetResult(null);
}
}
}
private sealed class StaticOptionsMonitor<TOptions> : IOptionsMonitor<TOptions>
where TOptions : class
{
private readonly TOptions _value;
public StaticOptionsMonitor(TOptions value)
{
_value = value;
}
public TOptions CurrentValue => _value;
public TOptions Get(string? name) => _value;
public IDisposable OnChange(Action<TOptions, string?> listener) => NullDisposable.Instance;
private sealed class NullDisposable : IDisposable
{
public static readonly NullDisposable Instance = new();
public void Dispose()
{
}
}
}
private sealed class TestLoggerProvider : ILoggerProvider
{
private readonly ConcurrentQueue<TestLogEntry> _entries = new();
public ILogger CreateLogger(string categoryName) => new TestLogger(categoryName, _entries);
public void Dispose()
{
}
public IEnumerable<TestLogEntry> GetEntriesForCategory(string categoryName)
=> _entries.Where(entry => entry.Category == categoryName);
private sealed class TestLogger : ILogger
{
private readonly string _category;
private readonly ConcurrentQueue<TestLogEntry> _entries;
public TestLogger(string category, ConcurrentQueue<TestLogEntry> entries)
{
_category = category;
_entries = entries;
}
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => NullDisposable.Instance;
public bool IsEnabled(LogLevel logLevel) => true;
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
{
_entries.Enqueue(new TestLogEntry(_category, logLevel, eventId, state, exception));
}
}
private sealed class NullDisposable : IDisposable
{
public static readonly NullDisposable Instance = new();
public void Dispose()
{
}
}
}
public sealed record TestLogEntry(string Category, LogLevel Level, EventId EventId, object? State, Exception? Exception)
{
public T? GetScopeProperty<T>(string name)
{
if (State is not IEnumerable<KeyValuePair<string, object?>> state)
{
return default;
}
foreach (var kvp in state)
{
if (string.Equals(kvp.Key, name, StringComparison.OrdinalIgnoreCase) && kvp.Value is T value)
{
return value;
}
}
return default;
}
public string ToFormattedString()
{
var properties = State is IEnumerable<KeyValuePair<string, object?>> kvps
? string.Join(", ", kvps.Select(kvp => $"{kvp.Key}={kvp.Value}"))
: State?.ToString() ?? string.Empty;
var exceptionPart = Exception is null ? string.Empty : $" Exception={Exception.GetType().Name}: {Exception.Message}";
return $"[{Level}] {Category} ({EventId.Id}) {properties}{exceptionPart}";
}
}
}
{
_scheduler = scheduler;
}
public List<string> Executions { get; } = new();
public async ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken)
{
Executions.Add(context.JobId);
await _scheduler.DelayAsync(TimeSpan.FromSeconds(45), cancellationToken);
}
}
private sealed class ControlledDelayScheduler : IDelayScheduler
{
private readonly object _lock = new();
private readonly SortedDictionary<double, List<ScheduledDelay>> _scheduled = new();
private double _currentMilliseconds;
public Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken)
{
if (delay <= TimeSpan.Zero)
{
return Task.CompletedTask;
}
var tcs = new TaskCompletionSource<object?>(TaskCreationOptions.RunContinuationsAsynchronously);
var scheduled = new ScheduledDelay(tcs, cancellationToken);
lock (_lock)
{
var due = _currentMilliseconds + delay.TotalMilliseconds;
if (!_scheduled.TryGetValue(due, out var list))
{
list = new List<ScheduledDelay>();
_scheduled.Add(due, list);
}
list.Add(scheduled);
}
return scheduled.Task;
}
public void AdvanceBy(TimeSpan delta)
{
lock (_lock)
{
_currentMilliseconds += delta.TotalMilliseconds;
var dueKeys = _scheduled.Keys.Where(key => key <= _currentMilliseconds).ToList();
foreach (var due in dueKeys)
{
foreach (var scheduled in _scheduled[due])
{
scheduled.Complete();
}
_scheduled.Remove(due);
}
}
}
private sealed class ScheduledDelay
{
private readonly TaskCompletionSource<object?> _tcs;
private readonly CancellationTokenRegistration _registration;
public ScheduledDelay(TaskCompletionSource<object?> tcs, CancellationToken cancellationToken)
{
_tcs = tcs;
if (cancellationToken.CanBeCanceled)
{
_registration = cancellationToken.Register(state =>
{
var source = (TaskCompletionSource<object?>)state!;
source.TrySetCanceled(cancellationToken);
}, tcs);
}
}
public Task Task => _tcs.Task;
public void Complete()
{
_registration.Dispose();
_tcs.TrySetResult(null);
}
}
}
private sealed class StaticOptionsMonitor<TOptions> : IOptionsMonitor<TOptions>
where TOptions : class
{
private readonly TOptions _value;
public StaticOptionsMonitor(TOptions value)
{
_value = value;
}
public TOptions CurrentValue => _value;
public TOptions Get(string? name) => _value;
public IDisposable OnChange(Action<TOptions, string?> listener) => NullDisposable.Instance;
private sealed class NullDisposable : IDisposable
{
public static readonly NullDisposable Instance = new();
public void Dispose()
{
}
}
}
private sealed class TestLoggerProvider : ILoggerProvider
{
private readonly ConcurrentQueue<TestLogEntry> _entries = new();
public ILogger CreateLogger(string categoryName) => new TestLogger(categoryName, _entries);
public void Dispose()
{
}
public IEnumerable<TestLogEntry> GetEntriesForCategory(string categoryName)
=> _entries.Where(entry => entry.Category == categoryName);
private sealed class TestLogger : ILogger
{
private readonly string _category;
private readonly ConcurrentQueue<TestLogEntry> _entries;
public TestLogger(string category, ConcurrentQueue<TestLogEntry> entries)
{
_category = category;
_entries = entries;
}
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => NullDisposable.Instance;
public bool IsEnabled(LogLevel logLevel) => true;
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
{
_entries.Enqueue(new TestLogEntry(_category, logLevel, eventId, state, exception));
}
}
private sealed class NullDisposable : IDisposable
{
public static readonly NullDisposable Instance = new();
public void Dispose()
{
}
}
}
public sealed record TestLogEntry(string Category, LogLevel Level, EventId EventId, object? State, Exception? Exception)
{
public T? GetScopeProperty<T>(string name)
{
if (State is not IEnumerable<KeyValuePair<string, object?>> state)
{
return default;
}
foreach (var kvp in state)
{
if (string.Equals(kvp.Key, name, StringComparison.OrdinalIgnoreCase) && kvp.Value is T value)
{
return value;
}
}
return default;
}
public string ToFormattedString()
{
var properties = State is IEnumerable<KeyValuePair<string, object?>> kvps
? string.Join(", ", kvps.Select(kvp => $"{kvp.Key}={kvp.Value}"))
: State?.ToString() ?? string.Empty;
var exceptionPart = Exception is null ? string.Empty : $" Exception={Exception.GetType().Name}: {Exception.Message}";
return $"[{Level}] {Category} ({EventId.Id}) {properties}{exceptionPart}";
}
}
}