up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Collections.ObjectModel;
|
||||
using System.Diagnostics.Metrics;
|
||||
@@ -13,6 +13,7 @@ using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Analyzers.Lang;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Plugin;
|
||||
using StellaOps.Scanner.Analyzers.OS;
|
||||
using StellaOps.Scanner.Analyzers.OS.Abstractions;
|
||||
using StellaOps.Scanner.Analyzers.OS.Plugin;
|
||||
using StellaOps.Scanner.Core.Contracts;
|
||||
@@ -24,12 +25,12 @@ using StellaOps.Scanner.Worker.Diagnostics;
|
||||
using StellaOps.Scanner.Worker.Processing;
|
||||
using StellaOps.Scanner.Worker.Tests.TestInfrastructure;
|
||||
using Xunit;
|
||||
using WorkerOptions = StellaOps.Scanner.Worker.Options.ScannerWorkerOptions;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Tests;
|
||||
|
||||
public sealed class CompositeScanAnalyzerDispatcherTests
|
||||
{
|
||||
using WorkerOptions = StellaOps.Scanner.Worker.Options.ScannerWorkerOptions;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Tests;
|
||||
|
||||
public sealed class CompositeScanAnalyzerDispatcherTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_RunsLanguageAnalyzers_StoresResults()
|
||||
{
|
||||
@@ -67,6 +68,7 @@ public sealed class CompositeScanAnalyzerDispatcherTests
|
||||
serviceCollection.AddSingleton(TimeProvider.System);
|
||||
serviceCollection.AddSurfaceEnvironment(options => options.ComponentName = "Scanner.Worker");
|
||||
serviceCollection.AddSurfaceValidation();
|
||||
serviceCollection.AddSingleton<ISurfaceValidatorRunner, NoopSurfaceValidatorRunner>();
|
||||
serviceCollection.AddSurfaceFileCache(options => options.RootDirectory = cacheRoot.Path);
|
||||
serviceCollection.AddSurfaceSecrets();
|
||||
|
||||
@@ -145,42 +147,203 @@ public sealed class CompositeScanAnalyzerDispatcherTests
|
||||
services?.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeOsCatalog : IOSAnalyzerPluginCatalog
|
||||
{
|
||||
public IReadOnlyCollection<IOSAnalyzerPlugin> Plugins => Array.Empty<IOSAnalyzerPlugin>();
|
||||
|
||||
public void LoadFromDirectory(string directory, bool seal = true)
|
||||
{
|
||||
}
|
||||
|
||||
public IReadOnlyList<IOSPackageAnalyzer> CreateAnalyzers(IServiceProvider services) => Array.Empty<IOSPackageAnalyzer>();
|
||||
}
|
||||
|
||||
private sealed class FakeLanguageCatalog : ILanguageAnalyzerPluginCatalog
|
||||
{
|
||||
private readonly IReadOnlyList<ILanguageAnalyzer> _analyzers;
|
||||
|
||||
public FakeLanguageCatalog(params ILanguageAnalyzer[] analyzers)
|
||||
{
|
||||
_analyzers = analyzers ?? Array.Empty<ILanguageAnalyzer>();
|
||||
}
|
||||
|
||||
public IReadOnlyCollection<ILanguageAnalyzerPlugin> Plugins => Array.Empty<ILanguageAnalyzerPlugin>();
|
||||
|
||||
public void LoadFromDirectory(string directory, bool seal = true)
|
||||
{
|
||||
}
|
||||
|
||||
public IReadOnlyList<ILanguageAnalyzer> CreateAnalyzers(IServiceProvider services) => _analyzers;
|
||||
}
|
||||
|
||||
private sealed class FakeLanguageAnalyzer : ILanguageAnalyzer
|
||||
{
|
||||
public string Id => "lang.fake";
|
||||
|
||||
public string DisplayName => "Fake Language Analyzer";
|
||||
|
||||
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_RunsOsAnalyzers_UsesSurfaceCache()
|
||||
{
|
||||
using var rootfs = new TempDirectory();
|
||||
using var cacheRoot = new TempDirectory();
|
||||
|
||||
Environment.SetEnvironmentVariable("SCANNER_SURFACE_FS_ENDPOINT", "https://surface.test");
|
||||
Environment.SetEnvironmentVariable("SCANNER_SURFACE_FS_BUCKET", "unit-test-bucket");
|
||||
Environment.SetEnvironmentVariable("SCANNER_SURFACE_CACHE_ROOT", cacheRoot.Path);
|
||||
Environment.SetEnvironmentVariable("SCANNER_SURFACE_SECRETS_PROVIDER", "inline");
|
||||
Environment.SetEnvironmentVariable("SCANNER_SURFACE_SECRETS_TENANT", "testtenant");
|
||||
Environment.SetEnvironmentVariable(
|
||||
"SURFACE_SECRET_TESTTENANT_SCANNERWORKEROSANALYZERS_REGISTRY_DEFAULT",
|
||||
Convert.ToBase64String(Encoding.UTF8.GetBytes("token-placeholder")));
|
||||
|
||||
var dpkgStatusPath = Path.Combine(rootfs.Path, "var", "lib", "dpkg", "status");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(dpkgStatusPath)!);
|
||||
await File.WriteAllTextAsync(dpkgStatusPath, "Package: demo\nStatus: install ok installed\n", CancellationToken.None);
|
||||
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
{ ScanMetadataKeys.RootFilesystemPath, rootfs.Path },
|
||||
{ ScanMetadataKeys.WorkspacePath, rootfs.Path },
|
||||
};
|
||||
|
||||
var analyzer = new FakeOsAnalyzer();
|
||||
var osCatalog = new FakeOsCatalog(analyzer);
|
||||
var languageCatalog = new FakeLanguageCatalog();
|
||||
|
||||
long hits = 0;
|
||||
long misses = 0;
|
||||
MeterListener? meterListener = null;
|
||||
ServiceProvider? services = null;
|
||||
try
|
||||
{
|
||||
var serviceCollection = new ServiceCollection();
|
||||
serviceCollection.AddSingleton<IConfiguration>(new ConfigurationBuilder().Build());
|
||||
serviceCollection.AddLogging(builder => builder.SetMinimumLevel(LogLevel.Debug));
|
||||
serviceCollection.AddSingleton(TimeProvider.System);
|
||||
serviceCollection.AddSurfaceEnvironment(options => options.ComponentName = "Scanner.Worker");
|
||||
serviceCollection.AddSurfaceValidation();
|
||||
serviceCollection.AddSingleton<ISurfaceValidatorRunner, NoopSurfaceValidatorRunner>();
|
||||
serviceCollection.AddSurfaceFileCache(options => options.RootDirectory = cacheRoot.Path);
|
||||
serviceCollection.AddSurfaceSecrets();
|
||||
|
||||
var metrics = new ScannerWorkerMetrics();
|
||||
serviceCollection.AddSingleton(metrics);
|
||||
|
||||
meterListener = new MeterListener();
|
||||
|
||||
meterListener.InstrumentPublished = (instrument, listener) =>
|
||||
{
|
||||
if (instrument.Meter.Name == ScannerWorkerInstrumentation.MeterName &&
|
||||
(instrument.Name == "scanner_worker_os_cache_hits_total" || instrument.Name == "scanner_worker_os_cache_misses_total"))
|
||||
{
|
||||
listener.EnableMeasurementEvents(instrument);
|
||||
}
|
||||
};
|
||||
|
||||
meterListener.SetMeasurementEventCallback<long>((instrument, measurement, tags, state) =>
|
||||
{
|
||||
if (instrument.Name == "scanner_worker_os_cache_hits_total")
|
||||
{
|
||||
Interlocked.Add(ref hits, measurement);
|
||||
}
|
||||
else if (instrument.Name == "scanner_worker_os_cache_misses_total")
|
||||
{
|
||||
Interlocked.Add(ref misses, measurement);
|
||||
}
|
||||
});
|
||||
|
||||
meterListener.Start();
|
||||
|
||||
services = serviceCollection.BuildServiceProvider();
|
||||
|
||||
var scopeFactory = services.GetRequiredService<IServiceScopeFactory>();
|
||||
var loggerFactory = services.GetRequiredService<ILoggerFactory>();
|
||||
var options = Microsoft.Extensions.Options.Options.Create(new WorkerOptions());
|
||||
var dispatcher = new CompositeScanAnalyzerDispatcher(
|
||||
scopeFactory,
|
||||
osCatalog,
|
||||
languageCatalog,
|
||||
options,
|
||||
loggerFactory.CreateLogger<CompositeScanAnalyzerDispatcher>(),
|
||||
metrics,
|
||||
new TestCryptoHash());
|
||||
|
||||
var lease = new TestJobLease(metadata);
|
||||
var context = new ScanJobContext(lease, TimeProvider.System, TimeProvider.System.GetUtcNow(), CancellationToken.None);
|
||||
|
||||
await dispatcher.ExecuteAsync(context, CancellationToken.None);
|
||||
|
||||
var leaseSecond = new TestJobLease(metadata);
|
||||
var contextSecond = new ScanJobContext(leaseSecond, TimeProvider.System, TimeProvider.System.GetUtcNow(), CancellationToken.None);
|
||||
await dispatcher.ExecuteAsync(contextSecond, CancellationToken.None);
|
||||
|
||||
Assert.Equal(1, analyzer.InvocationCount);
|
||||
Assert.True(context.Analysis.TryGet<Dictionary<string, OSPackageAnalyzerResult>>(ScanAnalysisKeys.OsPackageAnalyzers, out var results));
|
||||
Assert.Single(results);
|
||||
Assert.True(context.Analysis.TryGet<ImmutableArray<LayerComponentFragment>>(ScanAnalysisKeys.OsComponentFragments, out var fragments));
|
||||
Assert.False(fragments.IsDefaultOrEmpty);
|
||||
|
||||
Assert.Equal(1, hits);
|
||||
Assert.Equal(1, misses);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Environment.SetEnvironmentVariable("SCANNER_SURFACE_FS_ENDPOINT", null);
|
||||
Environment.SetEnvironmentVariable("SCANNER_SURFACE_FS_BUCKET", null);
|
||||
Environment.SetEnvironmentVariable("SCANNER_SURFACE_CACHE_ROOT", null);
|
||||
Environment.SetEnvironmentVariable("SCANNER_SURFACE_SECRETS_PROVIDER", null);
|
||||
Environment.SetEnvironmentVariable("SCANNER_SURFACE_SECRETS_TENANT", null);
|
||||
Environment.SetEnvironmentVariable("SURFACE_SECRET_TESTTENANT_SCANNERWORKEROSANALYZERS_REGISTRY_DEFAULT", null);
|
||||
meterListener?.Dispose();
|
||||
services?.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeOsCatalog : IOSAnalyzerPluginCatalog
|
||||
{
|
||||
private readonly IReadOnlyList<IOSPackageAnalyzer> _analyzers;
|
||||
|
||||
public FakeOsCatalog(params IOSPackageAnalyzer[] analyzers)
|
||||
{
|
||||
_analyzers = analyzers ?? Array.Empty<IOSPackageAnalyzer>();
|
||||
}
|
||||
|
||||
public IReadOnlyCollection<IOSAnalyzerPlugin> Plugins => Array.Empty<IOSAnalyzerPlugin>();
|
||||
|
||||
public void LoadFromDirectory(string directory, bool seal = true)
|
||||
{
|
||||
}
|
||||
|
||||
public IReadOnlyList<IOSPackageAnalyzer> CreateAnalyzers(IServiceProvider services) => _analyzers;
|
||||
}
|
||||
|
||||
private sealed class FakeLanguageCatalog : ILanguageAnalyzerPluginCatalog
|
||||
{
|
||||
private readonly IReadOnlyList<ILanguageAnalyzer> _analyzers;
|
||||
|
||||
public FakeLanguageCatalog(params ILanguageAnalyzer[] analyzers)
|
||||
{
|
||||
_analyzers = analyzers ?? Array.Empty<ILanguageAnalyzer>();
|
||||
}
|
||||
|
||||
public IReadOnlyCollection<ILanguageAnalyzerPlugin> Plugins => Array.Empty<ILanguageAnalyzerPlugin>();
|
||||
|
||||
public void LoadFromDirectory(string directory, bool seal = true)
|
||||
{
|
||||
}
|
||||
|
||||
public IReadOnlyList<ILanguageAnalyzer> CreateAnalyzers(IServiceProvider services) => _analyzers;
|
||||
}
|
||||
|
||||
private sealed class NoopSurfaceValidatorRunner : ISurfaceValidatorRunner
|
||||
{
|
||||
public ValueTask<SurfaceValidationResult> RunAllAsync(SurfaceValidationContext context, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.FromResult(SurfaceValidationResult.Success());
|
||||
|
||||
public ValueTask EnsureAsync(SurfaceValidationContext context, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
private sealed class FakeOsAnalyzer : IOSPackageAnalyzer
|
||||
{
|
||||
public string AnalyzerId => "dpkg";
|
||||
|
||||
public int InvocationCount { get; private set; }
|
||||
|
||||
private int _invocationCount;
|
||||
|
||||
public ValueTask<OSPackageAnalyzerResult> AnalyzeAsync(OSPackageAnalyzerContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref _invocationCount);
|
||||
InvocationCount = _invocationCount;
|
||||
|
||||
var package = new OSPackageRecord(
|
||||
analyzerId: AnalyzerId,
|
||||
packageUrl: "pkg:deb/debian/demo@1.0?arch=amd64",
|
||||
name: "demo",
|
||||
version: "1.0",
|
||||
architecture: "amd64",
|
||||
evidenceSource: PackageEvidenceSource.DpkgStatus,
|
||||
files: new[] { new OSPackageFileEvidence("/usr/bin/demo") });
|
||||
|
||||
var telemetry = new OSAnalyzerTelemetry(TimeSpan.Zero, 1, 1);
|
||||
return ValueTask.FromResult(new OSPackageAnalyzerResult(AnalyzerId, new[] { package }, telemetry));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeLanguageAnalyzer : ILanguageAnalyzer
|
||||
{
|
||||
public string Id => "lang.fake";
|
||||
|
||||
public string DisplayName => "Fake Language Analyzer";
|
||||
|
||||
public int InvocationCount { get; private set; }
|
||||
|
||||
public ValueTask AnalyzeAsync(LanguageAnalyzerContext context, LanguageComponentWriter writer, CancellationToken cancellationToken)
|
||||
@@ -198,69 +361,69 @@ public sealed class CompositeScanAnalyzerDispatcherTests
|
||||
|
||||
private int _invocationCount;
|
||||
}
|
||||
|
||||
private sealed class TestJobLease : IScanJobLease
|
||||
{
|
||||
private readonly Dictionary<string, string> _metadata;
|
||||
|
||||
public TestJobLease(Dictionary<string, string> metadata)
|
||||
{
|
||||
_metadata = metadata;
|
||||
JobId = Guid.NewGuid().ToString("n");
|
||||
ScanId = $"scan-{Guid.NewGuid():n}";
|
||||
Attempt = 1;
|
||||
EnqueuedAtUtc = DateTimeOffset.UtcNow.AddSeconds(-1);
|
||||
LeasedAtUtc = DateTimeOffset.UtcNow;
|
||||
LeaseDuration = TimeSpan.FromMinutes(5);
|
||||
}
|
||||
|
||||
public string JobId { get; }
|
||||
|
||||
public string ScanId { get; }
|
||||
|
||||
public int Attempt { get; }
|
||||
|
||||
public DateTimeOffset EnqueuedAtUtc { get; }
|
||||
|
||||
public DateTimeOffset LeasedAtUtc { get; }
|
||||
|
||||
public TimeSpan LeaseDuration { get; }
|
||||
|
||||
public IReadOnlyDictionary<string, string> Metadata => _metadata;
|
||||
|
||||
public ValueTask RenewAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask CompleteAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask AbandonAsync(string reason, CancellationToken cancellationToken) => ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask PoisonAsync(string reason, CancellationToken cancellationToken) => ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
private sealed class TempDirectory : IDisposable
|
||||
{
|
||||
public TempDirectory()
|
||||
{
|
||||
Path = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"stellaops-test-{Guid.NewGuid():n}");
|
||||
Directory.CreateDirectory(Path);
|
||||
}
|
||||
|
||||
public string Path { get; }
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(Path))
|
||||
{
|
||||
Directory.Delete(Path, recursive: true);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class TestJobLease : IScanJobLease
|
||||
{
|
||||
private readonly Dictionary<string, string> _metadata;
|
||||
|
||||
public TestJobLease(Dictionary<string, string> metadata)
|
||||
{
|
||||
_metadata = metadata;
|
||||
JobId = Guid.NewGuid().ToString("n");
|
||||
ScanId = $"scan-{Guid.NewGuid():n}";
|
||||
Attempt = 1;
|
||||
EnqueuedAtUtc = DateTimeOffset.UtcNow.AddSeconds(-1);
|
||||
LeasedAtUtc = DateTimeOffset.UtcNow;
|
||||
LeaseDuration = TimeSpan.FromMinutes(5);
|
||||
}
|
||||
|
||||
public string JobId { get; }
|
||||
|
||||
public string ScanId { get; }
|
||||
|
||||
public int Attempt { get; }
|
||||
|
||||
public DateTimeOffset EnqueuedAtUtc { get; }
|
||||
|
||||
public DateTimeOffset LeasedAtUtc { get; }
|
||||
|
||||
public TimeSpan LeaseDuration { get; }
|
||||
|
||||
public IReadOnlyDictionary<string, string> Metadata => _metadata;
|
||||
|
||||
public ValueTask RenewAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask CompleteAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask AbandonAsync(string reason, CancellationToken cancellationToken) => ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask PoisonAsync(string reason, CancellationToken cancellationToken) => ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
private sealed class TempDirectory : IDisposable
|
||||
{
|
||||
public TempDirectory()
|
||||
{
|
||||
Path = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"stellaops-test-{Guid.NewGuid():n}");
|
||||
Directory.CreateDirectory(Path);
|
||||
}
|
||||
|
||||
public string Path { get; }
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(Path))
|
||||
{
|
||||
Directory.Delete(Path, recursive: true);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Cryptography;
|
||||
using StellaOps.Scanner.Worker.Options;
|
||||
using StellaOps.Scanner.Worker.Processing.Surface;
|
||||
using Xunit;
|
||||
@@ -24,12 +25,16 @@ public sealed class HmacDsseEnvelopeSignerTests
|
||||
signing.KeyId = "scanner-hmac";
|
||||
});
|
||||
|
||||
var signer = new HmacDsseEnvelopeSigner(options, NullLogger<HmacDsseEnvelopeSigner>.Instance, new ServiceCollection().BuildServiceProvider());
|
||||
var signer = new HmacDsseEnvelopeSigner(
|
||||
options,
|
||||
DefaultCryptoHmac.CreateForTests(),
|
||||
NullLogger<HmacDsseEnvelopeSigner>.Instance,
|
||||
new ServiceCollection().BuildServiceProvider());
|
||||
var payload = Encoding.UTF8.GetBytes("{\"hello\":\"world\"}");
|
||||
|
||||
var envelope = await signer.SignAsync("application/json", payload, "test.kind", "root", view: null, CancellationToken.None);
|
||||
|
||||
var json = JsonDocument.Parse(envelope.Content.Span);
|
||||
var json = JsonDocument.Parse(envelope.Content);
|
||||
var sig = json.RootElement.GetProperty("signatures")[0].GetProperty("sig").GetString();
|
||||
|
||||
var expectedSig = ComputeExpectedSignature("application/json", payload, "a2V5LXNlY3JldA==");
|
||||
@@ -49,11 +54,15 @@ public sealed class HmacDsseEnvelopeSignerTests
|
||||
signing.AllowDeterministicFallback = true;
|
||||
});
|
||||
|
||||
var signer = new HmacDsseEnvelopeSigner(options, NullLogger<HmacDsseEnvelopeSigner>.Instance, new ServiceCollection().BuildServiceProvider());
|
||||
var signer = new HmacDsseEnvelopeSigner(
|
||||
options,
|
||||
DefaultCryptoHmac.CreateForTests(),
|
||||
NullLogger<HmacDsseEnvelopeSigner>.Instance,
|
||||
new ServiceCollection().BuildServiceProvider());
|
||||
var payload = Encoding.UTF8.GetBytes("abc");
|
||||
|
||||
var envelope = await signer.SignAsync("text/plain", payload, "kind", "root", view: null, CancellationToken.None);
|
||||
var json = JsonDocument.Parse(envelope.Content.Span);
|
||||
var json = JsonDocument.Parse(envelope.Content);
|
||||
var sig = json.RootElement.GetProperty("signatures")[0].GetProperty("sig").GetString();
|
||||
|
||||
// Deterministic signer encodes sha256 hex of payload as signature.
|
||||
|
||||
@@ -1,121 +1,128 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Worker.Options;
|
||||
using StellaOps.Scanner.Worker.Processing;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Tests;
|
||||
|
||||
public sealed class LeaseHeartbeatServiceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task RunAsync_RespectsSafetyFactorBudget()
|
||||
{
|
||||
var options = new ScannerWorkerOptions
|
||||
{
|
||||
MaxConcurrentJobs = 1,
|
||||
};
|
||||
options.Queue.HeartbeatSafetyFactor = 3.0;
|
||||
options.Queue.MinHeartbeatInterval = TimeSpan.FromSeconds(5);
|
||||
options.Queue.MaxHeartbeatInterval = TimeSpan.FromSeconds(60);
|
||||
options.Queue.SetHeartbeatRetryDelays(Array.Empty<TimeSpan>());
|
||||
options.Queue.MaxHeartbeatJitterMilliseconds = 750;
|
||||
|
||||
var optionsMonitor = new StaticOptionsMonitor<ScannerWorkerOptions>(options);
|
||||
using var cts = new CancellationTokenSource();
|
||||
var scheduler = new RecordingDelayScheduler(cts);
|
||||
var lease = new TestJobLease(TimeSpan.FromSeconds(90));
|
||||
|
||||
var service = new LeaseHeartbeatService(TimeProvider.System, scheduler, optionsMonitor, NullLogger<LeaseHeartbeatService>.Instance);
|
||||
|
||||
await service.RunAsync(lease, cts.Token);
|
||||
|
||||
var delay = Assert.Single(scheduler.Delays);
|
||||
var expectedMax = TimeSpan.FromTicks((long)(lease.LeaseDuration.Ticks / Math.Max(3.0, options.Queue.HeartbeatSafetyFactor)));
|
||||
Assert.True(delay <= expectedMax, $"Heartbeat delay {delay} should stay within safety factor budget {expectedMax}.");
|
||||
Assert.True(delay >= options.Queue.MinHeartbeatInterval, $"Heartbeat delay {delay} should respect minimum interval {options.Queue.MinHeartbeatInterval}.");
|
||||
}
|
||||
|
||||
private sealed class RecordingDelayScheduler : IDelayScheduler
|
||||
{
|
||||
private readonly CancellationTokenSource _cts;
|
||||
|
||||
public RecordingDelayScheduler(CancellationTokenSource cts)
|
||||
{
|
||||
_cts = cts ?? throw new ArgumentNullException(nameof(cts));
|
||||
}
|
||||
|
||||
public List<TimeSpan> Delays { get; } = new();
|
||||
|
||||
public Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken)
|
||||
{
|
||||
Delays.Add(delay);
|
||||
_cts.Cancel();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class TestJobLease : IScanJobLease
|
||||
{
|
||||
public TestJobLease(TimeSpan leaseDuration)
|
||||
{
|
||||
LeaseDuration = leaseDuration;
|
||||
EnqueuedAtUtc = DateTimeOffset.UtcNow - leaseDuration;
|
||||
LeasedAtUtc = DateTimeOffset.UtcNow;
|
||||
}
|
||||
|
||||
public string JobId { get; } = Guid.NewGuid().ToString("n");
|
||||
|
||||
public string ScanId { get; } = $"scan-{Guid.NewGuid():n}";
|
||||
|
||||
public int Attempt { get; } = 1;
|
||||
|
||||
public DateTimeOffset EnqueuedAtUtc { get; }
|
||||
|
||||
public DateTimeOffset LeasedAtUtc { get; }
|
||||
|
||||
public TimeSpan LeaseDuration { get; }
|
||||
|
||||
public IReadOnlyDictionary<string, string> Metadata { get; } = new Dictionary<string, string>();
|
||||
|
||||
public ValueTask RenewAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask CompleteAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask AbandonAsync(string reason, CancellationToken cancellationToken) => ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask PoisonAsync(string reason, CancellationToken cancellationToken) => ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
private sealed class StaticOptionsMonitor<TOptions> : IOptionsMonitor<TOptions>
|
||||
where TOptions : class
|
||||
{
|
||||
private readonly TOptions _value;
|
||||
|
||||
public StaticOptionsMonitor(TOptions value)
|
||||
{
|
||||
_value = value ?? throw new ArgumentNullException(nameof(value));
|
||||
}
|
||||
|
||||
public TOptions CurrentValue => _value;
|
||||
|
||||
public TOptions Get(string? name) => _value;
|
||||
|
||||
public IDisposable OnChange(Action<TOptions, string?> listener) => NullDisposable.Instance;
|
||||
|
||||
private sealed class NullDisposable : IDisposable
|
||||
{
|
||||
public static readonly NullDisposable Instance = new();
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Worker.Options;
|
||||
using StellaOps.Scanner.Worker.Determinism;
|
||||
using StellaOps.Scanner.Worker.Processing;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Tests;
|
||||
|
||||
public sealed class LeaseHeartbeatServiceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task RunAsync_RespectsSafetyFactorBudget()
|
||||
{
|
||||
var options = new ScannerWorkerOptions
|
||||
{
|
||||
MaxConcurrentJobs = 1,
|
||||
};
|
||||
options.Queue.HeartbeatSafetyFactor = 3.0;
|
||||
options.Queue.MinHeartbeatInterval = TimeSpan.FromSeconds(5);
|
||||
options.Queue.MaxHeartbeatInterval = TimeSpan.FromSeconds(60);
|
||||
options.Queue.SetHeartbeatRetryDelays(Array.Empty<TimeSpan>());
|
||||
options.Queue.MaxHeartbeatJitterMilliseconds = 750;
|
||||
|
||||
var optionsMonitor = new StaticOptionsMonitor<ScannerWorkerOptions>(options);
|
||||
using var cts = new CancellationTokenSource();
|
||||
var scheduler = new RecordingDelayScheduler(cts);
|
||||
var lease = new TestJobLease(TimeSpan.FromSeconds(90));
|
||||
var randomProvider = new DeterministicRandomProvider(seed: 1337);
|
||||
|
||||
var service = new LeaseHeartbeatService(
|
||||
TimeProvider.System,
|
||||
scheduler,
|
||||
optionsMonitor,
|
||||
randomProvider,
|
||||
NullLogger<LeaseHeartbeatService>.Instance);
|
||||
|
||||
await service.RunAsync(lease, cts.Token);
|
||||
|
||||
var delay = Assert.Single(scheduler.Delays);
|
||||
var expectedMax = TimeSpan.FromTicks((long)(lease.LeaseDuration.Ticks / Math.Max(3.0, options.Queue.HeartbeatSafetyFactor)));
|
||||
Assert.True(delay <= expectedMax, $"Heartbeat delay {delay} should stay within safety factor budget {expectedMax}.");
|
||||
Assert.True(delay >= options.Queue.MinHeartbeatInterval, $"Heartbeat delay {delay} should respect minimum interval {options.Queue.MinHeartbeatInterval}.");
|
||||
}
|
||||
|
||||
private sealed class RecordingDelayScheduler : IDelayScheduler
|
||||
{
|
||||
private readonly CancellationTokenSource _cts;
|
||||
|
||||
public RecordingDelayScheduler(CancellationTokenSource cts)
|
||||
{
|
||||
_cts = cts ?? throw new ArgumentNullException(nameof(cts));
|
||||
}
|
||||
|
||||
public List<TimeSpan> Delays { get; } = new();
|
||||
|
||||
public Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken)
|
||||
{
|
||||
Delays.Add(delay);
|
||||
_cts.Cancel();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class TestJobLease : IScanJobLease
|
||||
{
|
||||
public TestJobLease(TimeSpan leaseDuration)
|
||||
{
|
||||
LeaseDuration = leaseDuration;
|
||||
EnqueuedAtUtc = DateTimeOffset.UtcNow - leaseDuration;
|
||||
LeasedAtUtc = DateTimeOffset.UtcNow;
|
||||
}
|
||||
|
||||
public string JobId { get; } = Guid.NewGuid().ToString("n");
|
||||
|
||||
public string ScanId { get; } = $"scan-{Guid.NewGuid():n}";
|
||||
|
||||
public int Attempt { get; } = 1;
|
||||
|
||||
public DateTimeOffset EnqueuedAtUtc { get; }
|
||||
|
||||
public DateTimeOffset LeasedAtUtc { get; }
|
||||
|
||||
public TimeSpan LeaseDuration { get; }
|
||||
|
||||
public IReadOnlyDictionary<string, string> Metadata { get; } = new Dictionary<string, string>();
|
||||
|
||||
public ValueTask RenewAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask CompleteAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask AbandonAsync(string reason, CancellationToken cancellationToken) => ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask PoisonAsync(string reason, CancellationToken cancellationToken) => ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
private sealed class StaticOptionsMonitor<TOptions> : IOptionsMonitor<TOptions>
|
||||
where TOptions : class
|
||||
{
|
||||
private readonly TOptions _value;
|
||||
|
||||
public StaticOptionsMonitor(TOptions value)
|
||||
{
|
||||
_value = value ?? throw new ArgumentNullException(nameof(value));
|
||||
}
|
||||
|
||||
public TOptions CurrentValue => _value;
|
||||
|
||||
public TOptions Get(string? name) => _value;
|
||||
|
||||
public IDisposable OnChange(Action<TOptions, string?> listener) => NullDisposable.Instance;
|
||||
|
||||
private sealed class NullDisposable : IDisposable
|
||||
{
|
||||
public static readonly NullDisposable Instance = new();
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,245 +1,245 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Queue;
|
||||
using StellaOps.Scanner.Worker.Diagnostics;
|
||||
using StellaOps.Scanner.Worker.Hosting;
|
||||
using StellaOps.Scanner.Worker.Options;
|
||||
using StellaOps.Scanner.Worker.Processing;
|
||||
using StellaOps.Scanner.Worker.Tests.TestInfrastructure;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Tests;
|
||||
|
||||
public sealed class RedisWorkerSmokeTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Worker_CompletesJob_ViaRedisQueue()
|
||||
{
|
||||
var flag = Environment.GetEnvironmentVariable("STELLAOPS_REDIS_SMOKE");
|
||||
if (string.IsNullOrWhiteSpace(flag))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var redisConnection = Environment.GetEnvironmentVariable("STELLAOPS_REDIS_CONNECTION") ?? "localhost:6379";
|
||||
var streamName = $"scanner:jobs:{Guid.NewGuid():n}";
|
||||
var consumerGroup = $"worker-smoke-{Guid.NewGuid():n}";
|
||||
var configuration = BuildQueueConfiguration(redisConnection, streamName, consumerGroup);
|
||||
|
||||
var queueOptions = new ScannerQueueOptions();
|
||||
configuration.GetSection("scanner:queue").Bind(queueOptions);
|
||||
|
||||
var workerOptions = new ScannerWorkerOptions
|
||||
{
|
||||
MaxConcurrentJobs = 1,
|
||||
};
|
||||
workerOptions.Queue.HeartbeatSafetyFactor = 3.0;
|
||||
workerOptions.Queue.MinHeartbeatInterval = TimeSpan.FromSeconds(2);
|
||||
workerOptions.Queue.MaxHeartbeatInterval = TimeSpan.FromSeconds(8);
|
||||
workerOptions.Queue.SetHeartbeatRetryDelays(new[]
|
||||
{
|
||||
TimeSpan.FromMilliseconds(200),
|
||||
TimeSpan.FromMilliseconds(500),
|
||||
TimeSpan.FromSeconds(1),
|
||||
});
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging(builder =>
|
||||
{
|
||||
builder.SetMinimumLevel(LogLevel.Debug);
|
||||
builder.AddConsole();
|
||||
});
|
||||
services.AddSingleton(TimeProvider.System);
|
||||
services.AddScannerQueue(configuration, "scanner:queue");
|
||||
services.AddSingleton<IScanJobSource, QueueBackedScanJobSource>();
|
||||
services.AddSingleton<QueueBackedScanJobSourceDependencies>();
|
||||
services.AddSingleton(queueOptions);
|
||||
services.AddSingleton<IScanAnalyzerDispatcher, SmokeAnalyzerDispatcher>();
|
||||
services.AddSingleton<IScanStageExecutor, AnalyzerStageExecutor>();
|
||||
services.AddSingleton<ScannerWorkerMetrics>();
|
||||
services.AddSingleton<ScanProgressReporter>();
|
||||
services.AddSingleton<ScanJobProcessor>();
|
||||
services.AddSingleton<LeaseHeartbeatService>();
|
||||
services.AddSingleton<IDelayScheduler, SystemDelayScheduler>();
|
||||
services.AddSingleton<IOptionsMonitor<ScannerWorkerOptions>>(new StaticOptionsMonitor<ScannerWorkerOptions>(workerOptions));
|
||||
services.AddSingleton<ScannerWorkerHostedService>();
|
||||
|
||||
using var provider = services.BuildServiceProvider();
|
||||
var queue = provider.GetRequiredService<IScanQueue>();
|
||||
|
||||
var jobId = $"job-{Guid.NewGuid():n}";
|
||||
var scanId = $"scan-{Guid.NewGuid():n}";
|
||||
await queue.EnqueueAsync(new ScanQueueMessage(jobId, Encoding.UTF8.GetBytes("smoke"))
|
||||
{
|
||||
Attributes = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["scanId"] = scanId,
|
||||
["queue"] = "redis",
|
||||
}
|
||||
});
|
||||
|
||||
var hostedService = provider.GetRequiredService<ScannerWorkerHostedService>();
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
|
||||
|
||||
await hostedService.StartAsync(cts.Token);
|
||||
|
||||
var smokeObserver = provider.GetRequiredService<QueueBackedScanJobSourceDependencies>();
|
||||
await smokeObserver.JobCompleted.Task.WaitAsync(TimeSpan.FromSeconds(20));
|
||||
|
||||
await hostedService.StopAsync(CancellationToken.None);
|
||||
}
|
||||
|
||||
private static IConfiguration BuildQueueConfiguration(string connection, string stream, string consumerGroup)
|
||||
{
|
||||
return new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["scanner:queue:kind"] = "redis",
|
||||
["scanner:queue:defaultLeaseDuration"] = "00:00:30",
|
||||
["scanner:queue:redis:connectionString"] = connection,
|
||||
["scanner:queue:redis:streamName"] = stream,
|
||||
["scanner:queue:redis:consumerGroup"] = consumerGroup,
|
||||
["scanner:queue:redis:idempotencyKeyPrefix"] = $"{stream}:idemp:",
|
||||
["scanner:queue:redis:initializationTimeout"] = "00:00:10",
|
||||
})
|
||||
.Build();
|
||||
}
|
||||
|
||||
private sealed class SmokeAnalyzerDispatcher : IScanAnalyzerDispatcher
|
||||
{
|
||||
public ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class QueueBackedScanJobSourceDependencies
|
||||
{
|
||||
public QueueBackedScanJobSourceDependencies()
|
||||
{
|
||||
JobCompleted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
}
|
||||
|
||||
public TaskCompletionSource JobCompleted { get; }
|
||||
}
|
||||
|
||||
private sealed class QueueBackedScanJobSource : IScanJobSource
|
||||
{
|
||||
private readonly IScanQueue _queue;
|
||||
private readonly ScannerQueueOptions _queueOptions;
|
||||
private readonly QueueBackedScanJobSourceDependencies _deps;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly string _consumerName = $"worker-smoke-{Guid.NewGuid():n}";
|
||||
|
||||
public QueueBackedScanJobSource(
|
||||
IScanQueue queue,
|
||||
ScannerQueueOptions queueOptions,
|
||||
QueueBackedScanJobSourceDependencies deps,
|
||||
TimeProvider timeProvider)
|
||||
{
|
||||
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
|
||||
_queueOptions = queueOptions ?? throw new ArgumentNullException(nameof(queueOptions));
|
||||
_deps = deps ?? throw new ArgumentNullException(nameof(deps));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public async Task<IScanJobLease?> TryAcquireAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var request = new QueueLeaseRequest(_consumerName, 1, _queueOptions.DefaultLeaseDuration);
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Queue;
|
||||
using StellaOps.Scanner.Worker.Diagnostics;
|
||||
using StellaOps.Scanner.Worker.Hosting;
|
||||
using StellaOps.Scanner.Worker.Options;
|
||||
using StellaOps.Scanner.Worker.Processing;
|
||||
using StellaOps.Scanner.Worker.Tests.TestInfrastructure;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Tests;
|
||||
|
||||
public sealed class RedisWorkerSmokeTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Worker_CompletesJob_ViaRedisQueue()
|
||||
{
|
||||
var flag = Environment.GetEnvironmentVariable("STELLAOPS_REDIS_SMOKE");
|
||||
if (string.IsNullOrWhiteSpace(flag))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var redisConnection = Environment.GetEnvironmentVariable("STELLAOPS_REDIS_CONNECTION") ?? "localhost:6379";
|
||||
var streamName = $"scanner:jobs:{Guid.NewGuid():n}";
|
||||
var consumerGroup = $"worker-smoke-{Guid.NewGuid():n}";
|
||||
var configuration = BuildQueueConfiguration(redisConnection, streamName, consumerGroup);
|
||||
|
||||
var queueOptions = new ScannerQueueOptions();
|
||||
configuration.GetSection("scanner:queue").Bind(queueOptions);
|
||||
|
||||
var workerOptions = new ScannerWorkerOptions
|
||||
{
|
||||
MaxConcurrentJobs = 1,
|
||||
};
|
||||
workerOptions.Queue.HeartbeatSafetyFactor = 3.0;
|
||||
workerOptions.Queue.MinHeartbeatInterval = TimeSpan.FromSeconds(2);
|
||||
workerOptions.Queue.MaxHeartbeatInterval = TimeSpan.FromSeconds(8);
|
||||
workerOptions.Queue.SetHeartbeatRetryDelays(new[]
|
||||
{
|
||||
TimeSpan.FromMilliseconds(200),
|
||||
TimeSpan.FromMilliseconds(500),
|
||||
TimeSpan.FromSeconds(1),
|
||||
});
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging(builder =>
|
||||
{
|
||||
builder.SetMinimumLevel(LogLevel.Debug);
|
||||
builder.AddConsole();
|
||||
});
|
||||
services.AddSingleton(TimeProvider.System);
|
||||
services.AddScannerQueue(configuration, "scanner:queue");
|
||||
services.AddSingleton<IScanJobSource, QueueBackedScanJobSource>();
|
||||
services.AddSingleton<QueueBackedScanJobSourceDependencies>();
|
||||
services.AddSingleton(queueOptions);
|
||||
services.AddSingleton<IScanAnalyzerDispatcher, SmokeAnalyzerDispatcher>();
|
||||
services.AddSingleton<IScanStageExecutor, AnalyzerStageExecutor>();
|
||||
services.AddSingleton<ScannerWorkerMetrics>();
|
||||
services.AddSingleton<ScanProgressReporter>();
|
||||
services.AddSingleton<ScanJobProcessor>();
|
||||
services.AddSingleton<LeaseHeartbeatService>();
|
||||
services.AddSingleton<IDelayScheduler, SystemDelayScheduler>();
|
||||
services.AddSingleton<IOptionsMonitor<ScannerWorkerOptions>>(new StaticOptionsMonitor<ScannerWorkerOptions>(workerOptions));
|
||||
services.AddSingleton<ScannerWorkerHostedService>();
|
||||
|
||||
using var provider = services.BuildServiceProvider();
|
||||
var queue = provider.GetRequiredService<IScanQueue>();
|
||||
|
||||
var jobId = $"job-{Guid.NewGuid():n}";
|
||||
var scanId = $"scan-{Guid.NewGuid():n}";
|
||||
await queue.EnqueueAsync(new ScanQueueMessage(jobId, Encoding.UTF8.GetBytes("smoke"))
|
||||
{
|
||||
Attributes = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["scanId"] = scanId,
|
||||
["queue"] = "redis",
|
||||
}
|
||||
});
|
||||
|
||||
var hostedService = provider.GetRequiredService<ScannerWorkerHostedService>();
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
|
||||
|
||||
await hostedService.StartAsync(cts.Token);
|
||||
|
||||
var smokeObserver = provider.GetRequiredService<QueueBackedScanJobSourceDependencies>();
|
||||
await smokeObserver.JobCompleted.Task.WaitAsync(TimeSpan.FromSeconds(20));
|
||||
|
||||
await hostedService.StopAsync(CancellationToken.None);
|
||||
}
|
||||
|
||||
private static IConfiguration BuildQueueConfiguration(string connection, string stream, string consumerGroup)
|
||||
{
|
||||
return new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["scanner:queue:kind"] = "redis",
|
||||
["scanner:queue:defaultLeaseDuration"] = "00:00:30",
|
||||
["scanner:queue:redis:connectionString"] = connection,
|
||||
["scanner:queue:redis:streamName"] = stream,
|
||||
["scanner:queue:redis:consumerGroup"] = consumerGroup,
|
||||
["scanner:queue:redis:idempotencyKeyPrefix"] = $"{stream}:idemp:",
|
||||
["scanner:queue:redis:initializationTimeout"] = "00:00:10",
|
||||
})
|
||||
.Build();
|
||||
}
|
||||
|
||||
private sealed class SmokeAnalyzerDispatcher : IScanAnalyzerDispatcher
|
||||
{
|
||||
public ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class QueueBackedScanJobSourceDependencies
|
||||
{
|
||||
public QueueBackedScanJobSourceDependencies()
|
||||
{
|
||||
JobCompleted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
}
|
||||
|
||||
public TaskCompletionSource JobCompleted { get; }
|
||||
}
|
||||
|
||||
private sealed class QueueBackedScanJobSource : IScanJobSource
|
||||
{
|
||||
private readonly IScanQueue _queue;
|
||||
private readonly ScannerQueueOptions _queueOptions;
|
||||
private readonly QueueBackedScanJobSourceDependencies _deps;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly string _consumerName = $"worker-smoke-{Guid.NewGuid():n}";
|
||||
|
||||
public QueueBackedScanJobSource(
|
||||
IScanQueue queue,
|
||||
ScannerQueueOptions queueOptions,
|
||||
QueueBackedScanJobSourceDependencies deps,
|
||||
TimeProvider timeProvider)
|
||||
{
|
||||
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
|
||||
_queueOptions = queueOptions ?? throw new ArgumentNullException(nameof(queueOptions));
|
||||
_deps = deps ?? throw new ArgumentNullException(nameof(deps));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public async Task<IScanJobLease?> TryAcquireAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var request = new QueueLeaseRequest(_consumerName, 1, _queueOptions.DefaultLeaseDuration);
|
||||
var leases = await _queue.LeaseAsync(request, cancellationToken);
|
||||
if (leases.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return new QueueBackedScanJobLease(
|
||||
leases[0],
|
||||
_queueOptions,
|
||||
_deps,
|
||||
_timeProvider.GetUtcNow());
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class QueueBackedScanJobLease : IScanJobLease
|
||||
{
|
||||
private readonly IScanQueueLease _lease;
|
||||
private readonly ScannerQueueOptions _options;
|
||||
private readonly QueueBackedScanJobSourceDependencies _deps;
|
||||
private readonly DateTimeOffset _leasedAt;
|
||||
private readonly IReadOnlyDictionary<string, string> _metadata;
|
||||
|
||||
public QueueBackedScanJobLease(
|
||||
IScanQueueLease lease,
|
||||
ScannerQueueOptions options,
|
||||
QueueBackedScanJobSourceDependencies deps,
|
||||
DateTimeOffset leasedAt)
|
||||
{
|
||||
_lease = lease ?? throw new ArgumentNullException(nameof(lease));
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_deps = deps ?? throw new ArgumentNullException(nameof(deps));
|
||||
_leasedAt = leasedAt;
|
||||
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["queue"] = _options.Kind.ToString(),
|
||||
["queue.consumer"] = lease.Consumer,
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(lease.IdempotencyKey))
|
||||
{
|
||||
metadata["job.idempotency"] = lease.IdempotencyKey;
|
||||
}
|
||||
|
||||
foreach (var attribute in lease.Attributes)
|
||||
{
|
||||
metadata[attribute.Key] = attribute.Value;
|
||||
}
|
||||
|
||||
_metadata = metadata;
|
||||
}
|
||||
|
||||
public string JobId => _lease.JobId;
|
||||
|
||||
public string ScanId => _metadata.TryGetValue("scanId", out var scanId) ? scanId : _lease.JobId;
|
||||
|
||||
public int Attempt => _lease.Attempt;
|
||||
|
||||
public DateTimeOffset EnqueuedAtUtc => _lease.EnqueuedAt;
|
||||
|
||||
public DateTimeOffset LeasedAtUtc => _leasedAt;
|
||||
|
||||
public TimeSpan LeaseDuration => _lease.LeaseExpiresAt - _leasedAt;
|
||||
|
||||
public IReadOnlyDictionary<string, string> Metadata => _metadata;
|
||||
|
||||
public async ValueTask RenewAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (leases.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return new QueueBackedScanJobLease(
|
||||
leases[0],
|
||||
_queueOptions,
|
||||
_deps,
|
||||
_timeProvider.GetUtcNow());
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class QueueBackedScanJobLease : IScanJobLease
|
||||
{
|
||||
private readonly IScanQueueLease _lease;
|
||||
private readonly ScannerQueueOptions _options;
|
||||
private readonly QueueBackedScanJobSourceDependencies _deps;
|
||||
private readonly DateTimeOffset _leasedAt;
|
||||
private readonly IReadOnlyDictionary<string, string> _metadata;
|
||||
|
||||
public QueueBackedScanJobLease(
|
||||
IScanQueueLease lease,
|
||||
ScannerQueueOptions options,
|
||||
QueueBackedScanJobSourceDependencies deps,
|
||||
DateTimeOffset leasedAt)
|
||||
{
|
||||
_lease = lease ?? throw new ArgumentNullException(nameof(lease));
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_deps = deps ?? throw new ArgumentNullException(nameof(deps));
|
||||
_leasedAt = leasedAt;
|
||||
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["queue"] = _options.Kind.ToString(),
|
||||
["queue.consumer"] = lease.Consumer,
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(lease.IdempotencyKey))
|
||||
{
|
||||
metadata["job.idempotency"] = lease.IdempotencyKey;
|
||||
}
|
||||
|
||||
foreach (var attribute in lease.Attributes)
|
||||
{
|
||||
metadata[attribute.Key] = attribute.Value;
|
||||
}
|
||||
|
||||
_metadata = metadata;
|
||||
}
|
||||
|
||||
public string JobId => _lease.JobId;
|
||||
|
||||
public string ScanId => _metadata.TryGetValue("scanId", out var scanId) ? scanId : _lease.JobId;
|
||||
|
||||
public int Attempt => _lease.Attempt;
|
||||
|
||||
public DateTimeOffset EnqueuedAtUtc => _lease.EnqueuedAt;
|
||||
|
||||
public DateTimeOffset LeasedAtUtc => _leasedAt;
|
||||
|
||||
public TimeSpan LeaseDuration => _lease.LeaseExpiresAt - _leasedAt;
|
||||
|
||||
public IReadOnlyDictionary<string, string> Metadata => _metadata;
|
||||
|
||||
public async ValueTask RenewAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
await _lease.RenewAsync(_options.DefaultLeaseDuration, cancellationToken);
|
||||
}
|
||||
|
||||
public async ValueTask CompleteAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
}
|
||||
|
||||
public async ValueTask CompleteAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
await _lease.AcknowledgeAsync(cancellationToken);
|
||||
_deps.JobCompleted.TrySetResult();
|
||||
}
|
||||
|
||||
public async ValueTask AbandonAsync(string reason, CancellationToken cancellationToken)
|
||||
{
|
||||
_deps.JobCompleted.TrySetResult();
|
||||
}
|
||||
|
||||
public async ValueTask AbandonAsync(string reason, CancellationToken cancellationToken)
|
||||
{
|
||||
await _lease.ReleaseAsync(QueueReleaseDisposition.Retry, cancellationToken);
|
||||
}
|
||||
|
||||
public async ValueTask PoisonAsync(string reason, CancellationToken cancellationToken)
|
||||
{
|
||||
}
|
||||
|
||||
public async ValueTask PoisonAsync(string reason, CancellationToken cancellationToken)
|
||||
{
|
||||
await _lease.DeadLetterAsync(reason, cancellationToken);
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using StellaOps.Scanner.Core.Contracts;
|
||||
using StellaOps.Scanner.Worker.Processing;
|
||||
using StellaOps.Scanner.Worker.Processing.Replay;
|
||||
using Xunit;
|
||||
|
||||
@@ -14,27 +15,27 @@ public sealed class ReplaySealedBundleStageExecutorTests
|
||||
public async Task ExecuteAsync_SetsMetadata_WhenUriAndHashProvided()
|
||||
{
|
||||
var executor = new ReplaySealedBundleStageExecutor(NullLogger<ReplaySealedBundleStageExecutor>.Instance);
|
||||
var context = TestContexts.Create();
|
||||
context.Lease.Metadata["replay.bundle.uri"] = "cas://replay/input.tar.zst";
|
||||
context.Lease.Metadata["replay.bundle.sha256"] = "abc123";
|
||||
context.Lease.Metadata["determinism.policy"] = "rev-1";
|
||||
context.Lease.Metadata["determinism.feed"] = "feed-2";
|
||||
var context = TestContexts.Create(out var metadata);
|
||||
metadata["replay.bundle.uri"] = "cas://replay/input.tar.zst";
|
||||
metadata["replay.bundle.sha256"] = "abc123";
|
||||
metadata["determinism.policy"] = "rev-1";
|
||||
metadata["determinism.feed"] = "feed-2";
|
||||
|
||||
await executor.ExecuteAsync(context, CancellationToken.None);
|
||||
|
||||
Assert.True(context.Analysis.TryGet<ReplaySealedBundleMetadata>(ScanAnalysisKeys.ReplaySealedBundleMetadata, out var metadata));
|
||||
Assert.Equal("abc123", metadata.ManifestHash);
|
||||
Assert.Equal("cas://replay/input.tar.zst", metadata.BundleUri);
|
||||
Assert.Equal("rev-1", metadata.PolicySnapshotId);
|
||||
Assert.Equal("feed-2", metadata.FeedSnapshotId);
|
||||
Assert.True(context.Analysis.TryGet<ReplaySealedBundleMetadata>(ScanAnalysisKeys.ReplaySealedBundleMetadata, out var sealedMetadata));
|
||||
Assert.Equal("abc123", sealedMetadata.ManifestHash);
|
||||
Assert.Equal("cas://replay/input.tar.zst", sealedMetadata.BundleUri);
|
||||
Assert.Equal("rev-1", sealedMetadata.PolicySnapshotId);
|
||||
Assert.Equal("feed-2", sealedMetadata.FeedSnapshotId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_Skips_WhenHashMissing()
|
||||
{
|
||||
var executor = new ReplaySealedBundleStageExecutor(NullLogger<ReplaySealedBundleStageExecutor>.Instance);
|
||||
var context = TestContexts.Create();
|
||||
context.Lease.Metadata["replay.bundle.uri"] = "cas://replay/input.tar.zst";
|
||||
var context = TestContexts.Create(out var metadata);
|
||||
metadata["replay.bundle.uri"] = "cas://replay/input.tar.zst";
|
||||
|
||||
await executor.ExecuteAsync(context, CancellationToken.None);
|
||||
|
||||
@@ -44,9 +45,10 @@ public sealed class ReplaySealedBundleStageExecutorTests
|
||||
|
||||
internal static class TestContexts
|
||||
{
|
||||
public static ScanJobContext Create()
|
||||
public static ScanJobContext Create(out Dictionary<string, string> metadata)
|
||||
{
|
||||
var lease = new TestScanJobLease();
|
||||
metadata = lease.MutableMetadata;
|
||||
return new ScanJobContext(lease, TimeProvider.System, TimeProvider.System.GetUtcNow(), CancellationToken.None);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,26 +1,26 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using StellaOps.Scanner.Worker.Options;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Tests;
|
||||
|
||||
public sealed class ScannerWorkerOptionsValidatorTests
|
||||
{
|
||||
[Fact]
|
||||
public void Validate_Fails_WhenHeartbeatSafetyFactorBelowThree()
|
||||
{
|
||||
var options = new ScannerWorkerOptions();
|
||||
options.Queue.HeartbeatSafetyFactor = 2.5;
|
||||
|
||||
var validator = new ScannerWorkerOptionsValidator();
|
||||
var result = validator.Validate(string.Empty, options);
|
||||
|
||||
Assert.True(result.Failed, "Validation should fail when HeartbeatSafetyFactor < 3.");
|
||||
Assert.Contains(result.Failures, failure => failure.Contains("HeartbeatSafetyFactor", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
using System;
|
||||
using System.Linq;
|
||||
using StellaOps.Scanner.Worker.Options;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Tests;
|
||||
|
||||
public sealed class ScannerWorkerOptionsValidatorTests
|
||||
{
|
||||
[Fact]
|
||||
public void Validate_Fails_WhenHeartbeatSafetyFactorBelowThree()
|
||||
{
|
||||
var options = new ScannerWorkerOptions();
|
||||
options.Queue.HeartbeatSafetyFactor = 2.5;
|
||||
|
||||
var validator = new ScannerWorkerOptionsValidator();
|
||||
var result = validator.Validate(string.Empty, options);
|
||||
|
||||
Assert.True(result.Failed, "Validation should fail when HeartbeatSafetyFactor < 3.");
|
||||
Assert.Contains(result.Failures, failure => failure.Contains("HeartbeatSafetyFactor", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_Succeeds_WhenHeartbeatSafetyFactorAtLeastThree()
|
||||
{
|
||||
var options = new ScannerWorkerOptions();
|
||||
|
||||
@@ -43,14 +43,17 @@ public sealed class SurfaceManifestStageExecutorTests
|
||||
listener.Start();
|
||||
|
||||
var hash = CreateCryptoHash();
|
||||
var manifestWriter = new TestSurfaceManifestWriter();
|
||||
var executor = new SurfaceManifestStageExecutor(
|
||||
publisher,
|
||||
manifestWriter,
|
||||
cache,
|
||||
environment,
|
||||
metrics,
|
||||
NullLogger<SurfaceManifestStageExecutor>.Instance,
|
||||
hash,
|
||||
new NullRubyPackageInventoryStore(),
|
||||
new NullBunPackageInventoryStore(),
|
||||
new DeterminismContext(true, DateTimeOffset.Parse("2024-01-01T00:00:00Z"), 1337, true, 1),
|
||||
new DeterministicDsseEnvelopeSigner());
|
||||
|
||||
@@ -82,14 +85,17 @@ public sealed class SurfaceManifestStageExecutorTests
|
||||
listener.Start();
|
||||
|
||||
var hash = CreateCryptoHash();
|
||||
var manifestWriter = new TestSurfaceManifestWriter();
|
||||
var executor = new SurfaceManifestStageExecutor(
|
||||
publisher,
|
||||
manifestWriter,
|
||||
cache,
|
||||
environment,
|
||||
metrics,
|
||||
NullLogger<SurfaceManifestStageExecutor>.Instance,
|
||||
hash,
|
||||
new NullRubyPackageInventoryStore(),
|
||||
new NullBunPackageInventoryStore(),
|
||||
new DeterminismContext(false, DateTimeOffset.UnixEpoch, null, false, null),
|
||||
new DeterministicDsseEnvelopeSigner());
|
||||
|
||||
@@ -125,10 +131,14 @@ public sealed class SurfaceManifestStageExecutorTests
|
||||
var payloadMetrics = listener.Measurements
|
||||
.Where(m => m.InstrumentName == "scanner_worker_surface_payload_persisted_total")
|
||||
.ToArray();
|
||||
Assert.Equal(3, payloadMetrics.Length);
|
||||
Assert.Equal(7, payloadMetrics.Length);
|
||||
Assert.Contains(payloadMetrics, m => Equals("entrytrace.graph", m["surface.kind"]));
|
||||
Assert.Contains(payloadMetrics, m => Equals("entrytrace.ndjson", m["surface.kind"]));
|
||||
Assert.Contains(payloadMetrics, m => Equals("layer.fragments", m["surface.kind"]));
|
||||
Assert.Contains(payloadMetrics, m => Equals("composition.recipe", m["surface.kind"]));
|
||||
Assert.Contains(payloadMetrics, m => Equals("composition.recipe.dsse", m["surface.kind"]));
|
||||
Assert.Contains(payloadMetrics, m => Equals("layer.fragments.dsse", m["surface.kind"]));
|
||||
Assert.Contains(payloadMetrics, m => Equals("determinism.json", m["surface.kind"]));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -148,22 +158,28 @@ public sealed class SurfaceManifestStageExecutorTests
|
||||
|
||||
var executor = new SurfaceManifestStageExecutor(
|
||||
publisher,
|
||||
new TestSurfaceManifestWriter(),
|
||||
cache,
|
||||
environment,
|
||||
metrics,
|
||||
NullLogger<SurfaceManifestStageExecutor>.Instance,
|
||||
hash,
|
||||
new NullRubyPackageInventoryStore(),
|
||||
determinism);
|
||||
new NullBunPackageInventoryStore(),
|
||||
determinism,
|
||||
new DeterministicDsseEnvelopeSigner());
|
||||
|
||||
var context = CreateContext();
|
||||
context.Lease.Metadata["determinism.feed"] = "feed-001";
|
||||
context.Lease.Metadata["determinism.policy"] = "rev-77";
|
||||
var context = CreateContext(new Dictionary<string, string>
|
||||
{
|
||||
["determinism.feed"] = "feed-001",
|
||||
["determinism.policy"] = "rev-77"
|
||||
});
|
||||
PopulateAnalysis(context);
|
||||
|
||||
await executor.ExecuteAsync(context, CancellationToken.None);
|
||||
|
||||
var determinismPayload = publisher.LastRequest!.Payloads.Single(p => p.Kind == "determinism.json");
|
||||
var json = JsonDocument.Parse(determinismPayload.Content.Span);
|
||||
var json = JsonDocument.Parse(determinismPayload.Content);
|
||||
|
||||
Assert.True(json.RootElement.GetProperty("fixedClock").GetBoolean());
|
||||
Assert.Equal(42, json.RootElement.GetProperty("rngSeed").GetInt32());
|
||||
@@ -188,13 +204,16 @@ public sealed class SurfaceManifestStageExecutorTests
|
||||
|
||||
var executor = new SurfaceManifestStageExecutor(
|
||||
publisher,
|
||||
new TestSurfaceManifestWriter(),
|
||||
cache,
|
||||
environment,
|
||||
metrics,
|
||||
NullLogger<SurfaceManifestStageExecutor>.Instance,
|
||||
hash,
|
||||
new NullRubyPackageInventoryStore(),
|
||||
new DeterminismContext(false, DateTimeOffset.UnixEpoch, null, false, null));
|
||||
new NullBunPackageInventoryStore(),
|
||||
new DeterminismContext(false, DateTimeOffset.UnixEpoch, null, false, null),
|
||||
new DeterministicDsseEnvelopeSigner());
|
||||
|
||||
var context = CreateContext();
|
||||
|
||||
@@ -230,8 +249,8 @@ public sealed class SurfaceManifestStageExecutorTests
|
||||
Assert.Contains(publisher.LastRequest!.Payloads, p => p.Kind == "entropy.report");
|
||||
Assert.Contains(publisher.LastRequest!.Payloads, p => p.Kind == "entropy.layer-summary");
|
||||
|
||||
// Two payloads + manifest persisted to cache.
|
||||
Assert.Equal(3, cache.Entries.Count);
|
||||
// Two payloads + determinism + manifest persisted to cache.
|
||||
Assert.Equal(6, cache.Entries.Count);
|
||||
}
|
||||
|
||||
private static ScanJobContext CreateContext(Dictionary<string, string>? metadata = null)
|
||||
@@ -310,12 +329,16 @@ public sealed class SurfaceManifestStageExecutorTests
|
||||
|
||||
var executor = new SurfaceManifestStageExecutor(
|
||||
publisher,
|
||||
new TestSurfaceManifestWriter(),
|
||||
cache,
|
||||
environment,
|
||||
metrics,
|
||||
NullLogger<SurfaceManifestStageExecutor>.Instance,
|
||||
hash,
|
||||
packageStore);
|
||||
packageStore,
|
||||
new NullBunPackageInventoryStore(),
|
||||
new DeterminismContext(false, DateTimeOffset.UnixEpoch, null, false, null),
|
||||
new DeterministicDsseEnvelopeSigner());
|
||||
|
||||
var context = CreateContext();
|
||||
PopulateAnalysis(context);
|
||||
@@ -340,12 +363,16 @@ public sealed class SurfaceManifestStageExecutorTests
|
||||
|
||||
var executor = new SurfaceManifestStageExecutor(
|
||||
publisher,
|
||||
new TestSurfaceManifestWriter(),
|
||||
cache,
|
||||
environment,
|
||||
metrics,
|
||||
NullLogger<SurfaceManifestStageExecutor>.Instance,
|
||||
hash,
|
||||
packageStore);
|
||||
packageStore,
|
||||
new NullBunPackageInventoryStore(),
|
||||
new DeterminismContext(false, DateTimeOffset.UnixEpoch, null, false, null),
|
||||
new DeterministicDsseEnvelopeSigner());
|
||||
|
||||
var context = CreateContext();
|
||||
PopulateAnalysis(context);
|
||||
@@ -407,15 +434,19 @@ public sealed class SurfaceManifestStageExecutorTests
|
||||
var cache = new RecordingSurfaceCache();
|
||||
var environment = new TestSurfaceEnvironment("tenant-a");
|
||||
var hash = CreateCryptoHash();
|
||||
var manifestWriter = new TestSurfaceManifestWriter();
|
||||
var executor = new SurfaceManifestStageExecutor(
|
||||
publisher,
|
||||
manifestWriter,
|
||||
cache,
|
||||
environment,
|
||||
metrics,
|
||||
NullLogger<SurfaceManifestStageExecutor>.Instance,
|
||||
hash,
|
||||
new NullRubyPackageInventoryStore(),
|
||||
new DeterminismContext(false, DateTimeOffset.UnixEpoch, null, false, null));
|
||||
new NullBunPackageInventoryStore(),
|
||||
new DeterminismContext(false, DateTimeOffset.UnixEpoch, null, false, null),
|
||||
new DeterministicDsseEnvelopeSigner());
|
||||
|
||||
var context = CreateContext();
|
||||
var observationBytes = Encoding.UTF8.GetBytes("{\"entrypoints\":[\"mod.ts\"]}");
|
||||
@@ -461,13 +492,16 @@ public sealed class SurfaceManifestStageExecutorTests
|
||||
|
||||
var executor = new SurfaceManifestStageExecutor(
|
||||
publisher,
|
||||
new TestSurfaceManifestWriter(),
|
||||
cache,
|
||||
environment,
|
||||
metrics,
|
||||
NullLogger<SurfaceManifestStageExecutor>.Instance,
|
||||
hash,
|
||||
new NullRubyPackageInventoryStore(),
|
||||
determinism);
|
||||
new NullBunPackageInventoryStore(),
|
||||
determinism,
|
||||
new DeterministicDsseEnvelopeSigner());
|
||||
|
||||
var leaseMetadata = new Dictionary<string, string>
|
||||
{
|
||||
@@ -553,6 +587,46 @@ public sealed class SurfaceManifestStageExecutorTests
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class TestSurfaceManifestWriter : ISurfaceManifestWriter
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = false,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
|
||||
};
|
||||
|
||||
public int PublishCalls { get; private set; }
|
||||
|
||||
public SurfaceManifestDocument? LastDocument { get; private set; }
|
||||
|
||||
public Task<SurfaceManifestPublishResult> PublishAsync(
|
||||
SurfaceManifestDocument document,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
PublishCalls++;
|
||||
LastDocument = document;
|
||||
|
||||
var json = JsonSerializer.SerializeToUtf8Bytes(document, SerializerOptions);
|
||||
var digest = ComputeDigest(json);
|
||||
|
||||
return Task.FromResult(new SurfaceManifestPublishResult(
|
||||
ManifestDigest: digest,
|
||||
ManifestUri: $"cas://test/surface.manifests/{digest}",
|
||||
ArtifactId: $"surface-manifest::{digest}",
|
||||
Document: document,
|
||||
DeterminismMerkleRoot: document.DeterminismMerkleRoot));
|
||||
}
|
||||
|
||||
private static string ComputeDigest(ReadOnlySpan<byte> content)
|
||||
{
|
||||
Span<byte> hash = stackalloc byte[32];
|
||||
SHA256.HashData(content, hash);
|
||||
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class TestSurfaceManifestPublisher : ISurfaceManifestPublisher
|
||||
{
|
||||
private readonly string _tenant;
|
||||
@@ -664,30 +738,7 @@ public sealed class SurfaceManifestStageExecutorTests
|
||||
}
|
||||
|
||||
private static ICryptoHash CreateCryptoHash()
|
||||
=> new DefaultCryptoHash(new StaticOptionsMonitor<CryptoHashOptions>(new CryptoHashOptions()), NullLogger<DefaultCryptoHash>.Instance);
|
||||
|
||||
private sealed class StaticOptionsMonitor<T> : IOptionsMonitor<T>
|
||||
{
|
||||
public StaticOptionsMonitor(T value)
|
||||
{
|
||||
CurrentValue = value;
|
||||
}
|
||||
|
||||
public T CurrentValue { get; }
|
||||
|
||||
public T Get(string? name) => CurrentValue;
|
||||
|
||||
public IDisposable OnChange(Action<T, string?> listener) => Disposable.Instance;
|
||||
|
||||
private sealed class Disposable : IDisposable
|
||||
{
|
||||
public static readonly Disposable Instance = new();
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
=> DefaultCryptoHash.CreateForTests();
|
||||
|
||||
private sealed class RecordingRubyPackageStore : IRubyPackageInventoryStore
|
||||
{
|
||||
|
||||
@@ -1,29 +1,29 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Tests.TestInfrastructure;
|
||||
|
||||
public sealed class StaticOptionsMonitor<TOptions> : IOptionsMonitor<TOptions>
|
||||
where TOptions : class
|
||||
{
|
||||
private readonly TOptions _value;
|
||||
|
||||
public StaticOptionsMonitor(TOptions value)
|
||||
{
|
||||
_value = value ?? throw new ArgumentNullException(nameof(value));
|
||||
}
|
||||
|
||||
public TOptions CurrentValue => _value;
|
||||
|
||||
public TOptions Get(string? name) => _value;
|
||||
|
||||
public IDisposable OnChange(Action<TOptions, string?> listener) => NullDisposable.Instance;
|
||||
|
||||
private sealed class NullDisposable : IDisposable
|
||||
{
|
||||
public static readonly NullDisposable Instance = new();
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Tests.TestInfrastructure;
|
||||
|
||||
public sealed class StaticOptionsMonitor<TOptions> : IOptionsMonitor<TOptions>
|
||||
where TOptions : class
|
||||
{
|
||||
private readonly TOptions _value;
|
||||
|
||||
public StaticOptionsMonitor(TOptions value)
|
||||
{
|
||||
_value = value ?? throw new ArgumentNullException(nameof(value));
|
||||
}
|
||||
|
||||
public TOptions CurrentValue => _value;
|
||||
|
||||
public TOptions Get(string? name) => _value;
|
||||
|
||||
public IDisposable OnChange(Action<TOptions, string?> listener) => NullDisposable.Instance;
|
||||
|
||||
private sealed class NullDisposable : IDisposable
|
||||
{
|
||||
public static readonly NullDisposable Instance = new();
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,6 +35,30 @@ internal sealed class TestCryptoHash : ICryptoHash
|
||||
return Convert.ToHexString(bytes).ToLowerInvariant();
|
||||
}
|
||||
|
||||
public byte[] ComputeHashForPurpose(ReadOnlySpan<byte> data, string purpose)
|
||||
=> ComputeHash(data, GetAlgorithmForPurpose(purpose));
|
||||
|
||||
public string ComputeHashHexForPurpose(ReadOnlySpan<byte> data, string purpose)
|
||||
=> ComputeHashHex(data, GetAlgorithmForPurpose(purpose));
|
||||
|
||||
public string ComputeHashBase64ForPurpose(ReadOnlySpan<byte> data, string purpose)
|
||||
=> ComputeHashBase64(data, GetAlgorithmForPurpose(purpose));
|
||||
|
||||
public ValueTask<byte[]> ComputeHashForPurposeAsync(Stream stream, string purpose, CancellationToken cancellationToken = default)
|
||||
=> ComputeHashAsync(stream, GetAlgorithmForPurpose(purpose), cancellationToken);
|
||||
|
||||
public ValueTask<string> ComputeHashHexForPurposeAsync(Stream stream, string purpose, CancellationToken cancellationToken = default)
|
||||
=> ComputeHashHexAsync(stream, GetAlgorithmForPurpose(purpose), cancellationToken);
|
||||
|
||||
public string GetAlgorithmForPurpose(string purpose)
|
||||
=> HashAlgorithms.Sha256;
|
||||
|
||||
public string GetHashPrefix(string purpose)
|
||||
=> "sha256:";
|
||||
|
||||
public string ComputePrefixedHashForPurpose(ReadOnlySpan<byte> data, string purpose)
|
||||
=> $"{GetHashPrefix(purpose)}{ComputeHashHexForPurpose(data, purpose)}";
|
||||
|
||||
private static HashAlgorithm CreateAlgorithm(string? algorithmId)
|
||||
{
|
||||
return algorithmId?.ToUpperInvariant() switch
|
||||
|
||||
@@ -1,69 +1,85 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Microsoft.Extensions.Time.Testing;
|
||||
using StellaOps.Cryptography;
|
||||
using StellaOps.Scanner.Reachability;
|
||||
using StellaOps.Scanner.Storage;
|
||||
using StellaOps.Scanner.Storage.ObjectStore;
|
||||
using StellaOps.Scanner.Worker.Diagnostics;
|
||||
using StellaOps.Scanner.Worker.Determinism;
|
||||
using StellaOps.Scanner.Worker.Hosting;
|
||||
using StellaOps.Scanner.Worker.Options;
|
||||
using StellaOps.Scanner.Worker.Processing;
|
||||
using StellaOps.Scanner.Worker.Processing.Replay;
|
||||
using StellaOps.Scanner.Worker.Tests.TestInfrastructure;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Tests;
|
||||
|
||||
public sealed class WorkerBasicScanScenarioTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task DelayAsync_CompletesAfterTimeAdvance()
|
||||
{
|
||||
var scheduler = new ControlledDelayScheduler();
|
||||
var delayTask = scheduler.DelayAsync(TimeSpan.FromSeconds(5), CancellationToken.None);
|
||||
scheduler.AdvanceBy(TimeSpan.FromSeconds(5));
|
||||
await delayTask.WaitAsync(TimeSpan.FromSeconds(1));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Worker_CompletesJob_RecordsTelemetry_And_Heartbeats()
|
||||
{
|
||||
var fakeTime = new FakeTimeProvider();
|
||||
fakeTime.SetUtcNow(DateTimeOffset.UtcNow);
|
||||
|
||||
var options = new ScannerWorkerOptions
|
||||
{
|
||||
MaxConcurrentJobs = 1,
|
||||
};
|
||||
options.Telemetry.EnableTelemetry = false;
|
||||
options.Telemetry.EnableMetrics = true;
|
||||
|
||||
var optionsMonitor = new StaticOptionsMonitor<ScannerWorkerOptions>(options);
|
||||
var testLoggerProvider = new TestLoggerProvider();
|
||||
var lease = new TestJobLease(fakeTime);
|
||||
var jobSource = new TestJobSource(lease);
|
||||
var scheduler = new ControlledDelayScheduler();
|
||||
var analyzer = new TestAnalyzerDispatcher(scheduler);
|
||||
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Tests;
|
||||
|
||||
public sealed class WorkerBasicScanScenarioTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task DelayAsync_CompletesAfterTimeAdvance()
|
||||
{
|
||||
var scheduler = new ControlledDelayScheduler();
|
||||
var delayTask = scheduler.DelayAsync(TimeSpan.FromSeconds(5), CancellationToken.None);
|
||||
scheduler.AdvanceBy(TimeSpan.FromSeconds(5));
|
||||
await delayTask.WaitAsync(TimeSpan.FromSeconds(1));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Worker_CompletesJob_RecordsTelemetry_And_Heartbeats()
|
||||
{
|
||||
var fakeTime = new FakeTimeProvider();
|
||||
fakeTime.SetUtcNow(DateTimeOffset.UtcNow);
|
||||
|
||||
var options = new ScannerWorkerOptions
|
||||
{
|
||||
MaxConcurrentJobs = 1,
|
||||
};
|
||||
options.Telemetry.EnableTelemetry = false;
|
||||
options.Telemetry.EnableMetrics = true;
|
||||
|
||||
var optionsMonitor = new StaticOptionsMonitor<ScannerWorkerOptions>(options);
|
||||
var testLoggerProvider = new TestLoggerProvider();
|
||||
var lease = new TestJobLease(fakeTime);
|
||||
var jobSource = new TestJobSource(lease);
|
||||
var scheduler = new ControlledDelayScheduler();
|
||||
var analyzer = new TestAnalyzerDispatcher(scheduler);
|
||||
|
||||
using var listener = new WorkerMeterListener();
|
||||
listener.Start();
|
||||
|
||||
using var services = new ServiceCollection()
|
||||
.AddLogging(builder =>
|
||||
{
|
||||
builder.ClearProviders();
|
||||
builder.AddProvider(testLoggerProvider);
|
||||
builder.SetMinimumLevel(LogLevel.Debug);
|
||||
})
|
||||
.AddSingleton(fakeTime)
|
||||
.AddSingleton<TimeProvider>(fakeTime)
|
||||
.AddSingleton<IOptionsMonitor<ScannerWorkerOptions>>(optionsMonitor)
|
||||
.AddSingleton<ScannerWorkerMetrics>()
|
||||
.AddSingleton<ScanProgressReporter>()
|
||||
|
||||
using var services = new ServiceCollection()
|
||||
.AddLogging(builder =>
|
||||
{
|
||||
builder.ClearProviders();
|
||||
builder.AddProvider(testLoggerProvider);
|
||||
builder.SetMinimumLevel(LogLevel.Debug);
|
||||
})
|
||||
.AddSingleton(fakeTime)
|
||||
.AddSingleton<TimeProvider>(fakeTime)
|
||||
.AddSingleton<IOptionsMonitor<ScannerWorkerOptions>>(optionsMonitor)
|
||||
.AddSingleton<ScannerWorkerMetrics>()
|
||||
.AddSingleton<ScanProgressReporter>()
|
||||
.AddSingleton<ScanJobProcessor>()
|
||||
.AddSingleton<IDeterministicRandomProvider>(new DeterministicRandomProvider(seed: 1337))
|
||||
.AddSingleton<DeterministicRandomService>()
|
||||
.AddSingleton<IReachabilityUnionPublisherService, NullReachabilityUnionPublisherService>()
|
||||
.AddSingleton<ReplayBundleFetcher>(_ => new ReplayBundleFetcher(
|
||||
new NullArtifactObjectStore(),
|
||||
DefaultCryptoHash.CreateForTests(),
|
||||
new ScannerStorageOptions(),
|
||||
NullLogger<ReplayBundleFetcher>.Instance))
|
||||
.AddSingleton<LeaseHeartbeatService>()
|
||||
.AddSingleton<IDelayScheduler>(scheduler)
|
||||
.AddSingleton<IScanJobSource>(_ => jobSource)
|
||||
@@ -72,147 +88,155 @@ public sealed class WorkerBasicScanScenarioTests
|
||||
.AddSingleton<IScanStageExecutor, AnalyzerStageExecutor>()
|
||||
.AddSingleton<ScannerWorkerHostedService>()
|
||||
.BuildServiceProvider();
|
||||
|
||||
var worker = services.GetRequiredService<ScannerWorkerHostedService>();
|
||||
|
||||
await worker.StartAsync(CancellationToken.None);
|
||||
|
||||
await jobSource.LeaseIssued.Task.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
await Task.Yield();
|
||||
|
||||
var spin = 0;
|
||||
while (!lease.Completed.Task.IsCompleted && spin++ < 24)
|
||||
{
|
||||
fakeTime.Advance(TimeSpan.FromSeconds(15));
|
||||
scheduler.AdvanceBy(TimeSpan.FromSeconds(15));
|
||||
await Task.Delay(1);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await lease.Completed.Task.WaitAsync(TimeSpan.FromSeconds(30));
|
||||
}
|
||||
catch (TimeoutException ex)
|
||||
{
|
||||
var stageLogs = string.Join(Environment.NewLine, testLoggerProvider
|
||||
.GetEntriesForCategory(typeof(ScanProgressReporter).FullName!)
|
||||
.Select(entry => entry.ToFormattedString()));
|
||||
|
||||
throw new TimeoutException($"Worker did not complete within timeout. Logs:{Environment.NewLine}{stageLogs}", ex);
|
||||
}
|
||||
|
||||
await worker.StopAsync(CancellationToken.None);
|
||||
|
||||
Assert.True(lease.Completed.Task.IsCompletedSuccessfully, "Job should complete successfully.");
|
||||
|
||||
var worker = services.GetRequiredService<ScannerWorkerHostedService>();
|
||||
|
||||
await worker.StartAsync(CancellationToken.None);
|
||||
|
||||
await jobSource.LeaseIssued.Task.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
await Task.Yield();
|
||||
|
||||
var spin = 0;
|
||||
while (!lease.Completed.Task.IsCompleted && spin++ < 24)
|
||||
{
|
||||
fakeTime.Advance(TimeSpan.FromSeconds(15));
|
||||
scheduler.AdvanceBy(TimeSpan.FromSeconds(15));
|
||||
await Task.Delay(1);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await lease.Completed.Task.WaitAsync(TimeSpan.FromSeconds(30));
|
||||
}
|
||||
catch (TimeoutException ex)
|
||||
{
|
||||
var stageLogs = string.Join(Environment.NewLine, testLoggerProvider
|
||||
.GetEntriesForCategory(typeof(ScanProgressReporter).FullName!)
|
||||
.Select(entry => entry.ToFormattedString()));
|
||||
|
||||
throw new TimeoutException($"Worker did not complete within timeout. Logs:{Environment.NewLine}{stageLogs}", ex);
|
||||
}
|
||||
|
||||
await worker.StopAsync(CancellationToken.None);
|
||||
|
||||
Assert.True(lease.Completed.Task.IsCompletedSuccessfully, "Job should complete successfully.");
|
||||
Assert.Single(analyzer.Executions);
|
||||
|
||||
var stageOrder = testLoggerProvider
|
||||
.GetEntriesForCategory(typeof(ScanProgressReporter).FullName!)
|
||||
.Where(entry => entry.EventId.Id == 1000)
|
||||
.Select(entry => entry.GetScopeProperty<string>("Stage"))
|
||||
.Where(stage => stage is not null)
|
||||
.Cast<string>()
|
||||
.ToArray();
|
||||
|
||||
Assert.Equal(ScanStageNames.Ordered, stageOrder);
|
||||
|
||||
var queueLatency = listener.Measurements.Where(m => m.InstrumentName == "scanner_worker_queue_latency_ms").ToArray();
|
||||
Assert.Single(queueLatency);
|
||||
Assert.True(queueLatency[0].Value > 0, "Queue latency should be positive.");
|
||||
|
||||
var jobDuration = listener.Measurements.Where(m => m.InstrumentName == "scanner_worker_job_duration_ms").ToArray();
|
||||
Assert.Single(jobDuration);
|
||||
|
||||
var queueLatency = listener.Measurements.Where(m => m.InstrumentName == "scanner_worker_queue_latency_ms").ToArray();
|
||||
Assert.Single(queueLatency);
|
||||
Assert.True(queueLatency[0].Value > 0, "Queue latency should be positive.");
|
||||
|
||||
var jobDuration = listener.Measurements.Where(m => m.InstrumentName == "scanner_worker_job_duration_ms").ToArray();
|
||||
Assert.Single(jobDuration);
|
||||
var jobDurationMs = jobDuration[0].Value;
|
||||
Assert.True(jobDurationMs > 0, "Job duration should be positive.");
|
||||
|
||||
var stageDurations = listener.Measurements.Where(m => m.InstrumentName == "scanner_worker_stage_duration_ms").ToArray();
|
||||
Assert.Contains(stageDurations, m => m.Tags.TryGetValue("stage", out var stage) && Equals(stage, ScanStageNames.ExecuteAnalyzers));
|
||||
}
|
||||
|
||||
private sealed class TestJobSource : IScanJobSource
|
||||
{
|
||||
private readonly TestJobLease _lease;
|
||||
private int _delivered;
|
||||
|
||||
public TestJobSource(TestJobLease lease)
|
||||
{
|
||||
_lease = lease;
|
||||
}
|
||||
|
||||
public TaskCompletionSource LeaseIssued { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public Task<IScanJobLease?> TryAcquireAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (Interlocked.Exchange(ref _delivered, 1) == 0)
|
||||
{
|
||||
LeaseIssued.TrySetResult();
|
||||
return Task.FromResult<IScanJobLease?>(_lease);
|
||||
}
|
||||
|
||||
return Task.FromResult<IScanJobLease?>(null);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class TestJobLease : IScanJobLease
|
||||
{
|
||||
private readonly FakeTimeProvider _timeProvider;
|
||||
private readonly Dictionary<string, string> _metadata = new()
|
||||
{
|
||||
{ "queue", "tests" },
|
||||
{ "job.kind", "basic" },
|
||||
};
|
||||
|
||||
public TestJobLease(FakeTimeProvider timeProvider)
|
||||
{
|
||||
_timeProvider = timeProvider;
|
||||
EnqueuedAtUtc = _timeProvider.GetUtcNow() - TimeSpan.FromSeconds(5);
|
||||
LeasedAtUtc = _timeProvider.GetUtcNow();
|
||||
}
|
||||
|
||||
public string JobId { get; } = Guid.NewGuid().ToString("n");
|
||||
|
||||
public string ScanId { get; } = $"scan-{Guid.NewGuid():n}";
|
||||
|
||||
public int Attempt { get; } = 1;
|
||||
|
||||
public DateTimeOffset EnqueuedAtUtc { get; }
|
||||
|
||||
public DateTimeOffset LeasedAtUtc { get; }
|
||||
|
||||
public TimeSpan LeaseDuration { get; } = TimeSpan.FromSeconds(90);
|
||||
|
||||
public IReadOnlyDictionary<string, string> Metadata => _metadata;
|
||||
|
||||
public TaskCompletionSource Completed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public int RenewalCount => _renewalCount;
|
||||
|
||||
public ValueTask RenewAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref _renewalCount);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask CompleteAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
Completed.TrySetResult();
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask AbandonAsync(string reason, CancellationToken cancellationToken)
|
||||
{
|
||||
Completed.TrySetException(new InvalidOperationException($"Abandoned: {reason}"));
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask PoisonAsync(string reason, CancellationToken cancellationToken)
|
||||
{
|
||||
Completed.TrySetException(new InvalidOperationException($"Poisoned: {reason}"));
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
|
||||
|
||||
|
||||
var stageDurations = listener.Measurements.Where(m => m.InstrumentName == "scanner_worker_stage_duration_ms").ToArray();
|
||||
Assert.Contains(stageDurations, m => m.Tags.TryGetValue("stage", out var stage) && Equals(stage, ScanStageNames.ExecuteAnalyzers));
|
||||
}
|
||||
|
||||
private sealed class NullReachabilityUnionPublisherService : IReachabilityUnionPublisherService
|
||||
{
|
||||
public Task<ReachabilityUnionPublishResult> PublishAsync(ReachabilityUnionGraph graph, string analysisId, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(new ReachabilityUnionPublishResult("none", "none", 0));
|
||||
}
|
||||
|
||||
private sealed class NullArtifactObjectStore : IArtifactObjectStore
|
||||
{
|
||||
public Task PutAsync(ArtifactObjectDescriptor descriptor, Stream content, CancellationToken cancellationToken)
|
||||
=> Task.CompletedTask;
|
||||
|
||||
public Task<Stream?> GetAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken)
|
||||
=> Task.FromResult<Stream?>(null);
|
||||
|
||||
public Task DeleteAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken)
|
||||
=> Task.CompletedTask;
|
||||
}
|
||||
|
||||
private sealed class TestJobSource : IScanJobSource
|
||||
{
|
||||
private readonly TestJobLease _lease;
|
||||
private int _delivered;
|
||||
|
||||
public TestJobSource(TestJobLease lease)
|
||||
{
|
||||
_lease = lease;
|
||||
}
|
||||
|
||||
public TaskCompletionSource LeaseIssued { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public Task<IScanJobLease?> TryAcquireAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (Interlocked.Exchange(ref _delivered, 1) == 0)
|
||||
{
|
||||
LeaseIssued.TrySetResult();
|
||||
return Task.FromResult<IScanJobLease?>(_lease);
|
||||
}
|
||||
|
||||
return Task.FromResult<IScanJobLease?>(null);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class TestJobLease : IScanJobLease
|
||||
{
|
||||
private readonly FakeTimeProvider _timeProvider;
|
||||
private readonly Dictionary<string, string> _metadata = new()
|
||||
{
|
||||
{ "queue", "tests" },
|
||||
{ "job.kind", "basic" },
|
||||
};
|
||||
|
||||
public TestJobLease(FakeTimeProvider timeProvider)
|
||||
{
|
||||
_timeProvider = timeProvider;
|
||||
EnqueuedAtUtc = _timeProvider.GetUtcNow() - TimeSpan.FromSeconds(5);
|
||||
LeasedAtUtc = _timeProvider.GetUtcNow();
|
||||
}
|
||||
|
||||
public string JobId { get; } = Guid.NewGuid().ToString("n");
|
||||
|
||||
public string ScanId { get; } = $"scan-{Guid.NewGuid():n}";
|
||||
|
||||
public int Attempt { get; } = 1;
|
||||
|
||||
public DateTimeOffset EnqueuedAtUtc { get; }
|
||||
|
||||
public DateTimeOffset LeasedAtUtc { get; }
|
||||
|
||||
public TimeSpan LeaseDuration { get; } = TimeSpan.FromSeconds(90);
|
||||
|
||||
public IReadOnlyDictionary<string, string> Metadata => _metadata;
|
||||
|
||||
public TaskCompletionSource Completed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public int RenewalCount => _renewalCount;
|
||||
|
||||
public ValueTask RenewAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref _renewalCount);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask CompleteAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
Completed.TrySetResult();
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask AbandonAsync(string reason, CancellationToken cancellationToken)
|
||||
{
|
||||
Completed.TrySetException(new InvalidOperationException($"Abandoned: {reason}"));
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask PoisonAsync(string reason, CancellationToken cancellationToken)
|
||||
{
|
||||
Completed.TrySetException(new InvalidOperationException($"Poisoned: {reason}"));
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
|
||||
|
||||
private int _renewalCount;
|
||||
}
|
||||
|
||||
@@ -227,191 +251,191 @@ public sealed class WorkerBasicScanScenarioTests
|
||||
private readonly IDelayScheduler _scheduler;
|
||||
|
||||
public TestAnalyzerDispatcher(IDelayScheduler scheduler)
|
||||
{
|
||||
_scheduler = scheduler;
|
||||
}
|
||||
|
||||
public List<string> Executions { get; } = new();
|
||||
|
||||
public async ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
Executions.Add(context.JobId);
|
||||
await _scheduler.DelayAsync(TimeSpan.FromSeconds(45), cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class ControlledDelayScheduler : IDelayScheduler
|
||||
{
|
||||
private readonly object _lock = new();
|
||||
private readonly SortedDictionary<double, List<ScheduledDelay>> _scheduled = new();
|
||||
private double _currentMilliseconds;
|
||||
|
||||
public Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken)
|
||||
{
|
||||
if (delay <= TimeSpan.Zero)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
var tcs = new TaskCompletionSource<object?>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var scheduled = new ScheduledDelay(tcs, cancellationToken);
|
||||
lock (_lock)
|
||||
{
|
||||
var due = _currentMilliseconds + delay.TotalMilliseconds;
|
||||
if (!_scheduled.TryGetValue(due, out var list))
|
||||
{
|
||||
list = new List<ScheduledDelay>();
|
||||
_scheduled.Add(due, list);
|
||||
}
|
||||
|
||||
list.Add(scheduled);
|
||||
}
|
||||
|
||||
return scheduled.Task;
|
||||
}
|
||||
|
||||
public void AdvanceBy(TimeSpan delta)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
_currentMilliseconds += delta.TotalMilliseconds;
|
||||
var dueKeys = _scheduled.Keys.Where(key => key <= _currentMilliseconds).ToList();
|
||||
foreach (var due in dueKeys)
|
||||
{
|
||||
foreach (var scheduled in _scheduled[due])
|
||||
{
|
||||
scheduled.Complete();
|
||||
}
|
||||
|
||||
_scheduled.Remove(due);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class ScheduledDelay
|
||||
{
|
||||
private readonly TaskCompletionSource<object?> _tcs;
|
||||
private readonly CancellationTokenRegistration _registration;
|
||||
|
||||
public ScheduledDelay(TaskCompletionSource<object?> tcs, CancellationToken cancellationToken)
|
||||
{
|
||||
_tcs = tcs;
|
||||
if (cancellationToken.CanBeCanceled)
|
||||
{
|
||||
_registration = cancellationToken.Register(state =>
|
||||
{
|
||||
var source = (TaskCompletionSource<object?>)state!;
|
||||
source.TrySetCanceled(cancellationToken);
|
||||
}, tcs);
|
||||
}
|
||||
}
|
||||
|
||||
public Task Task => _tcs.Task;
|
||||
|
||||
public void Complete()
|
||||
{
|
||||
_registration.Dispose();
|
||||
_tcs.TrySetResult(null);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StaticOptionsMonitor<TOptions> : IOptionsMonitor<TOptions>
|
||||
where TOptions : class
|
||||
{
|
||||
private readonly TOptions _value;
|
||||
|
||||
public StaticOptionsMonitor(TOptions value)
|
||||
{
|
||||
_value = value;
|
||||
}
|
||||
|
||||
public TOptions CurrentValue => _value;
|
||||
|
||||
public TOptions Get(string? name) => _value;
|
||||
|
||||
public IDisposable OnChange(Action<TOptions, string?> listener) => NullDisposable.Instance;
|
||||
|
||||
private sealed class NullDisposable : IDisposable
|
||||
{
|
||||
public static readonly NullDisposable Instance = new();
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class TestLoggerProvider : ILoggerProvider
|
||||
{
|
||||
private readonly ConcurrentQueue<TestLogEntry> _entries = new();
|
||||
|
||||
public ILogger CreateLogger(string categoryName) => new TestLogger(categoryName, _entries);
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
|
||||
public IEnumerable<TestLogEntry> GetEntriesForCategory(string categoryName)
|
||||
=> _entries.Where(entry => entry.Category == categoryName);
|
||||
|
||||
private sealed class TestLogger : ILogger
|
||||
{
|
||||
private readonly string _category;
|
||||
private readonly ConcurrentQueue<TestLogEntry> _entries;
|
||||
|
||||
public TestLogger(string category, ConcurrentQueue<TestLogEntry> entries)
|
||||
{
|
||||
_category = category;
|
||||
_entries = entries;
|
||||
}
|
||||
|
||||
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => NullDisposable.Instance;
|
||||
|
||||
public bool IsEnabled(LogLevel logLevel) => true;
|
||||
|
||||
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
|
||||
{
|
||||
_entries.Enqueue(new TestLogEntry(_category, logLevel, eventId, state, exception));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class NullDisposable : IDisposable
|
||||
{
|
||||
public static readonly NullDisposable Instance = new();
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record TestLogEntry(string Category, LogLevel Level, EventId EventId, object? State, Exception? Exception)
|
||||
{
|
||||
public T? GetScopeProperty<T>(string name)
|
||||
{
|
||||
if (State is not IEnumerable<KeyValuePair<string, object?>> state)
|
||||
{
|
||||
return default;
|
||||
}
|
||||
|
||||
foreach (var kvp in state)
|
||||
{
|
||||
if (string.Equals(kvp.Key, name, StringComparison.OrdinalIgnoreCase) && kvp.Value is T value)
|
||||
{
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
return default;
|
||||
}
|
||||
|
||||
public string ToFormattedString()
|
||||
{
|
||||
var properties = State is IEnumerable<KeyValuePair<string, object?>> kvps
|
||||
? string.Join(", ", kvps.Select(kvp => $"{kvp.Key}={kvp.Value}"))
|
||||
: State?.ToString() ?? string.Empty;
|
||||
|
||||
var exceptionPart = Exception is null ? string.Empty : $" Exception={Exception.GetType().Name}: {Exception.Message}";
|
||||
return $"[{Level}] {Category} ({EventId.Id}) {properties}{exceptionPart}";
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
_scheduler = scheduler;
|
||||
}
|
||||
|
||||
public List<string> Executions { get; } = new();
|
||||
|
||||
public async ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
Executions.Add(context.JobId);
|
||||
await _scheduler.DelayAsync(TimeSpan.FromSeconds(45), cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class ControlledDelayScheduler : IDelayScheduler
|
||||
{
|
||||
private readonly object _lock = new();
|
||||
private readonly SortedDictionary<double, List<ScheduledDelay>> _scheduled = new();
|
||||
private double _currentMilliseconds;
|
||||
|
||||
public Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken)
|
||||
{
|
||||
if (delay <= TimeSpan.Zero)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
var tcs = new TaskCompletionSource<object?>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var scheduled = new ScheduledDelay(tcs, cancellationToken);
|
||||
lock (_lock)
|
||||
{
|
||||
var due = _currentMilliseconds + delay.TotalMilliseconds;
|
||||
if (!_scheduled.TryGetValue(due, out var list))
|
||||
{
|
||||
list = new List<ScheduledDelay>();
|
||||
_scheduled.Add(due, list);
|
||||
}
|
||||
|
||||
list.Add(scheduled);
|
||||
}
|
||||
|
||||
return scheduled.Task;
|
||||
}
|
||||
|
||||
public void AdvanceBy(TimeSpan delta)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
_currentMilliseconds += delta.TotalMilliseconds;
|
||||
var dueKeys = _scheduled.Keys.Where(key => key <= _currentMilliseconds).ToList();
|
||||
foreach (var due in dueKeys)
|
||||
{
|
||||
foreach (var scheduled in _scheduled[due])
|
||||
{
|
||||
scheduled.Complete();
|
||||
}
|
||||
|
||||
_scheduled.Remove(due);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class ScheduledDelay
|
||||
{
|
||||
private readonly TaskCompletionSource<object?> _tcs;
|
||||
private readonly CancellationTokenRegistration _registration;
|
||||
|
||||
public ScheduledDelay(TaskCompletionSource<object?> tcs, CancellationToken cancellationToken)
|
||||
{
|
||||
_tcs = tcs;
|
||||
if (cancellationToken.CanBeCanceled)
|
||||
{
|
||||
_registration = cancellationToken.Register(state =>
|
||||
{
|
||||
var source = (TaskCompletionSource<object?>)state!;
|
||||
source.TrySetCanceled(cancellationToken);
|
||||
}, tcs);
|
||||
}
|
||||
}
|
||||
|
||||
public Task Task => _tcs.Task;
|
||||
|
||||
public void Complete()
|
||||
{
|
||||
_registration.Dispose();
|
||||
_tcs.TrySetResult(null);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StaticOptionsMonitor<TOptions> : IOptionsMonitor<TOptions>
|
||||
where TOptions : class
|
||||
{
|
||||
private readonly TOptions _value;
|
||||
|
||||
public StaticOptionsMonitor(TOptions value)
|
||||
{
|
||||
_value = value;
|
||||
}
|
||||
|
||||
public TOptions CurrentValue => _value;
|
||||
|
||||
public TOptions Get(string? name) => _value;
|
||||
|
||||
public IDisposable OnChange(Action<TOptions, string?> listener) => NullDisposable.Instance;
|
||||
|
||||
private sealed class NullDisposable : IDisposable
|
||||
{
|
||||
public static readonly NullDisposable Instance = new();
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class TestLoggerProvider : ILoggerProvider
|
||||
{
|
||||
private readonly ConcurrentQueue<TestLogEntry> _entries = new();
|
||||
|
||||
public ILogger CreateLogger(string categoryName) => new TestLogger(categoryName, _entries);
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
|
||||
public IEnumerable<TestLogEntry> GetEntriesForCategory(string categoryName)
|
||||
=> _entries.Where(entry => entry.Category == categoryName);
|
||||
|
||||
private sealed class TestLogger : ILogger
|
||||
{
|
||||
private readonly string _category;
|
||||
private readonly ConcurrentQueue<TestLogEntry> _entries;
|
||||
|
||||
public TestLogger(string category, ConcurrentQueue<TestLogEntry> entries)
|
||||
{
|
||||
_category = category;
|
||||
_entries = entries;
|
||||
}
|
||||
|
||||
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => NullDisposable.Instance;
|
||||
|
||||
public bool IsEnabled(LogLevel logLevel) => true;
|
||||
|
||||
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
|
||||
{
|
||||
_entries.Enqueue(new TestLogEntry(_category, logLevel, eventId, state, exception));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class NullDisposable : IDisposable
|
||||
{
|
||||
public static readonly NullDisposable Instance = new();
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record TestLogEntry(string Category, LogLevel Level, EventId EventId, object? State, Exception? Exception)
|
||||
{
|
||||
public T? GetScopeProperty<T>(string name)
|
||||
{
|
||||
if (State is not IEnumerable<KeyValuePair<string, object?>> state)
|
||||
{
|
||||
return default;
|
||||
}
|
||||
|
||||
foreach (var kvp in state)
|
||||
{
|
||||
if (string.Equals(kvp.Key, name, StringComparison.OrdinalIgnoreCase) && kvp.Value is T value)
|
||||
{
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
return default;
|
||||
}
|
||||
|
||||
public string ToFormattedString()
|
||||
{
|
||||
var properties = State is IEnumerable<KeyValuePair<string, object?>> kvps
|
||||
? string.Join(", ", kvps.Select(kvp => $"{kvp.Key}={kvp.Value}"))
|
||||
: State?.ToString() ?? string.Empty;
|
||||
|
||||
var exceptionPart = Exception is null ? string.Empty : $" Exception={Exception.GetType().Name}: {Exception.Message}";
|
||||
return $"[{Level}] {Category} ({EventId.Id}) {properties}{exceptionPart}";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user