git.stella-ops.org/src/Scanner/StellaOps.Scanner.Worker/Program.cs

using System.Diagnostics;
using System.IO;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Auth.Client;
using StellaOps.Configuration;
using StellaOps.Scanner.Cache;
using StellaOps.Scanner.Reachability;
using StellaOps.Scanner.Analyzers.OS.Plugin;
using StellaOps.Scanner.Analyzers.Lang.Plugin;
using StellaOps.Scanner.EntryTrace;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.Core.Security;
using StellaOps.Scanner.Surface.Env;
using StellaOps.Scanner.Surface.FS;
using StellaOps.Scanner.Surface.Secrets;
using StellaOps.Scanner.Surface.Validation;
using StellaOps.Scanner.Worker.Diagnostics;
using StellaOps.Scanner.Worker.Hosting;
using StellaOps.Scanner.Worker.Options;
using StellaOps.Scanner.Worker.Processing;
using StellaOps.Scanner.Worker.Processing.Entropy;
using StellaOps.Scanner.Worker.Determinism;
using StellaOps.Scanner.Worker.Processing.Surface;
using StellaOps.Scanner.Storage.Extensions;
using StellaOps.Scanner.Storage;
using Reachability = StellaOps.Scanner.Worker.Processing.Reachability;

var builder = Host.CreateApplicationBuilder(args);

builder.Services.AddOptions<ScannerWorkerOptions>()
    .BindConfiguration(ScannerWorkerOptions.SectionName)
    .ValidateOnStart()
    .PostConfigure(options =>
    {
        if (options.Determinism.ConcurrencyLimit is { } limit && limit > 0)
        {
            options.MaxConcurrentJobs = Math.Min(options.MaxConcurrentJobs, limit);
        }
    });

builder.Services.AddSingleton<IValidateOptions<ScannerWorkerOptions>, ScannerWorkerOptionsValidator>();
var workerOptions = builder.Configuration.GetSection(ScannerWorkerOptions.SectionName).Get<ScannerWorkerOptions>() ?? new ScannerWorkerOptions();

if (workerOptions.Determinism.FixedClock)
{
    builder.Services.AddSingleton<TimeProvider>(_ => new DeterministicTimeProvider(workerOptions.Determinism.FixedInstantUtc));
}
else
{
    builder.Services.AddSingleton(TimeProvider.System);
}

builder.Services.AddSingleton(new DeterminismContext(
    workerOptions.Determinism.FixedClock,
    workerOptions.Determinism.FixedInstantUtc,
    workerOptions.Determinism.RngSeed,
    workerOptions.Determinism.FilterLogs,
    workerOptions.Determinism.ConcurrencyLimit));
builder.Services.AddSingleton<IDeterministicRandomProvider>(_ => new DeterministicRandomProvider(workerOptions.Determinism.RngSeed));
builder.Services.AddScannerCache(builder.Configuration);
builder.Services.AddSurfaceEnvironment(options =>
{
    options.ComponentName = "Scanner.Worker";
});
builder.Services.AddSurfaceValidation();
builder.Services.AddSurfaceFileCache();
builder.Services.AddSurfaceManifestStore();
builder.Services.AddSurfaceSecrets();
builder.Services.AddSingleton<IConfigureOptions<SurfaceCacheOptions>>(sp =>
    new SurfaceCacheOptionsConfigurator(sp.GetRequiredService<ISurfaceEnvironment>()));
builder.Services.AddSingleton<IConfigureOptions<SurfaceManifestStoreOptions>>(sp =>
    new SurfaceManifestStoreOptionsConfigurator(
        sp.GetRequiredService<ISurfaceEnvironment>(),
        sp.GetRequiredService<IOptions<SurfaceCacheOptions>>()));
builder.Services.AddSingleton<ScannerWorkerMetrics>();
builder.Services.AddSingleton<ScanProgressReporter>();
builder.Services.AddSingleton<ScanJobProcessor>();
builder.Services.AddSingleton<LeaseHeartbeatService>();
builder.Services.AddSingleton<IDelayScheduler, SystemDelayScheduler>();

builder.Services.AddEntryTraceAnalyzer();
builder.Services.AddSingleton<IEntryTraceExecutionService, EntryTraceExecutionService>();
builder.Services.AddSingleton<ReachabilityUnionWriter>();
builder.Services.AddSingleton<ReachabilityUnionPublisher>();
builder.Services.AddSingleton<IReachabilityUnionPublisherService, ReachabilityUnionPublisherService>();
builder.Services.AddSingleton<RichGraphWriter>();
builder.Services.AddSingleton<IRichGraphPublisher, ReachabilityRichGraphPublisher>();
builder.Services.AddSingleton<IRichGraphPublisherService, ReachabilityRichGraphPublisherService>();
builder.Services.AddSingleton<IScanStageExecutor, StellaOps.Scanner.Worker.Processing.Replay.ReplaySealedBundleStageExecutor>();
builder.Services.AddSingleton<StellaOps.Scanner.Worker.Processing.Replay.ReplayBundleFetcher>();

var storageSection = builder.Configuration.GetSection("ScannerStorage");
var connectionString = storageSection.GetValue<string>("Mongo:ConnectionString");
if (!string.IsNullOrWhiteSpace(connectionString))
{
    builder.Services.AddScannerStorage(storageSection);
    builder.Services.AddSingleton<IConfigureOptions<ScannerStorageOptions>, ScannerStorageSurfaceSecretConfigurator>();
builder.Services.AddSingleton<ISurfaceManifestPublisher, SurfaceManifestPublisher>();
builder.Services.AddSingleton<IScanStageExecutor, SurfaceManifestStageExecutor>();
builder.Services.AddSingleton<IDsseEnvelopeSigner, HmacDsseEnvelopeSigner>();
}
else
{
    builder.Services.TryAddSingleton<IRubyPackageInventoryStore, NullRubyPackageInventoryStore>();
}

builder.Services.TryAddSingleton<IScanJobSource, NullScanJobSource>();
builder.Services.TryAddSingleton<IPluginCatalogGuard, RestartOnlyPluginGuard>();
builder.Services.AddSingleton<IOSAnalyzerPluginCatalog, OsAnalyzerPluginCatalog>();
builder.Services.AddSingleton<ILanguageAnalyzerPluginCatalog, LanguageAnalyzerPluginCatalog>();
builder.Services.AddSingleton<IScanAnalyzerDispatcher, CompositeScanAnalyzerDispatcher>();
builder.Services.AddSingleton<IScanStageExecutor, RegistrySecretStageExecutor>();
builder.Services.AddSingleton<IScanStageExecutor, AnalyzerStageExecutor>();
builder.Services.AddSingleton<IScanStageExecutor, Reachability.ReachabilityBuildStageExecutor>();
builder.Services.AddSingleton<IScanStageExecutor, Reachability.ReachabilityPublishStageExecutor>();
builder.Services.AddSingleton<IScanStageExecutor, EntropyStageExecutor>();

builder.Services.AddSingleton<ScannerWorkerHostedService>();
builder.Services.AddHostedService(sp => sp.GetRequiredService<ScannerWorkerHostedService>());

builder.Services.AddStellaOpsCrypto(workerOptions.Crypto);

builder.Services.Configure<HostOptions>(options =>
{
    options.ShutdownTimeout = workerOptions.Shutdown.Timeout;
});

builder.ConfigureScannerWorkerTelemetry(workerOptions);

if (workerOptions.Authority.Enabled)
{
    builder.Services.AddStellaOpsAuthClient(clientOptions =>
    {
        clientOptions.Authority = workerOptions.Authority.Issuer?.Trim() ?? string.Empty;
        clientOptions.ClientId = workerOptions.Authority.ClientId?.Trim() ?? string.Empty;
        clientOptions.ClientSecret = workerOptions.Authority.ClientSecret;
        clientOptions.EnableRetries = workerOptions.Authority.Resilience.EnableRetries ?? true;
        clientOptions.HttpTimeout = TimeSpan.FromSeconds(workerOptions.Authority.BackchannelTimeoutSeconds);

        clientOptions.DefaultScopes.Clear();
        foreach (var scope in workerOptions.Authority.Scopes)
        {
            if (string.IsNullOrWhiteSpace(scope))
            {
                continue;
            }

            clientOptions.DefaultScopes.Add(scope);
        }

        clientOptions.RetryDelays.Clear();
        foreach (var delay in workerOptions.Authority.Resilience.RetryDelays)
        {
            if (delay <= TimeSpan.Zero)
            {
                continue;
            }

            clientOptions.RetryDelays.Add(delay);
        }

        if (workerOptions.Authority.Resilience.AllowOfflineCacheFallback is bool allowOffline)
        {
            clientOptions.AllowOfflineCacheFallback = allowOffline;
        }

        if (workerOptions.Authority.Resilience.OfflineCacheTolerance is { } tolerance && tolerance > TimeSpan.Zero)
        {
            clientOptions.OfflineCacheTolerance = tolerance;
        }
    });
}

builder.Logging.Configure(options =>
{
    options.ActivityTrackingOptions = ActivityTrackingOptions.SpanId
        | ActivityTrackingOptions.TraceId
        | ActivityTrackingOptions.ParentId;
});

var host = builder.Build();

// Fail fast if surface configuration is invalid at startup.
using (var scope = host.Services.CreateScope())
{
    var services = scope.ServiceProvider;
    var env = services.GetRequiredService<ISurfaceEnvironment>();
    var runner = services.GetRequiredService<ISurfaceValidatorRunner>();
    await runner.EnsureAsync(
        SurfaceValidationContext.Create(services, "Scanner.Worker.Startup", env.Settings),
        host.Services.GetRequiredService<IHostApplicationLifetime>().ApplicationStopping)
        .ConfigureAwait(false);
}

await host.RunAsync();

public partial class Program;

public sealed class SurfaceCacheOptionsConfigurator : IConfigureOptions<SurfaceCacheOptions>
{
    private readonly ISurfaceEnvironment _surfaceEnvironment;

    public SurfaceCacheOptionsConfigurator(ISurfaceEnvironment surfaceEnvironment)
    {
        _surfaceEnvironment = surfaceEnvironment ?? throw new ArgumentNullException(nameof(surfaceEnvironment));
    }

    public void Configure(SurfaceCacheOptions options)
    {
        ArgumentNullException.ThrowIfNull(options);
        var settings = _surfaceEnvironment.Settings;
        options.RootDirectory = settings.CacheRoot.FullName;
    }
}

public sealed class SurfaceManifestStoreOptionsConfigurator : IConfigureOptions<SurfaceManifestStoreOptions>
{
    private readonly ISurfaceEnvironment _surfaceEnvironment;
    private readonly IOptions<SurfaceCacheOptions> _cacheOptions;

    public SurfaceManifestStoreOptionsConfigurator(
        ISurfaceEnvironment surfaceEnvironment,
        IOptions<SurfaceCacheOptions> cacheOptions)
    {
        _surfaceEnvironment = surfaceEnvironment ?? throw new ArgumentNullException(nameof(surfaceEnvironment));
        _cacheOptions = cacheOptions ?? throw new ArgumentNullException(nameof(cacheOptions));
    }

    public void Configure(SurfaceManifestStoreOptions options)
    {
        ArgumentNullException.ThrowIfNull(options);

        var settings = _surfaceEnvironment.Settings;
        options.Bucket = settings.SurfaceFsBucket;
        options.Scheme = settings.SurfaceFsEndpoint.Scheme;

        if (string.IsNullOrWhiteSpace(options.RootDirectory))
        {
            var cacheRoot = _cacheOptions.Value.RootDirectory ?? Path.Combine(Path.GetTempPath(), "stellaops", "surface-cache");
            options.RootDirectory = Path.Combine(cacheRoot, "manifests");
        }
    }
}