Files
git.stella-ops.org/src/Scanner/StellaOps.Scanner.WebService/Program.cs
StellaOps Bot 37cba83708
Some checks failed
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
devportal-offline / build-offline (push) Has been cancelled
up
2025-12-03 00:10:19 +02:00

469 lines
19 KiB
C#

using System.Collections.Generic;
using System.Diagnostics;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Diagnostics;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.AspNetCore.Authentication;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using Serilog;
using Serilog.Events;
using StellaOps.Auth.Client;
using StellaOps.Auth.ServerIntegration;
using StellaOps.Configuration;
using StellaOps.Plugin.DependencyInjection;
using StellaOps.Cryptography.DependencyInjection;
using StellaOps.Cryptography.Plugin.BouncyCastle;
using StellaOps.Concelier.Core.Linksets;
using StellaOps.Policy;
using StellaOps.Scanner.Cache;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.Surface.Env;
using StellaOps.Scanner.Surface.FS;
using StellaOps.Scanner.Surface.Secrets;
using StellaOps.Scanner.Surface.Validation;
using StellaOps.Scanner.WebService.Diagnostics;
using StellaOps.Scanner.WebService.Determinism;
using StellaOps.Scanner.WebService.Endpoints;
using StellaOps.Scanner.WebService.Extensions;
using StellaOps.Scanner.WebService.Hosting;
using StellaOps.Scanner.WebService.Options;
using StellaOps.Scanner.WebService.Options;
using StellaOps.Scanner.WebService.Services;
using StellaOps.Scanner.WebService.Security;
using StellaOps.Scanner.WebService.Replay;
using StellaOps.Scanner.Storage;
using StellaOps.Scanner.Storage.Extensions;
using StellaOps.Scanner.Storage.Mongo;
using StellaOps.Scanner.WebService.Endpoints;
using StellaOps.Scanner.WebService.Options;
var builder = WebApplication.CreateBuilder(args);
builder.Configuration.AddStellaOpsDefaults(options =>
{
options.BasePath = builder.Environment.ContentRootPath;
options.EnvironmentPrefix = "SCANNER_";
options.ConfigureBuilder = configurationBuilder =>
{
configurationBuilder.AddScannerYaml(Path.Combine(builder.Environment.ContentRootPath, "../etc/scanner.yaml"));
};
});
var contentRoot = builder.Environment.ContentRootPath;
var bootstrapOptions = builder.Configuration.BindOptions<ScannerWebServiceOptions>(
ScannerWebServiceOptions.SectionName,
(opts, _) =>
{
ScannerWebServiceOptionsPostConfigure.Apply(opts, contentRoot);
ScannerWebServiceOptionsValidator.Validate(opts);
});
builder.Services.AddStellaOpsCrypto(bootstrapOptions.Crypto);
builder.Services.AddOptions<ScannerWebServiceOptions>()
.Bind(builder.Configuration.GetSection(ScannerWebServiceOptions.SectionName))
.PostConfigure(options =>
{
ScannerWebServiceOptionsPostConfigure.Apply(options, contentRoot);
ScannerWebServiceOptionsValidator.Validate(options);
})
.ValidateOnStart();
builder.Host.UseSerilog((context, services, loggerConfiguration) =>
{
loggerConfiguration
.MinimumLevel.Information()
.MinimumLevel.Override("Microsoft.AspNetCore", LogEventLevel.Warning)
.Enrich.FromLogContext()
.WriteTo.Console();
});
if (bootstrapOptions.Determinism.FixedClock)
{
builder.Services.AddSingleton<TimeProvider>(_ => new DeterministicTimeProvider(bootstrapOptions.Determinism.FixedInstantUtc));
}
else
{
builder.Services.AddSingleton(TimeProvider.System);
}
builder.Services.AddScannerCache(builder.Configuration);
builder.Services.AddSingleton<ServiceStatus>();
builder.Services.AddHttpContextAccessor();
builder.Services.AddSingleton<ScanProgressStream>();
builder.Services.AddSingleton<IScanProgressPublisher>(sp => sp.GetRequiredService<ScanProgressStream>());
builder.Services.AddSingleton<IScanProgressReader>(sp => sp.GetRequiredService<ScanProgressStream>());
builder.Services.AddSingleton<IScanCoordinator, InMemoryScanCoordinator>();
builder.Services.AddSingleton<IPolicySnapshotRepository, InMemoryPolicySnapshotRepository>();
builder.Services.AddSingleton<IPolicyAuditRepository, InMemoryPolicyAuditRepository>();
builder.Services.AddSingleton<PolicySnapshotStore>();
builder.Services.AddSingleton<PolicyPreviewService>();
builder.Services.AddSingleton<IRecordModeService, RecordModeService>();
builder.Services.AddStellaOpsCrypto();
builder.Services.AddBouncyCastleEd25519Provider();
builder.Services.AddSingleton<IReportSigner, ReportSigner>();
builder.Services.AddSurfaceEnvironment(options =>
{
options.ComponentName = "Scanner.WebService";
options.AddPrefix("SCANNER");
});
builder.Services.AddSurfaceValidation();
builder.Services.AddSurfaceFileCache();
builder.Services.AddSurfaceManifestStore();
builder.Services.AddSurfaceSecrets();
builder.Services.AddSingleton<IConfigureOptions<ScannerWebServiceOptions>, ScannerSurfaceSecretConfigurator>();
builder.Services.AddSingleton<IConfigureOptions<ScannerWebServiceOptions>, SurfaceFeatureFlagsConfigurator>();
builder.Services.AddSingleton<IConfigureOptions<SurfaceCacheOptions>>(sp =>
new SurfaceCacheOptionsConfigurator(sp.GetRequiredService<ISurfaceEnvironment>()));
builder.Services.AddSingleton<IConfigureOptions<SurfaceManifestStoreOptions>>(sp =>
new SurfaceManifestStoreOptionsConfigurator(
sp.GetRequiredService<ISurfaceEnvironment>(),
sp.GetRequiredService<IOptions<SurfaceCacheOptions>>()));
builder.Services.AddSingleton<ISurfacePointerService, SurfacePointerService>();
builder.Services.AddSingleton<IRedisConnectionFactory, RedisConnectionFactory>();
if (bootstrapOptions.Events is { Enabled: true } eventsOptions
&& string.Equals(eventsOptions.Driver, "redis", StringComparison.OrdinalIgnoreCase))
{
builder.Services.AddSingleton<IPlatformEventPublisher, RedisPlatformEventPublisher>();
}
else
{
builder.Services.AddSingleton<IPlatformEventPublisher, NullPlatformEventPublisher>();
}
builder.Services.AddSingleton<IReportEventDispatcher, ReportEventDispatcher>();
builder.Services.AddScannerStorage(storageOptions =>
{
storageOptions.Mongo.ConnectionString = bootstrapOptions.Storage.Dsn;
if (!string.IsNullOrWhiteSpace(bootstrapOptions.Storage.Database))
{
storageOptions.Mongo.DatabaseName = bootstrapOptions.Storage.Database;
}
storageOptions.Mongo.CommandTimeout = TimeSpan.FromSeconds(bootstrapOptions.Storage.CommandTimeoutSeconds);
storageOptions.Mongo.UseMajorityReadConcern = true;
storageOptions.Mongo.UseMajorityWriteConcern = true;
storageOptions.ObjectStore.Headers.Clear();
foreach (var header in bootstrapOptions.ArtifactStore.Headers)
{
storageOptions.ObjectStore.Headers[header.Key] = header.Value;
}
if (!string.IsNullOrWhiteSpace(bootstrapOptions.ArtifactStore.Bucket))
{
storageOptions.ObjectStore.BucketName = bootstrapOptions.ArtifactStore.Bucket;
}
if (!string.IsNullOrWhiteSpace(bootstrapOptions.ArtifactStore.RootPrefix))
{
storageOptions.ObjectStore.RootPrefix = bootstrapOptions.ArtifactStore.RootPrefix;
}
var artifactDriver = bootstrapOptions.ArtifactStore.Driver?.Trim() ?? string.Empty;
if (string.Equals(artifactDriver, ScannerStorageDefaults.ObjectStoreProviders.RustFs, StringComparison.OrdinalIgnoreCase))
{
storageOptions.ObjectStore.Driver = ScannerStorageDefaults.ObjectStoreProviders.RustFs;
storageOptions.ObjectStore.RustFs.BaseUrl = bootstrapOptions.ArtifactStore.Endpoint;
storageOptions.ObjectStore.RustFs.AllowInsecureTls = bootstrapOptions.ArtifactStore.AllowInsecureTls;
storageOptions.ObjectStore.RustFs.Timeout = TimeSpan.FromSeconds(Math.Max(1, bootstrapOptions.ArtifactStore.TimeoutSeconds));
storageOptions.ObjectStore.RustFs.ApiKey = bootstrapOptions.ArtifactStore.ApiKey;
storageOptions.ObjectStore.RustFs.ApiKeyHeader = bootstrapOptions.ArtifactStore.ApiKeyHeader ?? string.Empty;
storageOptions.ObjectStore.EnableObjectLock = false;
storageOptions.ObjectStore.ComplianceRetention = null;
}
else
{
var resolvedDriver = string.Equals(artifactDriver, ScannerStorageDefaults.ObjectStoreProviders.Minio, StringComparison.OrdinalIgnoreCase)
? ScannerStorageDefaults.ObjectStoreProviders.Minio
: ScannerStorageDefaults.ObjectStoreProviders.S3;
storageOptions.ObjectStore.Driver = resolvedDriver;
if (!string.IsNullOrWhiteSpace(bootstrapOptions.ArtifactStore.Endpoint))
{
storageOptions.ObjectStore.ServiceUrl = bootstrapOptions.ArtifactStore.Endpoint;
}
if (!string.IsNullOrWhiteSpace(bootstrapOptions.ArtifactStore.Region))
{
storageOptions.ObjectStore.Region = bootstrapOptions.ArtifactStore.Region;
}
storageOptions.ObjectStore.EnableObjectLock = bootstrapOptions.ArtifactStore.EnableObjectLock;
storageOptions.ObjectStore.ForcePathStyle = true;
storageOptions.ObjectStore.ComplianceRetention = bootstrapOptions.ArtifactStore.EnableObjectLock
? TimeSpan.FromDays(Math.Max(1, bootstrapOptions.ArtifactStore.ObjectLockRetentionDays))
: null;
storageOptions.ObjectStore.RustFs.ApiKey = null;
storageOptions.ObjectStore.RustFs.ApiKeyHeader = string.Empty;
storageOptions.ObjectStore.RustFs.BaseUrl = string.Empty;
}
});
builder.Services.AddSingleton<IPostConfigureOptions<ScannerStorageOptions>, ScannerStorageOptionsPostConfigurator>();
builder.Services.AddSingleton<RuntimeEventRateLimiter>();
builder.Services.AddSingleton<IRuntimeEventIngestionService, RuntimeEventIngestionService>();
builder.Services.AddSingleton<IRuntimeAttestationVerifier, RuntimeAttestationVerifier>();
builder.Services.AddSingleton<ILinksetResolver, LinksetResolver>();
builder.Services.AddSingleton<IRuntimePolicyService, RuntimePolicyService>();
var pluginHostOptions = ScannerPluginHostFactory.Build(bootstrapOptions, contentRoot);
builder.Services.RegisterPluginRoutines(builder.Configuration, pluginHostOptions);
builder.Services.AddOpenApiIfAvailable();
if (bootstrapOptions.Authority.Enabled)
{
builder.Services.AddStellaOpsAuthClient(clientOptions =>
{
clientOptions.Authority = bootstrapOptions.Authority.Issuer;
clientOptions.ClientId = bootstrapOptions.Authority.ClientId ?? string.Empty;
clientOptions.ClientSecret = bootstrapOptions.Authority.ClientSecret;
clientOptions.HttpTimeout = TimeSpan.FromSeconds(bootstrapOptions.Authority.BackchannelTimeoutSeconds);
clientOptions.DefaultScopes.Clear();
foreach (var scope in bootstrapOptions.Authority.ClientScopes)
{
clientOptions.DefaultScopes.Add(scope);
}
var resilience = bootstrapOptions.Authority.Resilience ?? new ScannerWebServiceOptions.AuthorityOptions.ResilienceOptions();
if (resilience.EnableRetries.HasValue)
{
clientOptions.EnableRetries = resilience.EnableRetries.Value;
}
if (resilience.RetryDelays is { Count: > 0 })
{
clientOptions.RetryDelays.Clear();
foreach (var delay in resilience.RetryDelays)
{
clientOptions.RetryDelays.Add(delay);
}
}
if (resilience.AllowOfflineCacheFallback.HasValue)
{
clientOptions.AllowOfflineCacheFallback = resilience.AllowOfflineCacheFallback.Value;
}
if (resilience.OfflineCacheTolerance.HasValue)
{
clientOptions.OfflineCacheTolerance = resilience.OfflineCacheTolerance.Value;
}
});
builder.Services.AddStellaOpsResourceServerAuthentication(
builder.Configuration,
configurationSection: null,
configure: resourceOptions =>
{
resourceOptions.Authority = bootstrapOptions.Authority.Issuer;
resourceOptions.RequireHttpsMetadata = bootstrapOptions.Authority.RequireHttpsMetadata;
resourceOptions.MetadataAddress = bootstrapOptions.Authority.MetadataAddress;
resourceOptions.BackchannelTimeout = TimeSpan.FromSeconds(bootstrapOptions.Authority.BackchannelTimeoutSeconds);
resourceOptions.TokenClockSkew = TimeSpan.FromSeconds(bootstrapOptions.Authority.TokenClockSkewSeconds);
resourceOptions.Audiences.Clear();
foreach (var audience in bootstrapOptions.Authority.Audiences)
{
resourceOptions.Audiences.Add(audience);
}
resourceOptions.RequiredScopes.Clear();
foreach (var scope in bootstrapOptions.Authority.RequiredScopes)
{
resourceOptions.RequiredScopes.Add(scope);
}
resourceOptions.BypassNetworks.Clear();
foreach (var network in bootstrapOptions.Authority.BypassNetworks)
{
resourceOptions.BypassNetworks.Add(network);
}
});
builder.Services.AddAuthorization(options =>
{
options.AddStellaOpsScopePolicy(ScannerPolicies.ScansEnqueue, bootstrapOptions.Authority.RequiredScopes.ToArray());
options.AddStellaOpsScopePolicy(ScannerPolicies.ScansRead, ScannerAuthorityScopes.ScansRead);
options.AddStellaOpsScopePolicy(ScannerPolicies.Reports, ScannerAuthorityScopes.ReportsRead);
options.AddStellaOpsScopePolicy(ScannerPolicies.RuntimeIngest, ScannerAuthorityScopes.RuntimeIngest);
});
}
else
{
builder.Services.AddAuthentication(options =>
{
options.DefaultAuthenticateScheme = "Anonymous";
options.DefaultChallengeScheme = "Anonymous";
})
.AddScheme<AuthenticationSchemeOptions, AnonymousAuthenticationHandler>("Anonymous", _ => { });
builder.Services.AddAuthorization(options =>
{
options.AddPolicy(ScannerPolicies.ScansEnqueue, policy => policy.RequireAssertion(_ => true));
options.AddPolicy(ScannerPolicies.ScansRead, policy => policy.RequireAssertion(_ => true));
options.AddPolicy(ScannerPolicies.Reports, policy => policy.RequireAssertion(_ => true));
options.AddPolicy(ScannerPolicies.RuntimeIngest, policy => policy.RequireAssertion(_ => true));
});
}
var app = builder.Build();
// Fail fast if surface configuration is invalid at startup.
using (var validationScope = app.Services.CreateScope())
{
var services = validationScope.ServiceProvider;
var env = services.GetRequiredService<ISurfaceEnvironment>();
var runner = services.GetRequiredService<ISurfaceValidatorRunner>();
await runner.EnsureAsync(
SurfaceValidationContext.Create(services, "Scanner.WebService.Startup", env.Settings),
app.Lifetime.ApplicationStopping)
.ConfigureAwait(false);
}
var resolvedOptions = app.Services.GetRequiredService<IOptions<ScannerWebServiceOptions>>().Value;
var authorityConfigured = resolvedOptions.Authority.Enabled;
if (authorityConfigured && resolvedOptions.Authority.AllowAnonymousFallback)
{
app.Logger.LogWarning(
"Scanner authority authentication is enabled but anonymous fallback remains allowed. Disable fallback before production rollout.");
}
using (var scope = app.Services.CreateScope())
{
var bootstrapper = scope.ServiceProvider.GetRequiredService<MongoBootstrapper>();
await bootstrapper.InitializeAsync(CancellationToken.None).ConfigureAwait(false);
}
if (resolvedOptions.Telemetry.EnableLogging && resolvedOptions.Telemetry.EnableRequestLogging)
{
app.UseSerilogRequestLogging(options =>
{
options.GetLevel = (httpContext, elapsed, exception) =>
exception is null ? LogEventLevel.Information : LogEventLevel.Error;
options.EnrichDiagnosticContext = (diagnosticContext, httpContext) =>
{
diagnosticContext.Set("RequestId", httpContext.TraceIdentifier);
diagnosticContext.Set("UserAgent", httpContext.Request.Headers.UserAgent.ToString());
if (Activity.Current is { TraceId: var traceId } && traceId != default)
{
diagnosticContext.Set("TraceId", traceId.ToString());
}
};
});
}
app.UseExceptionHandler(errorApp =>
{
errorApp.Run(async context =>
{
context.Response.ContentType = "application/problem+json";
var feature = context.Features.Get<IExceptionHandlerFeature>();
var error = feature?.Error;
var extensions = new Dictionary<string, object?>(StringComparer.Ordinal)
{
["traceId"] = Activity.Current?.TraceId.ToString() ?? context.TraceIdentifier,
};
var problem = Results.Problem(
detail: error?.Message,
instance: context.Request.Path,
statusCode: StatusCodes.Status500InternalServerError,
title: "Unexpected server error",
type: "https://stellaops.org/problems/internal-error",
extensions: extensions);
await problem.ExecuteAsync(context).ConfigureAwait(false);
});
});
if (authorityConfigured)
{
app.UseAuthentication();
app.UseAuthorization();
}
app.MapHealthEndpoints();
var apiGroup = app.MapGroup(resolvedOptions.Api.BasePath);
if (app.Environment.IsEnvironment("Testing"))
{
apiGroup.MapGet("/__auth-probe", () => Results.Ok("ok"))
.RequireAuthorization(ScannerPolicies.ScansEnqueue)
.WithName("scanner.auth-probe");
}
apiGroup.MapScanEndpoints(resolvedOptions.Api.ScansSegment);
apiGroup.MapReplayEndpoints();
if (resolvedOptions.Features.EnablePolicyPreview)
{
apiGroup.MapPolicyEndpoints(resolvedOptions.Api.PolicySegment);
}
apiGroup.MapReportEndpoints(resolvedOptions.Api.ReportsSegment);
apiGroup.MapRuntimeEndpoints(resolvedOptions.Api.RuntimeSegment);
app.MapOpenApiIfAvailable();
await app.RunAsync().ConfigureAwait(false);
public partial class Program;
internal sealed class SurfaceCacheOptionsConfigurator : IConfigureOptions<SurfaceCacheOptions>
{
private readonly ISurfaceEnvironment _surfaceEnvironment;
public SurfaceCacheOptionsConfigurator(ISurfaceEnvironment surfaceEnvironment)
{
_surfaceEnvironment = surfaceEnvironment ?? throw new ArgumentNullException(nameof(surfaceEnvironment));
}
public void Configure(SurfaceCacheOptions options)
{
ArgumentNullException.ThrowIfNull(options);
var settings = _surfaceEnvironment.Settings;
options.RootDirectory = settings.CacheRoot.FullName;
}
}
builder.Services.Configure<ConcelierLinksetOptions>(builder.Configuration.GetSection(ConcelierLinksetOptions.SectionName));
builder.Services.AddHttpClient<ConcelierHttpLinksetQueryService>((sp, client) =>
{
var options = sp.GetRequiredService<IOptions<ConcelierLinksetOptions>>().Value;
if (!string.IsNullOrWhiteSpace(options.BaseUrl))
{
client.BaseAddress = new Uri(options.BaseUrl);
}
client.Timeout = TimeSpan.FromSeconds(Math.Max(1, options.TimeoutSeconds));
if (!string.IsNullOrWhiteSpace(options.ApiKey))
{
var header = string.IsNullOrWhiteSpace(options.ApiKeyHeader) ? "Authorization" : options.ApiKeyHeader;
client.DefaultRequestHeaders.TryAddWithoutValidation(header, options.ApiKey);
}
})
.ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler
{
AutomaticDecompression = System.Net.DecompressionMethods.All
});
builder.Services.AddSingleton<IAdvisoryLinksetQueryService>(sp =>
{
var options = sp.GetRequiredService<IOptions<ConcelierLinksetOptions>>().Value;
if (options.Enabled && !string.IsNullOrWhiteSpace(options.BaseUrl))
{
return sp.GetRequiredService<ConcelierHttpLinksetQueryService>();
}
return new NullAdvisoryLinksetQueryService();
});