feat: Implement ScannerSurfaceSecretConfigurator for web service options

- Added ScannerSurfaceSecretConfigurator to configure ScannerWebServiceOptions using surface secrets.
- Integrated ISurfaceSecretProvider to fetch and apply secrets for artifact store configuration.
- Enhanced logging for secret retrieval and application processes.

feat: Implement ScannerStorageSurfaceSecretConfigurator for worker options

- Introduced ScannerStorageSurfaceSecretConfigurator to configure ScannerStorageOptions with surface secrets.
- Utilized ISurfaceSecretProvider to retrieve and apply secrets for object store settings.
- Improved logging for secret handling and configuration.

feat: Create SurfaceManifestPublisher for publishing surface manifests

- Developed SurfaceManifestPublisher to handle the creation and storage of surface manifests.
- Implemented methods for serializing manifest documents and storing payloads in the object store.
- Added dual write functionality for mirror storage of manifests.

feat: Add SurfaceManifestStageExecutor for processing scan stages

- Created SurfaceManifestStageExecutor to execute the manifest publishing stage in scan jobs.
- Integrated with SurfaceManifestPublisher to publish manifests based on collected payloads.
- Enhanced logging for job processing and manifest storage.

feat: Define SurfaceManifest models for manifest structure

- Established SurfaceManifestDocument, SurfaceManifestSource, SurfaceManifestArtifact, and SurfaceManifestStorage records.
- Implemented serialization attributes for JSON handling of manifest models.

feat: Implement CasAccessSecret and SurfaceSecretParser for secret handling

- Created CasAccessSecret record to represent surface access secrets.
- Developed SurfaceSecretParser to parse and validate surface secrets from JSON payloads.

test: Add unit tests for CasAccessSecretParser

- Implemented tests for parsing CasAccessSecret from JSON payloads and metadata fallbacks.
- Verified expected values and behavior for secret parsing logic.

test: Add unit tests for ScannerSurfaceSecretConfigurator

- Created tests for ScannerSurfaceSecretConfigurator to ensure correct application of surface secrets to web service options.
- Validated artifact store settings after configuration.

test: Add unit tests for ScannerStorageSurfaceSecretConfigurator

- Implemented tests for ScannerStorageSurfaceSecretConfigurator to verify correct application of surface secrets to storage options.
- Ensured accurate configuration of object store settings.

src/Scanner/StellaOps.Scanner.WebService/Options/ScannerStorageOptionsPostConfigurator.cs

@@ -0,0 +1,118 @@
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using StellaOps.Scanner.Storage;
+
+namespace StellaOps.Scanner.WebService.Options;
+
+internal sealed class ScannerStorageOptionsPostConfigurator : IPostConfigureOptions<ScannerStorageOptions>
+{
+    private readonly IOptionsMonitor<ScannerWebServiceOptions> _webOptions;
+    private readonly ILogger<ScannerStorageOptionsPostConfigurator> _logger;
+
+    public ScannerStorageOptionsPostConfigurator(
+        IOptionsMonitor<ScannerWebServiceOptions> webOptions,
+        ILogger<ScannerStorageOptionsPostConfigurator> logger)
+    {
+        _webOptions = webOptions ?? throw new ArgumentNullException(nameof(webOptions));
+        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+    }
+
+    public void PostConfigure(string? name, ScannerStorageOptions options)
+    {
+        ArgumentNullException.ThrowIfNull(options);
+
+        var source = _webOptions.CurrentValue?.ArtifactStore;
+        if (source is null)
+        {
+            return;
+        }
+
+        var target = options.ObjectStore ??= new ObjectStoreOptions();
+
+        if (!string.IsNullOrWhiteSpace(source.Driver))
+        {
+            target.Driver = source.Driver;
+        }
+
+        if (!string.IsNullOrWhiteSpace(source.Region))
+        {
+            target.Region = source.Region!;
+        }
+
+        if (!string.IsNullOrWhiteSpace(source.Bucket))
+        {
+            target.BucketName = source.Bucket!;
+        }
+
+        if (!string.IsNullOrWhiteSpace(source.RootPrefix))
+        {
+            target.RootPrefix = source.RootPrefix;
+        }
+
+        if (!string.IsNullOrWhiteSpace(source.Endpoint))
+        {
+            if (target.IsRustFsDriver())
+            {
+                target.RustFs ??= new RustFsOptions();
+                target.RustFs.BaseUrl = source.Endpoint;
+            }
+            else
+            {
+                target.ServiceUrl = source.Endpoint;
+            }
+        }
+
+        if (target.IsRustFsDriver())
+        {
+            if (target.RustFs is null)
+            {
+                target.RustFs = new RustFsOptions();
+            }
+
+            target.RustFs.AllowInsecureTls = source.AllowInsecureTls;
+
+            if (!string.IsNullOrWhiteSpace(source.ApiKeyHeader))
+            {
+                target.RustFs.ApiKeyHeader = source.ApiKeyHeader!;
+            }
+
+            if (!string.IsNullOrWhiteSpace(source.ApiKey))
+            {
+                target.RustFs.ApiKey = source.ApiKey;
+            }
+
+            if (!string.IsNullOrWhiteSpace(source.Endpoint))
+            {
+                target.RustFs.BaseUrl = source.Endpoint!;
+            }
+        }
+
+        if (!string.IsNullOrWhiteSpace(source.AccessKey))
+        {
+            target.AccessKeyId = source.AccessKey;
+        }
+
+        if (!string.IsNullOrWhiteSpace(source.SecretKey))
+        {
+            target.SecretAccessKey = source.SecretKey;
+        }
+
+        if (source.Headers is { Count: > 0 })
+        {
+            foreach (var (key, value) in source.Headers)
+            {
+                if (string.IsNullOrWhiteSpace(key) || string.IsNullOrWhiteSpace(value))
+                {
+                    continue;
+                }
+
+                target.Headers[key] = value;
+            }
+        }
+
+        _logger.LogDebug(
+            "Mirrored artifact store settings into scanner storage options (driver: {Driver}, bucket: {Bucket}).",
+            target.Driver,
+            target.BucketName);
+    }
+}

src/Scanner/StellaOps.Scanner.WebService/Options/ScannerSurfaceSecretConfigurator.cs

@@ -0,0 +1,124 @@
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using StellaOps.Scanner.Surface.Env;
+using StellaOps.Scanner.Surface.Secrets;
+
+namespace StellaOps.Scanner.WebService.Options;
+
+internal sealed class ScannerSurfaceSecretConfigurator : IConfigureOptions<ScannerWebServiceOptions>
+{
+    private const string ComponentName = "Scanner.WebService";
+
+    private readonly ISurfaceSecretProvider _secretProvider;
+    private readonly ISurfaceEnvironment _surfaceEnvironment;
+    private readonly ILogger<ScannerSurfaceSecretConfigurator> _logger;
+
+    public ScannerSurfaceSecretConfigurator(
+        ISurfaceSecretProvider secretProvider,
+        ISurfaceEnvironment surfaceEnvironment,
+        ILogger<ScannerSurfaceSecretConfigurator> logger)
+    {
+        _secretProvider = secretProvider ?? throw new ArgumentNullException(nameof(secretProvider));
+        _surfaceEnvironment = surfaceEnvironment ?? throw new ArgumentNullException(nameof(surfaceEnvironment));
+        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+    }
+
+    public void Configure(ScannerWebServiceOptions options)
+    {
+        ArgumentNullException.ThrowIfNull(options);
+
+        var tenant = _surfaceEnvironment.Settings.Secrets.Tenant;
+        var request = new SurfaceSecretRequest(
+            Tenant: tenant,
+            Component: ComponentName,
+            SecretType: "cas-access");
+
+        CasAccessSecret? secret = null;
+        try
+        {
+            using var handle = _secretProvider.GetAsync(request).AsTask().GetAwaiter().GetResult();
+            secret = SurfaceSecretParser.ParseCasAccessSecret(handle);
+        }
+        catch (SurfaceSecretNotFoundException)
+        {
+            _logger.LogDebug("Surface secret 'cas-access' not found for {Component}; retaining configured artifact store settings.", ComponentName);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Failed to resolve surface secret 'cas-access' for {Component}.", ComponentName);
+        }
+
+        if (secret is null)
+        {
+            return;
+        }
+
+        ApplySecret(options.ArtifactStore ??= new ScannerWebServiceOptions.ArtifactStoreOptions(), secret);
+    }
+
+    private void ApplySecret(ScannerWebServiceOptions.ArtifactStoreOptions artifactStore, CasAccessSecret secret)
+    {
+        if (!string.IsNullOrWhiteSpace(secret.Driver))
+        {
+            artifactStore.Driver = secret.Driver;
+        }
+
+        if (!string.IsNullOrWhiteSpace(secret.Endpoint))
+        {
+            artifactStore.Endpoint = secret.Endpoint!;
+        }
+
+        if (secret.AllowInsecureTls is { } insecure)
+        {
+            artifactStore.AllowInsecureTls = insecure;
+            artifactStore.UseTls = !insecure;
+        }
+
+        if (!string.IsNullOrWhiteSpace(secret.Region))
+        {
+            artifactStore.Region = secret.Region;
+        }
+
+        if (!string.IsNullOrWhiteSpace(secret.Bucket))
+        {
+            artifactStore.Bucket = secret.Bucket!;
+        }
+
+        if (!string.IsNullOrWhiteSpace(secret.RootPrefix))
+        {
+            artifactStore.RootPrefix = secret.RootPrefix!;
+        }
+
+        if (!string.IsNullOrWhiteSpace(secret.ApiKeyHeader))
+        {
+            artifactStore.ApiKeyHeader = secret.ApiKeyHeader!;
+        }
+
+        if (!string.IsNullOrWhiteSpace(secret.ApiKey))
+        {
+            artifactStore.ApiKey = secret.ApiKey;
+        }
+
+        if (!string.IsNullOrWhiteSpace(secret.AccessKeyId) && !string.IsNullOrWhiteSpace(secret.SecretAccessKey))
+        {
+            artifactStore.AccessKey = secret.AccessKeyId!;
+            artifactStore.SecretKey = secret.SecretAccessKey!;
+        }
+
+        foreach (var header in secret.Headers)
+        {
+            if (string.IsNullOrWhiteSpace(header.Key) || string.IsNullOrWhiteSpace(header.Value))
+            {
+                continue;
+            }
+
+            artifactStore.Headers[header.Key] = header.Value;
+        }
+
+        _logger.LogInformation(
+            "Surface secret 'cas-access' applied for {Component} (driver: {Driver}, bucket: {Bucket}).",
+            ComponentName,
+            artifactStore.Driver,
+            artifactStore.Bucket);
+    }
+}

src/Scanner/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptionsPostConfigure.cs

@@ -30,13 +30,7 @@ public static class ScannerWebServiceOptionsPostConfigure
 
         options.ArtifactStore ??= new ScannerWebServiceOptions.ArtifactStoreOptions();
         var artifactStore = options.ArtifactStore;
-        if (string.IsNullOrWhiteSpace(artifactStore.SecretKey)
-            && !string.IsNullOrWhiteSpace(artifactStore.SecretKeyFile))
-        {
-            artifactStore.SecretKey = ReadSecretFile(artifactStore.SecretKeyFile!, contentRootPath);
-        }
-
-        options.Signing ??= new ScannerWebServiceOptions.SigningOptions();
+        options.Signing ??= new ScannerWebServiceOptions.SigningOptions();
         var signing = options.Signing;
         if (string.IsNullOrWhiteSpace(signing.KeyPem)
             && !string.IsNullOrWhiteSpace(signing.KeyPemFile))