feat: Implement ScannerSurfaceSecretConfigurator for web service options

- Added ScannerSurfaceSecretConfigurator to configure ScannerWebServiceOptions using surface secrets.
- Integrated ISurfaceSecretProvider to fetch and apply secrets for artifact store configuration.
- Enhanced logging for secret retrieval and application processes.

feat: Implement ScannerStorageSurfaceSecretConfigurator for worker options

- Introduced ScannerStorageSurfaceSecretConfigurator to configure ScannerStorageOptions with surface secrets.
- Utilized ISurfaceSecretProvider to retrieve and apply secrets for object store settings.
- Improved logging for secret handling and configuration.

feat: Create SurfaceManifestPublisher for publishing surface manifests

- Developed SurfaceManifestPublisher to handle the creation and storage of surface manifests.
- Implemented methods for serializing manifest documents and storing payloads in the object store.
- Added dual write functionality for mirror storage of manifests.

feat: Add SurfaceManifestStageExecutor for processing scan stages

- Created SurfaceManifestStageExecutor to execute the manifest publishing stage in scan jobs.
- Integrated with SurfaceManifestPublisher to publish manifests based on collected payloads.
- Enhanced logging for job processing and manifest storage.

feat: Define SurfaceManifest models for manifest structure

- Established SurfaceManifestDocument, SurfaceManifestSource, SurfaceManifestArtifact, and SurfaceManifestStorage records.
- Implemented serialization attributes for JSON handling of manifest models.

feat: Implement CasAccessSecret and SurfaceSecretParser for secret handling

- Created CasAccessSecret record to represent surface access secrets.
- Developed SurfaceSecretParser to parse and validate surface secrets from JSON payloads.

test: Add unit tests for CasAccessSecretParser

- Implemented tests for parsing CasAccessSecret from JSON payloads and metadata fallbacks.
- Verified expected values and behavior for secret parsing logic.

test: Add unit tests for ScannerSurfaceSecretConfigurator

- Created tests for ScannerSurfaceSecretConfigurator to ensure correct application of surface secrets to web service options.
- Validated artifact store settings after configuration.

test: Add unit tests for ScannerStorageSurfaceSecretConfigurator

- Implemented tests for ScannerStorageSurfaceSecretConfigurator to verify correct application of surface secrets to storage options.
- Ensured accurate configuration of object store settings.

src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang/Core/Internal/LanguageAnalyzerSurfaceCache.cs

@@ -4,6 +4,8 @@ using System.Text;
 using Microsoft.Extensions.Logging;
 using StellaOps.Scanner.Surface.FS;
 
+public readonly record struct LanguageAnalyzerSurfaceCacheEntry(LanguageAnalyzerResult Result, bool IsHit);
+
 public sealed class LanguageAnalyzerSurfaceCache
 {
     private const string CacheNamespace = "scanner/lang/analyzers";
@@ -24,6 +26,17 @@ public sealed class LanguageAnalyzerSurfaceCache
         string fingerprint,
         Func<CancellationToken, ValueTask<LanguageAnalyzerResult>> factory,
         CancellationToken cancellationToken)
+    {
+        var entry = await GetOrCreateEntryAsync(logger, analyzerId, fingerprint, factory, cancellationToken).ConfigureAwait(false);
+        return entry.Result;
+    }
+
+    public async ValueTask<LanguageAnalyzerSurfaceCacheEntry> GetOrCreateEntryAsync(
+        ILogger logger,
+        string analyzerId,
+        string fingerprint,
+        Func<CancellationToken, ValueTask<LanguageAnalyzerResult>> factory,
+        CancellationToken cancellationToken)
     {
         ArgumentNullException.ThrowIfNull(logger);
         ArgumentNullException.ThrowIfNull(factory);
@@ -62,7 +75,7 @@ public sealed class LanguageAnalyzerSurfaceCache
                 fingerprint);
 
             result = await factory(cancellationToken).ConfigureAwait(false);
-            return result;
+            return new LanguageAnalyzerSurfaceCacheEntry(result, false);
         }
 
         if (cacheHit)
@@ -82,7 +95,7 @@ public sealed class LanguageAnalyzerSurfaceCache
                 fingerprint);
         }
 
-        return result;
+        return new LanguageAnalyzerSurfaceCacheEntry(result, cacheHit);
     }
 
     private static ReadOnlyMemory<byte> Serialize(LanguageAnalyzerResult result)

src/Scanner/__Libraries/StellaOps.Scanner.Core/Contracts/ScanAnalysisKeys.cs

@@ -15,4 +15,6 @@ public static class ScanAnalysisKeys
     public const string EntryTraceGraph = "analysis.entrytrace.graph";
 
     public const string EntryTraceNdjson = "analysis.entrytrace.ndjson";
+
+    public const string SurfaceManifest = "analysis.surface.manifest";
 }

src/Scanner/__Libraries/StellaOps.Scanner.Storage/Catalog/ArtifactDocument.cs

@@ -2,23 +2,30 @@ using MongoDB.Bson.Serialization.Attributes;
 
 namespace StellaOps.Scanner.Storage.Catalog;
 
-public enum ArtifactDocumentType
-{
-    LayerBom,
-    ImageBom,
-    Diff,
-    Index,
-    Attestation,
-}
-
-public enum ArtifactDocumentFormat
-{
-    CycloneDxJson,
-    CycloneDxProtobuf,
-    SpdxJson,
-    BomIndex,
-    DsseJson,
-}
+public enum ArtifactDocumentType
+{
+    LayerBom,
+    ImageBom,
+    Diff,
+    Index,
+    Attestation,
+    SurfaceManifest,
+    SurfaceEntryTrace,
+    SurfaceLayerFragment,
+}
+
+public enum ArtifactDocumentFormat
+{
+    CycloneDxJson,
+    CycloneDxProtobuf,
+    SpdxJson,
+    BomIndex,
+    DsseJson,
+    SurfaceManifestJson,
+    EntryTraceNdjson,
+    EntryTraceGraphJson,
+    ComponentFragmentJson,
+}
 
 [BsonIgnoreExtraElements]
 public sealed class ArtifactDocument

src/Scanner/__Libraries/StellaOps.Scanner.Storage/Extensions/ServiceCollectionExtensions.cs

@@ -2,6 +2,7 @@ using System;
 using System.Net.Http;
 using Amazon;
 using Amazon.S3;
+using Amazon.Runtime;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.DependencyInjection.Extensions;
@@ -150,14 +151,22 @@ public static class ServiceCollectionExtensions
         var options = provider.GetRequiredService<IOptions<ScannerStorageOptions>>().Value.ObjectStore;
         var config = new AmazonS3Config
         {
-            RegionEndpoint = RegionEndpoint.GetBySystemName(options.Region),
-            ForcePathStyle = options.ForcePathStyle,
-        };
-
-        if (!string.IsNullOrWhiteSpace(options.ServiceUrl))
-        {
-            config.ServiceURL = options.ServiceUrl;
-        }
+            RegionEndpoint = RegionEndpoint.GetBySystemName(options.Region),
+            ForcePathStyle = options.ForcePathStyle,
+        };
+
+        if (!string.IsNullOrWhiteSpace(options.ServiceUrl))
+        {
+            config.ServiceURL = options.ServiceUrl;
+        }
+
+        if (!string.IsNullOrWhiteSpace(options.AccessKeyId) && !string.IsNullOrWhiteSpace(options.SecretAccessKey))
+        {
+            AWSCredentials credentials = string.IsNullOrWhiteSpace(options.SessionToken)
+                ? new BasicAWSCredentials(options.AccessKeyId, options.SecretAccessKey)
+                : new SessionAWSCredentials(options.AccessKeyId, options.SecretAccessKey, options.SessionToken);
+            return new AmazonS3Client(credentials, config);
+        }
 
         return new AmazonS3Client(config);
     }

src/Scanner/__Libraries/StellaOps.Scanner.Storage/ObjectStore/ArtifactObjectKeyBuilder.cs

@@ -33,6 +33,9 @@ public static class ArtifactObjectKeyBuilder
             ArtifactDocumentType.ImageBom => ScannerStorageDefaults.ObjectPrefixes.Images,
             ArtifactDocumentType.Index => ScannerStorageDefaults.ObjectPrefixes.Indexes,
             ArtifactDocumentType.Attestation => ScannerStorageDefaults.ObjectPrefixes.Attestations,
+            ArtifactDocumentType.SurfaceManifest => ScannerStorageDefaults.ObjectPrefixes.SurfaceManifests,
+            ArtifactDocumentType.SurfaceEntryTrace => ScannerStorageDefaults.ObjectPrefixes.SurfaceEntryTrace,
+            ArtifactDocumentType.SurfaceLayerFragment => ScannerStorageDefaults.ObjectPrefixes.SurfaceLayerFragments,
             ArtifactDocumentType.Diff => "diffs",
             _ => ScannerStorageDefaults.ObjectPrefixes.Images,
         };
@@ -44,6 +47,10 @@ public static class ArtifactObjectKeyBuilder
             ArtifactDocumentFormat.SpdxJson => "sbom.spdx.json",
             ArtifactDocumentFormat.BomIndex => "bom-index.bin",
             ArtifactDocumentFormat.DsseJson => "artifact.dsse.json",
+            ArtifactDocumentFormat.SurfaceManifestJson => "surface.manifest.json",
+            ArtifactDocumentFormat.EntryTraceNdjson => "entrytrace.ndjson",
+            ArtifactDocumentFormat.EntryTraceGraphJson => "entrytrace.graph.json",
+            ArtifactDocumentFormat.ComponentFragmentJson => "layer-fragments.json",
             _ => "artifact.bin",
         };
 

src/Scanner/__Libraries/StellaOps.Scanner.Storage/ScannerStorageDefaults.cs

@@ -26,11 +26,14 @@ public static class ScannerStorageDefaults
         public const string Migrations = "schema_migrations";
     }
 
-    public static class ObjectPrefixes
-    {
-        public const string Layers = "layers";
-        public const string Images = "images";
-        public const string Indexes = "indexes";
-        public const string Attestations = "attest";
-    }
-}
+    public static class ObjectPrefixes
+    {
+        public const string Layers = "layers";
+        public const string Images = "images";
+        public const string Indexes = "indexes";
+        public const string Attestations = "attest";
+        public const string SurfaceManifests = "surface/manifests";
+        public const string SurfaceEntryTrace = "surface/payloads/entrytrace";
+        public const string SurfaceLayerFragments = "surface/payloads/layer-fragments";
+    }
+}

src/Scanner/__Libraries/StellaOps.Scanner.Storage/ScannerStorageOptions.cs

@@ -102,6 +102,15 @@ public sealed class ObjectStoreOptions
     public TimeSpan? ComplianceRetention { get; set; }
         = TimeSpan.FromDays(90);
 
+    public string? AccessKeyId { get; set; }
+        = null;
+
+    public string? SecretAccessKey { get; set; }
+        = null;
+
+    public string? SessionToken { get; set; }
+        = null;
+
     public IDictionary<string, string> Headers { get; } = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
 
     public RustFsOptions RustFs { get; set; } = new();

src/Scanner/__Libraries/StellaOps.Scanner.Surface.FS/SurfaceManifestModels.cs

@@ -0,0 +1,142 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Text.Json.Serialization;
+
+namespace StellaOps.Scanner.Surface.FS;
+
+/// <summary>
+/// Canonical manifest describing surface artefacts produced for a scan.
+/// </summary>
+public sealed record SurfaceManifestDocument
+{
+    public const string DefaultSchema = "stellaops.surface.manifest@1";
+
+    [JsonPropertyName("schema")]
+    public string Schema { get; init; } = DefaultSchema;
+
+    [JsonPropertyName("tenant")]
+    public string Tenant { get; init; } = string.Empty;
+
+    [JsonPropertyName("imageDigest")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? ImageDigest { get; init; }
+        = null;
+
+    [JsonPropertyName("scanId")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? ScanId { get; init; }
+        = null;
+
+    [JsonPropertyName("generatedAt")]
+    public DateTimeOffset GeneratedAt { get; init; }
+        = DateTimeOffset.UtcNow;
+
+    [JsonPropertyName("source")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public SurfaceManifestSource? Source { get; init; }
+        = null;
+
+    [JsonPropertyName("artifacts")]
+    public IReadOnlyList<SurfaceManifestArtifact> Artifacts { get; init; }
+        = ImmutableArray<SurfaceManifestArtifact>.Empty;
+}
+
+/// <summary>
+/// Identifies the producer of the manifest.
+/// </summary>
+public sealed record SurfaceManifestSource
+{
+    [JsonPropertyName("component")]
+    public string Component { get; init; } = "scanner.worker";
+
+    [JsonPropertyName("version")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Version { get; init; }
+        = null;
+
+    [JsonPropertyName("workerInstance")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? WorkerInstance { get; init; }
+        = null;
+
+    [JsonPropertyName("attempt")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public int? Attempt { get; init; }
+        = null;
+}
+
+/// <summary>
+/// Describes a surface artefact referenced by the manifest.
+/// </summary>
+public sealed record SurfaceManifestArtifact
+{
+    [JsonPropertyName("kind")]
+    public string Kind { get; init; } = string.Empty;
+
+    [JsonPropertyName("uri")]
+    public string Uri { get; init; } = string.Empty;
+
+    [JsonPropertyName("digest")]
+    public string Digest { get; init; } = string.Empty;
+
+    [JsonPropertyName("mediaType")]
+    public string MediaType { get; init; } = string.Empty;
+
+    [JsonPropertyName("format")]
+    public string Format { get; init; } = string.Empty;
+
+    [JsonPropertyName("sizeBytes")]
+    public long SizeBytes { get; init; }
+        = 0;
+
+    [JsonPropertyName("view")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? View { get; init; }
+        = null;
+
+    [JsonPropertyName("storage")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public SurfaceManifestStorage? Storage { get; init; }
+        = null;
+
+    [JsonPropertyName("metadata")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public IReadOnlyDictionary<string, string>? Metadata { get; init; }
+        = null;
+}
+
+/// <summary>
+/// Storage descriptor for an artefact.
+/// </summary>
+public sealed record SurfaceManifestStorage
+{
+    [JsonPropertyName("bucket")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Bucket { get; init; }
+        = null;
+
+    [JsonPropertyName("objectKey")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? ObjectKey { get; init; }
+        = null;
+
+    [JsonPropertyName("sizeBytes")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public long? SizeBytes { get; init; }
+        = null;
+
+    [JsonPropertyName("contentType")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? ContentType { get; init; }
+        = null;
+}
+
+/// <summary>
+/// Result from publishing a surface manifest.
+/// </summary>
+public sealed record SurfaceManifestPublishResult(
+    string ManifestDigest,
+    string ManifestUri,
+    string ArtifactId,
+    SurfaceManifestDocument Document);

src/Scanner/__Libraries/StellaOps.Scanner.Surface.Secrets/CasAccessSecret.cs

@@ -0,0 +1,207 @@
+using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using System.Text;
+using System.Text.Json;
+
+namespace StellaOps.Scanner.Surface.Secrets;
+
+public sealed record CasAccessSecret(
+    string Driver,
+    string? Endpoint,
+    string? Region,
+    string? Bucket,
+    string? RootPrefix,
+    string? ApiKey,
+    string? ApiKeyHeader,
+    IReadOnlyDictionary<string, string> Headers,
+    string? AccessKeyId,
+    string? SecretAccessKey,
+    string? SessionToken,
+    bool? AllowInsecureTls);
+
+public static class SurfaceSecretParser
+{
+    public static CasAccessSecret ParseCasAccessSecret(SurfaceSecretHandle handle)
+    {
+        ArgumentNullException.ThrowIfNull(handle);
+        var payload = handle.AsBytes();
+        if (payload.IsEmpty)
+        {
+            throw new InvalidOperationException("Surface secret payload is empty.");
+        }
+
+        var jsonText = DecodeUtf8(payload);
+        using var document = JsonDocument.Parse(jsonText);
+        var root = document.RootElement;
+
+        string driver = GetString(root, "driver") ?? GetMetadataValue(handle.Metadata, "driver") ?? "s3";
+        string? endpoint = GetString(root, "endpoint") ?? GetMetadataValue(handle.Metadata, "endpoint");
+        string? region = GetString(root, "region") ?? GetMetadataValue(handle.Metadata, "region");
+        string? bucket = GetString(root, "bucket") ?? GetMetadataValue(handle.Metadata, "bucket");
+        string? rootPrefix = GetString(root, "rootPrefix") ?? GetMetadataValue(handle.Metadata, "rootPrefix");
+        string? apiKey = GetString(root, "apiKey") ?? GetMetadataValue(handle.Metadata, "apiKey");
+        string? apiKeyHeader = GetString(root, "apiKeyHeader") ?? GetMetadataValue(handle.Metadata, "apiKeyHeader");
+        string? accessKeyId = GetString(root, "accessKeyId") ?? GetMetadataValue(handle.Metadata, "accessKeyId");
+        string? secretAccessKey = GetString(root, "secretAccessKey") ?? GetMetadataValue(handle.Metadata, "secretAccessKey");
+        string? sessionToken = GetString(root, "sessionToken") ?? GetMetadataValue(handle.Metadata, "sessionToken");
+        bool? allowInsecureTls = GetBoolean(root, "allowInsecureTls");
+
+        var headers = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+        PopulateHeaders(root, headers);
+        PopulateMetadataHeaders(handle.Metadata, headers);
+
+        return new CasAccessSecret(
+            driver.Trim(),
+            endpoint?.Trim(),
+            region?.Trim(),
+            bucket?.Trim(),
+            rootPrefix?.Trim(),
+            apiKey?.Trim(),
+            apiKeyHeader?.Trim(),
+            new ReadOnlyDictionary<string, string>(headers),
+            accessKeyId?.Trim(),
+            secretAccessKey?.Trim(),
+            sessionToken?.Trim(),
+            allowInsecureTls);
+    }
+
+    private static string DecodeUtf8(ReadOnlyMemory<byte> payload)
+    {
+        if (payload.IsEmpty)
+        {
+            return string.Empty;
+        }
+
+        try
+        {
+            return Encoding.UTF8.GetString(payload.Span);
+        }
+        catch (DecoderFallbackException ex)
+        {
+            throw new InvalidOperationException("Surface secret payload is not valid UTF-8 JSON.", ex);
+        }
+    }
+
+    private static string? GetString(JsonElement element, string propertyName)
+    {
+        if (TryGetPropertyIgnoreCase(element, propertyName, out var property))
+        {
+            return property.ValueKind switch
+            {
+                JsonValueKind.String => property.GetString(),
+                JsonValueKind.Number => property.GetRawText(),
+                JsonValueKind.True => bool.TrueString,
+                JsonValueKind.False => bool.FalseString,
+                _ => null
+            };
+        }
+
+        return null;
+    }
+
+    private static bool? GetBoolean(JsonElement element, string propertyName)
+    {
+        if (TryGetPropertyIgnoreCase(element, propertyName, out var property))
+        {
+            return property.ValueKind switch
+            {
+                JsonValueKind.True => true,
+                JsonValueKind.False => false,
+                JsonValueKind.String when bool.TryParse(property.GetString(), out var parsed) => parsed,
+                _ => null
+            };
+        }
+
+        return null;
+    }
+
+    private static bool TryGetPropertyIgnoreCase(JsonElement element, string propertyName, out JsonElement property)
+    {
+        if (element.ValueKind != JsonValueKind.Object)
+        {
+            property = default;
+            return false;
+        }
+
+        if (element.TryGetProperty(propertyName, out property))
+        {
+            return true;
+        }
+
+        foreach (var candidate in element.EnumerateObject())
+        {
+            if (string.Equals(candidate.Name, propertyName, StringComparison.OrdinalIgnoreCase))
+            {
+                property = candidate.Value;
+                return true;
+            }
+        }
+
+        property = default;
+        return false;
+    }
+
+    private static void PopulateHeaders(JsonElement element, IDictionary<string, string> headers)
+    {
+        if (!TryGetPropertyIgnoreCase(element, "headers", out var headersElement))
+        {
+            return;
+        }
+
+        if (headersElement.ValueKind != JsonValueKind.Object)
+        {
+            return;
+        }
+
+        foreach (var property in headersElement.EnumerateObject())
+        {
+            var value = property.Value.ValueKind switch
+            {
+                JsonValueKind.String => property.Value.GetString(),
+                JsonValueKind.Number => property.Value.GetRawText(),
+                JsonValueKind.True => bool.TrueString,
+                JsonValueKind.False => bool.FalseString,
+                _ => null
+            };
+
+            if (string.IsNullOrWhiteSpace(value))
+            {
+                continue;
+            }
+
+            headers[property.Name] = value.Trim();
+        }
+    }
+
+    private static void PopulateMetadataHeaders(IReadOnlyDictionary<string, string> metadata, IDictionary<string, string> headers)
+    {
+        foreach (var (key, value) in metadata)
+        {
+            if (!key.StartsWith("header:", StringComparison.OrdinalIgnoreCase))
+            {
+                continue;
+            }
+
+            var headerName = key["header:".Length..];
+            if (string.IsNullOrWhiteSpace(headerName) || string.IsNullOrWhiteSpace(value))
+            {
+                continue;
+            }
+
+            headers[headerName] = value;
+        }
+    }
+
+    private static string? GetMetadataValue(IReadOnlyDictionary<string, string> metadata, string key)
+    {
+        foreach (var (metadataKey, metadataValue) in metadata)
+        {
+            if (string.Equals(metadataKey, key, StringComparison.OrdinalIgnoreCase))
+            {
+                return metadataValue;
+            }
+        }
+
+        return null;
+    }
+}