feat: Implement ScannerSurfaceSecretConfigurator for web service options

- Added ScannerSurfaceSecretConfigurator to configure ScannerWebServiceOptions using surface secrets.
- Integrated ISurfaceSecretProvider to fetch and apply secrets for artifact store configuration.
- Enhanced logging for secret retrieval and application processes.

feat: Implement ScannerStorageSurfaceSecretConfigurator for worker options

- Introduced ScannerStorageSurfaceSecretConfigurator to configure ScannerStorageOptions with surface secrets.
- Utilized ISurfaceSecretProvider to retrieve and apply secrets for object store settings.
- Improved logging for secret handling and configuration.

feat: Create SurfaceManifestPublisher for publishing surface manifests

- Developed SurfaceManifestPublisher to handle the creation and storage of surface manifests.
- Implemented methods for serializing manifest documents and storing payloads in the object store.
- Added dual write functionality for mirror storage of manifests.

feat: Add SurfaceManifestStageExecutor for processing scan stages

- Created SurfaceManifestStageExecutor to execute the manifest publishing stage in scan jobs.
- Integrated with SurfaceManifestPublisher to publish manifests based on collected payloads.
- Enhanced logging for job processing and manifest storage.

feat: Define SurfaceManifest models for manifest structure

- Established SurfaceManifestDocument, SurfaceManifestSource, SurfaceManifestArtifact, and SurfaceManifestStorage records.
- Implemented serialization attributes for JSON handling of manifest models.

feat: Implement CasAccessSecret and SurfaceSecretParser for secret handling

- Created CasAccessSecret record to represent surface access secrets.
- Developed SurfaceSecretParser to parse and validate surface secrets from JSON payloads.

test: Add unit tests for CasAccessSecretParser

- Implemented tests for parsing CasAccessSecret from JSON payloads and metadata fallbacks.
- Verified expected values and behavior for secret parsing logic.

test: Add unit tests for ScannerSurfaceSecretConfigurator

- Created tests for ScannerSurfaceSecretConfigurator to ensure correct application of surface secrets to web service options.
- Validated artifact store settings after configuration.

test: Add unit tests for ScannerStorageSurfaceSecretConfigurator

- Implemented tests for ScannerStorageSurfaceSecretConfigurator to verify correct application of surface secrets to storage options.
- Ensured accurate configuration of object store settings.

src/Scanner/__Libraries/StellaOps.Scanner.Storage/Catalog/ArtifactDocument.cs

@@ -2,23 +2,30 @@ using MongoDB.Bson.Serialization.Attributes;
 
 namespace StellaOps.Scanner.Storage.Catalog;
 
-public enum ArtifactDocumentType
-{
-    LayerBom,
-    ImageBom,
-    Diff,
-    Index,
-    Attestation,
-}
-
-public enum ArtifactDocumentFormat
-{
-    CycloneDxJson,
-    CycloneDxProtobuf,
-    SpdxJson,
-    BomIndex,
-    DsseJson,
-}
+public enum ArtifactDocumentType
+{
+    LayerBom,
+    ImageBom,
+    Diff,
+    Index,
+    Attestation,
+    SurfaceManifest,
+    SurfaceEntryTrace,
+    SurfaceLayerFragment,
+}
+
+public enum ArtifactDocumentFormat
+{
+    CycloneDxJson,
+    CycloneDxProtobuf,
+    SpdxJson,
+    BomIndex,
+    DsseJson,
+    SurfaceManifestJson,
+    EntryTraceNdjson,
+    EntryTraceGraphJson,
+    ComponentFragmentJson,
+}
 
 [BsonIgnoreExtraElements]
 public sealed class ArtifactDocument

src/Scanner/__Libraries/StellaOps.Scanner.Storage/Extensions/ServiceCollectionExtensions.cs

@@ -2,6 +2,7 @@ using System;
 using System.Net.Http;
 using Amazon;
 using Amazon.S3;
+using Amazon.Runtime;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.DependencyInjection.Extensions;
@@ -150,14 +151,22 @@ public static class ServiceCollectionExtensions
         var options = provider.GetRequiredService<IOptions<ScannerStorageOptions>>().Value.ObjectStore;
         var config = new AmazonS3Config
         {
-            RegionEndpoint = RegionEndpoint.GetBySystemName(options.Region),
-            ForcePathStyle = options.ForcePathStyle,
-        };
-
-        if (!string.IsNullOrWhiteSpace(options.ServiceUrl))
-        {
-            config.ServiceURL = options.ServiceUrl;
-        }
+            RegionEndpoint = RegionEndpoint.GetBySystemName(options.Region),
+            ForcePathStyle = options.ForcePathStyle,
+        };
+
+        if (!string.IsNullOrWhiteSpace(options.ServiceUrl))
+        {
+            config.ServiceURL = options.ServiceUrl;
+        }
+
+        if (!string.IsNullOrWhiteSpace(options.AccessKeyId) && !string.IsNullOrWhiteSpace(options.SecretAccessKey))
+        {
+            AWSCredentials credentials = string.IsNullOrWhiteSpace(options.SessionToken)
+                ? new BasicAWSCredentials(options.AccessKeyId, options.SecretAccessKey)
+                : new SessionAWSCredentials(options.AccessKeyId, options.SecretAccessKey, options.SessionToken);
+            return new AmazonS3Client(credentials, config);
+        }
 
         return new AmazonS3Client(config);
     }

src/Scanner/__Libraries/StellaOps.Scanner.Storage/ObjectStore/ArtifactObjectKeyBuilder.cs

@@ -33,6 +33,9 @@ public static class ArtifactObjectKeyBuilder
             ArtifactDocumentType.ImageBom => ScannerStorageDefaults.ObjectPrefixes.Images,
             ArtifactDocumentType.Index => ScannerStorageDefaults.ObjectPrefixes.Indexes,
             ArtifactDocumentType.Attestation => ScannerStorageDefaults.ObjectPrefixes.Attestations,
+            ArtifactDocumentType.SurfaceManifest => ScannerStorageDefaults.ObjectPrefixes.SurfaceManifests,
+            ArtifactDocumentType.SurfaceEntryTrace => ScannerStorageDefaults.ObjectPrefixes.SurfaceEntryTrace,
+            ArtifactDocumentType.SurfaceLayerFragment => ScannerStorageDefaults.ObjectPrefixes.SurfaceLayerFragments,
             ArtifactDocumentType.Diff => "diffs",
             _ => ScannerStorageDefaults.ObjectPrefixes.Images,
         };
@@ -44,6 +47,10 @@ public static class ArtifactObjectKeyBuilder
             ArtifactDocumentFormat.SpdxJson => "sbom.spdx.json",
             ArtifactDocumentFormat.BomIndex => "bom-index.bin",
             ArtifactDocumentFormat.DsseJson => "artifact.dsse.json",
+            ArtifactDocumentFormat.SurfaceManifestJson => "surface.manifest.json",
+            ArtifactDocumentFormat.EntryTraceNdjson => "entrytrace.ndjson",
+            ArtifactDocumentFormat.EntryTraceGraphJson => "entrytrace.graph.json",
+            ArtifactDocumentFormat.ComponentFragmentJson => "layer-fragments.json",
             _ => "artifact.bin",
         };
 

src/Scanner/__Libraries/StellaOps.Scanner.Storage/ScannerStorageDefaults.cs

@@ -26,11 +26,14 @@ public static class ScannerStorageDefaults
         public const string Migrations = "schema_migrations";
     }
 
-    public static class ObjectPrefixes
-    {
-        public const string Layers = "layers";
-        public const string Images = "images";
-        public const string Indexes = "indexes";
-        public const string Attestations = "attest";
-    }
-}
+    public static class ObjectPrefixes
+    {
+        public const string Layers = "layers";
+        public const string Images = "images";
+        public const string Indexes = "indexes";
+        public const string Attestations = "attest";
+        public const string SurfaceManifests = "surface/manifests";
+        public const string SurfaceEntryTrace = "surface/payloads/entrytrace";
+        public const string SurfaceLayerFragments = "surface/payloads/layer-fragments";
+    }
+}

src/Scanner/__Libraries/StellaOps.Scanner.Storage/ScannerStorageOptions.cs

@@ -102,6 +102,15 @@ public sealed class ObjectStoreOptions
     public TimeSpan? ComplianceRetention { get; set; }
         = TimeSpan.FromDays(90);
 
+    public string? AccessKeyId { get; set; }
+        = null;
+
+    public string? SecretAccessKey { get; set; }
+        = null;
+
+    public string? SessionToken { get; set; }
+        = null;
+
     public IDictionary<string, string> Headers { get; } = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
 
     public RustFsOptions RustFs { get; set; } = new();