feat: Implement ScannerSurfaceSecretConfigurator for web service options
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
- Added ScannerSurfaceSecretConfigurator to configure ScannerWebServiceOptions using surface secrets. - Integrated ISurfaceSecretProvider to fetch and apply secrets for artifact store configuration. - Enhanced logging for secret retrieval and application processes. feat: Implement ScannerStorageSurfaceSecretConfigurator for worker options - Introduced ScannerStorageSurfaceSecretConfigurator to configure ScannerStorageOptions with surface secrets. - Utilized ISurfaceSecretProvider to retrieve and apply secrets for object store settings. - Improved logging for secret handling and configuration. feat: Create SurfaceManifestPublisher for publishing surface manifests - Developed SurfaceManifestPublisher to handle the creation and storage of surface manifests. - Implemented methods for serializing manifest documents and storing payloads in the object store. - Added dual write functionality for mirror storage of manifests. feat: Add SurfaceManifestStageExecutor for processing scan stages - Created SurfaceManifestStageExecutor to execute the manifest publishing stage in scan jobs. - Integrated with SurfaceManifestPublisher to publish manifests based on collected payloads. - Enhanced logging for job processing and manifest storage. feat: Define SurfaceManifest models for manifest structure - Established SurfaceManifestDocument, SurfaceManifestSource, SurfaceManifestArtifact, and SurfaceManifestStorage records. - Implemented serialization attributes for JSON handling of manifest models. feat: Implement CasAccessSecret and SurfaceSecretParser for secret handling - Created CasAccessSecret record to represent surface access secrets. - Developed SurfaceSecretParser to parse and validate surface secrets from JSON payloads. test: Add unit tests for CasAccessSecretParser - Implemented tests for parsing CasAccessSecret from JSON payloads and metadata fallbacks. - Verified expected values and behavior for secret parsing logic. test: Add unit tests for ScannerSurfaceSecretConfigurator - Created tests for ScannerSurfaceSecretConfigurator to ensure correct application of surface secrets to web service options. - Validated artifact store settings after configuration. test: Add unit tests for ScannerStorageSurfaceSecretConfigurator - Implemented tests for ScannerStorageSurfaceSecretConfigurator to verify correct application of surface secrets to storage options. - Ensured accurate configuration of object store settings.
This commit is contained in:
master
@@ -0,0 +1,141 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Storage;
|
||||
using StellaOps.Scanner.Surface.Env;
|
||||
using StellaOps.Scanner.Surface.Secrets;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Options;
|
||||
|
||||
internal sealed class ScannerStorageSurfaceSecretConfigurator : IConfigureOptions<ScannerStorageOptions>
|
||||
{
|
||||
private static readonly string ComponentName = "Scanner.Worker";
|
||||
|
||||
private readonly ISurfaceSecretProvider _secretProvider;
|
||||
private readonly ISurfaceEnvironment _surfaceEnvironment;
|
||||
private readonly ILogger<ScannerStorageSurfaceSecretConfigurator> _logger;
|
||||
|
||||
public ScannerStorageSurfaceSecretConfigurator(
|
||||
ISurfaceSecretProvider secretProvider,
|
||||
ISurfaceEnvironment surfaceEnvironment,
|
||||
ILogger<ScannerStorageSurfaceSecretConfigurator> logger)
|
||||
{
|
||||
_secretProvider = secretProvider ?? throw new ArgumentNullException(nameof(secretProvider));
|
||||
_surfaceEnvironment = surfaceEnvironment ?? throw new ArgumentNullException(nameof(surfaceEnvironment));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public void Configure(ScannerStorageOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
var tenant = _surfaceEnvironment.Settings.Secrets.Tenant;
|
||||
var request = new SurfaceSecretRequest(
|
||||
Tenant: tenant,
|
||||
Component: ComponentName,
|
||||
SecretType: "cas-access");
|
||||
|
||||
CasAccessSecret? secret = null;
|
||||
try
|
||||
{
|
||||
using var handle = _secretProvider.GetAsync(request).AsTask().GetAwaiter().GetResult();
|
||||
secret = SurfaceSecretParser.ParseCasAccessSecret(handle);
|
||||
}
|
||||
catch (SurfaceSecretNotFoundException)
|
||||
{
|
||||
_logger.LogDebug("Surface secret 'cas-access' not found for {Component}; using configured storage settings.", ComponentName);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to resolve surface secret 'cas-access' for {Component}.", ComponentName);
|
||||
}
|
||||
|
||||
if (secret is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
ApplySecret(options, secret);
|
||||
}
|
||||
|
||||
private void ApplySecret(ScannerStorageOptions options, CasAccessSecret secret)
|
||||
{
|
||||
var objectStore = options.ObjectStore ??= new ObjectStoreOptions();
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(secret.Driver))
|
||||
{
|
||||
objectStore.Driver = secret.Driver;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(secret.Region))
|
||||
{
|
||||
objectStore.Region = secret.Region;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(secret.Bucket))
|
||||
{
|
||||
objectStore.BucketName = secret.Bucket;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(secret.RootPrefix))
|
||||
{
|
||||
objectStore.RootPrefix = secret.RootPrefix;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(secret.Endpoint))
|
||||
{
|
||||
if (objectStore.IsRustFsDriver())
|
||||
{
|
||||
objectStore.RustFs ??= new RustFsOptions();
|
||||
objectStore.RustFs.BaseUrl = secret.Endpoint!;
|
||||
}
|
||||
else
|
||||
{
|
||||
objectStore.ServiceUrl = secret.Endpoint;
|
||||
}
|
||||
}
|
||||
|
||||
if (objectStore.IsRustFsDriver())
|
||||
{
|
||||
objectStore.RustFs ??= new RustFsOptions();
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(secret.ApiKeyHeader))
|
||||
{
|
||||
objectStore.RustFs.ApiKeyHeader = secret.ApiKeyHeader!;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(secret.ApiKey))
|
||||
{
|
||||
objectStore.RustFs.ApiKey = secret.ApiKey;
|
||||
}
|
||||
|
||||
if (secret.AllowInsecureTls is { } insecure)
|
||||
{
|
||||
objectStore.RustFs.AllowInsecureTls = insecure;
|
||||
}
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(secret.AccessKeyId) && !string.IsNullOrWhiteSpace(secret.SecretAccessKey))
|
||||
{
|
||||
objectStore.AccessKeyId = secret.AccessKeyId;
|
||||
objectStore.SecretAccessKey = secret.SecretAccessKey;
|
||||
objectStore.SessionToken = secret.SessionToken;
|
||||
}
|
||||
|
||||
foreach (var kvp in secret.Headers)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(kvp.Key) || string.IsNullOrWhiteSpace(kvp.Value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
objectStore.Headers[kvp.Key] = kvp.Value;
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Surface secret 'cas-access' applied for {Component} (driver: {Driver}, bucket: {Bucket}, region: {Region}).",
|
||||
ComponentName,
|
||||
objectStore.Driver,
|
||||
objectStore.BucketName,
|
||||
objectStore.Region);
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,7 @@ using StellaOps.Scanner.Surface.Env;
|
||||
using StellaOps.Scanner.Surface.FS;
|
||||
using StellaOps.Scanner.Surface.Validation;
|
||||
using StellaOps.Scanner.Worker.Options;
|
||||
using StellaOps.Scanner.Worker.Diagnostics;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Processing;
|
||||
|
||||
@@ -206,7 +207,7 @@ internal sealed class CompositeScanAnalyzerDispatcher : IScanAnalyzerDispatcher
|
||||
try
|
||||
{
|
||||
var engine = new LanguageAnalyzerEngine(new[] { analyzer });
|
||||
var cacheEntry = await cacheAdapter.GetOrCreateAsync(
|
||||
var cacheEntry = await cacheAdapter.GetOrCreateEntryAsync(
|
||||
_logger,
|
||||
analyzer.Id,
|
||||
workspaceFingerprint,
|
||||
|
||||
@@ -0,0 +1,264 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.IO;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Core.Contracts;
|
||||
using StellaOps.Scanner.Surface.FS;
|
||||
using StellaOps.Scanner.Storage;
|
||||
using StellaOps.Scanner.Storage.Catalog;
|
||||
using StellaOps.Scanner.Storage.ObjectStore;
|
||||
using StellaOps.Scanner.Storage.Repositories;
|
||||
using StellaOps.Scanner.Storage.Services;
|
||||
using StellaOps.Scanner.Surface.Env;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Processing.Surface;
|
||||
|
||||
internal sealed record SurfaceManifestPayload(
|
||||
ArtifactDocumentType ArtifactType,
|
||||
ArtifactDocumentFormat ArtifactFormat,
|
||||
string Kind,
|
||||
string MediaType,
|
||||
ReadOnlyMemory<byte> Content,
|
||||
string? View = null,
|
||||
IReadOnlyDictionary<string, string>? Metadata = null,
|
||||
bool RegisterArtifact = false);
|
||||
|
||||
internal sealed record SurfaceManifestRequest(
|
||||
string ScanId,
|
||||
string ImageDigest,
|
||||
int Attempt,
|
||||
IReadOnlyDictionary<string, string> Metadata,
|
||||
IReadOnlyList<SurfaceManifestPayload> Payloads,
|
||||
string Component,
|
||||
string? Version,
|
||||
string? WorkerInstance);
|
||||
|
||||
internal sealed class SurfaceManifestPublisher
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = false,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
|
||||
};
|
||||
|
||||
private readonly IArtifactObjectStore _objectStore;
|
||||
private readonly ArtifactRepository _artifactRepository;
|
||||
private readonly LinkRepository _linkRepository;
|
||||
private readonly ScannerStorageOptions _storageOptions;
|
||||
private readonly ISurfaceEnvironment _surfaceEnvironment;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<SurfaceManifestPublisher> _logger;
|
||||
|
||||
public SurfaceManifestPublisher(
|
||||
IArtifactObjectStore objectStore,
|
||||
ArtifactRepository artifactRepository,
|
||||
LinkRepository linkRepository,
|
||||
IOptions<ScannerStorageOptions> storageOptions,
|
||||
ISurfaceEnvironment surfaceEnvironment,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<SurfaceManifestPublisher> logger)
|
||||
{
|
||||
_objectStore = objectStore ?? throw new ArgumentNullException(nameof(objectStore));
|
||||
_artifactRepository = artifactRepository ?? throw new ArgumentNullException(nameof(artifactRepository));
|
||||
_linkRepository = linkRepository ?? throw new ArgumentNullException(nameof(linkRepository));
|
||||
_storageOptions = (storageOptions ?? throw new ArgumentNullException(nameof(storageOptions))).Value;
|
||||
_surfaceEnvironment = surfaceEnvironment ?? throw new ArgumentNullException(nameof(surfaceEnvironment));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<SurfaceManifestPublishResult> PublishAsync(SurfaceManifestRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
if (request.Payloads.Count == 0)
|
||||
{
|
||||
throw new ArgumentException("At least one payload must be provided.", nameof(request));
|
||||
}
|
||||
|
||||
var tenant = _surfaceEnvironment.Settings.Tenant;
|
||||
var generatedAt = _timeProvider.GetUtcNow();
|
||||
var artifacts = new List<SurfaceManifestArtifact>(request.Payloads.Count);
|
||||
|
||||
foreach (var payload in request.Payloads)
|
||||
{
|
||||
var artifact = await StorePayloadAsync(payload, tenant, cancellationToken).ConfigureAwait(false);
|
||||
artifacts.Add(artifact);
|
||||
}
|
||||
|
||||
var manifestDocument = new SurfaceManifestDocument
|
||||
{
|
||||
Tenant = tenant,
|
||||
ImageDigest = NormalizeDigest(request.ImageDigest),
|
||||
ScanId = request.ScanId,
|
||||
GeneratedAt = generatedAt,
|
||||
Source = new SurfaceManifestSource
|
||||
{
|
||||
Component = request.Component,
|
||||
Version = request.Version,
|
||||
WorkerInstance = request.WorkerInstance,
|
||||
Attempt = request.Attempt
|
||||
},
|
||||
Artifacts = artifacts.ToImmutableArray()
|
||||
};
|
||||
|
||||
var manifestBytes = JsonSerializer.SerializeToUtf8Bytes(manifestDocument, SerializerOptions);
|
||||
var manifestDigest = ComputeDigest(manifestBytes);
|
||||
var manifestKey = ArtifactObjectKeyBuilder.Build(
|
||||
ArtifactDocumentType.SurfaceManifest,
|
||||
ArtifactDocumentFormat.SurfaceManifestJson,
|
||||
manifestDigest,
|
||||
_storageOptions.ObjectStore.RootPrefix);
|
||||
var manifestDescriptor = new ArtifactObjectDescriptor(
|
||||
_storageOptions.ObjectStore.BucketName,
|
||||
manifestKey,
|
||||
Immutable: true,
|
||||
RetainFor: _storageOptions.ObjectStore.ComplianceRetention);
|
||||
|
||||
await using (var stream = new MemoryStream(manifestBytes, writable: false))
|
||||
{
|
||||
await _objectStore.PutAsync(manifestDescriptor, stream, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (_storageOptions.DualWrite.Enabled && !string.IsNullOrWhiteSpace(_storageOptions.DualWrite.MirrorBucket))
|
||||
{
|
||||
await using var mirrorStream = new MemoryStream(manifestBytes, writable: false);
|
||||
var mirrorDescriptor = manifestDescriptor with { Bucket = _storageOptions.DualWrite.MirrorBucket! };
|
||||
await _objectStore.PutAsync(mirrorDescriptor, mirrorStream, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var nowUtc = generatedAt.UtcDateTime;
|
||||
var artifactId = CatalogIdFactory.CreateArtifactId(ArtifactDocumentType.SurfaceManifest, manifestDigest);
|
||||
var manifestDocumentRecord = new ArtifactDocument
|
||||
{
|
||||
Id = artifactId,
|
||||
Type = ArtifactDocumentType.SurfaceManifest,
|
||||
Format = ArtifactDocumentFormat.SurfaceManifestJson,
|
||||
MediaType = "application/vnd.stellaops.surface.manifest+json",
|
||||
BytesSha256 = manifestDigest,
|
||||
SizeBytes = manifestBytes.Length,
|
||||
Immutable = true,
|
||||
RefCount = 1,
|
||||
CreatedAtUtc = nowUtc,
|
||||
UpdatedAtUtc = nowUtc,
|
||||
TtlClass = "surface.manifest"
|
||||
};
|
||||
|
||||
await _artifactRepository.UpsertAsync(manifestDocumentRecord, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var link = new LinkDocument
|
||||
{
|
||||
Id = CatalogIdFactory.CreateLinkId(LinkSourceType.Image, manifestDocument.ImageDigest ?? request.ScanId, artifactId),
|
||||
FromType = LinkSourceType.Image,
|
||||
FromDigest = manifestDocument.ImageDigest ?? request.ScanId,
|
||||
ArtifactId = artifactId,
|
||||
CreatedAtUtc = nowUtc
|
||||
};
|
||||
|
||||
await _linkRepository.UpsertAsync(link, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var manifestUri = BuildCasUri(_storageOptions.ObjectStore.BucketName, manifestKey);
|
||||
_logger.LogInformation("Published surface manifest {Manifest} for image {ImageDigest}.", artifactId, manifestDocument.ImageDigest);
|
||||
|
||||
return new SurfaceManifestPublishResult(
|
||||
ManifestDigest: manifestDigest,
|
||||
ManifestUri: manifestUri,
|
||||
ArtifactId: artifactId,
|
||||
Document: manifestDocument);
|
||||
}
|
||||
|
||||
private async Task<SurfaceManifestArtifact> StorePayloadAsync(SurfaceManifestPayload payload, string tenant, CancellationToken cancellationToken)
|
||||
{
|
||||
var digest = ComputeDigest(payload.Content.Span);
|
||||
var key = ArtifactObjectKeyBuilder.Build(
|
||||
payload.ArtifactType,
|
||||
payload.ArtifactFormat,
|
||||
digest,
|
||||
_storageOptions.ObjectStore.RootPrefix);
|
||||
|
||||
await using (var stream = new MemoryStream(payload.Content.ToArray(), writable: false))
|
||||
{
|
||||
var descriptor = new ArtifactObjectDescriptor(
|
||||
_storageOptions.ObjectStore.BucketName,
|
||||
key,
|
||||
Immutable: true,
|
||||
RetainFor: _storageOptions.ObjectStore.ComplianceRetention);
|
||||
|
||||
await _objectStore.PutAsync(descriptor, stream, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (_storageOptions.DualWrite.Enabled && !string.IsNullOrWhiteSpace(_storageOptions.DualWrite.MirrorBucket))
|
||||
{
|
||||
await using var mirrorStream = new MemoryStream(payload.Content.ToArray(), writable: false);
|
||||
var mirrorDescriptor = descriptor with { Bucket = _storageOptions.DualWrite.MirrorBucket! };
|
||||
await _objectStore.PutAsync(mirrorDescriptor, mirrorStream, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
return new SurfaceManifestArtifact
|
||||
{
|
||||
Kind = payload.Kind,
|
||||
Uri = BuildCasUri(_storageOptions.ObjectStore.BucketName, key),
|
||||
Digest = digest,
|
||||
MediaType = payload.MediaType,
|
||||
Format = MapFormat(payload.ArtifactFormat),
|
||||
SizeBytes = payload.Content.Length,
|
||||
View = payload.View,
|
||||
Storage = new SurfaceManifestStorage
|
||||
{
|
||||
Bucket = _storageOptions.ObjectStore.BucketName,
|
||||
ObjectKey = key,
|
||||
SizeBytes = payload.Content.Length,
|
||||
ContentType = payload.MediaType
|
||||
},
|
||||
Metadata = payload.Metadata
|
||||
};
|
||||
}
|
||||
|
||||
private static string BuildCasUri(string bucket, string key)
|
||||
{
|
||||
var normalizedKey = string.IsNullOrWhiteSpace(key) ? string.Empty : key.Trim().TrimStart('/');
|
||||
return $"cas://{bucket}/{normalizedKey}";
|
||||
}
|
||||
|
||||
private static string MapFormat(ArtifactDocumentFormat format)
|
||||
=> format switch
|
||||
{
|
||||
ArtifactDocumentFormat.EntryTraceNdjson => "entrytrace.ndjson",
|
||||
ArtifactDocumentFormat.EntryTraceGraphJson => "entrytrace.graph",
|
||||
ArtifactDocumentFormat.ComponentFragmentJson => "layer.fragments",
|
||||
ArtifactDocumentFormat.SurfaceManifestJson => "surface.manifest",
|
||||
ArtifactDocumentFormat.CycloneDxJson => "cdx-json",
|
||||
ArtifactDocumentFormat.CycloneDxProtobuf => "cdx-protobuf",
|
||||
ArtifactDocumentFormat.SpdxJson => "spdx-json",
|
||||
ArtifactDocumentFormat.BomIndex => "bom-index",
|
||||
ArtifactDocumentFormat.DsseJson => "dsse-json",
|
||||
_ => format.ToString().ToLowerInvariant()
|
||||
};
|
||||
|
||||
private static string ComputeDigest(ReadOnlySpan<byte> content)
|
||||
{
|
||||
Span<byte> hash = stackalloc byte[32];
|
||||
SHA256.HashData(content, hash);
|
||||
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
|
||||
}
|
||||
|
||||
private static string ComputeDigest(byte[] content)
|
||||
=> ComputeDigest(content.AsSpan());
|
||||
|
||||
private static string NormalizeDigest(string digest)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var trimmed = digest.Trim();
|
||||
return trimmed.Contains(':', StringComparison.Ordinal)
|
||||
? trimmed
|
||||
: $"sha256:{trimmed}";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using System.Reflection;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Scanner.Core.Contracts;
|
||||
using StellaOps.Scanner.EntryTrace;
|
||||
using StellaOps.Scanner.EntryTrace.Serialization;
|
||||
using StellaOps.Scanner.Surface.FS;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Processing.Surface;
|
||||
|
||||
internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = false,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
|
||||
};
|
||||
|
||||
private readonly SurfaceManifestPublisher _publisher;
|
||||
private readonly ILogger<SurfaceManifestStageExecutor> _logger;
|
||||
private readonly string _componentVersion;
|
||||
|
||||
public SurfaceManifestStageExecutor(
|
||||
SurfaceManifestPublisher publisher,
|
||||
ILogger<SurfaceManifestStageExecutor> logger)
|
||||
{
|
||||
_publisher = publisher ?? throw new ArgumentNullException(nameof(publisher));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_componentVersion = Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "unknown";
|
||||
}
|
||||
|
||||
public string StageName => ScanStageNames.ComposeArtifacts;
|
||||
|
||||
public async ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
|
||||
var payloads = CollectPayloads(context);
|
||||
if (payloads.Count == 0)
|
||||
{
|
||||
_logger.LogDebug("No surface payloads available for job {JobId}; skipping manifest publish.", context.JobId);
|
||||
return;
|
||||
}
|
||||
|
||||
var request = new SurfaceManifestRequest(
|
||||
ScanId: context.ScanId,
|
||||
ImageDigest: ResolveImageDigest(context),
|
||||
Attempt: context.Lease.Attempt,
|
||||
Metadata: context.Lease.Metadata,
|
||||
Payloads: payloads,
|
||||
Component: "scanner.worker",
|
||||
Version: _componentVersion,
|
||||
WorkerInstance: Environment.MachineName);
|
||||
|
||||
var result = await _publisher.PublishAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
context.Analysis.Set(ScanAnalysisKeys.SurfaceManifest, result);
|
||||
_logger.LogInformation("Surface manifest stored for job {JobId} with digest {Digest}.", context.JobId, result.ManifestDigest);
|
||||
}
|
||||
|
||||
private List<SurfaceManifestPayload> CollectPayloads(ScanJobContext context)
|
||||
{
|
||||
var payloads = new List<SurfaceManifestPayload>();
|
||||
|
||||
if (context.Analysis.TryGet<EntryTraceGraph>(ScanAnalysisKeys.EntryTraceGraph, out var graph) && graph is not null)
|
||||
{
|
||||
var graphJson = EntryTraceGraphSerializer.Serialize(graph);
|
||||
payloads.Add(new SurfaceManifestPayload(
|
||||
ArtifactDocumentType.SurfaceEntryTrace,
|
||||
ArtifactDocumentFormat.EntryTraceGraphJson,
|
||||
Kind: "entrytrace.graph",
|
||||
MediaType: "application/json",
|
||||
Content: Encoding.UTF8.GetBytes(graphJson),
|
||||
Metadata: new Dictionary<string, string>
|
||||
{
|
||||
["artifact"] = "entrytrace.graph",
|
||||
["nodes"] = graph.Nodes.Length.ToString(CultureInfoInvariant),
|
||||
["edges"] = graph.Edges.Length.ToString(CultureInfoInvariant)
|
||||
}));
|
||||
}
|
||||
|
||||
if (context.Analysis.TryGet(ScanAnalysisKeys.EntryTraceNdjson, out ImmutableArray<string> ndjson) && !ndjson.IsDefaultOrEmpty)
|
||||
{
|
||||
var builder = new StringBuilder();
|
||||
for (var i = 0; i < ndjson.Length; i++)
|
||||
{
|
||||
builder.Append(ndjson[i]);
|
||||
if (!ndjson[i].EndsWith('\n'))
|
||||
{
|
||||
builder.Append('\n');
|
||||
}
|
||||
}
|
||||
|
||||
payloads.Add(new SurfaceManifestPayload(
|
||||
ArtifactDocumentType.SurfaceEntryTrace,
|
||||
ArtifactDocumentFormat.EntryTraceNdjson,
|
||||
Kind: "entrytrace.ndjson",
|
||||
MediaType: "application/x-ndjson",
|
||||
Content: Encoding.UTF8.GetBytes(builder.ToString())));
|
||||
}
|
||||
|
||||
var fragments = context.Analysis.GetLayerFragments();
|
||||
if (!fragments.IsDefaultOrEmpty && fragments.Length > 0)
|
||||
{
|
||||
var fragmentsJson = JsonSerializer.Serialize(fragments, JsonOptions);
|
||||
payloads.Add(new SurfaceManifestPayload(
|
||||
ArtifactDocumentType.SurfaceLayerFragment,
|
||||
ArtifactDocumentFormat.ComponentFragmentJson,
|
||||
Kind: "layer.fragments",
|
||||
MediaType: "application/json",
|
||||
Content: Encoding.UTF8.GetBytes(fragmentsJson),
|
||||
View: "inventory"));
|
||||
}
|
||||
|
||||
return payloads;
|
||||
}
|
||||
|
||||
private static string ResolveImageDigest(ScanJobContext context)
|
||||
{
|
||||
static bool TryGet(IReadOnlyDictionary<string, string> metadata, string key, out string value)
|
||||
{
|
||||
if (metadata.TryGetValue(key, out var found) && !string.IsNullOrWhiteSpace(found))
|
||||
{
|
||||
value = found.Trim();
|
||||
return true;
|
||||
}
|
||||
|
||||
value = string.Empty;
|
||||
return false;
|
||||
}
|
||||
|
||||
var metadata = context.Lease.Metadata;
|
||||
if (TryGet(metadata, "image.digest", out var digest) ||
|
||||
TryGet(metadata, "imageDigest", out digest) ||
|
||||
TryGet(metadata, "scanner.image.digest", out digest))
|
||||
{
|
||||
return digest;
|
||||
}
|
||||
|
||||
return context.ScanId;
|
||||
}
|
||||
|
||||
private static readonly IFormatProvider CultureInfoInvariant = System.Globalization.CultureInfo.InvariantCulture;
|
||||
}
|
||||
@@ -18,7 +18,9 @@ using StellaOps.Scanner.Worker.Diagnostics;
|
||||
using StellaOps.Scanner.Worker.Hosting;
|
||||
using StellaOps.Scanner.Worker.Options;
|
||||
using StellaOps.Scanner.Worker.Processing;
|
||||
using StellaOps.Scanner.Worker.Processing.Surface;
|
||||
using StellaOps.Scanner.Storage.Extensions;
|
||||
using StellaOps.Scanner.Storage;
|
||||
|
||||
var builder = Host.CreateApplicationBuilder(args);
|
||||
|
||||
@@ -52,6 +54,9 @@ var connectionString = storageSection.GetValue<string>("Mongo:ConnectionString")
|
||||
if (!string.IsNullOrWhiteSpace(connectionString))
|
||||
{
|
||||
builder.Services.AddScannerStorage(storageSection);
|
||||
builder.Services.AddSingleton<IConfigureOptions<ScannerStorageOptions>, ScannerStorageSurfaceSecretConfigurator>();
|
||||
builder.Services.AddSingleton<SurfaceManifestPublisher>();
|
||||
builder.Services.AddSingleton<IScanStageExecutor, SurfaceManifestStageExecutor>();
|
||||
}
|
||||
|
||||
builder.Services.TryAddSingleton<IScanJobSource, NullScanJobSource>();
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| SCAN-REPLAY-186-002 | TODO | Scanner Worker Guild | REPLAY-CORE-185-001 | Enforce deterministic analyzer execution when consuming replay input bundles, emit layer Merkle metadata, and author `docs/modules/scanner/deterministic-execution.md` summarising invariants from `docs/replay/DETERMINISTIC_REPLAY.md` Section 4. | Replay mode analyzers pass determinism tests; new doc merged; integration fixtures updated. |
|
||||
| SCANNER-SURFACE-01 | DOING (2025-11-02) | Scanner Worker Guild | SURFACE-FS-02 | Persist Surface.FS manifests after analyzer stages, including layer CAS metadata and EntryTrace fragments.<br>2025-11-02: Draft Surface.FS manifests emitted for sample scans; telemetry counters under review. | Integration tests prove cache entries exist; telemetry counters exported. |
|
||||
| SCANNER-SURFACE-01 | DOING (2025-11-06) | Scanner Worker Guild | SURFACE-FS-02 | Persist Surface.FS manifests after analyzer stages, including layer CAS metadata and EntryTrace fragments.<br>2025-11-02: Draft Surface.FS manifests emitted for sample scans; telemetry counters under review.<br>2025-11-06: Resuming with manifest writer abstraction, rotation metadata, and telemetry counters for Surface.FS persistence. | Integration tests prove cache entries exist; telemetry counters exported. |
|
||||
| SCANNER-ENV-01 | TODO (2025-11-06) | Scanner Worker Guild | SURFACE-ENV-02 | Replace ad-hoc environment reads with `StellaOps.Scanner.Surface.Env` helpers for cache roots and CAS endpoints.<br>2025-11-02: Worker bootstrap now resolves cache roots via helper; warning path documented; smoke tests running.<br>2025-11-05 14:55Z: Extending helper usage into cache/secrets configuration, updating worker validator wiring, and drafting docs/tests for new Surface.Env outputs.<br>2025-11-06 17:05Z: README/design docs updated with warning catalogue; startup logging guidance captured for ops runbooks.<br>2025-11-06 07:45Z: Helm/Compose env profiles (dev/stage/prod/airgap/mirror) now seed `SCANNER_SURFACE_*` defaults to keep worker cache roots aligned with Surface.Env helpers.<br>2025-11-06 07:55Z: Paused; pending automation tracked via `DEVOPS-OPENSSL-11-001/002` and Surface.Env test fixtures. | Worker boots with helper; misconfiguration warnings documented; smoke tests updated. |
|
||||
> 2025-11-05 19:18Z: Bound `SurfaceCacheOptions` root directory to resolved Surface.Env settings and added unit coverage around the configurator.
|
||||
| SCANNER-SECRETS-01 | DOING (2025-11-02) | Scanner Worker Guild, Security Guild | SURFACE-SECRETS-02 | Adopt `StellaOps.Scanner.Surface.Secrets` for registry/CAS credentials during scan execution.<br>2025-11-02: Surface.Secrets provider wired for CAS token retrieval; integration tests added. | Secrets fetched via shared provider; legacy secret code removed; integration tests cover rotation. |
|
||||
| SCANNER-SECRETS-01 | DOING (2025-11-06) | Scanner Worker Guild, Security Guild | SURFACE-SECRETS-02 | Adopt `StellaOps.Scanner.Surface.Secrets` for registry/CAS credentials during scan execution.<br>2025-11-02: Surface.Secrets provider wired for CAS token retrieval; integration tests added.<br>2025-11-06: Continuing to replace legacy registry credential plumbing and extend rotation metrics/fixtures.<br>2025-11-06 21:35Z: Introduced `ScannerStorageSurfaceSecretConfigurator` mapping `cas-access` secrets into storage options plus unit coverage. | Secrets fetched via shared provider; legacy secret code removed; integration tests cover rotation. |
|
||||
|
||||
Reference in New Issue
Block a user