feat: Implement ScannerSurfaceSecretConfigurator for web service options
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
- Added ScannerSurfaceSecretConfigurator to configure ScannerWebServiceOptions using surface secrets. - Integrated ISurfaceSecretProvider to fetch and apply secrets for artifact store configuration. - Enhanced logging for secret retrieval and application processes. feat: Implement ScannerStorageSurfaceSecretConfigurator for worker options - Introduced ScannerStorageSurfaceSecretConfigurator to configure ScannerStorageOptions with surface secrets. - Utilized ISurfaceSecretProvider to retrieve and apply secrets for object store settings. - Improved logging for secret handling and configuration. feat: Create SurfaceManifestPublisher for publishing surface manifests - Developed SurfaceManifestPublisher to handle the creation and storage of surface manifests. - Implemented methods for serializing manifest documents and storing payloads in the object store. - Added dual write functionality for mirror storage of manifests. feat: Add SurfaceManifestStageExecutor for processing scan stages - Created SurfaceManifestStageExecutor to execute the manifest publishing stage in scan jobs. - Integrated with SurfaceManifestPublisher to publish manifests based on collected payloads. - Enhanced logging for job processing and manifest storage. feat: Define SurfaceManifest models for manifest structure - Established SurfaceManifestDocument, SurfaceManifestSource, SurfaceManifestArtifact, and SurfaceManifestStorage records. - Implemented serialization attributes for JSON handling of manifest models. feat: Implement CasAccessSecret and SurfaceSecretParser for secret handling - Created CasAccessSecret record to represent surface access secrets. - Developed SurfaceSecretParser to parse and validate surface secrets from JSON payloads. test: Add unit tests for CasAccessSecretParser - Implemented tests for parsing CasAccessSecret from JSON payloads and metadata fallbacks. - Verified expected values and behavior for secret parsing logic. test: Add unit tests for ScannerSurfaceSecretConfigurator - Created tests for ScannerSurfaceSecretConfigurator to ensure correct application of surface secrets to web service options. - Validated artifact store settings after configuration. test: Add unit tests for ScannerStorageSurfaceSecretConfigurator - Implemented tests for ScannerStorageSurfaceSecretConfigurator to verify correct application of surface secrets to storage options. - Ensured accurate configuration of object store settings.
This commit is contained in:
master
@@ -21,6 +21,7 @@ using StellaOps.Scanner.Surface.Env;
|
||||
using StellaOps.Scanner.Surface.FS;
|
||||
using StellaOps.Scanner.Surface.Validation;
|
||||
using StellaOps.Scanner.Worker.Options;
|
||||
using StellaOps.Scanner.Worker.Diagnostics;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Processing;
|
||||
|
||||
@@ -206,7 +207,7 @@ internal sealed class CompositeScanAnalyzerDispatcher : IScanAnalyzerDispatcher
|
||||
try
|
||||
{
|
||||
var engine = new LanguageAnalyzerEngine(new[] { analyzer });
|
||||
var cacheEntry = await cacheAdapter.GetOrCreateAsync(
|
||||
var cacheEntry = await cacheAdapter.GetOrCreateEntryAsync(
|
||||
_logger,
|
||||
analyzer.Id,
|
||||
workspaceFingerprint,
|
||||
|
||||
@@ -0,0 +1,264 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.IO;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Core.Contracts;
|
||||
using StellaOps.Scanner.Surface.FS;
|
||||
using StellaOps.Scanner.Storage;
|
||||
using StellaOps.Scanner.Storage.Catalog;
|
||||
using StellaOps.Scanner.Storage.ObjectStore;
|
||||
using StellaOps.Scanner.Storage.Repositories;
|
||||
using StellaOps.Scanner.Storage.Services;
|
||||
using StellaOps.Scanner.Surface.Env;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Processing.Surface;
|
||||
|
||||
internal sealed record SurfaceManifestPayload(
|
||||
ArtifactDocumentType ArtifactType,
|
||||
ArtifactDocumentFormat ArtifactFormat,
|
||||
string Kind,
|
||||
string MediaType,
|
||||
ReadOnlyMemory<byte> Content,
|
||||
string? View = null,
|
||||
IReadOnlyDictionary<string, string>? Metadata = null,
|
||||
bool RegisterArtifact = false);
|
||||
|
||||
internal sealed record SurfaceManifestRequest(
|
||||
string ScanId,
|
||||
string ImageDigest,
|
||||
int Attempt,
|
||||
IReadOnlyDictionary<string, string> Metadata,
|
||||
IReadOnlyList<SurfaceManifestPayload> Payloads,
|
||||
string Component,
|
||||
string? Version,
|
||||
string? WorkerInstance);
|
||||
|
||||
internal sealed class SurfaceManifestPublisher
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = false,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
|
||||
};
|
||||
|
||||
private readonly IArtifactObjectStore _objectStore;
|
||||
private readonly ArtifactRepository _artifactRepository;
|
||||
private readonly LinkRepository _linkRepository;
|
||||
private readonly ScannerStorageOptions _storageOptions;
|
||||
private readonly ISurfaceEnvironment _surfaceEnvironment;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<SurfaceManifestPublisher> _logger;
|
||||
|
||||
public SurfaceManifestPublisher(
|
||||
IArtifactObjectStore objectStore,
|
||||
ArtifactRepository artifactRepository,
|
||||
LinkRepository linkRepository,
|
||||
IOptions<ScannerStorageOptions> storageOptions,
|
||||
ISurfaceEnvironment surfaceEnvironment,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<SurfaceManifestPublisher> logger)
|
||||
{
|
||||
_objectStore = objectStore ?? throw new ArgumentNullException(nameof(objectStore));
|
||||
_artifactRepository = artifactRepository ?? throw new ArgumentNullException(nameof(artifactRepository));
|
||||
_linkRepository = linkRepository ?? throw new ArgumentNullException(nameof(linkRepository));
|
||||
_storageOptions = (storageOptions ?? throw new ArgumentNullException(nameof(storageOptions))).Value;
|
||||
_surfaceEnvironment = surfaceEnvironment ?? throw new ArgumentNullException(nameof(surfaceEnvironment));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<SurfaceManifestPublishResult> PublishAsync(SurfaceManifestRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
if (request.Payloads.Count == 0)
|
||||
{
|
||||
throw new ArgumentException("At least one payload must be provided.", nameof(request));
|
||||
}
|
||||
|
||||
var tenant = _surfaceEnvironment.Settings.Tenant;
|
||||
var generatedAt = _timeProvider.GetUtcNow();
|
||||
var artifacts = new List<SurfaceManifestArtifact>(request.Payloads.Count);
|
||||
|
||||
foreach (var payload in request.Payloads)
|
||||
{
|
||||
var artifact = await StorePayloadAsync(payload, tenant, cancellationToken).ConfigureAwait(false);
|
||||
artifacts.Add(artifact);
|
||||
}
|
||||
|
||||
var manifestDocument = new SurfaceManifestDocument
|
||||
{
|
||||
Tenant = tenant,
|
||||
ImageDigest = NormalizeDigest(request.ImageDigest),
|
||||
ScanId = request.ScanId,
|
||||
GeneratedAt = generatedAt,
|
||||
Source = new SurfaceManifestSource
|
||||
{
|
||||
Component = request.Component,
|
||||
Version = request.Version,
|
||||
WorkerInstance = request.WorkerInstance,
|
||||
Attempt = request.Attempt
|
||||
},
|
||||
Artifacts = artifacts.ToImmutableArray()
|
||||
};
|
||||
|
||||
var manifestBytes = JsonSerializer.SerializeToUtf8Bytes(manifestDocument, SerializerOptions);
|
||||
var manifestDigest = ComputeDigest(manifestBytes);
|
||||
var manifestKey = ArtifactObjectKeyBuilder.Build(
|
||||
ArtifactDocumentType.SurfaceManifest,
|
||||
ArtifactDocumentFormat.SurfaceManifestJson,
|
||||
manifestDigest,
|
||||
_storageOptions.ObjectStore.RootPrefix);
|
||||
var manifestDescriptor = new ArtifactObjectDescriptor(
|
||||
_storageOptions.ObjectStore.BucketName,
|
||||
manifestKey,
|
||||
Immutable: true,
|
||||
RetainFor: _storageOptions.ObjectStore.ComplianceRetention);
|
||||
|
||||
await using (var stream = new MemoryStream(manifestBytes, writable: false))
|
||||
{
|
||||
await _objectStore.PutAsync(manifestDescriptor, stream, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (_storageOptions.DualWrite.Enabled && !string.IsNullOrWhiteSpace(_storageOptions.DualWrite.MirrorBucket))
|
||||
{
|
||||
await using var mirrorStream = new MemoryStream(manifestBytes, writable: false);
|
||||
var mirrorDescriptor = manifestDescriptor with { Bucket = _storageOptions.DualWrite.MirrorBucket! };
|
||||
await _objectStore.PutAsync(mirrorDescriptor, mirrorStream, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var nowUtc = generatedAt.UtcDateTime;
|
||||
var artifactId = CatalogIdFactory.CreateArtifactId(ArtifactDocumentType.SurfaceManifest, manifestDigest);
|
||||
var manifestDocumentRecord = new ArtifactDocument
|
||||
{
|
||||
Id = artifactId,
|
||||
Type = ArtifactDocumentType.SurfaceManifest,
|
||||
Format = ArtifactDocumentFormat.SurfaceManifestJson,
|
||||
MediaType = "application/vnd.stellaops.surface.manifest+json",
|
||||
BytesSha256 = manifestDigest,
|
||||
SizeBytes = manifestBytes.Length,
|
||||
Immutable = true,
|
||||
RefCount = 1,
|
||||
CreatedAtUtc = nowUtc,
|
||||
UpdatedAtUtc = nowUtc,
|
||||
TtlClass = "surface.manifest"
|
||||
};
|
||||
|
||||
await _artifactRepository.UpsertAsync(manifestDocumentRecord, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var link = new LinkDocument
|
||||
{
|
||||
Id = CatalogIdFactory.CreateLinkId(LinkSourceType.Image, manifestDocument.ImageDigest ?? request.ScanId, artifactId),
|
||||
FromType = LinkSourceType.Image,
|
||||
FromDigest = manifestDocument.ImageDigest ?? request.ScanId,
|
||||
ArtifactId = artifactId,
|
||||
CreatedAtUtc = nowUtc
|
||||
};
|
||||
|
||||
await _linkRepository.UpsertAsync(link, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var manifestUri = BuildCasUri(_storageOptions.ObjectStore.BucketName, manifestKey);
|
||||
_logger.LogInformation("Published surface manifest {Manifest} for image {ImageDigest}.", artifactId, manifestDocument.ImageDigest);
|
||||
|
||||
return new SurfaceManifestPublishResult(
|
||||
ManifestDigest: manifestDigest,
|
||||
ManifestUri: manifestUri,
|
||||
ArtifactId: artifactId,
|
||||
Document: manifestDocument);
|
||||
}
|
||||
|
||||
private async Task<SurfaceManifestArtifact> StorePayloadAsync(SurfaceManifestPayload payload, string tenant, CancellationToken cancellationToken)
|
||||
{
|
||||
var digest = ComputeDigest(payload.Content.Span);
|
||||
var key = ArtifactObjectKeyBuilder.Build(
|
||||
payload.ArtifactType,
|
||||
payload.ArtifactFormat,
|
||||
digest,
|
||||
_storageOptions.ObjectStore.RootPrefix);
|
||||
|
||||
await using (var stream = new MemoryStream(payload.Content.ToArray(), writable: false))
|
||||
{
|
||||
var descriptor = new ArtifactObjectDescriptor(
|
||||
_storageOptions.ObjectStore.BucketName,
|
||||
key,
|
||||
Immutable: true,
|
||||
RetainFor: _storageOptions.ObjectStore.ComplianceRetention);
|
||||
|
||||
await _objectStore.PutAsync(descriptor, stream, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (_storageOptions.DualWrite.Enabled && !string.IsNullOrWhiteSpace(_storageOptions.DualWrite.MirrorBucket))
|
||||
{
|
||||
await using var mirrorStream = new MemoryStream(payload.Content.ToArray(), writable: false);
|
||||
var mirrorDescriptor = descriptor with { Bucket = _storageOptions.DualWrite.MirrorBucket! };
|
||||
await _objectStore.PutAsync(mirrorDescriptor, mirrorStream, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
return new SurfaceManifestArtifact
|
||||
{
|
||||
Kind = payload.Kind,
|
||||
Uri = BuildCasUri(_storageOptions.ObjectStore.BucketName, key),
|
||||
Digest = digest,
|
||||
MediaType = payload.MediaType,
|
||||
Format = MapFormat(payload.ArtifactFormat),
|
||||
SizeBytes = payload.Content.Length,
|
||||
View = payload.View,
|
||||
Storage = new SurfaceManifestStorage
|
||||
{
|
||||
Bucket = _storageOptions.ObjectStore.BucketName,
|
||||
ObjectKey = key,
|
||||
SizeBytes = payload.Content.Length,
|
||||
ContentType = payload.MediaType
|
||||
},
|
||||
Metadata = payload.Metadata
|
||||
};
|
||||
}
|
||||
|
||||
private static string BuildCasUri(string bucket, string key)
|
||||
{
|
||||
var normalizedKey = string.IsNullOrWhiteSpace(key) ? string.Empty : key.Trim().TrimStart('/');
|
||||
return $"cas://{bucket}/{normalizedKey}";
|
||||
}
|
||||
|
||||
private static string MapFormat(ArtifactDocumentFormat format)
|
||||
=> format switch
|
||||
{
|
||||
ArtifactDocumentFormat.EntryTraceNdjson => "entrytrace.ndjson",
|
||||
ArtifactDocumentFormat.EntryTraceGraphJson => "entrytrace.graph",
|
||||
ArtifactDocumentFormat.ComponentFragmentJson => "layer.fragments",
|
||||
ArtifactDocumentFormat.SurfaceManifestJson => "surface.manifest",
|
||||
ArtifactDocumentFormat.CycloneDxJson => "cdx-json",
|
||||
ArtifactDocumentFormat.CycloneDxProtobuf => "cdx-protobuf",
|
||||
ArtifactDocumentFormat.SpdxJson => "spdx-json",
|
||||
ArtifactDocumentFormat.BomIndex => "bom-index",
|
||||
ArtifactDocumentFormat.DsseJson => "dsse-json",
|
||||
_ => format.ToString().ToLowerInvariant()
|
||||
};
|
||||
|
||||
private static string ComputeDigest(ReadOnlySpan<byte> content)
|
||||
{
|
||||
Span<byte> hash = stackalloc byte[32];
|
||||
SHA256.HashData(content, hash);
|
||||
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
|
||||
}
|
||||
|
||||
private static string ComputeDigest(byte[] content)
|
||||
=> ComputeDigest(content.AsSpan());
|
||||
|
||||
private static string NormalizeDigest(string digest)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var trimmed = digest.Trim();
|
||||
return trimmed.Contains(':', StringComparison.Ordinal)
|
||||
? trimmed
|
||||
: $"sha256:{trimmed}";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using System.Reflection;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Scanner.Core.Contracts;
|
||||
using StellaOps.Scanner.EntryTrace;
|
||||
using StellaOps.Scanner.EntryTrace.Serialization;
|
||||
using StellaOps.Scanner.Surface.FS;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Processing.Surface;
|
||||
|
||||
internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = false,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
|
||||
};
|
||||
|
||||
private readonly SurfaceManifestPublisher _publisher;
|
||||
private readonly ILogger<SurfaceManifestStageExecutor> _logger;
|
||||
private readonly string _componentVersion;
|
||||
|
||||
public SurfaceManifestStageExecutor(
|
||||
SurfaceManifestPublisher publisher,
|
||||
ILogger<SurfaceManifestStageExecutor> logger)
|
||||
{
|
||||
_publisher = publisher ?? throw new ArgumentNullException(nameof(publisher));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_componentVersion = Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "unknown";
|
||||
}
|
||||
|
||||
public string StageName => ScanStageNames.ComposeArtifacts;
|
||||
|
||||
public async ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
|
||||
var payloads = CollectPayloads(context);
|
||||
if (payloads.Count == 0)
|
||||
{
|
||||
_logger.LogDebug("No surface payloads available for job {JobId}; skipping manifest publish.", context.JobId);
|
||||
return;
|
||||
}
|
||||
|
||||
var request = new SurfaceManifestRequest(
|
||||
ScanId: context.ScanId,
|
||||
ImageDigest: ResolveImageDigest(context),
|
||||
Attempt: context.Lease.Attempt,
|
||||
Metadata: context.Lease.Metadata,
|
||||
Payloads: payloads,
|
||||
Component: "scanner.worker",
|
||||
Version: _componentVersion,
|
||||
WorkerInstance: Environment.MachineName);
|
||||
|
||||
var result = await _publisher.PublishAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
context.Analysis.Set(ScanAnalysisKeys.SurfaceManifest, result);
|
||||
_logger.LogInformation("Surface manifest stored for job {JobId} with digest {Digest}.", context.JobId, result.ManifestDigest);
|
||||
}
|
||||
|
||||
private List<SurfaceManifestPayload> CollectPayloads(ScanJobContext context)
|
||||
{
|
||||
var payloads = new List<SurfaceManifestPayload>();
|
||||
|
||||
if (context.Analysis.TryGet<EntryTraceGraph>(ScanAnalysisKeys.EntryTraceGraph, out var graph) && graph is not null)
|
||||
{
|
||||
var graphJson = EntryTraceGraphSerializer.Serialize(graph);
|
||||
payloads.Add(new SurfaceManifestPayload(
|
||||
ArtifactDocumentType.SurfaceEntryTrace,
|
||||
ArtifactDocumentFormat.EntryTraceGraphJson,
|
||||
Kind: "entrytrace.graph",
|
||||
MediaType: "application/json",
|
||||
Content: Encoding.UTF8.GetBytes(graphJson),
|
||||
Metadata: new Dictionary<string, string>
|
||||
{
|
||||
["artifact"] = "entrytrace.graph",
|
||||
["nodes"] = graph.Nodes.Length.ToString(CultureInfoInvariant),
|
||||
["edges"] = graph.Edges.Length.ToString(CultureInfoInvariant)
|
||||
}));
|
||||
}
|
||||
|
||||
if (context.Analysis.TryGet(ScanAnalysisKeys.EntryTraceNdjson, out ImmutableArray<string> ndjson) && !ndjson.IsDefaultOrEmpty)
|
||||
{
|
||||
var builder = new StringBuilder();
|
||||
for (var i = 0; i < ndjson.Length; i++)
|
||||
{
|
||||
builder.Append(ndjson[i]);
|
||||
if (!ndjson[i].EndsWith('\n'))
|
||||
{
|
||||
builder.Append('\n');
|
||||
}
|
||||
}
|
||||
|
||||
payloads.Add(new SurfaceManifestPayload(
|
||||
ArtifactDocumentType.SurfaceEntryTrace,
|
||||
ArtifactDocumentFormat.EntryTraceNdjson,
|
||||
Kind: "entrytrace.ndjson",
|
||||
MediaType: "application/x-ndjson",
|
||||
Content: Encoding.UTF8.GetBytes(builder.ToString())));
|
||||
}
|
||||
|
||||
var fragments = context.Analysis.GetLayerFragments();
|
||||
if (!fragments.IsDefaultOrEmpty && fragments.Length > 0)
|
||||
{
|
||||
var fragmentsJson = JsonSerializer.Serialize(fragments, JsonOptions);
|
||||
payloads.Add(new SurfaceManifestPayload(
|
||||
ArtifactDocumentType.SurfaceLayerFragment,
|
||||
ArtifactDocumentFormat.ComponentFragmentJson,
|
||||
Kind: "layer.fragments",
|
||||
MediaType: "application/json",
|
||||
Content: Encoding.UTF8.GetBytes(fragmentsJson),
|
||||
View: "inventory"));
|
||||
}
|
||||
|
||||
return payloads;
|
||||
}
|
||||
|
||||
private static string ResolveImageDigest(ScanJobContext context)
|
||||
{
|
||||
static bool TryGet(IReadOnlyDictionary<string, string> metadata, string key, out string value)
|
||||
{
|
||||
if (metadata.TryGetValue(key, out var found) && !string.IsNullOrWhiteSpace(found))
|
||||
{
|
||||
value = found.Trim();
|
||||
return true;
|
||||
}
|
||||
|
||||
value = string.Empty;
|
||||
return false;
|
||||
}
|
||||
|
||||
var metadata = context.Lease.Metadata;
|
||||
if (TryGet(metadata, "image.digest", out var digest) ||
|
||||
TryGet(metadata, "imageDigest", out digest) ||
|
||||
TryGet(metadata, "scanner.image.digest", out digest))
|
||||
{
|
||||
return digest;
|
||||
}
|
||||
|
||||
return context.ScanId;
|
||||
}
|
||||
|
||||
private static readonly IFormatProvider CultureInfoInvariant = System.Globalization.CultureInfo.InvariantCulture;
|
||||
}
|
||||
Reference in New Issue
Block a user