Implement MongoDB-based storage for Pack Run approval, artifact, log, and state management
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Added MongoPackRunApprovalStore for managing approval states with MongoDB.
- Introduced MongoPackRunArtifactUploader for uploading and storing artifacts.
- Created MongoPackRunLogStore to handle logging of pack run events.
- Developed MongoPackRunStateStore for persisting and retrieving pack run states.
- Implemented unit tests for MongoDB stores to ensure correct functionality.
- Added MongoTaskRunnerTestContext for setting up MongoDB test environment.
- Enhanced PackRunStateFactory to correctly initialize state with gate reasons.
This commit is contained in:
master
2025-11-07 10:01:35 +02:00
parent e5ffcd6535
commit a1ce3f74fa
122 changed files with 8730 additions and 914 deletions

View File

@@ -12,6 +12,7 @@ using StellaOps.Scanner.Sbomer.BuildXPlugin.Attestation;
using StellaOps.Scanner.Sbomer.BuildXPlugin.Cas;
using StellaOps.Scanner.Sbomer.BuildXPlugin.Descriptor;
using StellaOps.Scanner.Sbomer.BuildXPlugin.Manifest;
using StellaOps.Scanner.Sbomer.BuildXPlugin.Surface;
namespace StellaOps.Scanner.Sbomer.BuildXPlugin;
@@ -131,6 +132,12 @@ internal static class Program
Console.WriteLine(" --attestor <url> (descriptor) Optional Attestor endpoint for provenance placeholders.");
Console.WriteLine(" --attestor-token <token> Bearer token for Attestor requests (or STELLAOPS_ATTESTOR_TOKEN).");
Console.WriteLine(" --attestor-insecure Skip TLS verification for Attestor requests (dev/test only).");
Console.WriteLine(" --surface-layer-fragments <path> Persist layer fragments JSON into Surface.FS.");
Console.WriteLine(" --surface-entrytrace-graph <path> Persist EntryTrace graph JSON into Surface.FS.");
Console.WriteLine(" --surface-entrytrace-ndjson <path> Persist EntryTrace NDJSON into Surface.FS.");
Console.WriteLine(" --surface-cache-root <path> Override Surface cache root (defaults to CAS root).");
Console.WriteLine(" --surface-bucket <name> Bucket name used in Surface CAS URIs (default scanner-artifacts).");
Console.WriteLine(" --surface-tenant <tenant> Tenant identifier recorded in the Surface manifest.");
return 0;
}
@@ -186,6 +193,11 @@ internal static class Program
private static async Task<int> RunDescriptorAsync(string[] args, CancellationToken cancellationToken)
{
var manifestDirectory = ResolveManifestDirectory(args);
var loader = new BuildxPluginManifestLoader(manifestDirectory);
var manifest = await loader.LoadDefaultAsync(cancellationToken).ConfigureAwait(false);
var casRoot = ResolveCasRoot(args, manifest);
var imageDigest = RequireOption(args, "--image");
var sbomPath = RequireOption(args, "--sbom");
@@ -244,11 +256,110 @@ internal static class Program
await attestorClient.SendPlaceholderAsync(attestorUri, document, cancellationToken).ConfigureAwait(false);
}
await TryPublishSurfaceArtifactsAsync(args, request, casRoot, version, cancellationToken).ConfigureAwait(false);
var json = JsonSerializer.Serialize(document, DescriptorJsonOptions);
Console.WriteLine(json);
return 0;
}
private static async Task TryPublishSurfaceArtifactsAsync(
string[] args,
DescriptorRequest descriptorRequest,
string casRoot,
string generatorVersion,
CancellationToken cancellationToken)
{
var surfaceOptions = ResolveSurfaceOptions(args, descriptorRequest, casRoot, generatorVersion);
if (surfaceOptions is null || !surfaceOptions.HasArtifacts)
{
return;
}
var writer = new SurfaceManifestWriter(TimeProvider.System);
var result = await writer.WriteAsync(surfaceOptions, cancellationToken).ConfigureAwait(false);
if (result is null)
{
return;
}
Console.Error.WriteLine($"surface manifest stored: {result.ManifestUri} ({result.Document.Artifacts.Count} artefacts)");
}
private static SurfaceOptions? ResolveSurfaceOptions(
string[] args,
DescriptorRequest descriptorRequest,
string casRoot,
string generatorVersion)
{
var layerFragmentsPath = GetOption(args, "--surface-layer-fragments")
?? Environment.GetEnvironmentVariable("STELLAOPS_SURFACE_LAYER_FRAGMENTS");
var entryTraceGraphPath = GetOption(args, "--surface-entrytrace-graph")
?? Environment.GetEnvironmentVariable("STELLAOPS_SURFACE_ENTRYTRACE_GRAPH");
var entryTraceNdjsonPath = GetOption(args, "--surface-entrytrace-ndjson")
?? Environment.GetEnvironmentVariable("STELLAOPS_SURFACE_ENTRYTRACE_NDJSON");
if (string.IsNullOrWhiteSpace(layerFragmentsPath) &&
string.IsNullOrWhiteSpace(entryTraceGraphPath) &&
string.IsNullOrWhiteSpace(entryTraceNdjsonPath))
{
return null;
}
var cacheRoot = GetOption(args, "--surface-cache-root")
?? Environment.GetEnvironmentVariable("STELLAOPS_SURFACE_CACHE_ROOT")
?? casRoot;
var bucket = GetOption(args, "--surface-bucket")
?? Environment.GetEnvironmentVariable("STELLAOPS_SURFACE_BUCKET")
?? SurfaceCasLayout.DefaultBucket;
var rootPrefix = GetOption(args, "--surface-root-prefix")
?? Environment.GetEnvironmentVariable("STELLAOPS_SURFACE_ROOT_PREFIX")
?? SurfaceCasLayout.DefaultRootPrefix;
var tenant = GetOption(args, "--surface-tenant")
?? Environment.GetEnvironmentVariable("STELLAOPS_SURFACE_TENANT")
?? "default";
var component = GetOption(args, "--surface-component")
?? Environment.GetEnvironmentVariable("STELLAOPS_SURFACE_COMPONENT")
?? "scanner.buildx";
var componentVersion = GetOption(args, "--surface-component-version")
?? Environment.GetEnvironmentVariable("STELLAOPS_SURFACE_COMPONENT_VERSION")
?? generatorVersion;
var workerInstance = GetOption(args, "--surface-worker-instance")
?? Environment.GetEnvironmentVariable("STELLAOPS_SURFACE_WORKER_INSTANCE")
?? Environment.MachineName;
var attemptValue = GetOption(args, "--surface-attempt")
?? Environment.GetEnvironmentVariable("STELLAOPS_SURFACE_ATTEMPT");
var attempt = 1;
if (!string.IsNullOrWhiteSpace(attemptValue) && int.TryParse(attemptValue, out var parsedAttempt) && parsedAttempt > 0)
{
attempt = parsedAttempt;
}
var scanId = GetOption(args, "--surface-scan-id")
?? Environment.GetEnvironmentVariable("STELLAOPS_SURFACE_SCAN_ID")
?? descriptorRequest.SbomName
?? descriptorRequest.ImageDigest;
var manifestOutput = GetOption(args, "--surface-manifest-output")
?? Environment.GetEnvironmentVariable("STELLAOPS_SURFACE_MANIFEST_OUTPUT");
return new SurfaceOptions(
CacheRoot: cacheRoot,
CacheBucket: bucket,
RootPrefix: rootPrefix,
Tenant: tenant,
Component: component,
ComponentVersion: componentVersion,
WorkerInstance: workerInstance,
Attempt: attempt,
ImageDigest: descriptorRequest.ImageDigest,
ScanId: scanId,
LayerFragmentsPath: layerFragmentsPath,
EntryTraceGraphPath: entryTraceGraphPath,
EntryTraceNdjsonPath: entryTraceNdjsonPath,
ManifestOutputPath: manifestOutput);
}
private static string? GetOption(string[] args, string optionName)
{
for (var i = 0; i < args.Length; i++)

View File

@@ -0,0 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Scanner.Sbomer.BuildXPlugin.Tests")]

View File

@@ -12,9 +12,13 @@
<InformationalVersion>0.1.0-alpha</InformationalVersion>
</PropertyGroup>
<ItemGroup>
<Content Include="stellaops.sbom-indexer.manifest.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
</Project>
<ItemGroup>
<Content Include="stellaops.sbom-indexer.manifest.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\\__Libraries\\StellaOps.Scanner.Surface.FS\\StellaOps.Scanner.Surface.FS.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,112 @@
using System;
using System.IO;
using System.Security.Cryptography;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Surface;
internal static class SurfaceCasLayout
{
internal const string DefaultBucket = "scanner-artifacts";
internal const string DefaultRootPrefix = "scanner";
private const string Sha256 = "sha256";
public static string NormalizeDigest(string? digest)
{
if (string.IsNullOrWhiteSpace(digest))
{
throw new BuildxPluginException("Surface artefact digest cannot be empty.");
}
var trimmed = digest.Trim();
return trimmed.Contains(':', StringComparison.Ordinal)
? trimmed
: $"{Sha256}:{trimmed}";
}
public static string ExtractDigestValue(string normalizedDigest)
{
var parts = normalizedDigest.Split(':', 2, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);
return parts.Length == 2 ? parts[1] : normalizedDigest;
}
public static string BuildObjectKey(string rootPrefix, SurfaceCasKind kind, string normalizedDigest)
{
var digestValue = ExtractDigestValue(normalizedDigest);
var prefix = kind switch
{
SurfaceCasKind.LayerFragments => "surface/payloads/layer-fragments",
SurfaceCasKind.EntryTraceGraph => "surface/payloads/entrytrace",
SurfaceCasKind.EntryTraceNdjson => "surface/payloads/entrytrace",
SurfaceCasKind.Manifest => "surface/manifests",
_ => "surface/unknown"
};
var extension = kind switch
{
SurfaceCasKind.LayerFragments => "layer-fragments.json",
SurfaceCasKind.EntryTraceGraph => "entrytrace.graph.json",
SurfaceCasKind.EntryTraceNdjson => "entrytrace.ndjson",
SurfaceCasKind.Manifest => "surface.manifest.json",
_ => "artifact.bin"
};
var normalizedRoot = string.IsNullOrWhiteSpace(rootPrefix)
? string.Empty
: rootPrefix.Trim().Trim('/');
var relative = $"{prefix}/{digestValue}/{extension}";
return string.IsNullOrWhiteSpace(normalizedRoot) ? relative : $"{normalizedRoot}/{relative}";
}
public static string BuildCasUri(string bucket, string objectKey)
{
var normalizedBucket = string.IsNullOrWhiteSpace(bucket) ? DefaultBucket : bucket.Trim();
var normalizedKey = string.IsNullOrWhiteSpace(objectKey) ? string.Empty : objectKey.Trim().TrimStart('/');
return $"cas://{normalizedBucket}/{normalizedKey}";
}
public static string ComputeDigest(ReadOnlySpan<byte> content)
{
Span<byte> hash = stackalloc byte[32];
SHA256.HashData(content, hash);
return $"{Sha256}:{Convert.ToHexString(hash).ToLowerInvariant()}";
}
public static async Task<string> WriteBytesAsync(string rootDirectory, string objectKey, byte[] bytes, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(rootDirectory))
{
throw new BuildxPluginException("Surface cache root must be provided.");
}
var normalizedRoot = Path.GetFullPath(rootDirectory);
var relativePath = objectKey.Replace('/', Path.DirectorySeparatorChar);
var fullPath = Path.Combine(normalizedRoot, relativePath);
var directory = Path.GetDirectoryName(fullPath);
if (!string.IsNullOrWhiteSpace(directory))
{
Directory.CreateDirectory(directory);
}
await using var stream = new FileStream(
fullPath,
FileMode.Create,
FileAccess.Write,
FileShare.Read,
bufferSize: 64 * 1024,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await stream.WriteAsync(bytes.AsMemory(0, bytes.Length), cancellationToken).ConfigureAwait(false);
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
return fullPath;
}
}
internal enum SurfaceCasKind
{
LayerFragments,
EntryTraceGraph,
EntryTraceNdjson,
Manifest
}

View File

@@ -0,0 +1,227 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Scanner.Surface.FS;
namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Surface;
internal sealed class SurfaceManifestWriter
{
private static readonly JsonSerializerOptions ManifestSerializerOptions = new(JsonSerializerDefaults.Web)
{
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
WriteIndented = false
};
private readonly TimeProvider _timeProvider;
public SurfaceManifestWriter(TimeProvider timeProvider)
{
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
}
public async Task<SurfaceManifestWriteResult?> WriteAsync(SurfaceOptions options, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(options);
if (!options.HasArtifacts)
{
return null;
}
var cacheRoot = EnsurePath(options.CacheRoot, "Surface cache root must be provided.");
var bucket = string.IsNullOrWhiteSpace(options.CacheBucket)
? SurfaceCasLayout.DefaultBucket
: options.CacheBucket.Trim();
var rootPrefix = string.IsNullOrWhiteSpace(options.RootPrefix)
? SurfaceCasLayout.DefaultRootPrefix
: options.RootPrefix.Trim();
var tenant = string.IsNullOrWhiteSpace(options.Tenant)
? "default"
: options.Tenant.Trim();
var component = string.IsNullOrWhiteSpace(options.Component)
? "scanner.buildx"
: options.Component.Trim();
var componentVersion = string.IsNullOrWhiteSpace(options.ComponentVersion)
? null
: options.ComponentVersion.Trim();
var workerInstance = string.IsNullOrWhiteSpace(options.WorkerInstance)
? Environment.MachineName
: options.WorkerInstance.Trim();
var attempt = options.Attempt <= 0 ? 1 : options.Attempt;
var scanId = string.IsNullOrWhiteSpace(options.ScanId)
? options.ImageDigest
: options.ScanId!.Trim();
Directory.CreateDirectory(cacheRoot);
var artifacts = new List<SurfaceArtifactWriteResult>();
if (!string.IsNullOrWhiteSpace(options.EntryTraceGraphPath))
{
var descriptor = new SurfaceArtifactDescriptor(
Kind: "entrytrace.graph",
Format: "entrytrace.graph",
MediaType: "application/json",
View: null,
CasKind: SurfaceCasKind.EntryTraceGraph,
FilePath: EnsurePath(options.EntryTraceGraphPath!, "EntryTrace graph path is required."));
artifacts.Add(await PersistArtifactAsync(descriptor, cacheRoot, bucket, rootPrefix, cancellationToken).ConfigureAwait(false));
}
if (!string.IsNullOrWhiteSpace(options.EntryTraceNdjsonPath))
{
var descriptor = new SurfaceArtifactDescriptor(
Kind: "entrytrace.ndjson",
Format: "entrytrace.ndjson",
MediaType: "application/x-ndjson",
View: null,
CasKind: SurfaceCasKind.EntryTraceNdjson,
FilePath: EnsurePath(options.EntryTraceNdjsonPath!, "EntryTrace NDJSON path is required."));
artifacts.Add(await PersistArtifactAsync(descriptor, cacheRoot, bucket, rootPrefix, cancellationToken).ConfigureAwait(false));
}
if (!string.IsNullOrWhiteSpace(options.LayerFragmentsPath))
{
var descriptor = new SurfaceArtifactDescriptor(
Kind: "layer.fragments",
Format: "layer.fragments",
MediaType: "application/json",
View: "inventory",
CasKind: SurfaceCasKind.LayerFragments,
FilePath: EnsurePath(options.LayerFragmentsPath!, "Layer fragments path is required."));
artifacts.Add(await PersistArtifactAsync(descriptor, cacheRoot, bucket, rootPrefix, cancellationToken).ConfigureAwait(false));
}
if (artifacts.Count == 0)
{
return null;
}
var orderedArtifacts = artifacts
.Select(a => a.ManifestArtifact)
.OrderBy(a => a.Kind, StringComparer.Ordinal)
.ThenBy(a => a.Format, StringComparer.Ordinal)
.ToImmutableArray();
var timestamp = _timeProvider.GetUtcNow();
var manifestDocument = new SurfaceManifestDocument
{
Tenant = tenant,
ImageDigest = SurfaceCasLayout.NormalizeDigest(options.ImageDigest),
ScanId = scanId,
GeneratedAt = timestamp,
Source = new SurfaceManifestSource
{
Component = component,
Version = componentVersion,
WorkerInstance = workerInstance,
Attempt = attempt
},
Artifacts = orderedArtifacts
};
var manifestBytes = JsonSerializer.SerializeToUtf8Bytes(manifestDocument, ManifestSerializerOptions);
var manifestDigest = SurfaceCasLayout.ComputeDigest(manifestBytes);
var manifestKey = SurfaceCasLayout.BuildObjectKey(rootPrefix, SurfaceCasKind.Manifest, manifestDigest);
var manifestPath = await SurfaceCasLayout.WriteBytesAsync(cacheRoot, manifestKey, manifestBytes, cancellationToken).ConfigureAwait(false);
var manifestUri = SurfaceCasLayout.BuildCasUri(bucket, manifestKey);
if (!string.IsNullOrWhiteSpace(options.ManifestOutputPath))
{
var manifestOutputPath = Path.GetFullPath(options.ManifestOutputPath);
var manifestOutputDirectory = Path.GetDirectoryName(manifestOutputPath);
if (!string.IsNullOrWhiteSpace(manifestOutputDirectory))
{
Directory.CreateDirectory(manifestOutputDirectory);
}
await File.WriteAllBytesAsync(manifestOutputPath, manifestBytes, cancellationToken).ConfigureAwait(false);
}
return new SurfaceManifestWriteResult(
manifestDigest,
manifestUri,
manifestPath,
manifestDocument,
artifacts);
}
private static async Task<SurfaceArtifactWriteResult> PersistArtifactAsync(
SurfaceArtifactDescriptor descriptor,
string cacheRoot,
string bucket,
string rootPrefix,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
if (!File.Exists(descriptor.FilePath))
{
throw new BuildxPluginException($"Surface artefact file {descriptor.FilePath} was not found.");
}
var content = await File.ReadAllBytesAsync(descriptor.FilePath, cancellationToken).ConfigureAwait(false);
var digest = SurfaceCasLayout.ComputeDigest(content);
var objectKey = SurfaceCasLayout.BuildObjectKey(rootPrefix, descriptor.CasKind, digest);
var filePath = await SurfaceCasLayout.WriteBytesAsync(cacheRoot, objectKey, content, cancellationToken).ConfigureAwait(false);
var uri = SurfaceCasLayout.BuildCasUri(bucket, objectKey);
var storage = new SurfaceManifestStorage
{
Bucket = bucket,
ObjectKey = objectKey,
SizeBytes = content.Length,
ContentType = descriptor.MediaType
};
var artifact = new SurfaceManifestArtifact
{
Kind = descriptor.Kind,
Uri = uri,
Digest = digest,
MediaType = descriptor.MediaType,
Format = descriptor.Format,
SizeBytes = content.Length,
View = descriptor.View,
Storage = storage
};
return new SurfaceArtifactWriteResult(objectKey, filePath, artifact);
}
private static string EnsurePath(string value, string message)
{
if (string.IsNullOrWhiteSpace(value))
{
throw new BuildxPluginException(message);
}
return Path.GetFullPath(value.Trim());
}
}
internal sealed record SurfaceArtifactDescriptor(
string Kind,
string Format,
string MediaType,
string? View,
SurfaceCasKind CasKind,
string FilePath);
internal sealed record SurfaceArtifactWriteResult(
string ObjectKey,
string FilePath,
SurfaceManifestArtifact ManifestArtifact);
internal sealed record SurfaceManifestWriteResult(
string ManifestDigest,
string ManifestUri,
string ManifestPath,
SurfaceManifestDocument Document,
IReadOnlyList<SurfaceArtifactWriteResult> Artifacts);

View File

@@ -0,0 +1,25 @@
using System;
namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Surface;
internal sealed record SurfaceOptions(
string CacheRoot,
string CacheBucket,
string RootPrefix,
string Tenant,
string Component,
string ComponentVersion,
string WorkerInstance,
int Attempt,
string ImageDigest,
string? ScanId,
string? LayerFragmentsPath,
string? EntryTraceGraphPath,
string? EntryTraceNdjsonPath,
string? ManifestOutputPath)
{
public bool HasArtifacts =>
!string.IsNullOrWhiteSpace(LayerFragmentsPath) ||
!string.IsNullOrWhiteSpace(EntryTraceGraphPath) ||
!string.IsNullOrWhiteSpace(EntryTraceNdjsonPath);
}

View File

@@ -2,6 +2,6 @@
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|----|--------|----------|------------|-------------|---------------|
| SCANNER-SURFACE-03 | DOING (2025-11-06) | BuildX Plugin Guild | SURFACE-FS-02 | Push layer manifests and entry fragments into Surface.FS during build-time SBOM generation.<br>2025-11-06: Kicked off manifest emitter wiring within BuildX export pipeline and outlined test fixtures targeting Surface.FS client mock. | BuildX integration tests confirm cache population; CLI docs updated. |
| SCANNER-SURFACE-03 | DONE (2025-11-07) | BuildX Plugin Guild | SURFACE-FS-02 | Push layer manifests and entry fragments into Surface.FS during build-time SBOM generation.<br>2025-11-06: Kicked off manifest emitter wiring within BuildX export pipeline and outlined test fixtures targeting Surface.FS client mock.<br>2025-11-07: Resumed work; reviewing Surface.FS models, CAS integration, and test harness approach before coding.<br>2025-11-07 22:10Z: Implemented Surface manifest writer + CLI plumbing, wired CAS persistence, documented the workflow, and added BuildX plug-in tests + Grafana fixture updates. | BuildX integration tests confirm cache population; CLI docs updated. |
| SCANNER-ENV-03 | TODO | BuildX Plugin Guild | SURFACE-ENV-02 | Adopt Surface.Env helpers for plugin configuration (cache roots, CAS endpoints, feature toggles). | Plugin loads helper; misconfig errors logged; README updated. |
| SCANNER-SECRETS-03 | TODO | BuildX Plugin Guild, Security Guild | SURFACE-SECRETS-02 | Use Surface.Secrets to retrieve registry credentials when interacting with CAS/referrers. | Secrets retrieved via shared library; e2e tests cover rotation; operations guide refreshed. |