Implement MongoDB-based storage for Pack Run approval, artifact, log, and state management

- Added MongoPackRunApprovalStore for managing approval states with MongoDB. - Introduced MongoPackRunArtifactUploader for uploading and storing artifacts. - Created MongoPackRunLogStore to handle logging of pack run events. - Developed MongoPackRunStateStore for persisting and retrieving pack run states. - Implemented unit tests for MongoDB stores to ensure correct functionality. - Added MongoTaskRunnerTestContext for setting up MongoDB test environment. - Enhanced PackRunStateFactory to correctly initialize state with gate reasons.
2025-11-07 10:01:35 +02:00
parent e5ffcd6535
commit a1ce3f74fa
122 changed files with 8730 additions and 914 deletions
--- a/src/Scanner/StellaOps.Scanner.Sbomer.BuildXPlugin/Surface/SurfaceCasLayout.cs
+++ b/src/Scanner/StellaOps.Scanner.Sbomer.BuildXPlugin/Surface/SurfaceCasLayout.cs
@@ -0,0 +1,112 @@
+using System;
+using System.IO;
+using System.Security.Cryptography;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Surface;
+
+internal static class SurfaceCasLayout
+{
+    internal const string DefaultBucket = "scanner-artifacts";
+    internal const string DefaultRootPrefix = "scanner";
+    private const string Sha256 = "sha256";
+
+    public static string NormalizeDigest(string? digest)
+    {
+        if (string.IsNullOrWhiteSpace(digest))
+        {
+            throw new BuildxPluginException("Surface artefact digest cannot be empty.");
+        }
+
+        var trimmed = digest.Trim();
+        return trimmed.Contains(':', StringComparison.Ordinal)
+            ? trimmed
+            : $"{Sha256}:{trimmed}";
+    }
+
+    public static string ExtractDigestValue(string normalizedDigest)
+    {
+        var parts = normalizedDigest.Split(':', 2, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);
+        return parts.Length == 2 ? parts[1] : normalizedDigest;
+    }
+
+    public static string BuildObjectKey(string rootPrefix, SurfaceCasKind kind, string normalizedDigest)
+    {
+        var digestValue = ExtractDigestValue(normalizedDigest);
+        var prefix = kind switch
+        {
+            SurfaceCasKind.LayerFragments => "surface/payloads/layer-fragments",
+            SurfaceCasKind.EntryTraceGraph => "surface/payloads/entrytrace",
+            SurfaceCasKind.EntryTraceNdjson => "surface/payloads/entrytrace",
+            SurfaceCasKind.Manifest => "surface/manifests",
+            _ => "surface/unknown"
+        };
+
+        var extension = kind switch
+        {
+            SurfaceCasKind.LayerFragments => "layer-fragments.json",
+            SurfaceCasKind.EntryTraceGraph => "entrytrace.graph.json",
+            SurfaceCasKind.EntryTraceNdjson => "entrytrace.ndjson",
+            SurfaceCasKind.Manifest => "surface.manifest.json",
+            _ => "artifact.bin"
+        };
+
+        var normalizedRoot = string.IsNullOrWhiteSpace(rootPrefix)
+            ? string.Empty
+            : rootPrefix.Trim().Trim('/');
+
+        var relative = $"{prefix}/{digestValue}/{extension}";
+        return string.IsNullOrWhiteSpace(normalizedRoot) ? relative : $"{normalizedRoot}/{relative}";
+    }
+
+    public static string BuildCasUri(string bucket, string objectKey)
+    {
+        var normalizedBucket = string.IsNullOrWhiteSpace(bucket) ? DefaultBucket : bucket.Trim();
+        var normalizedKey = string.IsNullOrWhiteSpace(objectKey) ? string.Empty : objectKey.Trim().TrimStart('/');
+        return $"cas://{normalizedBucket}/{normalizedKey}";
+    }
+
+    public static string ComputeDigest(ReadOnlySpan<byte> content)
+    {
+        Span<byte> hash = stackalloc byte[32];
+        SHA256.HashData(content, hash);
+        return $"{Sha256}:{Convert.ToHexString(hash).ToLowerInvariant()}";
+    }
+
+    public static async Task<string> WriteBytesAsync(string rootDirectory, string objectKey, byte[] bytes, CancellationToken cancellationToken)
+    {
+        if (string.IsNullOrWhiteSpace(rootDirectory))
+        {
+            throw new BuildxPluginException("Surface cache root must be provided.");
+        }
+
+        var normalizedRoot = Path.GetFullPath(rootDirectory);
+        var relativePath = objectKey.Replace('/', Path.DirectorySeparatorChar);
+        var fullPath = Path.Combine(normalizedRoot, relativePath);
+        var directory = Path.GetDirectoryName(fullPath);
+        if (!string.IsNullOrWhiteSpace(directory))
+        {
+            Directory.CreateDirectory(directory);
+        }
+
+        await using var stream = new FileStream(
+            fullPath,
+            FileMode.Create,
+            FileAccess.Write,
+            FileShare.Read,
+            bufferSize: 64 * 1024,
+            options: FileOptions.Asynchronous | FileOptions.SequentialScan);
+        await stream.WriteAsync(bytes.AsMemory(0, bytes.Length), cancellationToken).ConfigureAwait(false);
+        await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
+        return fullPath;
+    }
+}
+
+internal enum SurfaceCasKind
+{
+    LayerFragments,
+    EntryTraceGraph,
+    EntryTraceNdjson,
+    Manifest
+}
--- a/src/Scanner/StellaOps.Scanner.Sbomer.BuildXPlugin/Surface/SurfaceManifestWriter.cs
+++ b/src/Scanner/StellaOps.Scanner.Sbomer.BuildXPlugin/Surface/SurfaceManifestWriter.cs
@@ -0,0 +1,227 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.IO;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using System.Threading;
+using System.Threading.Tasks;
+using StellaOps.Scanner.Surface.FS;
+
+namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Surface;
+
+internal sealed class SurfaceManifestWriter
+{
+    private static readonly JsonSerializerOptions ManifestSerializerOptions = new(JsonSerializerDefaults.Web)
+    {
+        DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
+        WriteIndented = false
+    };
+
+    private readonly TimeProvider _timeProvider;
+
+    public SurfaceManifestWriter(TimeProvider timeProvider)
+    {
+        _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
+    }
+
+    public async Task<SurfaceManifestWriteResult?> WriteAsync(SurfaceOptions options, CancellationToken cancellationToken)
+    {
+        ArgumentNullException.ThrowIfNull(options);
+
+        if (!options.HasArtifacts)
+        {
+            return null;
+        }
+
+        var cacheRoot = EnsurePath(options.CacheRoot, "Surface cache root must be provided.");
+        var bucket = string.IsNullOrWhiteSpace(options.CacheBucket)
+            ? SurfaceCasLayout.DefaultBucket
+            : options.CacheBucket.Trim();
+        var rootPrefix = string.IsNullOrWhiteSpace(options.RootPrefix)
+            ? SurfaceCasLayout.DefaultRootPrefix
+            : options.RootPrefix.Trim();
+        var tenant = string.IsNullOrWhiteSpace(options.Tenant)
+            ? "default"
+            : options.Tenant.Trim();
+        var component = string.IsNullOrWhiteSpace(options.Component)
+            ? "scanner.buildx"
+            : options.Component.Trim();
+        var componentVersion = string.IsNullOrWhiteSpace(options.ComponentVersion)
+            ? null
+            : options.ComponentVersion.Trim();
+        var workerInstance = string.IsNullOrWhiteSpace(options.WorkerInstance)
+            ? Environment.MachineName
+            : options.WorkerInstance.Trim();
+        var attempt = options.Attempt <= 0 ? 1 : options.Attempt;
+        var scanId = string.IsNullOrWhiteSpace(options.ScanId)
+            ? options.ImageDigest
+            : options.ScanId!.Trim();
+
+        Directory.CreateDirectory(cacheRoot);
+
+        var artifacts = new List<SurfaceArtifactWriteResult>();
+
+        if (!string.IsNullOrWhiteSpace(options.EntryTraceGraphPath))
+        {
+            var descriptor = new SurfaceArtifactDescriptor(
+                Kind: "entrytrace.graph",
+                Format: "entrytrace.graph",
+                MediaType: "application/json",
+                View: null,
+                CasKind: SurfaceCasKind.EntryTraceGraph,
+                FilePath: EnsurePath(options.EntryTraceGraphPath!, "EntryTrace graph path is required."));
+            artifacts.Add(await PersistArtifactAsync(descriptor, cacheRoot, bucket, rootPrefix, cancellationToken).ConfigureAwait(false));
+        }
+
+        if (!string.IsNullOrWhiteSpace(options.EntryTraceNdjsonPath))
+        {
+            var descriptor = new SurfaceArtifactDescriptor(
+                Kind: "entrytrace.ndjson",
+                Format: "entrytrace.ndjson",
+                MediaType: "application/x-ndjson",
+                View: null,
+                CasKind: SurfaceCasKind.EntryTraceNdjson,
+                FilePath: EnsurePath(options.EntryTraceNdjsonPath!, "EntryTrace NDJSON path is required."));
+            artifacts.Add(await PersistArtifactAsync(descriptor, cacheRoot, bucket, rootPrefix, cancellationToken).ConfigureAwait(false));
+        }
+
+        if (!string.IsNullOrWhiteSpace(options.LayerFragmentsPath))
+        {
+            var descriptor = new SurfaceArtifactDescriptor(
+                Kind: "layer.fragments",
+                Format: "layer.fragments",
+                MediaType: "application/json",
+                View: "inventory",
+                CasKind: SurfaceCasKind.LayerFragments,
+                FilePath: EnsurePath(options.LayerFragmentsPath!, "Layer fragments path is required."));
+            artifacts.Add(await PersistArtifactAsync(descriptor, cacheRoot, bucket, rootPrefix, cancellationToken).ConfigureAwait(false));
+        }
+
+        if (artifacts.Count == 0)
+        {
+            return null;
+        }
+
+        var orderedArtifacts = artifacts
+            .Select(a => a.ManifestArtifact)
+            .OrderBy(a => a.Kind, StringComparer.Ordinal)
+            .ThenBy(a => a.Format, StringComparer.Ordinal)
+            .ToImmutableArray();
+
+        var timestamp = _timeProvider.GetUtcNow();
+        var manifestDocument = new SurfaceManifestDocument
+        {
+            Tenant = tenant,
+            ImageDigest = SurfaceCasLayout.NormalizeDigest(options.ImageDigest),
+            ScanId = scanId,
+            GeneratedAt = timestamp,
+            Source = new SurfaceManifestSource
+            {
+                Component = component,
+                Version = componentVersion,
+                WorkerInstance = workerInstance,
+                Attempt = attempt
+            },
+            Artifacts = orderedArtifacts
+        };
+
+        var manifestBytes = JsonSerializer.SerializeToUtf8Bytes(manifestDocument, ManifestSerializerOptions);
+        var manifestDigest = SurfaceCasLayout.ComputeDigest(manifestBytes);
+        var manifestKey = SurfaceCasLayout.BuildObjectKey(rootPrefix, SurfaceCasKind.Manifest, manifestDigest);
+        var manifestPath = await SurfaceCasLayout.WriteBytesAsync(cacheRoot, manifestKey, manifestBytes, cancellationToken).ConfigureAwait(false);
+        var manifestUri = SurfaceCasLayout.BuildCasUri(bucket, manifestKey);
+
+        if (!string.IsNullOrWhiteSpace(options.ManifestOutputPath))
+        {
+            var manifestOutputPath = Path.GetFullPath(options.ManifestOutputPath);
+            var manifestOutputDirectory = Path.GetDirectoryName(manifestOutputPath);
+            if (!string.IsNullOrWhiteSpace(manifestOutputDirectory))
+            {
+                Directory.CreateDirectory(manifestOutputDirectory);
+            }
+
+            await File.WriteAllBytesAsync(manifestOutputPath, manifestBytes, cancellationToken).ConfigureAwait(false);
+        }
+
+        return new SurfaceManifestWriteResult(
+            manifestDigest,
+            manifestUri,
+            manifestPath,
+            manifestDocument,
+            artifacts);
+    }
+
+    private static async Task<SurfaceArtifactWriteResult> PersistArtifactAsync(
+        SurfaceArtifactDescriptor descriptor,
+        string cacheRoot,
+        string bucket,
+        string rootPrefix,
+        CancellationToken cancellationToken)
+    {
+        cancellationToken.ThrowIfCancellationRequested();
+
+        if (!File.Exists(descriptor.FilePath))
+        {
+            throw new BuildxPluginException($"Surface artefact file {descriptor.FilePath} was not found.");
+        }
+
+        var content = await File.ReadAllBytesAsync(descriptor.FilePath, cancellationToken).ConfigureAwait(false);
+        var digest = SurfaceCasLayout.ComputeDigest(content);
+        var objectKey = SurfaceCasLayout.BuildObjectKey(rootPrefix, descriptor.CasKind, digest);
+        var filePath = await SurfaceCasLayout.WriteBytesAsync(cacheRoot, objectKey, content, cancellationToken).ConfigureAwait(false);
+        var uri = SurfaceCasLayout.BuildCasUri(bucket, objectKey);
+
+        var storage = new SurfaceManifestStorage
+        {
+            Bucket = bucket,
+            ObjectKey = objectKey,
+            SizeBytes = content.Length,
+            ContentType = descriptor.MediaType
+        };
+
+        var artifact = new SurfaceManifestArtifact
+        {
+            Kind = descriptor.Kind,
+            Uri = uri,
+            Digest = digest,
+            MediaType = descriptor.MediaType,
+            Format = descriptor.Format,
+            SizeBytes = content.Length,
+            View = descriptor.View,
+            Storage = storage
+        };
+
+        return new SurfaceArtifactWriteResult(objectKey, filePath, artifact);
+    }
+
+    private static string EnsurePath(string value, string message)
+    {
+        if (string.IsNullOrWhiteSpace(value))
+        {
+            throw new BuildxPluginException(message);
+        }
+
+        return Path.GetFullPath(value.Trim());
+    }
+}
+
+internal sealed record SurfaceArtifactDescriptor(
+    string Kind,
+    string Format,
+    string MediaType,
+    string? View,
+    SurfaceCasKind CasKind,
+    string FilePath);
+
+internal sealed record SurfaceArtifactWriteResult(
+    string ObjectKey,
+    string FilePath,
+    SurfaceManifestArtifact ManifestArtifact);
+
+internal sealed record SurfaceManifestWriteResult(
+    string ManifestDigest,
+    string ManifestUri,
+    string ManifestPath,
+    SurfaceManifestDocument Document,
+    IReadOnlyList<SurfaceArtifactWriteResult> Artifacts);
--- a/src/Scanner/StellaOps.Scanner.Sbomer.BuildXPlugin/Surface/SurfaceOptions.cs
+++ b/src/Scanner/StellaOps.Scanner.Sbomer.BuildXPlugin/Surface/SurfaceOptions.cs
@@ -0,0 +1,25 @@
+using System;
+
+namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Surface;
+
+internal sealed record SurfaceOptions(
+    string CacheRoot,
+    string CacheBucket,
+    string RootPrefix,
+    string Tenant,
+    string Component,
+    string ComponentVersion,
+    string WorkerInstance,
+    int Attempt,
+    string ImageDigest,
+    string? ScanId,
+    string? LayerFragmentsPath,
+    string? EntryTraceGraphPath,
+    string? EntryTraceNdjsonPath,
+    string? ManifestOutputPath)
+{
+    public bool HasArtifacts =>
+        !string.IsNullOrWhiteSpace(LayerFragmentsPath) ||
+        !string.IsNullOrWhiteSpace(EntryTraceGraphPath) ||
+        !string.IsNullOrWhiteSpace(EntryTraceNdjsonPath);
+}