stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

Implement MongoDB-based storage for Pack Run approval, artifact, log, and state management
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Details

Browse Source 
- Added MongoPackRunApprovalStore for managing approval states with MongoDB.
- Introduced MongoPackRunArtifactUploader for uploading and storing artifacts.
- Created MongoPackRunLogStore to handle logging of pack run events.
- Developed MongoPackRunStateStore for persisting and retrieving pack run states.
- Implemented unit tests for MongoDB stores to ensure correct functionality.
- Added MongoTaskRunnerTestContext for setting up MongoDB test environment.
- Enhanced PackRunStateFactory to correctly initialize state with gate reasons.
This commit is contained in:
master
2025-11-07 10:01:35 +02:00
parent e5ffcd6535
commit a1ce3f74fa
122 changed files with 8730 additions and 914 deletions
Download Patch File Download Diff File Expand all files Collapse all files

111
src/Scanner/StellaOps.Scanner.Sbomer.BuildXPlugin/Program.cs
View File

@@ -12,6 +12,7 @@ using StellaOps.Scanner.Sbomer.BuildXPlugin.Attestation;
using StellaOps.Scanner.Sbomer.BuildXPlugin.Cas;
using StellaOps.Scanner.Sbomer.BuildXPlugin.Descriptor;
using StellaOps.Scanner.Sbomer.BuildXPlugin.Manifest;
using StellaOps.Scanner.Sbomer.BuildXPlugin.Surface;
namespace StellaOps.Scanner.Sbomer.BuildXPlugin;
@@ -131,6 +132,12 @@ internal static class Program
Console.WriteLine(" --attestor <url> (descriptor) Optional Attestor endpoint for provenance placeholders.");
Console.WriteLine(" --attestor-token <token> Bearer token for Attestor requests (or STELLAOPS_ATTESTOR_TOKEN).");
Console.WriteLine(" --attestor-insecure Skip TLS verification for Attestor requests (dev/test only).");
Console.WriteLine(" --surface-layer-fragments <path> Persist layer fragments JSON into Surface.FS.");
Console.WriteLine(" --surface-entrytrace-graph <path> Persist EntryTrace graph JSON into Surface.FS.");
Console.WriteLine(" --surface-entrytrace-ndjson <path> Persist EntryTrace NDJSON into Surface.FS.");
Console.WriteLine(" --surface-cache-root <path> Override Surface cache root (defaults to CAS root).");
Console.WriteLine(" --surface-bucket <name> Bucket name used in Surface CAS URIs (default scanner-artifacts).");
Console.WriteLine(" --surface-tenant <tenant> Tenant identifier recorded in the Surface manifest.");
return 0;
}
@@ -186,6 +193,11 @@ internal static class Program
private static async Task<int> RunDescriptorAsync(string[] args, CancellationToken cancellationToken)
{
var manifestDirectory = ResolveManifestDirectory(args);
var loader = new BuildxPluginManifestLoader(manifestDirectory);
var manifest = await loader.LoadDefaultAsync(cancellationToken).ConfigureAwait(false);
var casRoot = ResolveCasRoot(args, manifest);
var imageDigest = RequireOption(args, "--image");
var sbomPath = RequireOption(args, "--sbom");
@@ -244,11 +256,110 @@ internal static class Program
await attestorClient.SendPlaceholderAsync(attestorUri, document, cancellationToken).ConfigureAwait(false);
}
await TryPublishSurfaceArtifactsAsync(args, request, casRoot, version, cancellationToken).ConfigureAwait(false);
var json = JsonSerializer.Serialize(document, DescriptorJsonOptions);
Console.WriteLine(json);
return 0;
}
private static async Task TryPublishSurfaceArtifactsAsync(
string[] args,
DescriptorRequest descriptorRequest,
string casRoot,
string generatorVersion,
CancellationToken cancellationToken)
{
var surfaceOptions = ResolveSurfaceOptions(args, descriptorRequest, casRoot, generatorVersion);
if (surfaceOptions is null || !surfaceOptions.HasArtifacts)
{
return;
}
var writer = new SurfaceManifestWriter(TimeProvider.System);
var result = await writer.WriteAsync(surfaceOptions, cancellationToken).ConfigureAwait(false);
if (result is null)
{
return;
}
Console.Error.WriteLine($"surface manifest stored: {result.ManifestUri} ({result.Document.Artifacts.Count} artefacts)");
}
private static SurfaceOptions? ResolveSurfaceOptions(
string[] args,
DescriptorRequest descriptorRequest,
string casRoot,
string generatorVersion)
{
var layerFragmentsPath = GetOption(args, "--surface-layer-fragments")
?? Environment.GetEnvironmentVariable("STELLAOPS_SURFACE_LAYER_FRAGMENTS");
var entryTraceGraphPath = GetOption(args, "--surface-entrytrace-graph")
?? Environment.GetEnvironmentVariable("STELLAOPS_SURFACE_ENTRYTRACE_GRAPH");
var entryTraceNdjsonPath = GetOption(args, "--surface-entrytrace-ndjson")
?? Environment.GetEnvironmentVariable("STELLAOPS_SURFACE_ENTRYTRACE_NDJSON");
if (string.IsNullOrWhiteSpace(layerFragmentsPath) &&
string.IsNullOrWhiteSpace(entryTraceGraphPath) &&
string.IsNullOrWhiteSpace(entryTraceNdjsonPath))
{
return null;
}
var cacheRoot = GetOption(args, "--surface-cache-root")
?? Environment.GetEnvironmentVariable("STELLAOPS_SURFACE_CACHE_ROOT")
?? casRoot;
var bucket = GetOption(args, "--surface-bucket")
?? Environment.GetEnvironmentVariable("STELLAOPS_SURFACE_BUCKET")
?? SurfaceCasLayout.DefaultBucket;
var rootPrefix = GetOption(args, "--surface-root-prefix")
?? Environment.GetEnvironmentVariable("STELLAOPS_SURFACE_ROOT_PREFIX")
?? SurfaceCasLayout.DefaultRootPrefix;
var tenant = GetOption(args, "--surface-tenant")
?? Environment.GetEnvironmentVariable("STELLAOPS_SURFACE_TENANT")
?? "default";
var component = GetOption(args, "--surface-component")
?? Environment.GetEnvironmentVariable("STELLAOPS_SURFACE_COMPONENT")
?? "scanner.buildx";
var componentVersion = GetOption(args, "--surface-component-version")
?? Environment.GetEnvironmentVariable("STELLAOPS_SURFACE_COMPONENT_VERSION")
?? generatorVersion;
var workerInstance = GetOption(args, "--surface-worker-instance")
?? Environment.GetEnvironmentVariable("STELLAOPS_SURFACE_WORKER_INSTANCE")
?? Environment.MachineName;
var attemptValue = GetOption(args, "--surface-attempt")
?? Environment.GetEnvironmentVariable("STELLAOPS_SURFACE_ATTEMPT");
var attempt = 1;
if (!string.IsNullOrWhiteSpace(attemptValue) && int.TryParse(attemptValue, out var parsedAttempt) && parsedAttempt > 0)
{
attempt = parsedAttempt;
}
var scanId = GetOption(args, "--surface-scan-id")
?? Environment.GetEnvironmentVariable("STELLAOPS_SURFACE_SCAN_ID")
?? descriptorRequest.SbomName
?? descriptorRequest.ImageDigest;
var manifestOutput = GetOption(args, "--surface-manifest-output")
?? Environment.GetEnvironmentVariable("STELLAOPS_SURFACE_MANIFEST_OUTPUT");
return new SurfaceOptions(
CacheRoot: cacheRoot,
CacheBucket: bucket,
RootPrefix: rootPrefix,
Tenant: tenant,
Component: component,
ComponentVersion: componentVersion,
WorkerInstance: workerInstance,
Attempt: attempt,
ImageDigest: descriptorRequest.ImageDigest,
ScanId: scanId,
LayerFragmentsPath: layerFragmentsPath,
EntryTraceGraphPath: entryTraceGraphPath,
EntryTraceNdjsonPath: entryTraceNdjsonPath,
ManifestOutputPath: manifestOutput);
}
private static string? GetOption(string[] args, string optionName)
{
for (var i = 0; i < args.Length; i++)

3
src/Scanner/StellaOps.Scanner.Sbomer.BuildXPlugin/Properties/AssemblyInfo.cs Normal file
View File

@@ -0,0 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Scanner.Sbomer.BuildXPlugin.Tests")]

16
src/Scanner/StellaOps.Scanner.Sbomer.BuildXPlugin/StellaOps.Scanner.Sbomer.BuildXPlugin.csproj
View File

@@ -12,9 +12,13 @@
<InformationalVersion>0.1.0-alpha</InformationalVersion>
</PropertyGroup>
<ItemGroup>
<Content Include="stellaops.sbom-indexer.manifest.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
</Project>
<ItemGroup>
<Content Include="stellaops.sbom-indexer.manifest.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\\__Libraries\\StellaOps.Scanner.Surface.FS\\StellaOps.Scanner.Surface.FS.csproj" />
</ItemGroup>
</Project>

112
src/Scanner/StellaOps.Scanner.Sbomer.BuildXPlugin/Surface/SurfaceCasLayout.cs Normal file
View File

@@ -0,0 +1,112 @@
using System;
using System.IO;
using System.Security.Cryptography;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Surface;
internal static class SurfaceCasLayout
{
internal const string DefaultBucket = "scanner-artifacts";
internal const string DefaultRootPrefix = "scanner";
private const string Sha256 = "sha256";
public static string NormalizeDigest(string? digest)
{
if (string.IsNullOrWhiteSpace(digest))
{
throw new BuildxPluginException("Surface artefact digest cannot be empty.");
}
var trimmed = digest.Trim();
return trimmed.Contains(':', StringComparison.Ordinal)
? trimmed
: $"{Sha256}:{trimmed}";
}
public static string ExtractDigestValue(string normalizedDigest)
{
var parts = normalizedDigest.Split(':', 2, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);
return parts.Length == 2 ? parts[1] : normalizedDigest;
}
public static string BuildObjectKey(string rootPrefix, SurfaceCasKind kind, string normalizedDigest)
{
var digestValue = ExtractDigestValue(normalizedDigest);
var prefix = kind switch
{
SurfaceCasKind.LayerFragments => "surface/payloads/layer-fragments",
SurfaceCasKind.EntryTraceGraph => "surface/payloads/entrytrace",
SurfaceCasKind.EntryTraceNdjson => "surface/payloads/entrytrace",
SurfaceCasKind.Manifest => "surface/manifests",
_ => "surface/unknown"
};
var extension = kind switch
{
SurfaceCasKind.LayerFragments => "layer-fragments.json",
SurfaceCasKind.EntryTraceGraph => "entrytrace.graph.json",
SurfaceCasKind.EntryTraceNdjson => "entrytrace.ndjson",
SurfaceCasKind.Manifest => "surface.manifest.json",
_ => "artifact.bin"
};
var normalizedRoot = string.IsNullOrWhiteSpace(rootPrefix)
? string.Empty
: rootPrefix.Trim().Trim('/');
var relative = $"{prefix}/{digestValue}/{extension}";
return string.IsNullOrWhiteSpace(normalizedRoot) ? relative : $"{normalizedRoot}/{relative}";
}
public static string BuildCasUri(string bucket, string objectKey)
{
var normalizedBucket = string.IsNullOrWhiteSpace(bucket) ? DefaultBucket : bucket.Trim();
var normalizedKey = string.IsNullOrWhiteSpace(objectKey) ? string.Empty : objectKey.Trim().TrimStart('/');
return $"cas://{normalizedBucket}/{normalizedKey}";
}
public static string ComputeDigest(ReadOnlySpan<byte> content)
{
Span<byte> hash = stackalloc byte[32];
SHA256.HashData(content, hash);
return $"{Sha256}:{Convert.ToHexString(hash).ToLowerInvariant()}";
}
public static async Task<string> WriteBytesAsync(string rootDirectory, string objectKey, byte[] bytes, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(rootDirectory))
{
throw new BuildxPluginException("Surface cache root must be provided.");
}
var normalizedRoot = Path.GetFullPath(rootDirectory);
var relativePath = objectKey.Replace('/', Path.DirectorySeparatorChar);
var fullPath = Path.Combine(normalizedRoot, relativePath);
var directory = Path.GetDirectoryName(fullPath);
if (!string.IsNullOrWhiteSpace(directory))
{
Directory.CreateDirectory(directory);
}
await using var stream = new FileStream(
fullPath,
FileMode.Create,
FileAccess.Write,
FileShare.Read,
bufferSize: 64 * 1024,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await stream.WriteAsync(bytes.AsMemory(0, bytes.Length), cancellationToken).ConfigureAwait(false);
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
return fullPath;
}
}
internal enum SurfaceCasKind
{
LayerFragments,
EntryTraceGraph,
EntryTraceNdjson,
Manifest
}

227
src/Scanner/StellaOps.Scanner.Sbomer.BuildXPlugin/Surface/SurfaceManifestWriter.cs Normal file
View File

@@ -0,0 +1,227 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Scanner.Surface.FS;
namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Surface;
internal sealed class SurfaceManifestWriter
{
private static readonly JsonSerializerOptions ManifestSerializerOptions = new(JsonSerializerDefaults.Web)
{
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
WriteIndented = false
};
private readonly TimeProvider _timeProvider;
public SurfaceManifestWriter(TimeProvider timeProvider)
{
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
}
public async Task<SurfaceManifestWriteResult?> WriteAsync(SurfaceOptions options, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(options);
if (!options.HasArtifacts)
{
return null;
}
var cacheRoot = EnsurePath(options.CacheRoot, "Surface cache root must be provided.");
var bucket = string.IsNullOrWhiteSpace(options.CacheBucket)
? SurfaceCasLayout.DefaultBucket
: options.CacheBucket.Trim();
var rootPrefix = string.IsNullOrWhiteSpace(options.RootPrefix)
? SurfaceCasLayout.DefaultRootPrefix
: options.RootPrefix.Trim();
var tenant = string.IsNullOrWhiteSpace(options.Tenant)
? "default"
: options.Tenant.Trim();
var component = string.IsNullOrWhiteSpace(options.Component)
? "scanner.buildx"
: options.Component.Trim();
var componentVersion = string.IsNullOrWhiteSpace(options.ComponentVersion)
? null
: options.ComponentVersion.Trim();
var workerInstance = string.IsNullOrWhiteSpace(options.WorkerInstance)
? Environment.MachineName
: options.WorkerInstance.Trim();
var attempt = options.Attempt <= 0 ? 1 : options.Attempt;
var scanId = string.IsNullOrWhiteSpace(options.ScanId)
? options.ImageDigest
: options.ScanId!.Trim();
Directory.CreateDirectory(cacheRoot);
var artifacts = new List<SurfaceArtifactWriteResult>();
if (!string.IsNullOrWhiteSpace(options.EntryTraceGraphPath))
{
var descriptor = new SurfaceArtifactDescriptor(
Kind: "entrytrace.graph",
Format: "entrytrace.graph",
MediaType: "application/json",
View: null,
CasKind: SurfaceCasKind.EntryTraceGraph,
FilePath: EnsurePath(options.EntryTraceGraphPath!, "EntryTrace graph path is required."));
artifacts.Add(await PersistArtifactAsync(descriptor, cacheRoot, bucket, rootPrefix, cancellationToken).ConfigureAwait(false));
}
if (!string.IsNullOrWhiteSpace(options.EntryTraceNdjsonPath))
{
var descriptor = new SurfaceArtifactDescriptor(
Kind: "entrytrace.ndjson",
Format: "entrytrace.ndjson",
MediaType: "application/x-ndjson",
View: null,
CasKind: SurfaceCasKind.EntryTraceNdjson,
FilePath: EnsurePath(options.EntryTraceNdjsonPath!, "EntryTrace NDJSON path is required."));
artifacts.Add(await PersistArtifactAsync(descriptor, cacheRoot, bucket, rootPrefix, cancellationToken).ConfigureAwait(false));
}
if (!string.IsNullOrWhiteSpace(options.LayerFragmentsPath))
{
var descriptor = new SurfaceArtifactDescriptor(
Kind: "layer.fragments",
Format: "layer.fragments",
MediaType: "application/json",
View: "inventory",
CasKind: SurfaceCasKind.LayerFragments,
FilePath: EnsurePath(options.LayerFragmentsPath!, "Layer fragments path is required."));
artifacts.Add(await PersistArtifactAsync(descriptor, cacheRoot, bucket, rootPrefix, cancellationToken).ConfigureAwait(false));
}
if (artifacts.Count == 0)
{
return null;
}
var orderedArtifacts = artifacts
.Select(a => a.ManifestArtifact)
.OrderBy(a => a.Kind, StringComparer.Ordinal)
.ThenBy(a => a.Format, StringComparer.Ordinal)
.ToImmutableArray();
var timestamp = _timeProvider.GetUtcNow();
var manifestDocument = new SurfaceManifestDocument
{
Tenant = tenant,
ImageDigest = SurfaceCasLayout.NormalizeDigest(options.ImageDigest),
ScanId = scanId,
GeneratedAt = timestamp,
Source = new SurfaceManifestSource
{
Component = component,
Version = componentVersion,
WorkerInstance = workerInstance,
Attempt = attempt
},
Artifacts = orderedArtifacts
};
var manifestBytes = JsonSerializer.SerializeToUtf8Bytes(manifestDocument, ManifestSerializerOptions);
var manifestDigest = SurfaceCasLayout.ComputeDigest(manifestBytes);
var manifestKey = SurfaceCasLayout.BuildObjectKey(rootPrefix, SurfaceCasKind.Manifest, manifestDigest);
var manifestPath = await SurfaceCasLayout.WriteBytesAsync(cacheRoot, manifestKey, manifestBytes, cancellationToken).ConfigureAwait(false);
var manifestUri = SurfaceCasLayout.BuildCasUri(bucket, manifestKey);
if (!string.IsNullOrWhiteSpace(options.ManifestOutputPath))
{
var manifestOutputPath = Path.GetFullPath(options.ManifestOutputPath);
var manifestOutputDirectory = Path.GetDirectoryName(manifestOutputPath);
if (!string.IsNullOrWhiteSpace(manifestOutputDirectory))
{
Directory.CreateDirectory(manifestOutputDirectory);
}
await File.WriteAllBytesAsync(manifestOutputPath, manifestBytes, cancellationToken).ConfigureAwait(false);
}
return new SurfaceManifestWriteResult(
manifestDigest,
manifestUri,
manifestPath,
manifestDocument,
artifacts);
}
private static async Task<SurfaceArtifactWriteResult> PersistArtifactAsync(
SurfaceArtifactDescriptor descriptor,
string cacheRoot,
string bucket,
string rootPrefix,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
if (!File.Exists(descriptor.FilePath))
{
throw new BuildxPluginException($"Surface artefact file {descriptor.FilePath} was not found.");
}
var content = await File.ReadAllBytesAsync(descriptor.FilePath, cancellationToken).ConfigureAwait(false);
var digest = SurfaceCasLayout.ComputeDigest(content);
var objectKey = SurfaceCasLayout.BuildObjectKey(rootPrefix, descriptor.CasKind, digest);
var filePath = await SurfaceCasLayout.WriteBytesAsync(cacheRoot, objectKey, content, cancellationToken).ConfigureAwait(false);
var uri = SurfaceCasLayout.BuildCasUri(bucket, objectKey);
var storage = new SurfaceManifestStorage
{
Bucket = bucket,
ObjectKey = objectKey,
SizeBytes = content.Length,
ContentType = descriptor.MediaType
};
var artifact = new SurfaceManifestArtifact
{
Kind = descriptor.Kind,
Uri = uri,
Digest = digest,
MediaType = descriptor.MediaType,
Format = descriptor.Format,
SizeBytes = content.Length,
View = descriptor.View,
Storage = storage
};
return new SurfaceArtifactWriteResult(objectKey, filePath, artifact);
}
private static string EnsurePath(string value, string message)
{
if (string.IsNullOrWhiteSpace(value))
{
throw new BuildxPluginException(message);
}
return Path.GetFullPath(value.Trim());
}
}
internal sealed record SurfaceArtifactDescriptor(
string Kind,
string Format,
string MediaType,
string? View,
SurfaceCasKind CasKind,
string FilePath);
internal sealed record SurfaceArtifactWriteResult(
string ObjectKey,
string FilePath,
SurfaceManifestArtifact ManifestArtifact);
internal sealed record SurfaceManifestWriteResult(
string ManifestDigest,
string ManifestUri,
string ManifestPath,
SurfaceManifestDocument Document,
IReadOnlyList<SurfaceArtifactWriteResult> Artifacts);

25
src/Scanner/StellaOps.Scanner.Sbomer.BuildXPlugin/Surface/SurfaceOptions.cs Normal file
View File

@@ -0,0 +1,25 @@
using System;
namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Surface;
internal sealed record SurfaceOptions(
string CacheRoot,
string CacheBucket,
string RootPrefix,
string Tenant,
string Component,
string ComponentVersion,
string WorkerInstance,
int Attempt,
string ImageDigest,
string? ScanId,
string? LayerFragmentsPath,
string? EntryTraceGraphPath,
string? EntryTraceNdjsonPath,
string? ManifestOutputPath)
{
public bool HasArtifacts =>
!string.IsNullOrWhiteSpace(LayerFragmentsPath) ||
!string.IsNullOrWhiteSpace(EntryTraceGraphPath) ||
!string.IsNullOrWhiteSpace(EntryTraceNdjsonPath);
}

2
src/Scanner/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md
View File

@@ -2,6 +2,6 @@
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|----|--------|----------|------------|-------------|---------------|
| SCANNER-SURFACE-03 | DOING (2025-11-06) | BuildX Plugin Guild | SURFACE-FS-02 | Push layer manifests and entry fragments into Surface.FS during build-time SBOM generation.<br>2025-11-06: Kicked off manifest emitter wiring within BuildX export pipeline and outlined test fixtures targeting Surface.FS client mock. | BuildX integration tests confirm cache population; CLI docs updated. |
| SCANNER-SURFACE-03 | DONE (2025-11-07) | BuildX Plugin Guild | SURFACE-FS-02 | Push layer manifests and entry fragments into Surface.FS during build-time SBOM generation.<br>2025-11-06: Kicked off manifest emitter wiring within BuildX export pipeline and outlined test fixtures targeting Surface.FS client mock.<br>2025-11-07: Resumed work; reviewing Surface.FS models, CAS integration, and test harness approach before coding.<br>2025-11-07 22:10Z: Implemented Surface manifest writer + CLI plumbing, wired CAS persistence, documented the workflow, and added BuildX plug-in tests + Grafana fixture updates. | BuildX integration tests confirm cache population; CLI docs updated. |
| SCANNER-ENV-03 | TODO | BuildX Plugin Guild | SURFACE-ENV-02 | Adopt Surface.Env helpers for plugin configuration (cache roots, CAS endpoints, feature toggles). | Plugin loads helper; misconfig errors logged; README updated. |
| SCANNER-SECRETS-03 | TODO | BuildX Plugin Guild, Security Guild | SURFACE-SECRETS-02 | Use Surface.Secrets to retrieve registry credentials when interacting with CAS/referrers. | Secrets retrieved via shared library; e2e tests cover rotation; operations guide refreshed. |

122
src/Scanner/StellaOps.Scanner.WebService/Options/ScannerSurfaceSecretConfigurator.cs
View File

@@ -1,3 +1,4 @@
using System.Collections.Generic;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Scanner.Surface.Env;
@@ -28,6 +29,13 @@ internal sealed class ScannerSurfaceSecretConfigurator : IConfigureOptions<Scann
ArgumentNullException.ThrowIfNull(options);
var tenant = _surfaceEnvironment.Settings.Secrets.Tenant;
ApplyCasAccessSecret(options, tenant);
ApplyRegistrySecret(options, tenant);
ApplyAttestationSecret(options, tenant);
}
private void ApplyCasAccessSecret(ScannerWebServiceOptions options, string tenant)
{
var request = new SurfaceSecretRequest(
Tenant: tenant,
Component: ComponentName,
@@ -56,6 +64,120 @@ internal sealed class ScannerSurfaceSecretConfigurator : IConfigureOptions<Scann
ApplySecret(options.ArtifactStore ??= new ScannerWebServiceOptions.ArtifactStoreOptions(), secret);
}
private void ApplyRegistrySecret(ScannerWebServiceOptions options, string tenant)
{
var request = new SurfaceSecretRequest(
Tenant: tenant,
Component: ComponentName,
SecretType: "registry");
RegistryAccessSecret? secret = null;
try
{
using var handle = _secretProvider.GetAsync(request).AsTask().GetAwaiter().GetResult();
secret = SurfaceSecretParser.ParseRegistryAccessSecret(handle);
}
catch (SurfaceSecretNotFoundException)
{
_logger.LogDebug("Surface secret 'registry' not found for {Component}; leaving registry credentials unchanged.", ComponentName);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to resolve surface secret 'registry' for {Component}.", ComponentName);
}
if (secret is null)
{
return;
}
options.Registry ??= new ScannerWebServiceOptions.RegistryOptions();
options.Registry.DefaultRegistry = secret.DefaultRegistry;
options.Registry.Credentials = new List<ScannerWebServiceOptions.RegistryCredentialOptions>();
foreach (var entry in secret.Entries)
{
var credential = new ScannerWebServiceOptions.RegistryCredentialOptions
{
Registry = entry.Registry,
Username = entry.Username,
Password = entry.Password,
IdentityToken = entry.IdentityToken,
RegistryToken = entry.RegistryToken,
RefreshToken = entry.RefreshToken,
ExpiresAt = entry.ExpiresAt,
AllowInsecureTls = entry.AllowInsecureTls,
Email = entry.Email
};
if (entry.Scopes.Count > 0)
{
credential.Scopes = new List<string>(entry.Scopes);
}
if (entry.Headers.Count > 0)
{
foreach (var (key, value) in entry.Headers)
{
credential.Headers[key] = value;
}
}
options.Registry.Credentials.Add(credential);
}
_logger.LogInformation(
"Surface secret 'registry' applied for {Component} (default: {DefaultRegistry}, entries: {Count}).",
ComponentName,
options.Registry.DefaultRegistry ?? "(none)",
options.Registry.Credentials.Count);
}
private void ApplyAttestationSecret(ScannerWebServiceOptions options, string tenant)
{
var request = new SurfaceSecretRequest(
Tenant: tenant,
Component: ComponentName,
SecretType: "attestation");
AttestationSecret? secret = null;
try
{
using var handle = _secretProvider.GetAsync(request).AsTask().GetAwaiter().GetResult();
secret = SurfaceSecretParser.ParseAttestationSecret(handle);
}
catch (SurfaceSecretNotFoundException)
{
_logger.LogDebug("Surface secret 'attestation' not found for {Component}; retaining signing configuration.", ComponentName);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to resolve surface secret 'attestation' for {Component}.", ComponentName);
}
if (secret is null)
{
return;
}
options.Signing ??= new ScannerWebServiceOptions.SigningOptions();
if (!string.IsNullOrWhiteSpace(secret.KeyPem))
{
options.Signing.KeyPem = secret.KeyPem;
}
if (!string.IsNullOrWhiteSpace(secret.CertificatePem))
{
options.Signing.CertificatePem = secret.CertificatePem;
}
if (!string.IsNullOrWhiteSpace(secret.CertificateChainPem))
{
options.Signing.CertificateChainPem = secret.CertificateChainPem;
}
}
private void ApplySecret(ScannerWebServiceOptions.ArtifactStoreOptions artifactStore, CasAccessSecret secret)
{
if (!string.IsNullOrWhiteSpace(secret.Driver))

77
src/Scanner/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptions.cs
View File

@@ -26,10 +26,15 @@ public sealed class ScannerWebServiceOptions
/// </summary>
public QueueOptions Queue { get; set; } = new();
/// <summary>
/// Object store configuration for SBOM artefacts.
/// </summary>
public ArtifactStoreOptions ArtifactStore { get; set; } = new();
/// <summary>
/// Object store configuration for SBOM artefacts.
/// </summary>
public ArtifactStoreOptions ArtifactStore { get; set; } = new();
/// <summary>
/// Registry credential configuration for report/export operations.
/// </summary>
public RegistryOptions Registry { get; set; } = new();
/// <summary>
/// Feature flags toggling optional behaviours.
@@ -144,11 +149,11 @@ public sealed class ScannerWebServiceOptions
public IDictionary<string, string> Headers { get; set; } = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
}
public sealed class FeatureFlagOptions
{
public bool AllowAnonymousScanSubmission { get; set; }
public bool EnableSignedReports { get; set; } = true;
public sealed class FeatureFlagOptions
{
public bool AllowAnonymousScanSubmission { get; set; }
public bool EnableSignedReports { get; set; } = true;
public bool EnablePolicyPreview { get; set; } = true;
@@ -233,11 +238,11 @@ public sealed class ScannerWebServiceOptions
}
}
public sealed class SigningOptions
{
public bool Enabled { get; set; } = false;
public string KeyId { get; set; } = string.Empty;
public sealed class SigningOptions
{
public bool Enabled { get; set; } = false;
public string KeyId { get; set; } = string.Empty;
public string Algorithm { get; set; } = "ed25519";
@@ -251,12 +256,44 @@ public sealed class ScannerWebServiceOptions
public string? CertificatePemFile { get; set; }
public string? CertificateChainPem { get; set; }
public string? CertificateChainPemFile { get; set; }
public int EnvelopeTtlSeconds { get; set; } = 600;
}
public string? CertificateChainPem { get; set; }
public string? CertificateChainPemFile { get; set; }
public int EnvelopeTtlSeconds { get; set; } = 600;
}
public sealed class RegistryOptions
{
public string? DefaultRegistry { get; set; }
public IList<RegistryCredentialOptions> Credentials { get; set; } = new List<RegistryCredentialOptions>();
}
public sealed class RegistryCredentialOptions
{
public string Registry { get; set; } = string.Empty;
public string? Username { get; set; }
public string? Password { get; set; }
public string? IdentityToken { get; set; }
public string? RegistryToken { get; set; }
public string? RefreshToken { get; set; }
public DateTimeOffset? ExpiresAt { get; set; }
public IList<string> Scopes { get; set; } = new List<string>();
public bool? AllowInsecureTls { get; set; }
public IDictionary<string, string> Headers { get; set; } = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
public string? Email { get; set; }
}
public sealed class ApiOptions
{

10
src/Scanner/StellaOps.Scanner.WebService/Services/SurfacePointerService.cs
View File

@@ -179,6 +179,10 @@ internal sealed class SurfacePointerService : ISurfacePointerService
ArtifactDocumentFormat.SpdxJson => "spdx-json",
ArtifactDocumentFormat.BomIndex => "bom-index",
ArtifactDocumentFormat.DsseJson => "dsse-json",
ArtifactDocumentFormat.SurfaceManifestJson => "surface.manifest",
ArtifactDocumentFormat.EntryTraceGraphJson => "entrytrace.graph",
ArtifactDocumentFormat.EntryTraceNdjson => "entrytrace.ndjson",
ArtifactDocumentFormat.ComponentFragmentJson => "layer.fragments",
_ => format.ToString().ToLowerInvariant()
};
@@ -199,6 +203,12 @@ internal sealed class SurfacePointerService : ISurfacePointerService
ArtifactDocumentType.Diff => ("diff", null),
ArtifactDocumentType.Attestation => ("attestation", null),
ArtifactDocumentType.Index => ("bom-index", null),
ArtifactDocumentType.SurfaceManifest => ("surface.manifest", null),
ArtifactDocumentType.SurfaceEntryTrace when document.Format == ArtifactDocumentFormat.EntryTraceGraphJson
=> ("entrytrace.graph", null),
ArtifactDocumentType.SurfaceEntryTrace when document.Format == ArtifactDocumentFormat.EntryTraceNdjson
=> ("entrytrace.ndjson", null),
ArtifactDocumentType.SurfaceLayerFragment => ("layer.fragments", "inventory"),
_ => (document.Type.ToString().ToLowerInvariant(), null)
};
}

2
src/Scanner/StellaOps.Scanner.WebService/TASKS.md
View File

@@ -6,7 +6,7 @@
| SCANNER-SURFACE-02 | DONE (2025-11-05) | Scanner WebService Guild | SURFACE-FS-02 | Publish Surface.FS pointers (CAS URIs, manifests) via scan/report APIs and update attestation metadata.<br>2025-11-05: Surface pointers projected through scan/report endpoints, orchestrator samples + DSSE fixtures refreshed with manifest block, readiness tests updated to use validator stub. | OpenAPI updated; clients regenerated; integration tests validate pointer presence and tenancy. |
| SCANNER-ENV-02 | TODO (2025-11-06) | Scanner WebService Guild, Ops Guild | SURFACE-ENV-02 | Wire Surface.Env helpers into WebService hosting (cache roots, feature flags) and document configuration.<br>2025-11-02: Cache root resolution switched to helper; feature flag bindings updated; Helm/Compose updates pending review.<br>2025-11-05 14:55Z: Aligning readiness checks, docs, and Helm/Compose templates with Surface.Env outputs and planning test coverage for configuration fallbacks.<br>2025-11-06 17:05Z: Surface.Env documentation/README refreshed; warning catalogue captured for ops handoff.<br>2025-11-06 07:45Z: Helm values (dev/stage/prod/airgap/mirror) and Compose examples updated with `SCANNER_SURFACE_*` defaults plus rollout warning note in `deploy/README.md`.<br>2025-11-06 07:55Z: Paused; follow-up automation captured under `DEVOPS-OPENSSL-11-001/002` and pending Surface.Env readiness tests. | Service uses helper; env table documented; helm/compose templates updated. |
> 2025-11-05 19:18Z: Added configurator to project wiring and unit test ensuring Surface.Env cache root is honoured.
| SCANNER-SECRETS-02 | DOING (2025-11-06) | Scanner WebService Guild, Security Guild | SURFACE-SECRETS-02 | Replace ad-hoc secret wiring with Surface.Secrets for report/export operations (registry and CAS tokens).<br>2025-11-02: Export/report flows now depend on Surface.Secrets stub; integration tests in progress.<br>2025-11-06: Restarting work to eliminate file-based secrets, plumb provider handles through report/export services, and extend failure/rotation tests.<br>2025-11-06 21:40Z: Added configurator + storage post-config to hydrate artifact/CAS credentials from `cas-access` secrets with unit coverage. | Secrets fetched through shared provider; unit/integration tests cover rotation + failure cases. |
| SCANNER-SECRETS-02 | DONE (2025-11-06) | Scanner WebService Guild, Security Guild | SURFACE-SECRETS-02 | Replace ad-hoc secret wiring with Surface.Secrets for report/export operations (registry and CAS tokens).<br>2025-11-02: Export/report flows now depend on Surface.Secrets stub; integration tests in progress.<br>2025-11-06: Restarting work to eliminate file-based secrets, plumb provider handles through report/export services, and extend failure/rotation tests.<br>2025-11-06 21:40Z: Added configurator + storage post-config to hydrate artifact/CAS credentials from `cas-access` secrets with unit coverage.<br>2025-11-06 23:58Z: Registry & attestation secrets now resolved via Surface.Secrets (options + tests updated); dotnet test suites executed with .NET 10 RC2 runtime where available. | Secrets fetched through shared provider; unit/integration tests cover rotation + failure cases. |
| SCANNER-EVENTS-16-301 | BLOCKED (2025-10-26) | Scanner WebService Guild | ORCH-SVC-38-101, NOTIFY-SVC-38-001 | Emit orchestrator-compatible envelopes (`scanner.event.*`) and update integration tests to verify Notifier ingestion (no Redis queue coupling). | Tests assert envelope schema + orchestrator publish; Notifier consumer harness passes; docs updated with new event contract. Blocked by .NET 10 preview OpenAPI/Auth dependency drift preventing `dotnet test` completion. |
| SCANNER-EVENTS-16-302 | DONE (2025-11-06) | Scanner WebService Guild | SCANNER-EVENTS-16-301 | Extend orchestrator event links (report/policy/attestation) once endpoints are finalised across gateway + console.<br>2025-11-06 22:55Z: Dispatcher now honours configurable API/console base segments, JSON samples/docs refreshed, and `ReportEventDispatcherTests` extended. Tests: `StellaOps.Scanner.WebService.Tests` build until pre-existing `SurfaceCacheOptionsConfiguratorTests` ctor signature drift (tracked separately). | Links section covers UI/API targets; downstream consumers validated; docs/samples updated. |

227
src/Scanner/StellaOps.Scanner.Worker/Diagnostics/ScannerWorkerMetrics.cs
View File

@@ -1,7 +1,8 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
using StellaOps.Scanner.Worker.Processing;
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
using StellaOps.Scanner.Surface.Secrets;
using StellaOps.Scanner.Worker.Processing;
namespace StellaOps.Scanner.Worker.Diagnostics;
@@ -9,11 +10,18 @@ public sealed class ScannerWorkerMetrics
{
private readonly Histogram<double> _queueLatencyMs;
private readonly Histogram<double> _jobDurationMs;
private readonly Histogram<double> _stageDurationMs;
private readonly Histogram<double> _stageDurationMs;
private readonly Counter<long> _jobsCompleted;
private readonly Counter<long> _jobsFailed;
private readonly Counter<long> _languageCacheHits;
private readonly Counter<long> _languageCacheMisses;
private readonly Counter<long> _registrySecretRequests;
private readonly Histogram<double> _registrySecretTtlSeconds;
private readonly Counter<long> _surfaceManifestsPublished;
private readonly Counter<long> _surfaceManifestSkipped;
private readonly Counter<long> _surfaceManifestFailures;
private readonly Counter<long> _surfacePayloadPersisted;
private readonly Histogram<double> _surfaceManifestPublishDurationMs;
public ScannerWorkerMetrics()
{
@@ -41,6 +49,29 @@ public sealed class ScannerWorkerMetrics
_languageCacheMisses = ScannerWorkerInstrumentation.Meter.CreateCounter<long>(
"scanner_worker_language_cache_misses_total",
description: "Number of language analyzer cache misses encountered by the worker.");
_registrySecretRequests = ScannerWorkerInstrumentation.Meter.CreateCounter<long>(
"scanner_worker_registry_secret_requests_total",
description: "Number of registry secret resolution attempts performed by the worker.");
_registrySecretTtlSeconds = ScannerWorkerInstrumentation.Meter.CreateHistogram<double>(
"scanner_worker_registry_secret_ttl_seconds",
unit: "s",
description: "Time-to-live in seconds for resolved registry secrets (earliest expiration).");
_surfaceManifestsPublished = ScannerWorkerInstrumentation.Meter.CreateCounter<long>(
"scanner_worker_surface_manifests_published_total",
description: "Number of surface manifests successfully published by the worker.");
_surfaceManifestSkipped = ScannerWorkerInstrumentation.Meter.CreateCounter<long>(
"scanner_worker_surface_manifests_skipped_total",
description: "Number of surface manifest publish attempts skipped due to missing payloads.");
_surfaceManifestFailures = ScannerWorkerInstrumentation.Meter.CreateCounter<long>(
"scanner_worker_surface_manifests_failed_total",
description: "Number of surface manifest publish attempts that failed.");
_surfacePayloadPersisted = ScannerWorkerInstrumentation.Meter.CreateCounter<long>(
"scanner_worker_surface_payload_persisted_total",
description: "Number of surface payload artefacts persisted to the local cache.");
_surfaceManifestPublishDurationMs = ScannerWorkerInstrumentation.Meter.CreateHistogram<double>(
"scanner_worker_surface_manifest_publish_duration_ms",
unit: "ms",
description: "Duration in milliseconds to persist and publish surface manifests.");
}
public void RecordQueueLatency(ScanJobContext context, TimeSpan latency)
@@ -63,15 +94,15 @@ public sealed class ScannerWorkerMetrics
_jobDurationMs.Record(duration.TotalMilliseconds, CreateTags(context));
}
public void RecordStageDuration(ScanJobContext context, string stage, TimeSpan duration)
{
if (duration <= TimeSpan.Zero)
{
return;
}
_stageDurationMs.Record(duration.TotalMilliseconds, CreateTags(context, stage: stage));
}
public void RecordStageDuration(ScanJobContext context, string stage, TimeSpan duration)
{
if (duration <= TimeSpan.Zero)
{
return;
}
_stageDurationMs.Record(duration.TotalMilliseconds, CreateTags(context, stage: stage));
}
public void IncrementJobCompleted(ScanJobContext context)
{
@@ -93,9 +124,130 @@ public sealed class ScannerWorkerMetrics
_languageCacheMisses.Add(1, CreateTags(context, analyzerId: analyzerId));
}
private static KeyValuePair<string, object?>[] CreateTags(ScanJobContext context, string? stage = null, string? failureReason = null, string? analyzerId = null)
public void RecordRegistrySecretResolved(
ScanJobContext context,
string secretName,
RegistryAccessSecret secret,
TimeProvider timeProvider)
{
var tags = new List<KeyValuePair<string, object?>>(stage is null ? 5 : 6)
var tags = CreateTags(
context,
secretName: secretName,
secretResult: "resolved",
secretEntryCount: secret.Entries.Count);
_registrySecretRequests.Add(1, tags);
if (ComputeTtlSeconds(secret, timeProvider) is double ttlSeconds)
{
_registrySecretTtlSeconds.Record(ttlSeconds, tags);
}
}
public void RecordRegistrySecretMissing(ScanJobContext context, string secretName)
{
var tags = CreateTags(context, secretName: secretName, secretResult: "missing");
_registrySecretRequests.Add(1, tags);
}
public void RecordRegistrySecretFailure(ScanJobContext context, string secretName)
{
var tags = CreateTags(context, secretName: secretName, secretResult: "failure");
_registrySecretRequests.Add(1, tags);
}
public void RecordSurfaceManifestPublished(ScanJobContext context, int payloadCount, TimeSpan duration)
{
if (payloadCount < 0)
{
payloadCount = 0;
}
var tags = CreateTags(
context,
surfaceAction: "manifest",
surfaceResult: "published",
surfacePayloadCount: payloadCount);
_surfaceManifestsPublished.Add(1, tags);
if (duration > TimeSpan.Zero)
{
_surfaceManifestPublishDurationMs.Record(duration.TotalMilliseconds, tags);
}
}
public void RecordSurfaceManifestSkipped(ScanJobContext context)
{
var tags = CreateTags(context, surfaceAction: "manifest", surfaceResult: "skipped");
_surfaceManifestSkipped.Add(1, tags);
}
public void RecordSurfaceManifestFailed(ScanJobContext context, string failureReason)
{
var tags = CreateTags(
context,
surfaceAction: "manifest",
surfaceResult: "failed",
failureReason: failureReason);
_surfaceManifestFailures.Add(1, tags);
}
public void RecordSurfacePayloadPersisted(ScanJobContext context, string surfaceKind)
{
var normalizedKind = string.IsNullOrWhiteSpace(surfaceKind)
? "unknown"
: surfaceKind.Trim().ToLowerInvariant();
var tags = CreateTags(
context,
surfaceAction: "payload",
surfaceKind: normalizedKind,
surfaceResult: "cached");
_surfacePayloadPersisted.Add(1, tags);
}
private static double? ComputeTtlSeconds(RegistryAccessSecret secret, TimeProvider timeProvider)
{
DateTimeOffset? earliest = null;
foreach (var entry in secret.Entries)
{
if (entry.ExpiresAt is null)
{
continue;
}
if (earliest is null || entry.ExpiresAt < earliest)
{
earliest = entry.ExpiresAt;
}
}
if (earliest is null)
{
return null;
}
var now = timeProvider.GetUtcNow();
var ttl = (earliest.Value - now).TotalSeconds;
return ttl < 0 ? 0 : ttl;
}
private static KeyValuePair<string, object?>[] CreateTags(
ScanJobContext context,
string? stage = null,
string? failureReason = null,
string? analyzerId = null,
string? secretName = null,
string? secretResult = null,
int? secretEntryCount = null,
string? surfaceAction = null,
string? surfaceKind = null,
string? surfaceResult = null,
int? surfacePayloadCount = null)
{
var tags = new List<KeyValuePair<string, object?>>(8)
{
new("job.id", context.JobId),
new("scan.id", context.ScanId),
@@ -113,10 +265,10 @@ public sealed class ScannerWorkerMetrics
}
if (!string.IsNullOrWhiteSpace(stage))
{
tags.Add(new KeyValuePair<string, object?>("stage", stage));
}
{
tags.Add(new KeyValuePair<string, object?>("stage", stage));
}
if (!string.IsNullOrWhiteSpace(failureReason))
{
tags.Add(new KeyValuePair<string, object?>("reason", failureReason));
@@ -127,6 +279,41 @@ public sealed class ScannerWorkerMetrics
tags.Add(new KeyValuePair<string, object?>("analyzer.id", analyzerId));
}
if (!string.IsNullOrWhiteSpace(secretName))
{
tags.Add(new KeyValuePair<string, object?>("secret.name", secretName));
}
if (!string.IsNullOrWhiteSpace(secretResult))
{
tags.Add(new KeyValuePair<string, object?>("secret.result", secretResult));
}
if (secretEntryCount is not null)
{
tags.Add(new KeyValuePair<string, object?>("secret.entries", secretEntryCount.Value));
}
if (!string.IsNullOrWhiteSpace(surfaceAction))
{
tags.Add(new KeyValuePair<string, object?>("surface.action", surfaceAction));
}
if (!string.IsNullOrWhiteSpace(surfaceKind))
{
tags.Add(new KeyValuePair<string, object?>("surface.kind", surfaceKind));
}
if (!string.IsNullOrWhiteSpace(surfaceResult))
{
tags.Add(new KeyValuePair<string, object?>("surface.result", surfaceResult));
}
if (surfacePayloadCount is not null)
{
tags.Add(new KeyValuePair<string, object?>("surface.payload_count", surfacePayloadCount.Value));
}
return tags.ToArray();
}
}

108
src/Scanner/StellaOps.Scanner.Worker/Processing/RegistrySecretStageExecutor.cs Normal file
View File

@@ -0,0 +1,108 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.Surface.Env;
using StellaOps.Scanner.Surface.Secrets;
using StellaOps.Scanner.Worker.Diagnostics;
namespace StellaOps.Scanner.Worker.Processing;
internal sealed class RegistrySecretStageExecutor : IScanStageExecutor
{
private const string ComponentName = "Scanner.Worker.Registry";
private const string SecretType = "registry";
private static readonly string[] SecretNameMetadataKeys =
{
"surface.registry.secret",
"scanner.registry.secret",
"registry.secret",
};
private readonly ISurfaceSecretProvider _secretProvider;
private readonly ISurfaceEnvironment _surfaceEnvironment;
private readonly ScannerWorkerMetrics _metrics;
private readonly TimeProvider _timeProvider;
private readonly ILogger<RegistrySecretStageExecutor> _logger;
public RegistrySecretStageExecutor(
ISurfaceSecretProvider secretProvider,
ISurfaceEnvironment surfaceEnvironment,
ScannerWorkerMetrics metrics,
TimeProvider timeProvider,
ILogger<RegistrySecretStageExecutor> logger)
{
_secretProvider = secretProvider ?? throw new ArgumentNullException(nameof(secretProvider));
_surfaceEnvironment = surfaceEnvironment ?? throw new ArgumentNullException(nameof(surfaceEnvironment));
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public string StageName => ScanStageNames.ResolveImage;
public async ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
var secretName = ResolveSecretName(context.Lease.Metadata);
var request = new SurfaceSecretRequest(
Tenant: _surfaceEnvironment.Settings.Secrets.Tenant,
Component: ComponentName,
SecretType: SecretType,
Name: secretName);
try
{
using var handle = await _secretProvider.GetAsync(request, cancellationToken).ConfigureAwait(false);
var secret = SurfaceSecretParser.ParseRegistryAccessSecret(handle);
context.Analysis.Set(ScanAnalysisKeys.RegistryCredentials, secret);
_metrics.RecordRegistrySecretResolved(
context,
secretName ?? "default",
secret,
_timeProvider);
_logger.LogInformation(
"Registry secret '{SecretName}' resolved with {EntryCount} entries for job {JobId}.",
secretName ?? "default",
secret.Entries.Count,
context.JobId);
}
catch (SurfaceSecretNotFoundException)
{
_metrics.RecordRegistrySecretMissing(context, secretName ?? "default");
_logger.LogDebug(
"Registry secret '{SecretName}' not found for job {JobId}; continuing without registry credentials.",
secretName ?? "default",
context.JobId);
}
catch (Exception ex)
{
_metrics.RecordRegistrySecretFailure(context, secretName ?? "default");
_logger.LogWarning(
ex,
"Failed to resolve registry secret '{SecretName}' for job {JobId}; continuing without registry credentials.",
secretName ?? "default",
context.JobId);
}
}
private static string? ResolveSecretName(IReadOnlyDictionary<string, string> metadata)
{
foreach (var key in SecretNameMetadataKeys)
{
if (metadata.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
{
return value.Trim();
}
}
return null;
}
}

8
src/Scanner/StellaOps.Scanner.Worker/Processing/Surface/SurfaceManifestPublisher.cs
View File

@@ -4,6 +4,7 @@ using System.IO;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Scanner.Core.Contracts;
@@ -37,7 +38,12 @@ internal sealed record SurfaceManifestRequest(
string? Version,
string? WorkerInstance);
internal sealed class SurfaceManifestPublisher
internal interface ISurfaceManifestPublisher
{
Task<SurfaceManifestPublishResult> PublishAsync(SurfaceManifestRequest request, CancellationToken cancellationToken);
}
internal sealed class SurfaceManifestPublisher : ISurfaceManifestPublisher
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{

166
src/Scanner/StellaOps.Scanner.Worker/Processing/Surface/SurfaceManifestStageExecutor.cs
View File

@@ -1,14 +1,20 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics;
using System.Globalization;
using System.Reflection;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.Logging;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.EntryTrace;
using StellaOps.Scanner.EntryTrace.Serialization;
using StellaOps.Scanner.Surface.Env;
using StellaOps.Scanner.Surface.FS;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Worker.Diagnostics;
namespace StellaOps.Scanner.Worker.Processing.Surface;
@@ -20,15 +26,30 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
private readonly SurfaceManifestPublisher _publisher;
private static readonly JsonSerializerOptions ManifestSerializerOptions = new(JsonSerializerDefaults.Web)
{
WriteIndented = false,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
private readonly ISurfaceManifestPublisher _publisher;
private readonly ISurfaceCache _surfaceCache;
private readonly ISurfaceEnvironment _surfaceEnvironment;
private readonly ScannerWorkerMetrics _metrics;
private readonly ILogger<SurfaceManifestStageExecutor> _logger;
private readonly string _componentVersion;
public SurfaceManifestStageExecutor(
SurfaceManifestPublisher publisher,
ISurfaceManifestPublisher publisher,
ISurfaceCache surfaceCache,
ISurfaceEnvironment surfaceEnvironment,
ScannerWorkerMetrics metrics,
ILogger<SurfaceManifestStageExecutor> logger)
{
_publisher = publisher ?? throw new ArgumentNullException(nameof(publisher));
_surfaceCache = surfaceCache ?? throw new ArgumentNullException(nameof(surfaceCache));
_surfaceEnvironment = surfaceEnvironment ?? throw new ArgumentNullException(nameof(surfaceEnvironment));
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_componentVersion = Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "unknown";
}
@@ -42,23 +63,49 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
var payloads = CollectPayloads(context);
if (payloads.Count == 0)
{
_metrics.RecordSurfaceManifestSkipped(context);
_logger.LogDebug("No surface payloads available for job {JobId}; skipping manifest publish.", context.JobId);
return;
}
var request = new SurfaceManifestRequest(
ScanId: context.ScanId,
ImageDigest: ResolveImageDigest(context),
Attempt: context.Lease.Attempt,
Metadata: context.Lease.Metadata,
Payloads: payloads,
Component: "scanner.worker",
Version: _componentVersion,
WorkerInstance: Environment.MachineName);
var tenant = _surfaceEnvironment.Settings?.Tenant ?? string.Empty;
var stopwatch = Stopwatch.StartNew();
var result = await _publisher.PublishAsync(request, cancellationToken).ConfigureAwait(false);
context.Analysis.Set(ScanAnalysisKeys.SurfaceManifest, result);
_logger.LogInformation("Surface manifest stored for job {JobId} with digest {Digest}.", context.JobId, result.ManifestDigest);
try
{
await PersistPayloadsToSurfaceCacheAsync(context, tenant, payloads, cancellationToken).ConfigureAwait(false);
var request = new SurfaceManifestRequest(
ScanId: context.ScanId,
ImageDigest: ResolveImageDigest(context),
Attempt: context.Lease.Attempt,
Metadata: context.Lease.Metadata,
Payloads: payloads,
Component: "scanner.worker",
Version: _componentVersion,
WorkerInstance: Environment.MachineName);
var result = await _publisher.PublishAsync(request, cancellationToken).ConfigureAwait(false);
await PersistManifestToSurfaceCacheAsync(context, tenant, result, cancellationToken).ConfigureAwait(false);
context.Analysis.Set(ScanAnalysisKeys.SurfaceManifest, result);
stopwatch.Stop();
_metrics.RecordSurfaceManifestPublished(context, payloads.Count, stopwatch.Elapsed);
_logger.LogInformation("Surface manifest stored for job {JobId} with digest {Digest}.", context.JobId, result.ManifestDigest);
}
catch (OperationCanceledException)
{
stopwatch.Stop();
throw;
}
catch (Exception ex)
{
stopwatch.Stop();
_metrics.RecordSurfaceManifestFailed(context, ex.GetType().Name);
_logger.LogError(ex, "Failed to persist surface manifest for job {JobId}.", context.JobId);
throw;
}
}
private List<SurfaceManifestPayload> CollectPayloads(ScanJobContext context)
@@ -118,6 +165,56 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
return payloads;
}
private async Task PersistPayloadsToSurfaceCacheAsync(
ScanJobContext context,
string tenant,
IReadOnlyList<SurfaceManifestPayload> payloads,
CancellationToken cancellationToken)
{
if (payloads.Count == 0)
{
return;
}
foreach (var payload in payloads)
{
cancellationToken.ThrowIfCancellationRequested();
var digest = ComputeDigest(payload.Content.Span);
var normalizedKind = NormalizeKind(payload.Kind);
var cacheKey = CreateArtifactCacheKey(tenant, normalizedKind, digest);
await _surfaceCache.SetAsync(cacheKey, payload.Content, cancellationToken).ConfigureAwait(false);
_logger.LogDebug(
"Cached surface payload {Kind} for job {JobId} with digest {Digest}.",
normalizedKind,
context.JobId,
digest);
_metrics.RecordSurfacePayloadPersisted(context, normalizedKind);
}
}
private async Task PersistManifestToSurfaceCacheAsync(
ScanJobContext context,
string tenant,
SurfaceManifestPublishResult result,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var manifestBytes = JsonSerializer.SerializeToUtf8Bytes(result.Document, ManifestSerializerOptions);
var cacheKey = CreateManifestCacheKey(tenant, result.ManifestDigest);
await _surfaceCache.SetAsync(cacheKey, manifestBytes, cancellationToken).ConfigureAwait(false);
_logger.LogDebug(
"Cached surface manifest for job {JobId} with digest {Digest}.",
context.JobId,
result.ManifestDigest);
}
private static string ResolveImageDigest(ScanJobContext context)
{
static bool TryGet(IReadOnlyDictionary<string, string> metadata, string key, out string value)
@@ -143,5 +240,46 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
return context.ScanId;
}
private static SurfaceCacheKey CreateArtifactCacheKey(string tenant, string kind, string digest)
{
var @namespace = $"surface.artifacts.{kind}";
var contentKey = NormalizeDigestForKey(digest);
return new SurfaceCacheKey(@namespace, tenant, contentKey);
}
private static SurfaceCacheKey CreateManifestCacheKey(string tenant, string digest)
{
var contentKey = NormalizeDigestForKey(digest);
return new SurfaceCacheKey("surface.manifests", tenant, contentKey);
}
private static string NormalizeKind(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return "unknown";
}
var trimmed = value.Trim();
return trimmed.ToLowerInvariant();
}
private static string NormalizeDigestForKey(string? digest)
{
if (string.IsNullOrWhiteSpace(digest))
{
return string.Empty;
}
return digest.Trim();
}
private static string ComputeDigest(ReadOnlySpan<byte> content)
{
Span<byte> hash = stackalloc byte[32];
SHA256.HashData(content, hash);
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
}
private static readonly IFormatProvider CultureInfoInvariant = System.Globalization.CultureInfo.InvariantCulture;
}

3
src/Scanner/StellaOps.Scanner.Worker/Program.cs
View File

@@ -55,7 +55,7 @@ if (!string.IsNullOrWhiteSpace(connectionString))
{
builder.Services.AddScannerStorage(storageSection);
builder.Services.AddSingleton<IConfigureOptions<ScannerStorageOptions>, ScannerStorageSurfaceSecretConfigurator>();
builder.Services.AddSingleton<SurfaceManifestPublisher>();
builder.Services.AddSingleton<ISurfaceManifestPublisher, SurfaceManifestPublisher>();
builder.Services.AddSingleton<IScanStageExecutor, SurfaceManifestStageExecutor>();
}
@@ -64,6 +64,7 @@ builder.Services.TryAddSingleton<IPluginCatalogGuard, RestartOnlyPluginGuard>();
builder.Services.AddSingleton<IOSAnalyzerPluginCatalog, OsAnalyzerPluginCatalog>();
builder.Services.AddSingleton<ILanguageAnalyzerPluginCatalog, LanguageAnalyzerPluginCatalog>();
builder.Services.AddSingleton<IScanAnalyzerDispatcher, CompositeScanAnalyzerDispatcher>();
builder.Services.AddSingleton<IScanStageExecutor, RegistrySecretStageExecutor>();
builder.Services.AddSingleton<IScanStageExecutor, AnalyzerStageExecutor>();
builder.Services.AddSingleton<ScannerWorkerHostedService>();

9
src/Scanner/StellaOps.Scanner.Worker/TASKS.md
View File

@@ -3,7 +3,10 @@
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|----|--------|----------|------------|-------------|---------------|
| SCAN-REPLAY-186-002 | TODO | Scanner Worker Guild | REPLAY-CORE-185-001 | Enforce deterministic analyzer execution when consuming replay input bundles, emit layer Merkle metadata, and author `docs/modules/scanner/deterministic-execution.md` summarising invariants from `docs/replay/DETERMINISTIC_REPLAY.md` Section 4. | Replay mode analyzers pass determinism tests; new doc merged; integration fixtures updated. |
| SCANNER-SURFACE-01 | DOING (2025-11-06) | Scanner Worker Guild | SURFACE-FS-02 | Persist Surface.FS manifests after analyzer stages, including layer CAS metadata and EntryTrace fragments.<br>2025-11-02: Draft Surface.FS manifests emitted for sample scans; telemetry counters under review.<br>2025-11-06: Resuming with manifest writer abstraction, rotation metadata, and telemetry counters for Surface.FS persistence. | Integration tests prove cache entries exist; telemetry counters exported. |
| SCANNER-SURFACE-01 | DONE (2025-11-06) | Scanner Worker Guild | SURFACE-FS-02 | Persist Surface.FS manifests after analyzer stages, including layer CAS metadata and EntryTrace fragments.<br>2025-11-02: Draft Surface.FS manifests emitted for sample scans; telemetry counters under review.<br>2025-11-06: Resuming with manifest writer abstraction, rotation metadata, and telemetry counters for Surface.FS persistence.<br>2025-11-06 21:05Z: Stage now persists manifest/payload caches, exports metrics to Prometheus/Grafana, and WebService pointer tests validate consumption. | Integration tests prove cache entries exist; telemetry counters exported. |
> 2025-11-05 19:18Z: Bound root directory to resolved Surface.Env settings and added unit coverage around the configurator.
> 2025-11-06 18:45Z: Resuming manifest persistence—planning publisher abstraction refactor, CAS storage wiring, and telemetry/test coverage.
> 2025-11-06 20:20Z: Hooked Surface metrics into Grafana (new dashboard JSON) and verified WebService consumption via end-to-end pointer test seeding manifest + payload entries.
> 2025-11-06 21:05Z: Completed Surface manifest cache + metrics work; tests/docs updated and task ready to close.
| SCANNER-ENV-01 | TODO (2025-11-06) | Scanner Worker Guild | SURFACE-ENV-02 | Replace ad-hoc environment reads with `StellaOps.Scanner.Surface.Env` helpers for cache roots and CAS endpoints.<br>2025-11-02: Worker bootstrap now resolves cache roots via helper; warning path documented; smoke tests running.<br>2025-11-05 14:55Z: Extending helper usage into cache/secrets configuration, updating worker validator wiring, and drafting docs/tests for new Surface.Env outputs.<br>2025-11-06 17:05Z: README/design docs updated with warning catalogue; startup logging guidance captured for ops runbooks.<br>2025-11-06 07:45Z: Helm/Compose env profiles (dev/stage/prod/airgap/mirror) now seed `SCANNER_SURFACE_*` defaults to keep worker cache roots aligned with Surface.Env helpers.<br>2025-11-06 07:55Z: Paused; pending automation tracked via `DEVOPS-OPENSSL-11-001/002` and Surface.Env test fixtures. | Worker boots with helper; misconfiguration warnings documented; smoke tests updated. |
> 2025-11-05 19:18Z: Bound `SurfaceCacheOptions` root directory to resolved Surface.Env settings and added unit coverage around the configurator.
| SCANNER-SECRETS-01 | DOING (2025-11-06) | Scanner Worker Guild, Security Guild | SURFACE-SECRETS-02 | Adopt `StellaOps.Scanner.Surface.Secrets` for registry/CAS credentials during scan execution.<br>2025-11-02: Surface.Secrets provider wired for CAS token retrieval; integration tests added.<br>2025-11-06: Continuing to replace legacy registry credential plumbing and extend rotation metrics/fixtures.<br>2025-11-06 21:35Z: Introduced `ScannerStorageSurfaceSecretConfigurator` mapping `cas-access` secrets into storage options plus unit coverage. | Secrets fetched via shared provider; legacy secret code removed; integration tests cover rotation. |
| SCANNER-SECRETS-01 | DONE (2025-11-06) | Scanner Worker Guild, Security Guild | SURFACE-SECRETS-02 | Adopt `StellaOps.Scanner.Surface.Secrets` for registry/CAS credentials during scan execution.<br>2025-11-02: Surface.Secrets provider wired for CAS token retrieval; integration tests added.<br>2025-11-06: Replaced registry credential plumbing with shared provider, added registry secret stage + metrics, and installed .NET 10 RC2 to validate parser/stage suites via targeted `dotnet test`. | Secrets fetched via shared provider; legacy secret code removed; integration tests cover rotation. |

2
src/Scanner/__Libraries/StellaOps.Scanner.Core/Contracts/ScanAnalysisKeys.cs
View File

@@ -17,4 +17,6 @@ public static class ScanAnalysisKeys
public const string EntryTraceNdjson = "analysis.entrytrace.ndjson";
public const string SurfaceManifest = "analysis.surface.manifest";
public const string RegistryCredentials = "analysis.registry.credentials";
}

53
src/Scanner/__Libraries/StellaOps.Scanner.Surface.Secrets/AttestationSecret.cs Normal file
View File

@@ -0,0 +1,53 @@
using System.Text.Json;
namespace StellaOps.Scanner.Surface.Secrets;
public sealed record AttestationSecret(
string KeyPem,
string? CertificatePem,
string? CertificateChainPem,
string? RekorApiToken);
public static partial class SurfaceSecretParser
{
public static AttestationSecret ParseAttestationSecret(SurfaceSecretHandle handle)
{
ArgumentNullException.ThrowIfNull(handle);
var payload = handle.AsBytes();
if (payload.IsEmpty)
{
throw new InvalidOperationException("Surface secret payload is empty.");
}
using var document = JsonDocument.Parse(DecodeUtf8(payload));
var root = document.RootElement;
var keyPem = GetString(root, "keyPem")
?? GetString(root, "pem")
?? GetMetadataValue(handle.Metadata, "keyPem")
?? GetMetadataValue(handle.Metadata, "pem");
if (string.IsNullOrWhiteSpace(keyPem))
{
throw new InvalidOperationException("Attestation secret must include a 'keyPem' value.");
}
var certificatePem = GetString(root, "certificatePem")
?? GetMetadataValue(handle.Metadata, "certificatePem");
var certificateChainPem = GetString(root, "certificateChainPem")
?? GetMetadataValue(handle.Metadata, "certificateChainPem");
var rekorToken = GetString(root, "rekorToken")
?? GetString(root, "rekorApiToken")
?? GetMetadataValue(handle.Metadata, "rekorToken")
?? GetMetadataValue(handle.Metadata, "rekorApiToken");
return new AttestationSecret(
keyPem.Trim(),
certificatePem?.Trim(),
certificateChainPem?.Trim(),
rekorToken?.Trim());
}
}

2
src/Scanner/__Libraries/StellaOps.Scanner.Surface.Secrets/CasAccessSecret.cs
View File

@@ -19,7 +19,7 @@ public sealed record CasAccessSecret(
string? SessionToken,
bool? AllowInsecureTls);
public static class SurfaceSecretParser
public static partial class SurfaceSecretParser
{
public static CasAccessSecret ParseCasAccessSecret(SurfaceSecretHandle handle)
{

347
src/Scanner/__Libraries/StellaOps.Scanner.Surface.Secrets/RegistryAccessSecret.cs Normal file
View File

@@ -0,0 +1,347 @@
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Globalization;
using System.Text;
using System.Text.Json;
namespace StellaOps.Scanner.Surface.Secrets;
public sealed record RegistryAccessSecret(
IReadOnlyList<RegistryCredential> Entries,
string? DefaultRegistry);
public sealed record RegistryCredential(
string Registry,
string? Username,
string? Password,
string? IdentityToken,
string? RegistryToken,
string? RefreshToken,
DateTimeOffset? ExpiresAt,
IReadOnlyCollection<string> Scopes,
bool? AllowInsecureTls,
IReadOnlyDictionary<string, string> Headers,
string? Email);
public static partial class SurfaceSecretParser
{
public static RegistryAccessSecret ParseRegistryAccessSecret(SurfaceSecretHandle handle)
{
ArgumentNullException.ThrowIfNull(handle);
var entries = new List<RegistryCredential>();
string? defaultRegistry = null;
var payload = handle.AsBytes();
if (!payload.IsEmpty)
{
var jsonText = DecodeUtf8(payload);
using var document = JsonDocument.Parse(jsonText);
var root = document.RootElement;
defaultRegistry = GetString(root, "defaultRegistry") ?? GetMetadataValue(handle.Metadata, "defaultRegistry");
if (TryParseRegistryEntries(root, handle.Metadata, entries) ||
TryParseAuthsObject(root, handle.Metadata, entries))
{
// entries already populated
}
else if (root.ValueKind == JsonValueKind.Object && root.GetRawText().Length > 2) // not empty object
{
entries.Add(ParseRegistryEntry(root, handle.Metadata, fallbackRegistry: null));
}
}
if (entries.Count == 0 && TryCreateRegistryEntryFromMetadata(handle.Metadata, out var metadataEntry))
{
entries.Add(metadataEntry);
}
if (entries.Count == 0)
{
throw new InvalidOperationException("Registry secret payload does not contain credentials.");
}
defaultRegistry ??= GetMetadataValue(handle.Metadata, "defaultRegistry")
?? entries[0].Registry;
return new RegistryAccessSecret(
new ReadOnlyCollection<RegistryCredential>(entries),
string.IsNullOrWhiteSpace(defaultRegistry) ? entries[0].Registry : defaultRegistry.Trim());
}
private static bool TryParseRegistryEntries(
JsonElement root,
IReadOnlyDictionary<string, string> metadata,
ICollection<RegistryCredential> entries)
{
if (!TryGetPropertyIgnoreCase(root, "entries", out var entriesElement) ||
entriesElement.ValueKind != JsonValueKind.Array)
{
return false;
}
foreach (var entryElement in entriesElement.EnumerateArray())
{
if (entryElement.ValueKind != JsonValueKind.Object)
{
continue;
}
entries.Add(ParseRegistryEntry(entryElement, metadata, fallbackRegistry: null));
}
return entries.Count > 0;
}
private static bool TryParseAuthsObject(
JsonElement root,
IReadOnlyDictionary<string, string> metadata,
ICollection<RegistryCredential> entries)
{
if (!TryGetPropertyIgnoreCase(root, "auths", out var authsElement) ||
authsElement.ValueKind != JsonValueKind.Object)
{
return false;
}
foreach (var property in authsElement.EnumerateObject())
{
if (property.Value.ValueKind != JsonValueKind.Object)
{
continue;
}
entries.Add(ParseRegistryEntry(property.Value, metadata, property.Name));
}
return entries.Count > 0;
}
private static RegistryCredential ParseRegistryEntry(
JsonElement element,
IReadOnlyDictionary<string, string> metadata,
string? fallbackRegistry)
{
var registry = GetString(element, "registry")
?? GetString(element, "server")
?? fallbackRegistry
?? GetMetadataValue(metadata, "registry")
?? throw new InvalidOperationException("Registry credential is missing a registry identifier.");
registry = registry.Trim();
var username = GetString(element, "username") ?? GetString(element, "user");
var password = GetString(element, "password") ?? GetString(element, "pass");
var token = GetString(element, "token") ?? GetString(element, "registryToken");
var identityToken = GetString(element, "identityToken") ?? GetString(element, "identitytoken");
var refreshToken = GetString(element, "refreshToken");
var email = GetString(element, "email");
var allowInsecure = GetBoolean(element, "allowInsecureTls");
var headers = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
PopulateHeaders(element, headers);
PopulateMetadataHeaders(metadata, headers);
var scopes = new List<string>();
PopulateScopes(element, scopes);
PopulateMetadataScopes(metadata, scopes);
var expiresAt = ParseDateTime(element, "expiresAt");
var auth = GetString(element, "auth");
if (!string.IsNullOrWhiteSpace(auth))
{
TryApplyBasicAuth(auth, ref username, ref password);
}
username ??= GetMetadataValue(metadata, "username");
password ??= GetMetadataValue(metadata, "password");
token ??= GetMetadataValue(metadata, "token") ?? GetMetadataValue(metadata, "registryToken");
identityToken ??= GetMetadataValue(metadata, "identityToken");
refreshToken ??= GetMetadataValue(metadata, "refreshToken");
email ??= GetMetadataValue(metadata, "email");
return new RegistryCredential(
registry,
username?.Trim(),
password,
identityToken,
token,
refreshToken,
expiresAt,
scopes.Count == 0 ? Array.Empty<string>() : new ReadOnlyCollection<string>(scopes),
allowInsecure,
new ReadOnlyDictionary<string, string>(headers),
email);
}
private static bool TryCreateRegistryEntryFromMetadata(
IReadOnlyDictionary<string, string> metadata,
out RegistryCredential entry)
{
var registry = GetMetadataValue(metadata, "registry");
var username = GetMetadataValue(metadata, "username");
var password = GetMetadataValue(metadata, "password");
var identityToken = GetMetadataValue(metadata, "identityToken");
var token = GetMetadataValue(metadata, "token") ?? GetMetadataValue(metadata, "registryToken");
if (string.IsNullOrWhiteSpace(registry) &&
string.IsNullOrWhiteSpace(username) &&
string.IsNullOrWhiteSpace(password) &&
string.IsNullOrWhiteSpace(identityToken) &&
string.IsNullOrWhiteSpace(token))
{
entry = null!;
return false;
}
var headers = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
PopulateMetadataHeaders(metadata, headers);
var scopes = new List<string>();
PopulateMetadataScopes(metadata, scopes);
entry = new RegistryCredential(
registry?.Trim() ?? "registry.local",
username?.Trim(),
password,
identityToken,
token,
GetMetadataValue(metadata, "refreshToken"),
ParseDateTime(metadata, "expiresAt"),
scopes.Count == 0 ? Array.Empty<string>() : new ReadOnlyCollection<string>(scopes),
ParseBoolean(metadata, "allowInsecureTls"),
new ReadOnlyDictionary<string, string>(headers),
GetMetadataValue(metadata, "email"));
return true;
}
private static void PopulateScopes(JsonElement element, ICollection<string> scopes)
{
if (!TryGetPropertyIgnoreCase(element, "scopes", out var scopesElement))
{
return;
}
switch (scopesElement.ValueKind)
{
case JsonValueKind.Array:
foreach (var scope in scopesElement.EnumerateArray())
{
if (scope.ValueKind == JsonValueKind.String)
{
var value = scope.GetString();
if (!string.IsNullOrWhiteSpace(value))
{
scopes.Add(value.Trim());
}
}
}
break;
case JsonValueKind.String:
var text = scopesElement.GetString();
if (!string.IsNullOrWhiteSpace(text))
{
foreach (var part in text.Split(new[] { ',', ' ' }, StringSplitOptions.RemoveEmptyEntries))
{
scopes.Add(part.Trim());
}
}
break;
}
}
private static void PopulateMetadataScopes(IReadOnlyDictionary<string, string> metadata, ICollection<string> scopes)
{
foreach (var (key, value) in metadata)
{
if (!key.StartsWith("scope", StringComparison.OrdinalIgnoreCase))
{
continue;
}
if (string.IsNullOrWhiteSpace(value))
{
continue;
}
scopes.Add(value.Trim());
}
}
private static void TryApplyBasicAuth(string auth, ref string? username, ref string? password)
{
try
{
var decoded = Encoding.UTF8.GetString(Convert.FromBase64String(auth));
var separator = decoded.IndexOf(':');
if (separator >= 0)
{
username ??= decoded[..separator];
password ??= decoded[(separator + 1)..];
}
}
catch (FormatException)
{
// ignore malformed auth; caller may still have explicit username/password fields
}
}
private static DateTimeOffset? ParseDateTime(JsonElement element, string propertyName)
{
if (!TryGetPropertyIgnoreCase(element, propertyName, out var value) ||
value.ValueKind != JsonValueKind.String)
{
return null;
}
var text = value.GetString();
if (string.IsNullOrWhiteSpace(text))
{
return null;
}
if (DateTimeOffset.TryParse(text, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed))
{
return parsed;
}
return null;
}
private static DateTimeOffset? ParseDateTime(IReadOnlyDictionary<string, string> metadata, string key)
{
var value = GetMetadataValue(metadata, key);
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed))
{
return parsed;
}
return null;
}
private static bool? ParseBoolean(IReadOnlyDictionary<string, string> metadata, string key)
{
var value = GetMetadataValue(metadata, key);
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
if (bool.TryParse(value, out var parsed))
{
return parsed;
}
return null;
}
}

122
src/Scanner/__Tests/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/Descriptor/DescriptorCommandSurfaceTests.cs Normal file
View File

@@ -0,0 +1,122 @@
using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Threading.Tasks;
using StellaOps.Scanner.Sbomer.BuildXPlugin.Descriptor;
using StellaOps.Scanner.Sbomer.BuildXPlugin.Manifest;
using StellaOps.Scanner.Sbomer.BuildXPlugin.Tests.TestUtilities;
using StellaOps.Scanner.Surface.FS;
using Xunit;
namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Tests.Descriptor;
public sealed class DescriptorCommandSurfaceTests
{
[Fact]
public async Task DescriptorCommand_PublishesSurfaceArtifacts()
{
await using var temp = new TempDirectory();
var casRoot = Path.Combine(temp.Path, "cas");
Directory.CreateDirectory(casRoot);
var sbomPath = Path.Combine(temp.Path, "sample.cdx.json");
await File.WriteAllTextAsync(sbomPath, "{\"bomFormat\":\"CycloneDX\",\"specVersion\":\"1.6\"}");
var layerFragmentsPath = Path.Combine(temp.Path, "layer-fragments.json");
await File.WriteAllTextAsync(layerFragmentsPath, "[]");
var entryTraceGraphPath = Path.Combine(temp.Path, "entrytrace-graph.json");
await File.WriteAllTextAsync(entryTraceGraphPath, "{\"nodes\":[],\"edges\":[]}");
var entryTraceNdjsonPath = Path.Combine(temp.Path, "entrytrace.ndjson");
await File.WriteAllTextAsync(entryTraceNdjsonPath, "{}\n{}");
var manifestOutputPath = Path.Combine(temp.Path, "out", "surface-manifest.json");
var repoRoot = TestPathHelper.FindRepositoryRoot();
var manifestDirectory = Path.Combine(repoRoot, "src", "Scanner", "StellaOps.Scanner.Sbomer.BuildXPlugin");
var pluginAssembly = typeof(BuildxPluginManifest).Assembly.Location;
var psi = new ProcessStartInfo("dotnet")
{
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
WorkingDirectory = repoRoot
};
psi.ArgumentList.Add(pluginAssembly);
psi.ArgumentList.Add("descriptor");
psi.ArgumentList.Add("--manifest");
psi.ArgumentList.Add(manifestDirectory);
psi.ArgumentList.Add("--cas");
psi.ArgumentList.Add(casRoot);
psi.ArgumentList.Add("--image");
psi.ArgumentList.Add("sha256:feedfacefeedfacefeedfacefeedfacefeedfacefeedfacefeedfacefeedface");
psi.ArgumentList.Add("--sbom");
psi.ArgumentList.Add(sbomPath);
psi.ArgumentList.Add("--sbom-name");
psi.ArgumentList.Add("sample.cdx.json");
psi.ArgumentList.Add("--surface-layer-fragments");
psi.ArgumentList.Add(layerFragmentsPath);
psi.ArgumentList.Add("--surface-entrytrace-graph");
psi.ArgumentList.Add(entryTraceGraphPath);
psi.ArgumentList.Add("--surface-entrytrace-ndjson");
psi.ArgumentList.Add(entryTraceNdjsonPath);
psi.ArgumentList.Add("--surface-cache-root");
psi.ArgumentList.Add(casRoot);
psi.ArgumentList.Add("--surface-tenant");
psi.ArgumentList.Add("test-tenant");
psi.ArgumentList.Add("--surface-manifest-output");
psi.ArgumentList.Add(manifestOutputPath);
var process = Process.Start(psi) ?? throw new InvalidOperationException("Failed to start BuildX plug-in process.");
var stdout = await process.StandardOutput.ReadToEndAsync();
var stderr = await process.StandardError.ReadToEndAsync();
await process.WaitForExitAsync();
Assert.True(process.ExitCode == 0, $"Descriptor command failed.\nSTDOUT: {stdout}\nSTDERR: {stderr}");
var descriptor = JsonSerializer.Deserialize<DescriptorDocument>(stdout, new JsonSerializerOptions(JsonSerializerDefaults.Web));
Assert.NotNull(descriptor);
Assert.Equal("stellaops.buildx.descriptor.v1", descriptor!.Schema);
Assert.Equal("sha256:d07d06ae82e1789a5b505731f3ec3add106e23a55395213c9a881c7e816c695c", descriptor.Artifact.Digest);
Assert.Contains("surface manifest stored", stderr, StringComparison.OrdinalIgnoreCase);
Assert.True(File.Exists(manifestOutputPath));
var surfaceManifestPath = Directory.GetFiles(Path.Combine(casRoot, "scanner", "surface", "manifests"), "*.json", SearchOption.AllDirectories).Single();
var manifestDocument = JsonSerializer.Deserialize<SurfaceManifestDocument>(await File.ReadAllTextAsync(surfaceManifestPath), new JsonSerializerOptions(JsonSerializerDefaults.Web));
Assert.NotNull(manifestDocument);
Assert.Equal("test-tenant", manifestDocument!.Tenant);
Assert.Equal(3, manifestDocument.Artifacts.Count);
foreach (var artifact in manifestDocument.Artifacts)
{
Assert.StartsWith("cas://", artifact.Uri, StringComparison.Ordinal);
var localPath = ResolveLocalPath(artifact.Uri, casRoot);
Assert.True(File.Exists(localPath), $"Missing CAS object for {artifact.Uri}");
}
}
private static string ResolveLocalPath(string casUri, string casRoot)
{
const string prefix = "cas://";
if (!casUri.StartsWith(prefix, StringComparison.Ordinal))
{
throw new InvalidOperationException($"Unsupported CAS URI {casUri}.");
}
var slashIndex = casUri.IndexOf(/, prefix.Length);
if (slashIndex < 0)
{
throw new InvalidOperationException($"CAS URI {casUri} does not contain a bucket path.");
}
var relative = casUri[(slashIndex + 1)..];
var localPath = Path.Combine(casRoot, relative.Replace(/, Path.DirectorySeparatorChar));
return localPath;
}
}

88
src/Scanner/__Tests/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/Fixtures/descriptor.baseline.json
View File

@@ -1,45 +1,45 @@
{
"schema": "stellaops.buildx.descriptor.v1",
"generatedAt": "2025-10-18T12:00:00\u002B00:00",
"generator": {
"name": "StellaOps.Scanner.Sbomer.BuildXPlugin",
"version": "1.2.3"
},
"subject": {
"mediaType": "application/vnd.oci.image.manifest.v1\u002Bjson",
"digest": "sha256:0123456789abcdef"
},
"artifact": {
"mediaType": "application/vnd.cyclonedx\u002Bjson",
"digest": "sha256:d07d06ae82e1789a5b505731f3ec3add106e23a55395213c9a881c7e816c695c",
"size": 45,
"annotations": {
"org.opencontainers.artifact.type": "application/vnd.stellaops.sbom.layer\u002Bjson",
"org.stellaops.scanner.version": "1.2.3",
"org.stellaops.sbom.kind": "inventory",
"org.stellaops.sbom.format": "cyclonedx-json",
"org.stellaops.provenance.status": "pending",
"org.stellaops.provenance.dsse.sha256": "sha256:1b364a6b888d580feb8565f7b6195b24535ca8201b4bcac58da063b32c47220d",
"org.stellaops.provenance.nonce": "a608acf859cd58a8389816b8d9eb2a07",
"org.stellaops.license.id": "lic-123",
"org.opencontainers.image.title": "sample.cdx.json",
"org.stellaops.repository": "git.stella-ops.org/stellaops"
}
},
"provenance": {
"status": "pending",
"expectedDsseSha256": "sha256:1b364a6b888d580feb8565f7b6195b24535ca8201b4bcac58da063b32c47220d",
"nonce": "a608acf859cd58a8389816b8d9eb2a07",
"attestorUri": "https://attestor.local/api/v1/provenance",
"predicateType": "https://slsa.dev/provenance/v1"
},
"metadata": {
"sbomDigest": "sha256:d07d06ae82e1789a5b505731f3ec3add106e23a55395213c9a881c7e816c695c",
"sbomPath": "sample.cdx.json",
"sbomMediaType": "application/vnd.cyclonedx\u002Bjson",
"subjectMediaType": "application/vnd.oci.image.manifest.v1\u002Bjson",
"repository": "git.stella-ops.org/stellaops",
"buildRef": "refs/heads/main",
"attestorUri": "https://attestor.local/api/v1/provenance"
}
{
"schema": "stellaops.buildx.descriptor.v1",
"generatedAt": "2025-10-18T12:00:00\u002B00:00",
"generator": {
"name": "StellaOps.Scanner.Sbomer.BuildXPlugin",
"version": "1.2.3"
},
"subject": {
"mediaType": "application/vnd.oci.image.manifest.v1\u002Bjson",
"digest": "sha256:0123456789abcdef"
},
"artifact": {
"mediaType": "application/vnd.cyclonedx\u002Bjson",
"digest": "sha256:d07d06ae82e1789a5b505731f3ec3add106e23a55395213c9a881c7e816c695c",
"size": 45,
"annotations": {
"org.opencontainers.artifact.type": "application/vnd.stellaops.sbom.layer\u002Bjson",
"org.stellaops.scanner.version": "1.2.3",
"org.stellaops.sbom.kind": "inventory",
"org.stellaops.sbom.format": "cyclonedx-json",
"org.stellaops.provenance.status": "pending",
"org.stellaops.provenance.dsse.sha256": "sha256:35ab4784f3bad40bb0063b522939ac729cf43d2012059947c0e56475d682c05e",
"org.stellaops.provenance.nonce": "5e13230e3dcbc8be996d8132d92e8826",
"org.stellaops.license.id": "lic-123",
"org.opencontainers.image.title": "sample.cdx.json",
"org.stellaops.repository": "git.stella-ops.org/stellaops"
}
},
"provenance": {
"status": "pending",
"expectedDsseSha256": "sha256:35ab4784f3bad40bb0063b522939ac729cf43d2012059947c0e56475d682c05e",
"nonce": "5e13230e3dcbc8be996d8132d92e8826",
"attestorUri": "https://attestor.local/api/v1/provenance",
"predicateType": "https://slsa.dev/provenance/v1"
},
"metadata": {
"sbomDigest": "sha256:d07d06ae82e1789a5b505731f3ec3add106e23a55395213c9a881c7e816c695c",
"sbomPath": "sample.cdx.json",
"sbomMediaType": "application/vnd.cyclonedx\u002Bjson",
"subjectMediaType": "application/vnd.oci.image.manifest.v1\u002Bjson",
"repository": "git.stella-ops.org/stellaops",
"buildRef": "refs/heads/main",
"attestorUri": "https://attestor.local/api/v1/provenance"
}
}

95
src/Scanner/__Tests/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/Surface/SurfaceManifestWriterTests.cs Normal file
View File

@@ -0,0 +1,95 @@
using System;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Scanner.Sbomer.BuildXPlugin.Surface;
using StellaOps.Scanner.Sbomer.BuildXPlugin.Tests.TestUtilities;
using Xunit;
namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Tests.Surface;
public sealed class SurfaceManifestWriterTests
{
[Fact]
public async Task WriteAsync_PersistsArtifactsAndManifest()
{
await using var temp = new TempDirectory();
var fragmentsPath = Path.Combine(temp.Path, "layer-fragments.json");
await File.WriteAllTextAsync(fragmentsPath, "[]");
var graphPath = Path.Combine(temp.Path, "entrytrace-graph.json");
await File.WriteAllTextAsync(graphPath, "{\"nodes\":[],\"edges\":[]}");
var ndjsonPath = Path.Combine(temp.Path, "entrytrace.ndjson");
await File.WriteAllTextAsync(ndjsonPath, "{}\n{}");
var manifestOutputPath = Path.Combine(temp.Path, "out", "surface-manifest.json");
var options = new SurfaceOptions(
CacheRoot: temp.Path,
CacheBucket: "scanner-artifacts",
RootPrefix: "scanner",
Tenant: "tenant-a",
Component: "scanner.buildx",
ComponentVersion: "1.2.3",
WorkerInstance: "builder-01",
Attempt: 2,
ImageDigest: "sha256:feedface",
ScanId: "scan-123",
LayerFragmentsPath: fragmentsPath,
EntryTraceGraphPath: graphPath,
EntryTraceNdjsonPath: ndjsonPath,
ManifestOutputPath: manifestOutputPath);
var writer = new SurfaceManifestWriter(TimeProvider.System);
var result = await writer.WriteAsync(options, CancellationToken.None);
Assert.NotNull(result);
Assert.NotNull(result!.Document.Source);
Assert.Equal("tenant-a", result.Document.Tenant);
Assert.Equal("scanner.buildx", result.Document.Source!.Component);
Assert.Equal("1.2.3", result.Document.Source.Version);
Assert.Equal(3, result.Document.Artifacts.Count);
var kinds = result.Document.Artifacts.Select(a => a.Kind).ToHashSet();
Assert.Contains("entrytrace.graph", kinds);
Assert.Contains("entrytrace.ndjson", kinds);
Assert.Contains("layer.fragments", kinds);
Assert.True(File.Exists(result.ManifestPath));
Assert.True(File.Exists(manifestOutputPath));
foreach (var artifact in result.Artifacts)
{
Assert.True(File.Exists(artifact.FilePath));
Assert.False(string.IsNullOrWhiteSpace(artifact.ManifestArtifact.Uri));
Assert.StartsWith("cas://scanner-artifacts/", artifact.ManifestArtifact.Uri, StringComparison.Ordinal);
}
}
[Fact]
public async Task WriteAsync_NoArtifacts_ReturnsNull()
{
await using var temp = new TempDirectory();
var options = new SurfaceOptions(
CacheRoot: temp.Path,
CacheBucket: "scanner-artifacts",
RootPrefix: "scanner",
Tenant: "tenant-a",
Component: "scanner.buildx",
ComponentVersion: "1.0",
WorkerInstance: "builder-01",
Attempt: 1,
ImageDigest: "sha256:deadbeef",
ScanId: "scan-1",
LayerFragmentsPath: null,
EntryTraceGraphPath: null,
EntryTraceNdjsonPath: null,
ManifestOutputPath: null);
var writer = new SurfaceManifestWriter(TimeProvider.System);
var result = await writer.WriteAsync(options, CancellationToken.None);
Assert.Null(result);
}
}

23
src/Scanner/__Tests/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/TestUtilities/TestPathHelper.cs Normal file
View File

@@ -0,0 +1,23 @@
using System;
using System.IO;
namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Tests.TestUtilities;
internal static class TestPathHelper
{
public static string FindRepositoryRoot()
{
var current = AppContext.BaseDirectory;
for (var i = 0; i < 15 && !string.IsNullOrWhiteSpace(current); i++)
{
if (File.Exists(Path.Combine(current, "global.json")))
{
return current;
}
current = Directory.GetParent(current)?.FullName;
}
throw new InvalidOperationException("Unable to locate repository root (global.json not found).");
}
}

112
src/Scanner/__Tests/StellaOps.Scanner.Surface.Secrets.Tests/RegistryAccessSecretParserTests.cs Normal file
View File

@@ -0,0 +1,112 @@
using System;
using System.Collections.Generic;
using System.Text;
using StellaOps.Scanner.Surface.Secrets;
using Xunit;
namespace StellaOps.Scanner.Surface.Secrets.Tests;
public sealed class RegistryAccessSecretParserTests
{
[Fact]
public void ParseRegistrySecret_WithEntriesArray_ReturnsCredential()
{
const string json = """
{
"defaultRegistry": "registry.example.com",
"entries": [
{
"registry": "registry.example.com",
"username": "demo",
"password": "s3cret",
"token": "token-123",
"identityToken": "identity-token",
"refreshToken": "refresh-token",
"expiresAt": "2025-12-01T10:00:00Z",
"allowInsecureTls": false,
"scopes": ["repo:sample:pull"],
"headers": {
"X-Test": "value"
},
"email": "demo@example.com"
}
]
}
""";
using var handle = SurfaceSecretHandle.FromBytes(Encoding.UTF8.GetBytes(json));
var secret = SurfaceSecretParser.ParseRegistryAccessSecret(handle);
Assert.Equal("registry.example.com", secret.DefaultRegistry);
var entry = Assert.Single(secret.Entries);
Assert.Equal("registry.example.com", entry.Registry);
Assert.Equal("demo", entry.Username);
Assert.Equal("s3cret", entry.Password);
Assert.Equal("token-123", entry.RegistryToken);
Assert.Equal("identity-token", entry.IdentityToken);
Assert.Equal("refresh-token", entry.RefreshToken);
Assert.Equal("demo@example.com", entry.Email);
Assert.Equal(new DateTimeOffset(2025, 12, 1, 10, 0, 0, TimeSpan.Zero), entry.ExpiresAt);
Assert.Equal(false, entry.AllowInsecureTls);
Assert.Contains("repo:sample:pull", entry.Scopes);
Assert.Equal("value", entry.Headers["X-Test"]);
}
[Fact]
public void ParseRegistrySecret_WithDockerAuthsObject_DecodesBasicAuth()
{
const string json = """
{
"auths": {
"ghcr.io": {
"auth": "ZGVtbzpwYXNz",
"identitytoken": "id-token"
}
}
}
""";
var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["token"] = "metadata-token"
};
using var handle = SurfaceSecretHandle.FromBytes(Encoding.UTF8.GetBytes(json), metadata);
var secret = SurfaceSecretParser.ParseRegistryAccessSecret(handle);
var entry = Assert.Single(secret.Entries);
Assert.Equal("ghcr.io", entry.Registry);
Assert.Equal("demo", entry.Username);
Assert.Equal("pass", entry.Password);
Assert.Equal("metadata-token", entry.RegistryToken);
Assert.Equal("id-token", entry.IdentityToken);
}
[Fact]
public void ParseRegistrySecret_MetadataFallback_ReturnsCredential()
{
var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["registry"] = "registry.internal",
["username"] = "meta-user",
["password"] = "meta-pass",
["scope:0"] = "repo:internal:pull",
["header:X-From"] = "metadata",
["defaultRegistry"] = "registry.internal",
["expiresAt"] = "2025-11-10T00:00:00Z",
["allowInsecureTls"] = "true"
};
using var handle = SurfaceSecretHandle.FromBytes(ReadOnlySpan<byte>.Empty, metadata);
var secret = SurfaceSecretParser.ParseRegistryAccessSecret(handle);
var entry = Assert.Single(secret.Entries);
Assert.Equal("registry.internal", entry.Registry);
Assert.Equal("meta-user", entry.Username);
Assert.Equal("meta-pass", entry.Password);
Assert.Contains("repo:internal:pull", entry.Scopes);
Assert.Equal("metadata", entry.Headers["X-From"]);
Assert.True(entry.AllowInsecureTls);
Assert.Equal("registry.internal", secret.DefaultRegistry);
}
}

97
src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/ScannerSurfaceSecretConfiguratorTests.cs
View File

@@ -30,7 +30,10 @@ public sealed class ScannerSurfaceSecretConfiguratorTests
""";
using var handle = SurfaceSecretHandle.FromBytes(Encoding.UTF8.GetBytes(json));
var secretProvider = new StubSecretProvider(handle);
var secretProvider = new StubSecretProvider(new Dictionary<string, SurfaceSecretHandle>(StringComparer.OrdinalIgnoreCase)
{
["cas-access"] = handle
});
var environment = new StubSurfaceEnvironment();
var options = new ScannerWebServiceOptions();
@@ -82,17 +85,101 @@ public sealed class ScannerSurfaceSecretConfiguratorTests
Assert.Equal("X-Sync", storageOptions.ObjectStore.RustFs.ApiKeyHeader);
}
[Fact]
public void Configure_AppliesAttestationSecretToSigning()
{
const string json = """
{
"keyPem": "-----BEGIN KEY-----\nYWJj\n-----END KEY-----",
"certificatePem": "CERT-PEM",
"certificateChainPem": "CHAIN-PEM"
}
""";
using var handle = SurfaceSecretHandle.FromBytes(Encoding.UTF8.GetBytes(json));
var secretProvider = new StubSecretProvider(new Dictionary<string, SurfaceSecretHandle>(StringComparer.OrdinalIgnoreCase)
{
["attestation"] = handle
});
var environment = new StubSurfaceEnvironment();
var options = new ScannerWebServiceOptions();
var configurator = new ScannerSurfaceSecretConfigurator(
secretProvider,
environment,
NullLogger<ScannerSurfaceSecretConfigurator>.Instance);
configurator.Configure(options);
Assert.Equal("-----BEGIN KEY-----\nYWJj\n-----END KEY-----", options.Signing.KeyPem);
Assert.Equal("CERT-PEM", options.Signing.CertificatePem);
Assert.Equal("CHAIN-PEM", options.Signing.CertificateChainPem);
}
[Fact]
public void Configure_AppliesRegistrySecretToOptions()
{
const string json = """
{
"defaultRegistry": "registry.example.com",
"entries": [
{
"registry": "registry.example.com",
"username": "demo",
"password": "secret",
"scopes": ["repo:sample:pull"],
"headers": { "X-Test": "value" },
"allowInsecureTls": true,
"email": "demo@example.com"
}
]
}
""";
using var handle = SurfaceSecretHandle.FromBytes(Encoding.UTF8.GetBytes(json));
var secretProvider = new StubSecretProvider(new Dictionary<string, SurfaceSecretHandle>(StringComparer.OrdinalIgnoreCase)
{
["registry"] = handle
});
var environment = new StubSurfaceEnvironment();
var options = new ScannerWebServiceOptions();
var configurator = new ScannerSurfaceSecretConfigurator(
secretProvider,
environment,
NullLogger<ScannerSurfaceSecretConfigurator>.Instance);
configurator.Configure(options);
Assert.Equal("registry.example.com", options.Registry.DefaultRegistry);
var credential = Assert.Single(options.Registry.Credentials);
Assert.Equal("registry.example.com", credential.Registry);
Assert.Equal("demo", credential.Username);
Assert.Equal("secret", credential.Password);
Assert.True(credential.AllowInsecureTls);
Assert.Contains("repo:sample:pull", credential.Scopes);
Assert.Equal("value", credential.Headers["X-Test"]);
Assert.Equal("demo@example.com", credential.Email);
}
private sealed class StubSecretProvider : ISurfaceSecretProvider
{
private readonly SurfaceSecretHandle _handle;
private readonly IDictionary<string, SurfaceSecretHandle> _handles;
public StubSecretProvider(SurfaceSecretHandle handle)
public StubSecretProvider(IDictionary<string, SurfaceSecretHandle> handles)
{
_handle = handle;
_handles = handles ?? throw new ArgumentNullException(nameof(handles));
}
public ValueTask<SurfaceSecretHandle> GetAsync(SurfaceSecretRequest request, CancellationToken cancellationToken = default)
=> ValueTask.FromResult(_handle);
{
if (_handles.TryGetValue(request.SecretType, out var handle))
{
return ValueTask.FromResult(handle);
}
throw new SurfaceSecretNotFoundException(request);
}
}
private sealed class StubSurfaceEnvironment : ISurfaceEnvironment

147
src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/ScansEndpointsTests.cs
View File

@@ -9,7 +9,7 @@ using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading.Tasks;
using System.Threading;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.AspNetCore.TestHost;
using Microsoft.Extensions.DependencyInjection;
@@ -17,6 +17,7 @@ using StellaOps.Scanner.EntryTrace;
using StellaOps.Scanner.EntryTrace.Serialization;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Repositories;
using StellaOps.Scanner.Storage.ObjectStore;
using StellaOps.Scanner.WebService.Contracts;
using StellaOps.Scanner.WebService.Domain;
using StellaOps.Scanner.WebService.Services;
@@ -92,39 +93,88 @@ public sealed class ScansEndpointsTests
using var factory = new ScannerApplicationFactory();
const string manifestDigest = "sha256:b2efc2d1f8b042b7f168bcb7d4e2f8e91d36b8306bd855382c5f847efc2c1111";
const string graphDigest = "sha256:9a0d4f8c7b6a5e4d3c2b1a0f9e8d7c6b5a4f3e2d1c0b9a8f7e6d5c4b3a291819";
const string ndjsonDigest = "sha256:3f2e1d0c9b8a7f6e5d4c3b2a1908f7e6d5c4b3a29181726354433221100ffeec";
const string fragmentsDigest = "sha256:aa55aa55aa55aa55aa55aa55aa55aa55aa55aa55aa55aa55aa55aa55aa55aa55";
using (var scope = factory.Services.CreateScope())
{
var artifactRepository = scope.ServiceProvider.GetRequiredService<ArtifactRepository>();
var linkRepository = scope.ServiceProvider.GetRequiredService<LinkRepository>();
var artifactId = CatalogIdFactory.CreateArtifactId(ArtifactDocumentType.ImageBom, digest);
var now = DateTime.UtcNow;
var artifact = new ArtifactDocument
async Task InsertAsync(
ArtifactDocumentType type,
ArtifactDocumentFormat format,
string artifactDigest,
string mediaType,
string ttlClass)
{
Id = artifactId,
Type = ArtifactDocumentType.ImageBom,
Format = ArtifactDocumentFormat.CycloneDxJson,
MediaType = "application/vnd.cyclonedx+json; version=1.6; view=inventory",
BytesSha256 = digest,
SizeBytes = 2048,
Immutable = true,
RefCount = 1,
TtlClass = "default",
CreatedAtUtc = DateTime.UtcNow,
UpdatedAtUtc = DateTime.UtcNow
};
var artifactId = CatalogIdFactory.CreateArtifactId(type, artifactDigest);
var document = new ArtifactDocument
{
Id = artifactId,
Type = type,
Format = format,
MediaType = mediaType,
BytesSha256 = artifactDigest,
SizeBytes = 2048,
Immutable = true,
RefCount = 1,
TtlClass = ttlClass,
CreatedAtUtc = now,
UpdatedAtUtc = now
};
await artifactRepository.UpsertAsync(artifact, CancellationToken.None).ConfigureAwait(false);
await artifactRepository.UpsertAsync(document, CancellationToken.None).ConfigureAwait(false);
var link = new LinkDocument
{
Id = CatalogIdFactory.CreateLinkId(LinkSourceType.Image, digest, artifactId),
FromType = LinkSourceType.Image,
FromDigest = digest,
ArtifactId = artifactId,
CreatedAtUtc = DateTime.UtcNow
};
var link = new LinkDocument
{
Id = CatalogIdFactory.CreateLinkId(LinkSourceType.Image, digest, artifactId),
FromType = LinkSourceType.Image,
FromDigest = digest,
ArtifactId = artifactId,
CreatedAtUtc = now
};
await linkRepository.UpsertAsync(link, CancellationToken.None).ConfigureAwait(false);
await linkRepository.UpsertAsync(link, CancellationToken.None).ConfigureAwait(false);
}
await InsertAsync(
ArtifactDocumentType.ImageBom,
ArtifactDocumentFormat.CycloneDxJson,
digest,
"application/vnd.cyclonedx+json; version=1.6; view=inventory",
"default").ConfigureAwait(false);
await InsertAsync(
ArtifactDocumentType.SurfaceManifest,
ArtifactDocumentFormat.SurfaceManifestJson,
manifestDigest,
"application/vnd.stellaops.surface.manifest+json",
"surface.manifest").ConfigureAwait(false);
await InsertAsync(
ArtifactDocumentType.SurfaceEntryTrace,
ArtifactDocumentFormat.EntryTraceGraphJson,
graphDigest,
"application/json",
"surface.payload").ConfigureAwait(false);
await InsertAsync(
ArtifactDocumentType.SurfaceEntryTrace,
ArtifactDocumentFormat.EntryTraceNdjson,
ndjsonDigest,
"application/x-ndjson",
"surface.payload").ConfigureAwait(false);
await InsertAsync(
ArtifactDocumentType.SurfaceLayerFragment,
ArtifactDocumentFormat.ComponentFragmentJson,
fragmentsDigest,
"application/json",
"surface.payload").ConfigureAwait(false);
}
using var client = factory.CreateClient();
@@ -160,15 +210,46 @@ public sealed class ScansEndpointsTests
Assert.Equal(digest, manifest.ImageDigest);
Assert.Equal(surface.Tenant, manifest.Tenant);
Assert.NotEqual(default, manifest.GeneratedAt);
var manifestArtifact = Assert.Single(manifest.Artifacts);
Assert.Equal("sbom-inventory", manifestArtifact.Kind);
Assert.Equal("cdx-json", manifestArtifact.Format);
Assert.Equal(digest, manifestArtifact.Digest);
Assert.Equal("application/vnd.cyclonedx+json; version=1.6; view=inventory", manifestArtifact.MediaType);
Assert.Equal("inventory", manifestArtifact.View);
var artifactsByKind = manifest.Artifacts.ToDictionary(a => a.Kind, StringComparer.Ordinal);
Assert.Equal(5, artifactsByKind.Count);
var expectedUri = $"cas://scanner-artifacts/scanner/images/{digestValue}/sbom.cdx.json";
Assert.Equal(expectedUri, manifestArtifact.Uri);
static string BuildUri(ArtifactDocumentType type, ArtifactDocumentFormat format, string digestValue)
=> $"cas://scanner-artifacts/{ArtifactObjectKeyBuilder.Build(type, format, digestValue, \"scanner\")}";
var inventory = artifactsByKind["sbom-inventory"];
Assert.Equal(digest, inventory.Digest);
Assert.Equal("cdx-json", inventory.Format);
Assert.Equal("application/vnd.cyclonedx+json; version=1.6; view=inventory", inventory.MediaType);
Assert.Equal("inventory", inventory.View);
Assert.Equal(BuildUri(ArtifactDocumentType.ImageBom, ArtifactDocumentFormat.CycloneDxJson, digest), inventory.Uri);
var manifestArtifact = artifactsByKind["surface.manifest"];
Assert.Equal(manifestDigest, manifestArtifact.Digest);
Assert.Equal("surface.manifest", manifestArtifact.Format);
Assert.Equal("application/vnd.stellaops.surface.manifest+json", manifestArtifact.MediaType);
Assert.Null(manifestArtifact.View);
Assert.Equal(BuildUri(ArtifactDocumentType.SurfaceManifest, ArtifactDocumentFormat.SurfaceManifestJson, manifestDigest), manifestArtifact.Uri);
var graphArtifact = artifactsByKind["entrytrace.graph"];
Assert.Equal(graphDigest, graphArtifact.Digest);
Assert.Equal("entrytrace.graph", graphArtifact.Format);
Assert.Equal("application/json", graphArtifact.MediaType);
Assert.Null(graphArtifact.View);
Assert.Equal(BuildUri(ArtifactDocumentType.SurfaceEntryTrace, ArtifactDocumentFormat.EntryTraceGraphJson, graphDigest), graphArtifact.Uri);
var ndjsonArtifact = artifactsByKind["entrytrace.ndjson"];
Assert.Equal(ndjsonDigest, ndjsonArtifact.Digest);
Assert.Equal("entrytrace.ndjson", ndjsonArtifact.Format);
Assert.Equal("application/x-ndjson", ndjsonArtifact.MediaType);
Assert.Null(ndjsonArtifact.View);
Assert.Equal(BuildUri(ArtifactDocumentType.SurfaceEntryTrace, ArtifactDocumentFormat.EntryTraceNdjson, ndjsonDigest), ndjsonArtifact.Uri);
var fragmentsArtifact = artifactsByKind["layer.fragments"];
Assert.Equal(fragmentsDigest, fragmentsArtifact.Digest);
Assert.Equal("layer.fragments", fragmentsArtifact.Format);
Assert.Equal("application/json", fragmentsArtifact.MediaType);
Assert.Equal("inventory", fragmentsArtifact.View);
Assert.Equal(BuildUri(ArtifactDocumentType.SurfaceLayerFragment, ArtifactDocumentFormat.ComponentFragmentJson, fragmentsDigest), fragmentsArtifact.Uri);
}
[Fact]

221
src/Scanner/__Tests/StellaOps.Scanner.Worker.Tests/RegistrySecretStageExecutorTests.cs Normal file
View File

@@ -0,0 +1,221 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging.Abstractions;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.Surface.Env;
using StellaOps.Scanner.Surface.Secrets;
using StellaOps.Scanner.Worker.Diagnostics;
using StellaOps.Scanner.Worker.Processing;
using Xunit;
namespace StellaOps.Scanner.Worker.Tests;
public sealed class RegistrySecretStageExecutorTests
{
[Fact]
public async Task ExecuteAsync_WithSecret_StoresCredentialsAndEmitsMetrics()
{
const string secretJson = """
{
"defaultRegistry": "registry.example.com",
"entries": [
{
"registry": "registry.example.com",
"username": "demo",
"password": "s3cret",
"expiresAt": "2099-01-01T00:00:00Z"
}
]
}
""";
var provider = new StubSecretProvider(secretJson);
var environment = new StubSurfaceEnvironment("tenant-eu");
var metrics = new ScannerWorkerMetrics();
var timeProvider = TimeProvider.System;
var executor = new RegistrySecretStageExecutor(
provider,
environment,
metrics,
timeProvider,
NullLogger<RegistrySecretStageExecutor>.Instance);
var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["surface.registry.secret"] = "primary"
};
var lease = new StubLease("job-1", "scan-1", metadata);
using var contextCancellation = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken.None);
var context = new ScanJobContext(lease, timeProvider, timeProvider.GetUtcNow(), contextCancellation.Token);
var measurements = new List<(long Value, KeyValuePair<string, object?>[] Tags)>();
using var listener = CreateCounterListener("scanner_worker_registry_secret_requests_total", measurements);
await executor.ExecuteAsync(context, CancellationToken.None);
listener.RecordObservableInstruments();
Assert.True(context.Analysis.TryGet<RegistryAccessSecret>(ScanAnalysisKeys.RegistryCredentials, out var secret));
Assert.NotNull(secret);
Assert.Single(secret!.Entries);
Assert.Contains(
measurements,
measurement => measurement.Value == 1 &&
HasTagValue(measurement.Tags, "secret.result", "resolved") &&
HasTagValue(measurement.Tags, "secret.name", "primary"));
}
[Fact]
public async Task ExecuteAsync_SecretMissing_RecordsMissingMetric()
{
var provider = new MissingSecretProvider();
var environment = new StubSurfaceEnvironment("tenant-eu");
var metrics = new ScannerWorkerMetrics();
var executor = new RegistrySecretStageExecutor(
provider,
environment,
metrics,
TimeProvider.System,
NullLogger<RegistrySecretStageExecutor>.Instance);
var lease = new StubLease("job-2", "scan-2", new Dictionary<string, string>());
var context = new ScanJobContext(lease, TimeProvider.System, TimeProvider.System.GetUtcNow(), CancellationToken.None);
var measurements = new List<(long Value, KeyValuePair<string, object?>[] Tags)>();
using var listener = CreateCounterListener("scanner_worker_registry_secret_requests_total", measurements);
await executor.ExecuteAsync(context, CancellationToken.None);
listener.RecordObservableInstruments();
Assert.False(context.Analysis.TryGet<RegistryAccessSecret>(ScanAnalysisKeys.RegistryCredentials, out _));
Assert.Contains(
measurements,
measurement => measurement.Value == 1 &&
HasTagValue(measurement.Tags, "secret.result", "missing") &&
HasTagValue(measurement.Tags, "secret.name", "default"));
}
private static MeterListener CreateCounterListener(
string instrumentName,
ICollection<(long Value, KeyValuePair<string, object?>[] Tags)> measurements)
{
var listener = new MeterListener
{
InstrumentPublished = (instrument, meterListener) =>
{
if (instrument.Meter.Name == ScannerWorkerInstrumentation.MeterName &&
instrument.Name == instrumentName)
{
meterListener.EnableMeasurementEvents(instrument);
}
}
};
listener.SetMeasurementEventCallback<long>((instrument, measurement, tags, state) =>
{
var copy = tags.ToArray();
measurements.Add((measurement, copy));
});
listener.Start();
return listener;
}
private static bool HasTagValue(IEnumerable<KeyValuePair<string, object?>> tags, string key, string expected)
=> tags.Any(tag => string.Equals(tag.Key, key, StringComparison.OrdinalIgnoreCase) &&
string.Equals(tag.Value?.ToString(), expected, StringComparison.OrdinalIgnoreCase));
private sealed class StubSecretProvider : ISurfaceSecretProvider
{
private readonly string _json;
public StubSecretProvider(string json)
{
_json = json;
}
public ValueTask<SurfaceSecretHandle> GetAsync(SurfaceSecretRequest request, CancellationToken cancellationToken = default)
{
var bytes = Encoding.UTF8.GetBytes(_json);
return ValueTask.FromResult(SurfaceSecretHandle.FromBytes(bytes));
}
}
private sealed class MissingSecretProvider : ISurfaceSecretProvider
{
public ValueTask<SurfaceSecretHandle> GetAsync(SurfaceSecretRequest request, CancellationToken cancellationToken = default)
=> throw new SurfaceSecretNotFoundException(request);
}
private sealed class StubSurfaceEnvironment : ISurfaceEnvironment
{
public StubSurfaceEnvironment(string tenant)
{
Settings = new SurfaceEnvironmentSettings(
new Uri("https://surface.example"),
"bucket",
"region",
new DirectoryInfo(Path.GetTempPath()),
1024,
false,
Array.Empty<string>(),
new SurfaceSecretsConfiguration("inline", tenant, null, null, null, AllowInline: true),
tenant,
new SurfaceTlsConfiguration(null, null, null))
{
CreatedAtUtc = DateTimeOffset.UtcNow
};
RawVariables = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
}
public SurfaceEnvironmentSettings Settings { get; }
public IReadOnlyDictionary<string, string> RawVariables { get; }
}
private sealed class StubLease : IScanJobLease
{
private readonly IReadOnlyDictionary<string, string> _metadata;
public StubLease(string jobId, string scanId, IReadOnlyDictionary<string, string> metadata)
{
JobId = jobId;
ScanId = scanId;
_metadata = metadata;
EnqueuedAtUtc = DateTimeOffset.UtcNow.AddMinutes(-1);
LeasedAtUtc = DateTimeOffset.UtcNow;
}
public string JobId { get; }
public string ScanId { get; }
public int Attempt { get; } = 1;
public DateTimeOffset EnqueuedAtUtc { get; }
public DateTimeOffset LeasedAtUtc { get; }
public TimeSpan LeaseDuration { get; } = TimeSpan.FromMinutes(5);
public IReadOnlyDictionary<string, string> Metadata => _metadata;
public ValueTask RenewAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask;
public ValueTask CompleteAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask;
public ValueTask AbandonAsync(string reason, CancellationToken cancellationToken) => ValueTask.CompletedTask;
public ValueTask PoisonAsync(string reason, CancellationToken cancellationToken) => ValueTask.CompletedTask;
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
}
}

349
src/Scanner/__Tests/StellaOps.Scanner.Worker.Tests/SurfaceManifestStageExecutorTests.cs Normal file
View File

@@ -0,0 +1,349 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using System.Security.Cryptography;
using Microsoft.Extensions.Logging.Abstractions;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.EntryTrace;
using StellaOps.Scanner.Surface.Env;
using StellaOps.Scanner.Surface.FS;
using StellaOps.Scanner.Worker.Diagnostics;
using StellaOps.Scanner.Worker.Processing;
using StellaOps.Scanner.Worker.Processing.Surface;
using StellaOps.Scanner.Worker.Tests.TestInfrastructure;
using Xunit;
namespace StellaOps.Scanner.Worker.Tests;
public sealed class SurfaceManifestStageExecutorTests
{
[Fact]
public async Task ExecuteAsync_WhenNoPayloads_SkipsPublishAndRecordsSkipMetric()
{
var metrics = new ScannerWorkerMetrics();
var publisher = new TestSurfaceManifestPublisher();
var cache = new RecordingSurfaceCache();
var environment = new TestSurfaceEnvironment("tenant-a");
using var listener = new WorkerMeterListener();
listener.Start();
var executor = new SurfaceManifestStageExecutor(
publisher,
cache,
environment,
metrics,
NullLogger<SurfaceManifestStageExecutor>.Instance);
var context = CreateContext();
await executor.ExecuteAsync(context, CancellationToken.None);
Assert.Equal(0, publisher.PublishCalls);
Assert.Empty(cache.Entries);
var skipMetrics = listener.Measurements
.Where(m => m.InstrumentName == "scanner_worker_surface_manifests_skipped_total")
.ToArray();
Assert.Single(skipMetrics);
Assert.Equal(1, skipMetrics[0].Value);
Assert.Equal("skipped", skipMetrics[0]["surface.result"]);
}
[Fact]
public async Task ExecuteAsync_PublishesPayloads_CachesArtifacts_AndRecordsMetrics()
{
var metrics = new ScannerWorkerMetrics();
var publisher = new TestSurfaceManifestPublisher("tenant-a");
var cache = new RecordingSurfaceCache();
var environment = new TestSurfaceEnvironment("tenant-a");
using var listener = new WorkerMeterListener();
listener.Start();
var executor = new SurfaceManifestStageExecutor(
publisher,
cache,
environment,
metrics,
NullLogger<SurfaceManifestStageExecutor>.Instance);
var context = CreateContext();
PopulateAnalysis(context);
await executor.ExecuteAsync(context, CancellationToken.None);
Assert.Equal(1, publisher.PublishCalls);
Assert.True(context.Analysis.TryGet<SurfaceManifestPublishResult>(ScanAnalysisKeys.SurfaceManifest, out var result));
Assert.NotNull(result);
Assert.Equal(publisher.LastManifestDigest, result!.ManifestDigest);
Assert.Equal(4, cache.Entries.Count);
Assert.Contains(cache.Entries.Keys, key => key.Namespace == "surface.artifacts.entrytrace.graph" && key.Tenant == "tenant-a");
Assert.Contains(cache.Entries.Keys, key => key.Namespace == "surface.artifacts.entrytrace.ndjson" && key.Tenant == "tenant-a");
Assert.Contains(cache.Entries.Keys, key => key.Namespace == "surface.artifacts.layer.fragments" && key.Tenant == "tenant-a");
Assert.Contains(cache.Entries.Keys, key => key.Namespace == "surface.manifests" && key.Tenant == "tenant-a");
var publishedMetrics = listener.Measurements
.Where(m => m.InstrumentName == "scanner_worker_surface_manifests_published_total")
.ToArray();
Assert.Single(publishedMetrics);
Assert.Equal(1, publishedMetrics[0].Value);
Assert.Equal("published", publishedMetrics[0]["surface.result"]);
Assert.Equal(3, Convert.ToInt32(publishedMetrics[0]["surface.payload_count"]));
var payloadMetrics = listener.Measurements
.Where(m => m.InstrumentName == "scanner_worker_surface_payload_persisted_total")
.ToArray();
Assert.Equal(3, payloadMetrics.Length);
Assert.Contains(payloadMetrics, m => Equals("entrytrace.graph", m["surface.kind"]));
Assert.Contains(payloadMetrics, m => Equals("entrytrace.ndjson", m["surface.kind"]));
Assert.Contains(payloadMetrics, m => Equals("layer.fragments", m["surface.kind"]));
}
private static ScanJobContext CreateContext()
{
var lease = new FakeJobLease();
return new ScanJobContext(lease, TimeProvider.System, DateTimeOffset.UtcNow, CancellationToken.None);
}
private static void PopulateAnalysis(ScanJobContext context)
{
var node = new EntryTraceNode(
Id: 1,
Kind: EntryTraceNodeKind.Command,
DisplayName: "/bin/entry",
Arguments: ImmutableArray<string>.Empty,
InterpreterKind: EntryTraceInterpreterKind.None,
Evidence: null,
Span: null,
Metadata: null);
var graph = new EntryTraceGraph(
Outcome: EntryTraceOutcome.Resolved,
Nodes: ImmutableArray.Create(node),
Edges: ImmutableArray<EntryTraceEdge>.Empty,
Diagnostics: ImmutableArray<EntryTraceDiagnostic>.Empty,
Plans: ImmutableArray<EntryTracePlan>.Empty,
Terminals: ImmutableArray<EntryTraceTerminal>.Empty);
context.Analysis.Set(ScanAnalysisKeys.EntryTraceGraph, graph);
var ndjson = ImmutableArray.Create("{\"entry\":\"/bin/entry\"}\n");
context.Analysis.Set(ScanAnalysisKeys.EntryTraceNdjson, ndjson);
var component = new ComponentRecord
{
Identity = ComponentIdentity.Create("pkg:test", "test", "1.0.0"),
LayerDigest = "sha256:layer-1",
Evidence = ImmutableArray<ComponentEvidence>.Empty,
Usage = ComponentUsage.Create(true, new[] { "/bin/entry" })
};
var fragment = LayerComponentFragment.Create("sha256:layer-1", new[] { component });
context.Analysis.Set(ScanAnalysisKeys.LayerComponentFragments, ImmutableArray.Create(fragment));
}
private sealed class RecordingSurfaceCache : ISurfaceCache
{
private readonly Dictionary<SurfaceCacheKey, byte[]> _entries = new();
public IReadOnlyDictionary<SurfaceCacheKey, byte[]> Entries => _entries;
public Task<T> GetOrCreateAsync<T>(
SurfaceCacheKey key,
Func<CancellationToken, Task<T>> factory,
Func<T, ReadOnlyMemory<byte>> serializer,
Func<ReadOnlyMemory<byte>, T> deserializer,
CancellationToken cancellationToken = default)
{
if (_entries.TryGetValue(key, out var payload))
{
return Task.FromResult(deserializer(payload));
}
return CreateAsync(key, factory, serializer, cancellationToken);
}
public Task<T?> TryGetAsync<T>(
SurfaceCacheKey key,
Func<ReadOnlyMemory<byte>, T> deserializer,
CancellationToken cancellationToken = default)
{
if (_entries.TryGetValue(key, out var payload))
{
return Task.FromResult<T?>(deserializer(payload));
}
return Task.FromResult<T?>(default);
}
public Task SetAsync(
SurfaceCacheKey key,
ReadOnlyMemory<byte> payload,
CancellationToken cancellationToken = default)
{
_entries[key] = payload.ToArray();
return Task.CompletedTask;
}
private async Task<T> CreateAsync<T>(
SurfaceCacheKey key,
Func<CancellationToken, Task<T>> factory,
Func<T, ReadOnlyMemory<byte>> serializer,
CancellationToken cancellationToken)
{
var value = await factory(cancellationToken).ConfigureAwait(false);
_entries[key] = serializer(value).ToArray();
return value;
}
}
private sealed class TestSurfaceManifestPublisher : ISurfaceManifestPublisher
{
private readonly string _tenant;
private readonly JsonSerializerOptions _options = new(JsonSerializerDefaults.Web)
{
WriteIndented = false,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
public TestSurfaceManifestPublisher(string tenant = "tenant-a")
{
_tenant = tenant;
}
public int PublishCalls { get; private set; }
public SurfaceManifestRequest? LastRequest { get; private set; }
public string? LastManifestDigest { get; private set; }
public Task<SurfaceManifestPublishResult> PublishAsync(SurfaceManifestRequest request, CancellationToken cancellationToken)
{
PublishCalls++;
LastRequest = request;
var artifacts = request.Payloads.Select(payload =>
{
var digest = ComputeDigest(payload.Content.Span);
return new SurfaceManifestArtifact
{
Kind = payload.Kind,
Uri = $"cas://test/{payload.Kind}/{digest}",
Digest = digest,
MediaType = payload.MediaType,
Format = payload.ArtifactFormat.ToString().ToLowerInvariant(),
SizeBytes = payload.Content.Length,
View = payload.View,
Metadata = payload.Metadata,
Storage = new SurfaceManifestStorage
{
Bucket = "test-bucket",
ObjectKey = $"objects/{digest}",
SizeBytes = payload.Content.Length,
ContentType = payload.MediaType
}
};
}).ToImmutableArray();
var document = new SurfaceManifestDocument
{
Tenant = _tenant,
ImageDigest = request.ImageDigest,
ScanId = request.ScanId,
GeneratedAt = DateTimeOffset.UtcNow,
Source = new SurfaceManifestSource
{
Component = request.Component,
Version = request.Version,
WorkerInstance = request.WorkerInstance,
Attempt = request.Attempt
},
Artifacts = artifacts
};
var manifestBytes = JsonSerializer.SerializeToUtf8Bytes(document, _options);
var manifestDigest = ComputeDigest(manifestBytes);
LastManifestDigest = manifestDigest;
var result = new SurfaceManifestPublishResult(
ManifestDigest: manifestDigest,
ManifestUri: $"cas://test/manifests/{manifestDigest}",
ArtifactId: $"surface-manifest::{manifestDigest}",
Document: document);
return Task.FromResult(result);
}
private static string ComputeDigest(ReadOnlySpan<byte> content)
{
Span<byte> hash = stackalloc byte[32];
SHA256.HashData(content, hash);
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
}
}
private sealed class TestSurfaceEnvironment : ISurfaceEnvironment
{
public TestSurfaceEnvironment(string tenant)
{
var cacheRoot = new DirectoryInfo(Path.Combine(Path.GetTempPath(), "surface-cache-test"));
Settings = new SurfaceEnvironmentSettings(
SurfaceFsEndpoint: new Uri("https://surface.local"),
SurfaceFsBucket: "test-bucket",
SurfaceFsRegion: null,
CacheRoot: cacheRoot,
CacheQuotaMegabytes: 512,
PrefetchEnabled: false,
FeatureFlags: Array.Empty<string>(),
Secrets: new SurfaceSecretsConfiguration("none", tenant, null, null, null, false),
Tenant: tenant,
Tls: new SurfaceTlsConfiguration(null, null, null));
}
public SurfaceEnvironmentSettings Settings { get; }
public IReadOnlyDictionary<string, string> RawVariables { get; } = new Dictionary<string, string>();
}
private sealed class FakeJobLease : IScanJobLease
{
private readonly Dictionary<string, string> _metadata = new()
{
["queue"] = "tests",
["job.kind"] = "unit"
};
public string JobId { get; } = Guid.NewGuid().ToString("n");
public string ScanId { get; } = $"scan-{Guid.NewGuid():n}";
public int Attempt { get; } = 1;
public DateTimeOffset EnqueuedAtUtc { get; } = DateTimeOffset.UtcNow.AddMinutes(-1);
public DateTimeOffset LeasedAtUtc { get; } = DateTimeOffset.UtcNow;
public TimeSpan LeaseDuration { get; } = TimeSpan.FromMinutes(5);
public IReadOnlyDictionary<string, string> Metadata => _metadata;
public ValueTask RenewAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask;
public ValueTask CompleteAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask;
public ValueTask AbandonAsync(string reason, CancellationToken cancellationToken) => ValueTask.CompletedTask;
public ValueTask PoisonAsync(string reason, CancellationToken cancellationToken) => ValueTask.CompletedTask;
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
}
}

61
src/Scanner/__Tests/StellaOps.Scanner.Worker.Tests/TestInfrastructure/WorkerMeterListener.cs Normal file
View File

@@ -0,0 +1,61 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
using System.Globalization;
using StellaOps.Scanner.Worker.Diagnostics;
namespace StellaOps.Scanner.Worker.Tests.TestInfrastructure;
public sealed class WorkerMeterListener : IDisposable
{
private readonly MeterListener _listener;
public ConcurrentBag<Measurement> Measurements { get; } = new();
public WorkerMeterListener()
{
_listener = new MeterListener
{
InstrumentPublished = (instrument, listener) =>
{
if (instrument.Meter.Name == ScannerWorkerInstrumentation.MeterName)
{
listener.EnableMeasurementEvents(instrument);
}
}
};
_listener.SetMeasurementEventCallback<double>((instrument, measurement, tags, state) =>
{
AddMeasurement(instrument, measurement, tags);
});
_listener.SetMeasurementEventCallback<long>((instrument, measurement, tags, state) =>
{
AddMeasurement(instrument, measurement, tags);
});
}
public void Start() => _listener.Start();
public void Dispose() => _listener.Dispose();
public sealed record Measurement(string InstrumentName, double Value, IReadOnlyDictionary<string, object?> Tags)
{
public object? this[string name] => Tags.TryGetValue(name, out var value) ? value : null;
}
private void AddMeasurement<T>(Instrument instrument, T measurement, ReadOnlySpan<KeyValuePair<string, object?>> tags)
where T : struct, IConvertible
{
var tagDictionary = new Dictionary<string, object?>(tags.Length, StringComparer.Ordinal);
foreach (var tag in tags)
{
tagDictionary[tag.Key] = tag.Value;
}
var value = Convert.ToDouble(measurement, System.Globalization.CultureInfo.InvariantCulture);
Measurements.Add(new Measurement(instrument.Name, value, tagDictionary));
}
}

70
src/Scanner/__Tests/StellaOps.Scanner.Worker.Tests/WorkerBasicScanScenarioTests.cs
View File

@@ -1,19 +1,19 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Microsoft.Extensions.Time.Testing;
using StellaOps.Scanner.Worker.Diagnostics;
using StellaOps.Scanner.Worker.Hosting;
using StellaOps.Scanner.Worker.Options;
using StellaOps.Scanner.Worker.Processing;
using Xunit;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Microsoft.Extensions.Time.Testing;
using StellaOps.Scanner.Worker.Diagnostics;
using StellaOps.Scanner.Worker.Hosting;
using StellaOps.Scanner.Worker.Options;
using StellaOps.Scanner.Worker.Processing;
using StellaOps.Scanner.Worker.Tests.TestInfrastructure;
using Xunit;
namespace StellaOps.Scanner.Worker.Tests;
@@ -48,8 +48,8 @@ public sealed class WorkerBasicScanScenarioTests
var scheduler = new ControlledDelayScheduler();
var analyzer = new TestAnalyzerDispatcher(scheduler);
using var listener = new WorkerMetricsListener();
listener.Start();
using var listener = new WorkerMeterListener();
listener.Start();
using var services = new ServiceCollection()
.AddLogging(builder =>
@@ -341,46 +341,6 @@ public sealed class WorkerBasicScanScenarioTests
}
}
private sealed class WorkerMetricsListener : IDisposable
{
private readonly MeterListener _listener;
public ConcurrentBag<Measurement> Measurements { get; } = new();
public WorkerMetricsListener()
{
_listener = new MeterListener
{
InstrumentPublished = (instrument, listener) =>
{
if (instrument.Meter.Name == ScannerWorkerInstrumentation.MeterName)
{
listener.EnableMeasurementEvents(instrument);
}
}
};
_listener.SetMeasurementEventCallback<double>((instrument, measurement, tags, state) =>
{
var tagDictionary = new Dictionary<string, object?>(tags.Length, StringComparer.Ordinal);
foreach (var tag in tags)
{
tagDictionary[tag.Key] = tag.Value;
}
Measurements.Add(new Measurement(instrument.Name, measurement, tagDictionary));
});
}
public void Start() => _listener.Start();
public void Dispose() => _listener.Dispose();
}
public sealed record Measurement(string InstrumentName, double Value, IReadOnlyDictionary<string, object?> Tags)
{
public object? this[string name] => Tags.TryGetValue(name, out var value) ? value : null;
}
private sealed class TestLoggerProvider : ILoggerProvider
{
private readonly ConcurrentQueue<TestLogEntry> _entries = new();