feat: Implement Policy Engine Evaluation Service and Cache with unit tests
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

Temp commit to debug
This commit is contained in:
master
2025-11-05 07:35:53 +00:00
parent 40e7f827da
commit 9253620833
125 changed files with 18735 additions and 17215 deletions

View File

@@ -55,9 +55,13 @@ public sealed record ReportDocumentDto
[JsonPropertyOrder(6)]
public IReadOnlyList<PolicyPreviewVerdictDto> Verdicts { get; init; } = Array.Empty<PolicyPreviewVerdictDto>();
[JsonPropertyName("issues")]
[JsonPropertyOrder(7)]
public IReadOnlyList<PolicyPreviewIssueDto> Issues { get; init; } = Array.Empty<PolicyPreviewIssueDto>();
[JsonPropertyName("issues")]
[JsonPropertyOrder(7)]
public IReadOnlyList<PolicyPreviewIssueDto> Issues { get; init; } = Array.Empty<PolicyPreviewIssueDto>();
[JsonPropertyName("surface")]
[JsonPropertyOrder(8)]
public SurfacePointersDto? Surface { get; init; }
}
public sealed record ReportPolicyDto

View File

@@ -1,13 +1,14 @@
namespace StellaOps.Scanner.WebService.Contracts;
public sealed record ScanStatusResponse(
string ScanId,
string Status,
ScanStatusTarget Image,
DateTimeOffset CreatedAt,
DateTimeOffset UpdatedAt,
string? FailureReason);
public sealed record ScanStatusTarget(
string? Reference,
string? Digest);
namespace StellaOps.Scanner.WebService.Contracts;
public sealed record ScanStatusResponse(
string ScanId,
string Status,
ScanStatusTarget Image,
DateTimeOffset CreatedAt,
DateTimeOffset UpdatedAt,
string? FailureReason,
SurfacePointersDto? Surface);
public sealed record ScanStatusTarget(
string? Reference,
string? Digest);

View File

@@ -0,0 +1,87 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Scanner.WebService.Contracts;
public sealed record SurfacePointersDto
{
[JsonPropertyName("tenant")]
[JsonPropertyOrder(0)]
public string Tenant { get; init; } = string.Empty;
[JsonPropertyName("generatedAt")]
[JsonPropertyOrder(1)]
public DateTimeOffset GeneratedAt { get; init; }
= DateTimeOffset.UtcNow;
[JsonPropertyName("manifestDigest")]
[JsonPropertyOrder(2)]
public string ManifestDigest { get; init; } = string.Empty;
[JsonPropertyName("manifestUri")]
[JsonPropertyOrder(3)]
public string? ManifestUri { get; init; }
= null;
[JsonPropertyName("manifest")]
[JsonPropertyOrder(4)]
public SurfaceManifestDocument Manifest { get; init; } = new();
}
public sealed record SurfaceManifestDocument
{
[JsonPropertyName("schema")]
[JsonPropertyOrder(0)]
public string Schema { get; init; } = "stellaops.surface.manifest@1";
[JsonPropertyName("tenant")]
[JsonPropertyOrder(1)]
public string Tenant { get; init; } = string.Empty;
[JsonPropertyName("imageDigest")]
[JsonPropertyOrder(2)]
public string ImageDigest { get; init; } = string.Empty;
[JsonPropertyName("generatedAt")]
[JsonPropertyOrder(3)]
public DateTimeOffset GeneratedAt { get; init; } = DateTimeOffset.UtcNow;
[JsonPropertyName("artifacts")]
[JsonPropertyOrder(4)]
public IReadOnlyList<SurfaceManifestArtifact> Artifacts { get; init; } = Array.Empty<SurfaceManifestArtifact>();
}
public sealed record SurfaceManifestArtifact
{
[JsonPropertyName("kind")]
[JsonPropertyOrder(0)]
public string Kind { get; init; } = string.Empty;
[JsonPropertyName("uri")]
[JsonPropertyOrder(1)]
public string Uri { get; init; } = string.Empty;
[JsonPropertyName("digest")]
[JsonPropertyOrder(2)]
public string Digest { get; init; } = string.Empty;
[JsonPropertyName("mediaType")]
[JsonPropertyOrder(3)]
public string MediaType { get; init; } = string.Empty;
[JsonPropertyName("format")]
[JsonPropertyOrder(4)]
public string Format { get; init; } = string.Empty;
[JsonPropertyName("sizeBytes")]
[JsonPropertyOrder(5)]
public long SizeBytes { get; init; }
= 0;
[JsonPropertyName("view")]
[JsonPropertyOrder(6)]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? View { get; init; }
= null;
}

View File

@@ -1,12 +1,16 @@
using System.Diagnostics;
using System.Collections.Generic;
using System.Diagnostics;
using System.Text;
using System.Text.Json;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Routing;
using Microsoft.Extensions.Options;
using StellaOps.Scanner.WebService.Diagnostics;
using StellaOps.Scanner.WebService.Options;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Scanner.WebService.Diagnostics;
using StellaOps.Scanner.WebService.Options;
using StellaOps.Scanner.Surface.Env;
using StellaOps.Scanner.Surface.Validation;
namespace StellaOps.Scanner.WebService.Endpoints;
@@ -56,27 +60,69 @@ internal static class HealthEndpoints
return Json(document, StatusCodes.Status200OK);
}
private static async Task<IResult> HandleReady(
ServiceStatus status,
HttpContext context,
CancellationToken cancellationToken)
{
ApplyNoCache(context.Response);
await Task.CompletedTask;
status.RecordReadyCheck(success: true, latency: TimeSpan.Zero, error: null);
var snapshot = status.CreateSnapshot();
var ready = snapshot.Ready;
var document = new ReadyDocument(
Status: ready.IsReady ? "ready" : "unready",
CheckedAt: ready.CheckedAt,
LatencyMs: ready.Latency?.TotalMilliseconds,
Error: ready.Error);
return Json(document, StatusCodes.Status200OK);
}
private static async Task<IResult> HandleReady(
ServiceStatus status,
ISurfaceValidatorRunner validatorRunner,
ISurfaceEnvironment surfaceEnvironment,
ILoggerFactory loggerFactory,
HttpContext context,
CancellationToken cancellationToken)
{
ApplyNoCache(context.Response);
ArgumentNullException.ThrowIfNull(loggerFactory);
var logger = loggerFactory.CreateLogger("Scanner.WebService.Health");
var stopwatch = Stopwatch.StartNew();
var success = true;
string? error = null;
try
{
var validationContext = SurfaceValidationContext.Create(
context.RequestServices,
"Scanner.WebService.ReadyCheck",
surfaceEnvironment.Settings,
properties: new Dictionary<string, object?>
{
["path"] = context.Request.Path.ToString()
});
await validatorRunner.EnsureAsync(validationContext, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (SurfaceValidationException ex)
{
success = false;
error = ex.Message;
}
catch (Exception ex)
{
success = false;
error = ex.Message;
logger.LogError(ex, "Surface validation failed during ready check.");
}
finally
{
stopwatch.Stop();
}
status.RecordReadyCheck(success, stopwatch.Elapsed, error);
var snapshot = status.CreateSnapshot();
var ready = snapshot.Ready;
var document = new ReadyDocument(
Status: ready.IsReady ? "ready" : "unready",
CheckedAt: ready.CheckedAt,
LatencyMs: ready.Latency?.TotalMilliseconds,
Error: ready.Error);
var statusCode = success ? StatusCodes.Status200OK : StatusCodes.Status503ServiceUnavailable;
return Json(document, statusCode);
}
private static void ApplyNoCache(HttpResponse response)
{

View File

@@ -1,17 +1,18 @@
using System.Collections.Generic;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Routing;
using StellaOps.Policy;
using StellaOps.Scanner.WebService.Constants;
using StellaOps.Scanner.WebService.Contracts;
using StellaOps.Scanner.WebService.Infrastructure;
using StellaOps.Scanner.WebService.Security;
using StellaOps.Scanner.WebService.Services;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Routing;
using Microsoft.Extensions.Logging;
using StellaOps.Policy;
using StellaOps.Scanner.WebService.Constants;
using StellaOps.Scanner.WebService.Contracts;
using StellaOps.Scanner.WebService.Infrastructure;
using StellaOps.Scanner.WebService.Security;
using StellaOps.Scanner.WebService.Services;
namespace StellaOps.Scanner.WebService.Endpoints;
@@ -49,25 +50,30 @@ internal static class ReportEndpoints
});
}
private static async Task<IResult> HandleCreateReportAsync(
ReportRequestDto request,
PolicyPreviewService previewService,
IReportSigner signer,
TimeProvider timeProvider,
IReportEventDispatcher eventDispatcher,
HttpContext context,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
ArgumentNullException.ThrowIfNull(previewService);
ArgumentNullException.ThrowIfNull(signer);
ArgumentNullException.ThrowIfNull(timeProvider);
ArgumentNullException.ThrowIfNull(eventDispatcher);
if (string.IsNullOrWhiteSpace(request.ImageDigest))
{
return ProblemResultFactory.Create(
context,
private static async Task<IResult> HandleCreateReportAsync(
ReportRequestDto request,
PolicyPreviewService previewService,
IReportSigner signer,
TimeProvider timeProvider,
IReportEventDispatcher eventDispatcher,
ISurfacePointerService surfacePointerService,
ILoggerFactory loggerFactory,
HttpContext context,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
ArgumentNullException.ThrowIfNull(previewService);
ArgumentNullException.ThrowIfNull(signer);
ArgumentNullException.ThrowIfNull(timeProvider);
ArgumentNullException.ThrowIfNull(eventDispatcher);
ArgumentNullException.ThrowIfNull(surfacePointerService);
ArgumentNullException.ThrowIfNull(loggerFactory);
var logger = loggerFactory.CreateLogger("Scanner.WebService.Reports");
if (string.IsNullOrWhiteSpace(request.ImageDigest))
{
return ProblemResultFactory.Create(
context,
ProblemTypes.Validation,
"Invalid report request",
StatusCodes.Status400BadRequest,
@@ -127,26 +133,46 @@ internal static class ReportEndpoints
.ToArray();
var issuesDto = preview.Issues.Select(PolicyDtoMapper.ToIssueDto).ToArray();
var summary = BuildSummary(projectedVerdicts);
var verdict = ComputeVerdict(projectedVerdicts);
var reportId = CreateReportId(request.ImageDigest!, preview.PolicyDigest);
var generatedAt = timeProvider.GetUtcNow();
var document = new ReportDocumentDto
{
ReportId = reportId,
ImageDigest = request.ImageDigest!,
GeneratedAt = generatedAt,
Verdict = verdict,
Policy = new ReportPolicyDto
{
RevisionId = preview.RevisionId,
Digest = preview.PolicyDigest
},
Summary = summary,
Verdicts = projectedVerdicts,
Issues = issuesDto
};
var summary = BuildSummary(projectedVerdicts);
var verdict = ComputeVerdict(projectedVerdicts);
var reportId = CreateReportId(request.ImageDigest!, preview.PolicyDigest);
var generatedAt = timeProvider.GetUtcNow();
SurfacePointersDto? surfacePointers = null;
try
{
surfacePointers = await surfacePointerService
.TryBuildAsync(request.ImageDigest!, context.RequestAborted)
.ConfigureAwait(false);
}
catch (OperationCanceledException) when (context.RequestAborted.IsCancellationRequested)
{
throw;
}
catch (Exception ex)
{
if (!context.RequestAborted.IsCancellationRequested)
{
logger.LogDebug(ex, "Failed to build surface pointers for digest {Digest}.", request.ImageDigest);
}
}
var document = new ReportDocumentDto
{
ReportId = reportId,
ImageDigest = request.ImageDigest!,
GeneratedAt = generatedAt,
Verdict = verdict,
Policy = new ReportPolicyDto
{
RevisionId = preview.RevisionId,
Digest = preview.PolicyDigest
},
Summary = summary,
Verdicts = projectedVerdicts,
Issues = issuesDto,
Surface = surfacePointers
};
var payloadBytes = JsonSerializer.SerializeToUtf8Bytes(document, SerializerOptions);
var signature = signer.Sign(payloadBytes);
@@ -169,11 +195,11 @@ internal static class ReportEndpoints
};
}
var response = new ReportResponseDto
{
Report = document,
Dsse = envelope
};
var response = new ReportResponseDto
{
Report = document,
Dsse = envelope
};
await eventDispatcher
.PublishAsync(request, preview, document, envelope, context, cancellationToken)

View File

@@ -140,10 +140,12 @@ internal static class ScanEndpoints
private static async Task<IResult> HandleStatusAsync(
string scanId,
IScanCoordinator coordinator,
ISurfacePointerService surfacePointerService,
HttpContext context,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(coordinator);
ArgumentNullException.ThrowIfNull(surfacePointerService);
if (!ScanId.TryParse(scanId, out var parsed))
{
@@ -163,7 +165,23 @@ internal static class ScanEndpoints
ProblemTypes.NotFound,
"Scan not found",
StatusCodes.Status404NotFound,
detail: "Requested scan could not be located.");
detail: "Requested scan could not be located.");
}
SurfacePointersDto? surfacePointers = null;
var digest = snapshot.Target.Digest;
if (!string.IsNullOrWhiteSpace(digest))
{
try
{
surfacePointers = await surfacePointerService
.TryBuildAsync(digest!, context.RequestAborted)
.ConfigureAwait(false);
}
catch (OperationCanceledException) when (context.RequestAborted.IsCancellationRequested)
{
throw;
}
}
var response = new ScanStatusResponse(
@@ -172,7 +190,8 @@ internal static class ScanEndpoints
Image: new ScanStatusTarget(snapshot.Target.Reference, snapshot.Target.Digest),
CreatedAt: snapshot.CreatedAt,
UpdatedAt: snapshot.UpdatedAt,
FailureReason: snapshot.FailureReason);
FailureReason: snapshot.FailureReason,
Surface: surfacePointers);
return Json(response, StatusCodes.Status200OK);
}

View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using StellaOps.Scanner.Storage;
namespace StellaOps.Scanner.WebService.Options;
@@ -129,6 +130,8 @@ public sealed class ScannerWebServiceOptions
public int ObjectLockRetentionDays { get; set; } = 30;
public string RootPrefix { get; set; } = ScannerStorageDefaults.DefaultRootPrefix;
public string? ApiKey { get; set; }
public string ApiKeyHeader { get; set; } = string.Empty;

View File

@@ -19,6 +19,10 @@ using StellaOps.Cryptography.DependencyInjection;
using StellaOps.Cryptography.Plugin.BouncyCastle;
using StellaOps.Policy;
using StellaOps.Scanner.Cache;
using StellaOps.Scanner.Surface.Env;
using StellaOps.Scanner.Surface.FS;
using StellaOps.Scanner.Surface.Secrets;
using StellaOps.Scanner.Surface.Validation;
using StellaOps.Scanner.WebService.Diagnostics;
using StellaOps.Scanner.WebService.Endpoints;
using StellaOps.Scanner.WebService.Extensions;
@@ -80,11 +84,20 @@ builder.Services.AddSingleton<IScanProgressReader>(sp => sp.GetRequiredService<S
builder.Services.AddSingleton<IScanCoordinator, InMemoryScanCoordinator>();
builder.Services.AddSingleton<IPolicySnapshotRepository, InMemoryPolicySnapshotRepository>();
builder.Services.AddSingleton<IPolicyAuditRepository, InMemoryPolicyAuditRepository>();
builder.Services.AddSingleton<PolicySnapshotStore>();
builder.Services.AddSingleton<PolicyPreviewService>();
builder.Services.AddStellaOpsCrypto();
builder.Services.AddBouncyCastleEd25519Provider();
builder.Services.AddSingleton<PolicySnapshotStore>();
builder.Services.AddSingleton<PolicyPreviewService>();
builder.Services.AddStellaOpsCrypto();
builder.Services.AddBouncyCastleEd25519Provider();
builder.Services.AddSingleton<IReportSigner, ReportSigner>();
builder.Services.AddSurfaceEnvironment(options =>
{
options.ComponentName = "Scanner.WebService";
options.AddPrefix("SCANNER");
});
builder.Services.AddSurfaceValidation();
builder.Services.AddSurfaceFileCache();
builder.Services.AddSurfaceSecrets();
builder.Services.AddSingleton<ISurfacePointerService, SurfacePointerService>();
builder.Services.AddSingleton<IRedisConnectionFactory, RedisConnectionFactory>();
if (bootstrapOptions.Events is { Enabled: true } eventsOptions
&& string.Equals(eventsOptions.Driver, "redis", StringComparison.OrdinalIgnoreCase))
@@ -119,6 +132,11 @@ builder.Services.AddScannerStorage(storageOptions =>
storageOptions.ObjectStore.BucketName = bootstrapOptions.ArtifactStore.Bucket;
}
if (!string.IsNullOrWhiteSpace(bootstrapOptions.ArtifactStore.RootPrefix))
{
storageOptions.ObjectStore.RootPrefix = bootstrapOptions.ArtifactStore.RootPrefix;
}
var artifactDriver = bootstrapOptions.ArtifactStore.Driver?.Trim() ?? string.Empty;
if (string.Equals(artifactDriver, ScannerStorageDefaults.ObjectStoreProviders.RustFs, StringComparison.OrdinalIgnoreCase))
{

View File

@@ -207,10 +207,7 @@ internal static class OrchestratorEventSerializer
return;
}
info.PolymorphismOptions ??= new JsonPolymorphismOptions
{
UnknownDerivedTypeHandling = JsonUnknownDerivedTypeHandling.Fail
};
info.PolymorphismOptions ??= new JsonPolymorphismOptions();
AddDerivedType(info.PolymorphismOptions, typeof(ReportReadyEventPayload));
AddDerivedType(info.PolymorphismOptions, typeof(ScanCompletedEventPayload));

View File

@@ -0,0 +1,279 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Security.Cryptography;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Scanner.Storage;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.ObjectStore;
using StellaOps.Scanner.Storage.Repositories;
using StellaOps.Scanner.Surface.Env;
using StellaOps.Scanner.WebService.Contracts;
using StellaOps.Scanner.WebService.Options;
namespace StellaOps.Scanner.WebService.Services;
internal interface ISurfacePointerService
{
Task<SurfacePointersDto?> TryBuildAsync(string imageDigest, CancellationToken cancellationToken);
}
internal sealed class SurfacePointerService : ISurfacePointerService
{
private static readonly JsonSerializerOptions ManifestSerializerOptions = new(JsonSerializerDefaults.Web)
{
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
WriteIndented = false
};
private readonly LinkRepository _linkRepository;
private readonly ArtifactRepository _artifactRepository;
private readonly IOptionsMonitor<ScannerWebServiceOptions> _optionsMonitor;
private readonly ISurfaceEnvironment _surfaceEnvironment;
private readonly TimeProvider _timeProvider;
private readonly ILogger<SurfacePointerService> _logger;
public SurfacePointerService(
LinkRepository linkRepository,
ArtifactRepository artifactRepository,
IOptionsMonitor<ScannerWebServiceOptions> optionsMonitor,
ISurfaceEnvironment surfaceEnvironment,
TimeProvider timeProvider,
ILogger<SurfacePointerService> logger)
{
_linkRepository = linkRepository ?? throw new ArgumentNullException(nameof(linkRepository));
_artifactRepository = artifactRepository ?? throw new ArgumentNullException(nameof(artifactRepository));
_optionsMonitor = optionsMonitor ?? throw new ArgumentNullException(nameof(optionsMonitor));
_surfaceEnvironment = surfaceEnvironment ?? throw new ArgumentNullException(nameof(surfaceEnvironment));
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<SurfacePointersDto?> TryBuildAsync(string imageDigest, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(imageDigest))
{
return null;
}
var normalizedDigest = imageDigest.Trim();
List<LinkDocument> links;
try
{
links = await _linkRepository.ListBySourceAsync(LinkSourceType.Image, normalizedDigest, cancellationToken)
.ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to load link documents for digest {Digest}.", normalizedDigest);
return null;
}
if (links.Count == 0)
{
return null;
}
var options = _optionsMonitor.CurrentValue ?? new ScannerWebServiceOptions();
var artifactStore = options.ArtifactStore ?? new ScannerWebServiceOptions.ArtifactStoreOptions();
var bucket = ResolveBucket(artifactStore);
var rootPrefix = artifactStore.RootPrefix ?? ScannerStorageDefaults.DefaultRootPrefix;
var tenant = _surfaceEnvironment.Settings.Tenant;
var generatedAt = _timeProvider.GetUtcNow();
var artifacts = ImmutableArray.CreateBuilder<SurfaceManifestArtifact>();
foreach (var link in links)
{
cancellationToken.ThrowIfCancellationRequested();
ArtifactDocument? artifactDocument;
try
{
artifactDocument = await _artifactRepository.GetAsync(link.ArtifactId, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to load artifact document {ArtifactId}.", link.ArtifactId);
continue;
}
if (artifactDocument is null)
{
continue;
}
var objectKey = ArtifactObjectKeyBuilder.Build(
artifactDocument.Type,
artifactDocument.Format,
artifactDocument.BytesSha256,
rootPrefix);
var uri = BuildCasUri(bucket, objectKey);
var (kind, view) = MapKindAndView(artifactDocument);
var format = MapFormat(artifactDocument.Format);
var artifact = new SurfaceManifestArtifact
{
Kind = kind,
Uri = uri,
Digest = artifactDocument.BytesSha256,
MediaType = artifactDocument.MediaType,
Format = format,
SizeBytes = artifactDocument.SizeBytes,
View = view
};
artifacts.Add(artifact);
}
if (artifacts.Count == 0)
{
return null;
}
var orderedArtifacts = artifacts.OrderBy(a => a.Kind, StringComparer.Ordinal)
.ThenBy(a => a.Format, StringComparer.Ordinal)
.ThenBy(a => a.Digest, StringComparer.Ordinal)
.ToImmutableArray();
var manifest = new SurfaceManifestDocument
{
Tenant = tenant,
ImageDigest = normalizedDigest,
GeneratedAt = generatedAt,
Artifacts = orderedArtifacts
};
var manifestJson = JsonSerializer.SerializeToUtf8Bytes(manifest, ManifestSerializerOptions);
var manifestDigest = ComputeDigest(manifestJson);
var manifestUri = BuildManifestUri(bucket, rootPrefix, tenant, manifestDigest);
return new SurfacePointersDto
{
Tenant = tenant,
GeneratedAt = generatedAt,
ManifestDigest = manifestDigest,
ManifestUri = manifestUri,
Manifest = manifest with { GeneratedAt = generatedAt }
};
}
private static string ResolveBucket(ScannerWebServiceOptions.ArtifactStoreOptions artifactStore)
{
if (!string.IsNullOrWhiteSpace(artifactStore.Bucket))
{
return artifactStore.Bucket.Trim();
}
return ScannerStorageDefaults.DefaultBucketName;
}
private static string MapFormat(ArtifactDocumentFormat format)
=> format switch
{
ArtifactDocumentFormat.CycloneDxJson => "cdx-json",
ArtifactDocumentFormat.CycloneDxProtobuf => "cdx-protobuf",
ArtifactDocumentFormat.SpdxJson => "spdx-json",
ArtifactDocumentFormat.BomIndex => "bom-index",
ArtifactDocumentFormat.DsseJson => "dsse-json",
_ => format.ToString().ToLowerInvariant()
};
private static (string Kind, string? View) MapKindAndView(ArtifactDocument document)
{
if (document.Type == ArtifactDocumentType.ImageBom)
{
var view = ResolveView(document.MediaType);
var kind = string.Equals(view, "usage", StringComparison.OrdinalIgnoreCase)
? "sbom-usage"
: "sbom-inventory";
return (kind, view);
}
return document.Type switch
{
ArtifactDocumentType.LayerBom => ("layer-sbom", null),
ArtifactDocumentType.Diff => ("diff", null),
ArtifactDocumentType.Attestation => ("attestation", null),
ArtifactDocumentType.Index => ("bom-index", null),
_ => (document.Type.ToString().ToLowerInvariant(), null)
};
}
private static string? ResolveView(string mediaType)
{
if (string.IsNullOrWhiteSpace(mediaType))
{
return null;
}
if (mediaType.Contains("view=usage", StringComparison.OrdinalIgnoreCase))
{
return "usage";
}
if (mediaType.Contains("view=inventory", StringComparison.OrdinalIgnoreCase))
{
return "inventory";
}
return null;
}
private static string BuildCasUri(string bucket, string key)
{
var normalizedKey = string.IsNullOrWhiteSpace(key) ? string.Empty : key.Trim().TrimStart('/');
return $"cas://{bucket}/{normalizedKey}";
}
private static string BuildManifestUri(string bucket, string rootPrefix, string tenant, string manifestDigest)
{
var (algorithm, digestValue) = SplitDigest(manifestDigest);
var prefix = string.IsNullOrWhiteSpace(rootPrefix)
? "surface/manifests"
: $"{TrimTrailingSlash(rootPrefix)}/surface/manifests";
var head = digestValue.Length >= 4
? $"{digestValue[..2]}/{digestValue[2..4]}"
: digestValue;
var key = $"{prefix}/{tenant}/{algorithm}/{head}/{digestValue}.json";
return $"cas://{bucket}/{key}";
}
private static (string Algorithm, string Digest) SplitDigest(string digest)
{
if (string.IsNullOrWhiteSpace(digest))
{
return ("sha256", digest ?? string.Empty);
}
var parts = digest.Split(':', 2, StringSplitOptions.TrimEntries);
if (parts.Length == 2)
{
return (parts[0], parts[1]);
}
return ("sha256", digest);
}
private static string TrimTrailingSlash(string value)
=> string.IsNullOrWhiteSpace(value)
? string.Empty
: value.Trim().TrimEnd('/');
private static string ComputeDigest(ReadOnlySpan<byte> payload)
{
Span<byte> hash = stackalloc byte[32];
if (!SHA256.TryHashData(payload, hash, out _))
{
using var sha = SHA256.Create();
hash = sha.ComputeHash(payload.ToArray());
}
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
}
}

View File

@@ -29,6 +29,10 @@
<ProjectReference Include="../../Notify/__Libraries/StellaOps.Notify.Models/StellaOps.Notify.Models.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Scanner.Cache/StellaOps.Scanner.Cache.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Scanner.Storage/StellaOps.Scanner.Storage.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Scanner.Surface.Env/StellaOps.Scanner.Surface.Env.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Scanner.Surface.Validation/StellaOps.Scanner.Surface.Validation.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Scanner.Surface.FS/StellaOps.Scanner.Surface.FS.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Scanner.Surface.Secrets/StellaOps.Scanner.Surface.Secrets.csproj" />
<ProjectReference Include="../../Zastava/__Libraries/StellaOps.Zastava.Core/StellaOps.Zastava.Core.csproj" />
</ItemGroup>
</Project>
</Project>

View File

@@ -3,7 +3,7 @@
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|----|--------|----------|------------|-------------|---------------|
| SCAN-REPLAY-186-001 | TODO | Scanner WebService Guild | REPLAY-CORE-185-001 | Implement scan `record` mode producing replay manifests/bundles, capture policy/feed/tool hashes, and update `docs/modules/scanner/architecture.md` referencing `docs/replay/DETERMINISTIC_REPLAY.md` Section 6. | API/worker integration tests cover record mode; docs merged; replay artifacts stored per spec. |
| SCANNER-SURFACE-02 | DOING (2025-11-02) | Scanner WebService Guild | SURFACE-FS-02 | Publish Surface.FS pointers (CAS URIs, manifests) via scan/report APIs and update attestation metadata.<br>2025-11-02: Scan/report API responses now include preview CAS URIs; attestation metadata draft published. | OpenAPI updated; clients regenerated; integration tests validate pointer presence and tenancy. |
| SCANNER-SURFACE-02 | DONE (2025-11-05) | Scanner WebService Guild | SURFACE-FS-02 | Publish Surface.FS pointers (CAS URIs, manifests) via scan/report APIs and update attestation metadata.<br>2025-11-05: Surface pointers projected through scan/report endpoints, orchestrator samples + DSSE fixtures refreshed with manifest block, readiness tests updated to use validator stub. | OpenAPI updated; clients regenerated; integration tests validate pointer presence and tenancy. |
| SCANNER-ENV-02 | DOING (2025-11-02) | Scanner WebService Guild, Ops Guild | SURFACE-ENV-02 | Wire Surface.Env helpers into WebService hosting (cache roots, feature flags) and document configuration.<br>2025-11-02: Cache root resolution switched to helper; feature flag bindings updated; Helm/Compose updates pending review. | Service uses helper; env table documented; helm/compose templates updated. |
| SCANNER-SECRETS-02 | DOING (2025-11-02) | Scanner WebService Guild, Security Guild | SURFACE-SECRETS-02 | Replace ad-hoc secret wiring with Surface.Secrets for report/export operations (registry and CAS tokens).<br>2025-11-02: Export/report flows now depend on Surface.Secrets stub; integration tests in progress. | Secrets fetched through shared provider; unit/integration tests cover rotation + failure cases. |
| SCANNER-EVENTS-16-301 | BLOCKED (2025-10-26) | Scanner WebService Guild | ORCH-SVC-38-101, NOTIFY-SVC-38-001 | Emit orchestrator-compatible envelopes (`scanner.event.*`) and update integration tests to verify Notifier ingestion (no Redis queue coupling). | Tests assert envelope schema + orchestrator publish; Notifier consumer harness passes; docs updated with new event contract. Blocked by .NET 10 preview OpenAPI/Auth dependency drift preventing `dotnet test` completion. |

View File

@@ -0,0 +1,75 @@
using System;
using StellaOps.Scanner.Storage.Catalog;
namespace StellaOps.Scanner.Storage.ObjectStore;
/// <summary>
/// Builds deterministic object keys for scanner artefacts stored in the backing object store.
/// </summary>
public static class ArtifactObjectKeyBuilder
{
/// <summary>
/// Builds an object key for the provided artefact metadata.
/// </summary>
/// <param name="type">Artefact type.</param>
/// <param name="format">Artefact format.</param>
/// <param name="digest">Content digest (with or without algorithm prefix).</param>
/// <param name="rootPrefix">Optional root prefix to prepend (defaults to <c>scanner</c>).</param>
/// <returns>Deterministic storage key.</returns>
public static string Build(
ArtifactDocumentType type,
ArtifactDocumentFormat format,
string digest,
string? rootPrefix = null)
{
ArgumentException.ThrowIfNullOrWhiteSpace(digest);
var normalizedDigest = NormalizeDigest(digest);
var digestValue = ExtractDigest(normalizedDigest);
var prefix = type switch
{
ArtifactDocumentType.LayerBom => ScannerStorageDefaults.ObjectPrefixes.Layers,
ArtifactDocumentType.ImageBom => ScannerStorageDefaults.ObjectPrefixes.Images,
ArtifactDocumentType.Index => ScannerStorageDefaults.ObjectPrefixes.Indexes,
ArtifactDocumentType.Attestation => ScannerStorageDefaults.ObjectPrefixes.Attestations,
ArtifactDocumentType.Diff => "diffs",
_ => ScannerStorageDefaults.ObjectPrefixes.Images,
};
var extension = format switch
{
ArtifactDocumentFormat.CycloneDxJson => "sbom.cdx.json",
ArtifactDocumentFormat.CycloneDxProtobuf => "sbom.cdx.pb",
ArtifactDocumentFormat.SpdxJson => "sbom.spdx.json",
ArtifactDocumentFormat.BomIndex => "bom-index.bin",
ArtifactDocumentFormat.DsseJson => "artifact.dsse.json",
_ => "artifact.bin",
};
var key = $"{prefix}/{digestValue}/{extension}";
if (string.IsNullOrWhiteSpace(rootPrefix))
{
return key;
}
return $"{TrimTrailingSlash(rootPrefix)}/{key}";
}
private static string NormalizeDigest(string digest)
=> digest.Contains(':', StringComparison.Ordinal)
? digest.Trim()
: $"sha256:{digest.Trim()}";
private static string ExtractDigest(string normalizedDigest)
{
var parts = normalizedDigest.Split(':', 2, StringSplitOptions.TrimEntries);
return parts.Length == 2 ? parts[1] : normalizedDigest;
}
private static string TrimTrailingSlash(string value)
=> string.IsNullOrWhiteSpace(value)
? string.Empty
: value.Trim().TrimEnd('/');
}

View File

@@ -50,8 +50,12 @@ public sealed class ArtifactStorageService
try
{
var normalizedDigest = $"sha256:{digestHex}";
var artifactId = CatalogIdFactory.CreateArtifactId(type, normalizedDigest);
var key = BuildObjectKey(type, format, normalizedDigest);
var artifactId = CatalogIdFactory.CreateArtifactId(type, normalizedDigest);
var key = ArtifactObjectKeyBuilder.Build(
type,
format,
normalizedDigest,
_options.ObjectStore.RootPrefix);
var descriptor = new ArtifactObjectDescriptor(
_options.ObjectStore.BucketName,
key,
@@ -137,45 +141,4 @@ public sealed class ArtifactStorageService
return (bufferStream, total, digestHex);
}
private string BuildObjectKey(ArtifactDocumentType type, ArtifactDocumentFormat format, string digest)
{
var normalizedDigest = digest.Split(':', 2, StringSplitOptions.TrimEntries)[^1];
var prefix = type switch
{
ArtifactDocumentType.LayerBom => ScannerStorageDefaults.ObjectPrefixes.Layers,
ArtifactDocumentType.ImageBom => ScannerStorageDefaults.ObjectPrefixes.Images,
ArtifactDocumentType.Diff => "diffs",
ArtifactDocumentType.Index => ScannerStorageDefaults.ObjectPrefixes.Indexes,
ArtifactDocumentType.Attestation => ScannerStorageDefaults.ObjectPrefixes.Attestations,
_ => ScannerStorageDefaults.ObjectPrefixes.Images,
};
var extension = format switch
{
ArtifactDocumentFormat.CycloneDxJson => "sbom.cdx.json",
ArtifactDocumentFormat.CycloneDxProtobuf => "sbom.cdx.pb",
ArtifactDocumentFormat.SpdxJson => "sbom.spdx.json",
ArtifactDocumentFormat.BomIndex => "bom-index.bin",
ArtifactDocumentFormat.DsseJson => "artifact.dsse.json",
_ => "artifact.bin",
};
var rootPrefix = _options.ObjectStore.RootPrefix;
if (string.IsNullOrWhiteSpace(rootPrefix))
{
return $"{prefix}/{normalizedDigest}/{extension}";
}
return $"{TrimTrailingSlash(rootPrefix)}/{prefix}/{normalizedDigest}/{extension}";
}
private static string TrimTrailingSlash(string prefix)
{
if (string.IsNullOrWhiteSpace(prefix))
{
return string.Empty;
}
return prefix.TrimEnd('/');
}
}
}

View File

@@ -1,10 +1,13 @@
using System.Collections.Generic;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.AspNetCore.TestHost;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Mongo2Go;
using System.Collections.Generic;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.AspNetCore.TestHost;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Mongo2Go;
using StellaOps.Scanner.Surface.Validation;
namespace StellaOps.Scanner.WebService.Tests;
@@ -56,14 +59,17 @@ internal sealed class ScannerApplicationFactory : WebApplicationFactory<Program>
Environment.SetEnvironmentVariable("SCANNER__AUTHORITY__CLIENTID", null);
Environment.SetEnvironmentVariable("SCANNER__AUTHORITY__CLIENTSECRET", null);
Environment.SetEnvironmentVariable("SCANNER__STORAGE__DSN", configuration["scanner:storage:dsn"]);
Environment.SetEnvironmentVariable("SCANNER__QUEUE__DSN", configuration["scanner:queue:dsn"]);
Environment.SetEnvironmentVariable("SCANNER__ARTIFACTSTORE__ENDPOINT", configuration["scanner:artifactStore:endpoint"]);
Environment.SetEnvironmentVariable("SCANNER__ARTIFACTSTORE__ACCESSKEY", configuration["scanner:artifactStore:accessKey"]);
Environment.SetEnvironmentVariable("SCANNER__ARTIFACTSTORE__SECRETKEY", configuration["scanner:artifactStore:secretKey"]);
if (configuration.TryGetValue("scanner:events:enabled", out var eventsEnabled))
{
Environment.SetEnvironmentVariable("SCANNER__EVENTS__ENABLED", eventsEnabled);
}
Environment.SetEnvironmentVariable("SCANNER__QUEUE__DSN", configuration["scanner:queue:dsn"]);
Environment.SetEnvironmentVariable("SCANNER__ARTIFACTSTORE__ENDPOINT", configuration["scanner:artifactStore:endpoint"]);
Environment.SetEnvironmentVariable("SCANNER__ARTIFACTSTORE__ACCESSKEY", configuration["scanner:artifactStore:accessKey"]);
Environment.SetEnvironmentVariable("SCANNER__ARTIFACTSTORE__SECRETKEY", configuration["scanner:artifactStore:secretKey"]);
Environment.SetEnvironmentVariable("SCANNER_SURFACE_FS_ENDPOINT", "https://surface.local");
Environment.SetEnvironmentVariable("SCANNER_SURFACE_FS_BUCKET", configuration["scanner:artifactStore:bucket"]);
Environment.SetEnvironmentVariable("SCANNER_SURFACE_PREFETCH_ENABLED", "false");
if (configuration.TryGetValue("scanner:events:enabled", out var eventsEnabled))
{
Environment.SetEnvironmentVariable("SCANNER__EVENTS__ENABLED", eventsEnabled);
}
if (configuration.TryGetValue("scanner:authority:enabled", out var authorityEnabled))
{
@@ -100,11 +106,13 @@ internal sealed class ScannerApplicationFactory : WebApplicationFactory<Program>
configBuilder.AddInMemoryCollection(configuration);
});
builder.ConfigureTestServices(services =>
{
configureServices?.Invoke(services);
});
}
builder.ConfigureTestServices(services =>
{
configureServices?.Invoke(services);
services.RemoveAll<ISurfaceValidatorRunner>();
services.AddSingleton<ISurfaceValidatorRunner, TestSurfaceValidatorRunner>();
});
}
protected override void Dispose(bool disposing)
{
@@ -163,6 +171,19 @@ internal sealed class ScannerApplicationFactory : WebApplicationFactory<Program>
current = parent.FullName;
}
return null;
}
}
return null;
}
private sealed class TestSurfaceValidatorRunner : ISurfaceValidatorRunner
{
public ValueTask<SurfaceValidationResult> RunAllAsync(
SurfaceValidationContext context,
CancellationToken cancellationToken = default)
=> ValueTask.FromResult(SurfaceValidationResult.Success());
public ValueTask EnsureAsync(
SurfaceValidationContext context,
CancellationToken cancellationToken = default)
=> ValueTask.CompletedTask;
}
}

View File

@@ -54,10 +54,10 @@ public sealed class ScansEndpointsTests
Assert.Equal(payload.ScanId, status!.ScanId);
Assert.Equal("Pending", status.Status);
Assert.Equal("ghcr.io/demo/app:1.0.0", status.Image.Reference);
}
[Fact]
public async Task SubmitScanIsDeterministicForIdenticalPayloads()
}
[Fact]
public async Task SubmitScanIsDeterministicForIdenticalPayloads()
{
using var factory = new ScannerApplicationFactory();
using var client = factory.CreateClient();
@@ -81,11 +81,98 @@ public sealed class ScansEndpointsTests
Assert.Equal(firstPayload!.ScanId, secondPayload!.ScanId);
Assert.True(firstPayload.Created);
Assert.False(secondPayload.Created);
}
[Fact]
public async Task SubmitScanValidatesImageDescriptor()
{
}
[Fact]
public async Task ScanStatusIncludesSurfacePointersWhenArtifactsExist()
{
const string digest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
var digestValue = digest.Split(':', 2)[1];
using var factory = new ScannerApplicationFactory();
using (var scope = factory.Services.CreateScope())
{
var artifactRepository = scope.ServiceProvider.GetRequiredService<ArtifactRepository>();
var linkRepository = scope.ServiceProvider.GetRequiredService<LinkRepository>();
var artifactId = CatalogIdFactory.CreateArtifactId(ArtifactDocumentType.ImageBom, digest);
var artifact = new ArtifactDocument
{
Id = artifactId,
Type = ArtifactDocumentType.ImageBom,
Format = ArtifactDocumentFormat.CycloneDxJson,
MediaType = "application/vnd.cyclonedx+json; version=1.6; view=inventory",
BytesSha256 = digest,
SizeBytes = 2048,
Immutable = true,
RefCount = 1,
TtlClass = "default",
CreatedAtUtc = DateTime.UtcNow,
UpdatedAtUtc = DateTime.UtcNow
};
await artifactRepository.UpsertAsync(artifact, CancellationToken.None).ConfigureAwait(false);
var link = new LinkDocument
{
Id = CatalogIdFactory.CreateLinkId(LinkSourceType.Image, digest, artifactId),
FromType = LinkSourceType.Image,
FromDigest = digest,
ArtifactId = artifactId,
CreatedAtUtc = DateTime.UtcNow
};
await linkRepository.UpsertAsync(link, CancellationToken.None).ConfigureAwait(false);
}
using var client = factory.CreateClient();
var submitRequest = new ScanSubmitRequest
{
Image = new ScanImageDescriptor
{
Digest = digest
}
};
var submitResponse = await client.PostAsJsonAsync("/api/v1/scans", submitRequest);
submitResponse.EnsureSuccessStatusCode();
var submission = await submitResponse.Content.ReadFromJsonAsync<ScanSubmitResponse>();
Assert.NotNull(submission);
var statusResponse = await client.GetAsync($"/api/v1/scans/{submission!.ScanId}");
statusResponse.EnsureSuccessStatusCode();
var status = await statusResponse.Content.ReadFromJsonAsync<ScanStatusResponse>();
Assert.NotNull(status);
Assert.NotNull(status!.Surface);
var surface = status.Surface!;
Assert.Equal("default", surface.Tenant);
Assert.False(string.IsNullOrWhiteSpace(surface.ManifestDigest));
Assert.NotNull(surface.ManifestUri);
Assert.Contains("cas://scanner-artifacts/", surface.ManifestUri, StringComparison.Ordinal);
var manifest = surface.Manifest;
Assert.Equal(digest, manifest.ImageDigest);
Assert.Equal(surface.Tenant, manifest.Tenant);
Assert.NotEqual(default, manifest.GeneratedAt);
var manifestArtifact = Assert.Single(manifest.Artifacts);
Assert.Equal("sbom-inventory", manifestArtifact.Kind);
Assert.Equal("cdx-json", manifestArtifact.Format);
Assert.Equal(digest, manifestArtifact.Digest);
Assert.Equal("application/vnd.cyclonedx+json; version=1.6; view=inventory", manifestArtifact.MediaType);
Assert.Equal("inventory", manifestArtifact.View);
var expectedUri = $"cas://scanner-artifacts/scanner/images/{digestValue}/sbom.cdx.json";
Assert.Equal(expectedUri, manifestArtifact.Uri);
}
[Fact]
public async Task SubmitScanValidatesImageDescriptor()
{
using var factory = new ScannerApplicationFactory();
using var client = factory.CreateClient();
@@ -462,7 +549,7 @@ public sealed class ScansEndpointsTests
var storedResult = new EntryTraceResult(scanId, "sha256:test", generatedAt, graph, ndjson);
using var factory = new ScannerApplicationFactory(
configuration: null,
configureConfiguration: null,
services =>
{
services.AddSingleton<IEntryTraceResultStore>(new StubEntryTraceResultStore(storedResult));
@@ -485,7 +572,7 @@ public sealed class ScansEndpointsTests
public async Task GetEntryTraceReturnsNotFoundWhenMissing()
{
using var factory = new ScannerApplicationFactory(
configuration: null,
configureConfiguration: null,
services =>
{
services.AddSingleton<IEntryTraceResultStore>(new StubEntryTraceResultStore(null));