doctor enhancements, setup, enhancements, ui functionality and design consolidation and , test projects fixes , product advisory attestation/rekor and delta verfications enhancements
This commit is contained in:
master
@@ -0,0 +1,343 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Diagnostics;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Scanner.Manifest.Resolution;
|
||||
|
||||
namespace StellaOps.Scanner.Delta;
|
||||
|
||||
/// <summary>
|
||||
/// Implementation of IDeltaLayerScanner that scans only changed layers between image versions.
|
||||
/// Reduces scan time and CVE churn by reusing cached per-layer SBOMs.
|
||||
/// </summary>
|
||||
public sealed class DeltaLayerScanner : IDeltaLayerScanner
|
||||
{
|
||||
private readonly ILayerDigestResolver _layerResolver;
|
||||
private readonly ILayerSbomCas _sbomCas;
|
||||
private readonly ILayerScannerPipeline _scannerPipeline;
|
||||
private readonly ISbomComposer _sbomComposer;
|
||||
private readonly ILogger<DeltaLayerScanner> _logger;
|
||||
|
||||
public DeltaLayerScanner(
|
||||
ILayerDigestResolver layerResolver,
|
||||
ILayerSbomCas sbomCas,
|
||||
ILayerScannerPipeline scannerPipeline,
|
||||
ISbomComposer sbomComposer,
|
||||
ILogger<DeltaLayerScanner> logger)
|
||||
{
|
||||
_layerResolver = layerResolver ?? throw new ArgumentNullException(nameof(layerResolver));
|
||||
_sbomCas = sbomCas ?? throw new ArgumentNullException(nameof(sbomCas));
|
||||
_scannerPipeline = scannerPipeline ?? throw new ArgumentNullException(nameof(scannerPipeline));
|
||||
_sbomComposer = sbomComposer ?? throw new ArgumentNullException(nameof(sbomComposer));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<DeltaScanResult> ScanDeltaAsync(
|
||||
string oldImage,
|
||||
string newImage,
|
||||
DeltaScanOptions? options = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(oldImage);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(newImage);
|
||||
|
||||
options ??= new DeltaScanOptions();
|
||||
var totalStopwatch = Stopwatch.StartNew();
|
||||
|
||||
_logger.LogInformation("Starting delta scan: {OldImage} -> {NewImage}", oldImage, newImage);
|
||||
|
||||
// Resolve layers for both images in parallel
|
||||
var resolveOptions = new LayerResolutionOptions
|
||||
{
|
||||
ComputeDiffIds = true,
|
||||
Platform = options.Platform
|
||||
};
|
||||
|
||||
var oldResolvedTask = _layerResolver.ResolveLayersAsync(oldImage, resolveOptions, cancellationToken);
|
||||
var newResolvedTask = _layerResolver.ResolveLayersAsync(newImage, resolveOptions, cancellationToken);
|
||||
|
||||
var oldResolved = await oldResolvedTask.ConfigureAwait(false);
|
||||
var newResolved = await newResolvedTask.ConfigureAwait(false);
|
||||
|
||||
// Identify layer changes using diffID (content hash)
|
||||
var oldDiffIds = oldResolved.Layers
|
||||
.Where(l => !string.IsNullOrWhiteSpace(l.DiffId))
|
||||
.Select(l => l.DiffId!)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var addedLayers = new List<LayerChangeInfo>();
|
||||
var unchangedLayers = new List<LayerChangeInfo>();
|
||||
var removedLayers = new List<LayerChangeInfo>();
|
||||
|
||||
// Categorize new image layers
|
||||
foreach (var layer in newResolved.Layers)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(layer.DiffId))
|
||||
{
|
||||
// Treat as added if diffID unknown
|
||||
addedLayers.Add(CreateLayerInfo(layer));
|
||||
}
|
||||
else if (oldDiffIds.Contains(layer.DiffId))
|
||||
{
|
||||
unchangedLayers.Add(CreateLayerInfo(layer));
|
||||
}
|
||||
else
|
||||
{
|
||||
addedLayers.Add(CreateLayerInfo(layer));
|
||||
}
|
||||
}
|
||||
|
||||
// Find removed layers
|
||||
var newDiffIds = newResolved.Layers
|
||||
.Where(l => !string.IsNullOrWhiteSpace(l.DiffId))
|
||||
.Select(l => l.DiffId!)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var layer in oldResolved.Layers)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(layer.DiffId) && !newDiffIds.Contains(layer.DiffId))
|
||||
{
|
||||
removedLayers.Add(CreateLayerInfo(layer));
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Layer analysis: {Added} added, {Removed} removed, {Unchanged} unchanged",
|
||||
addedLayers.Count, removedLayers.Count, unchangedLayers.Count);
|
||||
|
||||
// Collect SBOMs for unchanged layers from cache
|
||||
var layerSboms = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
var cachedComponentCount = 0;
|
||||
|
||||
if (options.UseCachedSboms && !options.ForceFullScan)
|
||||
{
|
||||
for (var i = 0; i < unchangedLayers.Count; i++)
|
||||
{
|
||||
var layer = unchangedLayers[i];
|
||||
var cachedSbom = await _sbomCas.GetAsync(layer.DiffId, options.SbomFormat, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(cachedSbom))
|
||||
{
|
||||
layerSboms[layer.DiffId] = cachedSbom;
|
||||
var componentCount = CountComponents(cachedSbom);
|
||||
unchangedLayers[i] = layer with { FromCache = true, ComponentCount = componentCount };
|
||||
cachedComponentCount += componentCount;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Layer not in cache - need to scan it
|
||||
_logger.LogDebug("Unchanged layer {DiffId} not in cache, will scan", layer.DiffId);
|
||||
addedLayers.Add(layer);
|
||||
unchangedLayers.RemoveAt(i);
|
||||
i--;
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogDebug("Retrieved {Count} cached layer SBOMs", layerSboms.Count);
|
||||
}
|
||||
|
||||
// Scan added layers
|
||||
var addedScanStopwatch = Stopwatch.StartNew();
|
||||
var addedComponentCount = 0;
|
||||
|
||||
for (var i = 0; i < addedLayers.Count; i++)
|
||||
{
|
||||
var layer = addedLayers[i];
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var layerSbom = await _scannerPipeline.ScanLayerAsync(
|
||||
newResolved.ImageReference,
|
||||
layer.DiffId,
|
||||
layer.LayerDigest,
|
||||
options.SbomFormat,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(layerSbom))
|
||||
{
|
||||
layerSboms[layer.DiffId] = layerSbom;
|
||||
|
||||
// Store in cache for future use
|
||||
await _sbomCas.StoreAsync(layer.DiffId, options.SbomFormat, layerSbom, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var componentCount = CountComponents(layerSbom);
|
||||
addedLayers[i] = layer with { ComponentCount = componentCount };
|
||||
addedComponentCount += componentCount;
|
||||
}
|
||||
}
|
||||
|
||||
addedScanStopwatch.Stop();
|
||||
_logger.LogInformation("Scanned {Count} added layers in {Duration:N2}s",
|
||||
addedLayers.Count, addedScanStopwatch.Elapsed.TotalSeconds);
|
||||
|
||||
// Compose final SBOM in layer order
|
||||
var orderedSboms = newResolved.Layers
|
||||
.Where(l => !string.IsNullOrWhiteSpace(l.DiffId) && layerSboms.ContainsKey(l.DiffId!))
|
||||
.Select(l => layerSboms[l.DiffId!])
|
||||
.ToList();
|
||||
|
||||
var compositeSbom = await _sbomComposer.ComposeAsync(
|
||||
orderedSboms,
|
||||
newResolved.ImageReference,
|
||||
options.SbomFormat,
|
||||
options.IncludeLayerAttribution,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
totalStopwatch.Stop();
|
||||
|
||||
var result = new DeltaScanResult
|
||||
{
|
||||
OldImage = oldImage,
|
||||
OldManifestDigest = oldResolved.ManifestDigest,
|
||||
NewImage = newImage,
|
||||
NewManifestDigest = newResolved.ManifestDigest,
|
||||
AddedLayers = [.. addedLayers],
|
||||
RemovedLayers = [.. removedLayers],
|
||||
UnchangedLayers = [.. unchangedLayers],
|
||||
CompositeSbom = compositeSbom,
|
||||
SbomFormat = options.SbomFormat,
|
||||
ScanDuration = totalStopwatch.Elapsed,
|
||||
AddedLayersScanDuration = addedScanStopwatch.Elapsed,
|
||||
AddedComponentCount = addedComponentCount,
|
||||
CachedComponentCount = cachedComponentCount,
|
||||
UsedCache = options.UseCachedSboms && !options.ForceFullScan
|
||||
};
|
||||
|
||||
_logger.LogInformation(
|
||||
"Delta scan complete: {LayerReuse:P1} reuse, {Duration:N2}s total, {AddedDuration:N2}s scanning added layers",
|
||||
result.LayerReuseRatio, result.ScanDuration.TotalSeconds, result.AddedLayersScanDuration.TotalSeconds);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public async Task<LayerChangeSummary> IdentifyLayerChangesAsync(
|
||||
string oldImage,
|
||||
string newImage,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(oldImage);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(newImage);
|
||||
|
||||
var resolveOptions = new LayerResolutionOptions { ComputeDiffIds = true };
|
||||
|
||||
var oldResolvedTask = _layerResolver.ResolveLayersAsync(oldImage, resolveOptions, cancellationToken);
|
||||
var newResolvedTask = _layerResolver.ResolveLayersAsync(newImage, resolveOptions, cancellationToken);
|
||||
|
||||
var oldResolved = await oldResolvedTask.ConfigureAwait(false);
|
||||
var newResolved = await newResolvedTask.ConfigureAwait(false);
|
||||
|
||||
var oldDiffIds = oldResolved.Layers
|
||||
.Where(l => !string.IsNullOrWhiteSpace(l.DiffId))
|
||||
.Select(l => l.DiffId!)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var newDiffIds = newResolved.Layers
|
||||
.Where(l => !string.IsNullOrWhiteSpace(l.DiffId))
|
||||
.Select(l => l.DiffId!)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var added = newDiffIds.Except(oldDiffIds, StringComparer.OrdinalIgnoreCase).ToList();
|
||||
var removed = oldDiffIds.Except(newDiffIds, StringComparer.OrdinalIgnoreCase).ToList();
|
||||
var unchanged = newDiffIds.Intersect(oldDiffIds, StringComparer.OrdinalIgnoreCase).ToList();
|
||||
|
||||
return new LayerChangeSummary
|
||||
{
|
||||
OldImage = oldImage,
|
||||
NewImage = newImage,
|
||||
OldLayerCount = oldResolved.LayerCount,
|
||||
NewLayerCount = newResolved.LayerCount,
|
||||
AddedCount = added.Count,
|
||||
RemovedCount = removed.Count,
|
||||
UnchangedCount = unchanged.Count,
|
||||
AddedDiffIds = [.. added],
|
||||
RemovedDiffIds = [.. removed],
|
||||
UnchangedDiffIds = [.. unchanged]
|
||||
};
|
||||
}
|
||||
|
||||
private static LayerChangeInfo CreateLayerInfo(LayerProvenance layer) => new()
|
||||
{
|
||||
LayerDigest = layer.LayerDigest,
|
||||
DiffId = layer.DiffId ?? string.Empty,
|
||||
LayerIndex = layer.LayerIndex,
|
||||
Size = layer.Size,
|
||||
MediaType = layer.MediaType
|
||||
};
|
||||
|
||||
private static int CountComponents(string sbom)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(sbom))
|
||||
return 0;
|
||||
|
||||
// Count component markers in SBOM
|
||||
var cycloneDxCount = CountOccurrences(sbom, "\"bom-ref\"");
|
||||
var spdxCount = CountOccurrences(sbom, "\"SPDXID\"");
|
||||
|
||||
return Math.Max(cycloneDxCount, spdxCount);
|
||||
}
|
||||
|
||||
private static int CountOccurrences(string text, string pattern)
|
||||
{
|
||||
var count = 0;
|
||||
var index = 0;
|
||||
while ((index = text.IndexOf(pattern, index, StringComparison.Ordinal)) != -1)
|
||||
{
|
||||
count++;
|
||||
index += pattern.Length;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Interface for per-layer SBOM content-addressable storage.
|
||||
/// </summary>
|
||||
public interface ILayerSbomCas
|
||||
{
|
||||
/// <summary>
|
||||
/// Stores a per-layer SBOM.
|
||||
/// </summary>
|
||||
Task StoreAsync(string diffId, string format, string sbom, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves a per-layer SBOM.
|
||||
/// </summary>
|
||||
Task<string?> GetAsync(string diffId, string format, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Checks if a per-layer SBOM exists.
|
||||
/// </summary>
|
||||
Task<bool> ExistsAsync(string diffId, string format, CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Interface for scanning individual layers.
|
||||
/// </summary>
|
||||
public interface ILayerScannerPipeline
|
||||
{
|
||||
/// <summary>
|
||||
/// Scans a single layer and produces an SBOM.
|
||||
/// </summary>
|
||||
Task<string?> ScanLayerAsync(
|
||||
string imageReference,
|
||||
string diffId,
|
||||
string layerDigest,
|
||||
string sbomFormat,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Interface for composing multiple layer SBOMs into a single image SBOM.
|
||||
/// </summary>
|
||||
public interface ISbomComposer
|
||||
{
|
||||
/// <summary>
|
||||
/// Composes multiple layer SBOMs into a single image SBOM.
|
||||
/// </summary>
|
||||
Task<string?> ComposeAsync(
|
||||
IReadOnlyList<string> layerSboms,
|
||||
string imageReference,
|
||||
string sbomFormat,
|
||||
bool includeLayerAttribution,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
Reference in New Issue
Block a user