345 lines
13 KiB
C#
345 lines
13 KiB
C#
|
|
using Microsoft.Extensions.Logging;
|
|
using StellaOps.Scanner.Manifest.Resolution;
|
|
using System.Collections.Immutable;
|
|
using System.Diagnostics;
|
|
|
|
namespace StellaOps.Scanner.Delta;
|
|
|
|
/// <summary>
|
|
/// Implementation of IDeltaLayerScanner that scans only changed layers between image versions.
|
|
/// Reduces scan time and CVE churn by reusing cached per-layer SBOMs.
|
|
/// </summary>
|
|
public sealed class DeltaLayerScanner : IDeltaLayerScanner
|
|
{
|
|
private readonly ILayerDigestResolver _layerResolver;
|
|
private readonly ILayerSbomCas _sbomCas;
|
|
private readonly ILayerScannerPipeline _scannerPipeline;
|
|
private readonly ISbomComposer _sbomComposer;
|
|
private readonly ILogger<DeltaLayerScanner> _logger;
|
|
|
|
public DeltaLayerScanner(
|
|
ILayerDigestResolver layerResolver,
|
|
ILayerSbomCas sbomCas,
|
|
ILayerScannerPipeline scannerPipeline,
|
|
ISbomComposer sbomComposer,
|
|
ILogger<DeltaLayerScanner> logger)
|
|
{
|
|
_layerResolver = layerResolver ?? throw new ArgumentNullException(nameof(layerResolver));
|
|
_sbomCas = sbomCas ?? throw new ArgumentNullException(nameof(sbomCas));
|
|
_scannerPipeline = scannerPipeline ?? throw new ArgumentNullException(nameof(scannerPipeline));
|
|
_sbomComposer = sbomComposer ?? throw new ArgumentNullException(nameof(sbomComposer));
|
|
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
|
}
|
|
|
|
public async Task<DeltaScanResult> ScanDeltaAsync(
|
|
string oldImage,
|
|
string newImage,
|
|
DeltaScanOptions? options = null,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
ArgumentException.ThrowIfNullOrWhiteSpace(oldImage);
|
|
ArgumentException.ThrowIfNullOrWhiteSpace(newImage);
|
|
|
|
options ??= new DeltaScanOptions();
|
|
var totalStopwatch = Stopwatch.StartNew();
|
|
|
|
_logger.LogInformation("Starting delta scan: {OldImage} -> {NewImage}", oldImage, newImage);
|
|
|
|
// Resolve layers for both images in parallel
|
|
var resolveOptions = new LayerResolutionOptions
|
|
{
|
|
ComputeDiffIds = true,
|
|
Platform = options.Platform
|
|
};
|
|
|
|
var oldResolvedTask = _layerResolver.ResolveLayersAsync(oldImage, resolveOptions, cancellationToken);
|
|
var newResolvedTask = _layerResolver.ResolveLayersAsync(newImage, resolveOptions, cancellationToken);
|
|
|
|
var oldResolved = await oldResolvedTask.ConfigureAwait(false);
|
|
var newResolved = await newResolvedTask.ConfigureAwait(false);
|
|
|
|
// Identify layer changes using diffID (content hash)
|
|
var oldDiffIds = oldResolved.Layers
|
|
.Where(l => !string.IsNullOrWhiteSpace(l.DiffId))
|
|
.Select(l => l.DiffId!)
|
|
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
|
|
|
var addedLayers = new List<LayerChangeInfo>();
|
|
var unchangedLayers = new List<LayerChangeInfo>();
|
|
var removedLayers = new List<LayerChangeInfo>();
|
|
|
|
// Categorize new image layers
|
|
foreach (var layer in newResolved.Layers)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(layer.DiffId))
|
|
{
|
|
// Treat as added if diffID unknown
|
|
addedLayers.Add(CreateLayerInfo(layer));
|
|
}
|
|
else if (oldDiffIds.Contains(layer.DiffId))
|
|
{
|
|
unchangedLayers.Add(CreateLayerInfo(layer));
|
|
}
|
|
else
|
|
{
|
|
addedLayers.Add(CreateLayerInfo(layer));
|
|
}
|
|
}
|
|
|
|
// Find removed layers
|
|
var newDiffIds = newResolved.Layers
|
|
.Where(l => !string.IsNullOrWhiteSpace(l.DiffId))
|
|
.Select(l => l.DiffId!)
|
|
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
|
|
|
foreach (var layer in oldResolved.Layers)
|
|
{
|
|
if (!string.IsNullOrWhiteSpace(layer.DiffId) && !newDiffIds.Contains(layer.DiffId))
|
|
{
|
|
removedLayers.Add(CreateLayerInfo(layer));
|
|
}
|
|
}
|
|
|
|
_logger.LogInformation(
|
|
"Layer analysis: {Added} added, {Removed} removed, {Unchanged} unchanged",
|
|
addedLayers.Count, removedLayers.Count, unchangedLayers.Count);
|
|
|
|
// Collect SBOMs for unchanged layers from cache
|
|
var layerSboms = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
|
var cachedComponentCount = 0;
|
|
|
|
if (options.UseCachedSboms && !options.ForceFullScan)
|
|
{
|
|
for (var i = 0; i < unchangedLayers.Count; i++)
|
|
{
|
|
var layer = unchangedLayers[i];
|
|
var cachedSbom = await _sbomCas.GetAsync(layer.DiffId, options.SbomFormat, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
|
|
if (!string.IsNullOrWhiteSpace(cachedSbom))
|
|
{
|
|
layerSboms[layer.DiffId] = cachedSbom;
|
|
var componentCount = CountComponents(cachedSbom);
|
|
unchangedLayers[i] = layer with { FromCache = true, ComponentCount = componentCount };
|
|
cachedComponentCount += componentCount;
|
|
}
|
|
else
|
|
{
|
|
// Layer not in cache - need to scan it
|
|
_logger.LogDebug("Unchanged layer {DiffId} not in cache, will scan", layer.DiffId);
|
|
addedLayers.Add(layer);
|
|
unchangedLayers.RemoveAt(i);
|
|
i--;
|
|
}
|
|
}
|
|
|
|
_logger.LogDebug("Retrieved {Count} cached layer SBOMs", layerSboms.Count);
|
|
}
|
|
|
|
// Scan added layers
|
|
var addedScanStopwatch = Stopwatch.StartNew();
|
|
var addedComponentCount = 0;
|
|
|
|
for (var i = 0; i < addedLayers.Count; i++)
|
|
{
|
|
var layer = addedLayers[i];
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
|
|
var layerSbom = await _scannerPipeline.ScanLayerAsync(
|
|
newResolved.ImageReference,
|
|
layer.DiffId,
|
|
layer.LayerDigest,
|
|
options.SbomFormat,
|
|
cancellationToken).ConfigureAwait(false);
|
|
|
|
if (!string.IsNullOrWhiteSpace(layerSbom))
|
|
{
|
|
layerSboms[layer.DiffId] = layerSbom;
|
|
|
|
// Store in cache for future use
|
|
await _sbomCas.StoreAsync(layer.DiffId, options.SbomFormat, layerSbom, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
|
|
var componentCount = CountComponents(layerSbom);
|
|
addedLayers[i] = layer with { ComponentCount = componentCount };
|
|
addedComponentCount += componentCount;
|
|
}
|
|
}
|
|
|
|
addedScanStopwatch.Stop();
|
|
_logger.LogInformation("Scanned {Count} added layers in {Duration:N2}s",
|
|
addedLayers.Count, addedScanStopwatch.Elapsed.TotalSeconds);
|
|
|
|
// Compose final SBOM in layer order
|
|
var orderedSboms = newResolved.Layers
|
|
.Where(l => !string.IsNullOrWhiteSpace(l.DiffId) && layerSboms.ContainsKey(l.DiffId!))
|
|
.Select(l => layerSboms[l.DiffId!])
|
|
.ToList();
|
|
|
|
var compositeSbom = await _sbomComposer.ComposeAsync(
|
|
orderedSboms,
|
|
newResolved.ImageReference,
|
|
options.SbomFormat,
|
|
options.IncludeLayerAttribution,
|
|
cancellationToken).ConfigureAwait(false);
|
|
|
|
totalStopwatch.Stop();
|
|
|
|
var result = new DeltaScanResult
|
|
{
|
|
OldImage = oldImage,
|
|
OldManifestDigest = oldResolved.ManifestDigest,
|
|
NewImage = newImage,
|
|
NewManifestDigest = newResolved.ManifestDigest,
|
|
AddedLayers = [.. addedLayers],
|
|
RemovedLayers = [.. removedLayers],
|
|
UnchangedLayers = [.. unchangedLayers],
|
|
CompositeSbom = compositeSbom,
|
|
SbomFormat = options.SbomFormat,
|
|
ScanDuration = totalStopwatch.Elapsed,
|
|
AddedLayersScanDuration = addedScanStopwatch.Elapsed,
|
|
AddedComponentCount = addedComponentCount,
|
|
CachedComponentCount = cachedComponentCount,
|
|
UsedCache = options.UseCachedSboms && !options.ForceFullScan
|
|
};
|
|
|
|
_logger.LogInformation(
|
|
"Delta scan complete: {LayerReuse:P1} reuse, {Duration:N2}s total, {AddedDuration:N2}s scanning added layers",
|
|
result.LayerReuseRatio, result.ScanDuration.TotalSeconds, result.AddedLayersScanDuration.TotalSeconds);
|
|
|
|
return result;
|
|
}
|
|
|
|
public async Task<LayerChangeSummary> IdentifyLayerChangesAsync(
|
|
string oldImage,
|
|
string newImage,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
ArgumentException.ThrowIfNullOrWhiteSpace(oldImage);
|
|
ArgumentException.ThrowIfNullOrWhiteSpace(newImage);
|
|
|
|
var resolveOptions = new LayerResolutionOptions { ComputeDiffIds = true };
|
|
|
|
var oldResolvedTask = _layerResolver.ResolveLayersAsync(oldImage, resolveOptions, cancellationToken);
|
|
var newResolvedTask = _layerResolver.ResolveLayersAsync(newImage, resolveOptions, cancellationToken);
|
|
|
|
var oldResolved = await oldResolvedTask.ConfigureAwait(false);
|
|
var newResolved = await newResolvedTask.ConfigureAwait(false);
|
|
|
|
var oldDiffIds = oldResolved.Layers
|
|
.Where(l => !string.IsNullOrWhiteSpace(l.DiffId))
|
|
.Select(l => l.DiffId!)
|
|
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
|
|
|
var newDiffIds = newResolved.Layers
|
|
.Where(l => !string.IsNullOrWhiteSpace(l.DiffId))
|
|
.Select(l => l.DiffId!)
|
|
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
|
|
|
var added = newDiffIds.Except(oldDiffIds, StringComparer.OrdinalIgnoreCase).ToList();
|
|
var removed = oldDiffIds.Except(newDiffIds, StringComparer.OrdinalIgnoreCase).ToList();
|
|
var unchanged = newDiffIds.Intersect(oldDiffIds, StringComparer.OrdinalIgnoreCase).ToList();
|
|
|
|
return new LayerChangeSummary
|
|
{
|
|
OldImage = oldImage,
|
|
NewImage = newImage,
|
|
OldLayerCount = oldResolved.LayerCount,
|
|
NewLayerCount = newResolved.LayerCount,
|
|
AddedCount = added.Count,
|
|
RemovedCount = removed.Count,
|
|
UnchangedCount = unchanged.Count,
|
|
AddedDiffIds = [.. added],
|
|
RemovedDiffIds = [.. removed],
|
|
UnchangedDiffIds = [.. unchanged]
|
|
};
|
|
}
|
|
|
|
private static LayerChangeInfo CreateLayerInfo(LayerProvenance layer) => new()
|
|
{
|
|
LayerDigest = layer.LayerDigest,
|
|
DiffId = layer.DiffId ?? string.Empty,
|
|
LayerIndex = layer.LayerIndex,
|
|
Size = layer.Size,
|
|
MediaType = layer.MediaType
|
|
};
|
|
|
|
private static int CountComponents(string sbom)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(sbom))
|
|
return 0;
|
|
|
|
// Count component markers in SBOM
|
|
var cycloneDxCount = CountOccurrences(sbom, "\"bom-ref\"");
|
|
var spdxCount = CountOccurrences(sbom, "\"SPDXID\"");
|
|
|
|
return Math.Max(cycloneDxCount, spdxCount);
|
|
}
|
|
|
|
private static int CountOccurrences(string text, string pattern)
|
|
{
|
|
var count = 0;
|
|
var index = 0;
|
|
while ((index = text.IndexOf(pattern, index, StringComparison.Ordinal)) != -1)
|
|
{
|
|
count++;
|
|
index += pattern.Length;
|
|
}
|
|
return count;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Interface for per-layer SBOM content-addressable storage.
|
|
/// </summary>
|
|
public interface ILayerSbomCas
|
|
{
|
|
/// <summary>
|
|
/// Stores a per-layer SBOM.
|
|
/// </summary>
|
|
Task StoreAsync(string diffId, string format, string sbom, CancellationToken cancellationToken = default);
|
|
|
|
/// <summary>
|
|
/// Retrieves a per-layer SBOM.
|
|
/// </summary>
|
|
Task<string?> GetAsync(string diffId, string format, CancellationToken cancellationToken = default);
|
|
|
|
/// <summary>
|
|
/// Checks if a per-layer SBOM exists.
|
|
/// </summary>
|
|
Task<bool> ExistsAsync(string diffId, string format, CancellationToken cancellationToken = default);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Interface for scanning individual layers.
|
|
/// </summary>
|
|
public interface ILayerScannerPipeline
|
|
{
|
|
/// <summary>
|
|
/// Scans a single layer and produces an SBOM.
|
|
/// </summary>
|
|
Task<string?> ScanLayerAsync(
|
|
string imageReference,
|
|
string diffId,
|
|
string layerDigest,
|
|
string sbomFormat,
|
|
CancellationToken cancellationToken = default);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Interface for composing multiple layer SBOMs into a single image SBOM.
|
|
/// </summary>
|
|
public interface ISbomComposer
|
|
{
|
|
/// <summary>
|
|
/// Composes multiple layer SBOMs into a single image SBOM.
|
|
/// </summary>
|
|
Task<string?> ComposeAsync(
|
|
IReadOnlyList<string> layerSboms,
|
|
string imageReference,
|
|
string sbomFormat,
|
|
bool includeLayerAttribution,
|
|
CancellationToken cancellationToken = default);
|
|
}
|