doctor enhancements, setup, enhancements, ui functionality and design consolidation and , test projects fixes , product advisory attestation/rekor and delta verfications enhancements

2026-01-19 09:02:59 +02:00
parent 8c4bf54aed
commit 17419ba7c4
809 changed files with 170738 additions and 12244 deletions
--- a/src/Scanner/__Libraries/StellaOps.Scanner.Cache/LayerSbomCas/ILayerSbomCas.cs
+++ b/src/Scanner/__Libraries/StellaOps.Scanner.Cache/LayerSbomCas/ILayerSbomCas.cs
@@ -0,0 +1,173 @@
+namespace StellaOps.Scanner.Cache.LayerSbomCas;
+
+/// <summary>
+/// Content-addressable storage for per-layer SBOMs keyed by diffID.
+/// Layer SBOMs are immutable (same diffID = same content = same SBOM),
+/// so entries can be cached indefinitely.
+/// </summary>
+public interface ILayerSbomCas
+{
+    /// <summary>
+    /// Stores a per-layer SBOM.
+    /// </summary>
+    /// <param name="diffId">Layer diffID (sha256:...).</param>
+    /// <param name="format">SBOM format (cyclonedx or spdx).</param>
+    /// <param name="sbom">SBOM content.</param>
+    /// <param name="cancellationToken">Cancellation token.</param>
+    Task StoreAsync(string diffId, string format, string sbom, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Retrieves a per-layer SBOM.
+    /// </summary>
+    /// <param name="diffId">Layer diffID.</param>
+    /// <param name="format">SBOM format.</param>
+    /// <param name="cancellationToken">Cancellation token.</param>
+    /// <returns>SBOM content or null if not found.</returns>
+    Task<string?> GetAsync(string diffId, string format, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Checks if a per-layer SBOM exists.
+    /// </summary>
+    /// <param name="diffId">Layer diffID.</param>
+    /// <param name="format">SBOM format.</param>
+    /// <param name="cancellationToken">Cancellation token.</param>
+    /// <returns>True if exists.</returns>
+    Task<bool> ExistsAsync(string diffId, string format, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Deletes a per-layer SBOM.
+    /// </summary>
+    /// <param name="diffId">Layer diffID.</param>
+    /// <param name="format">SBOM format.</param>
+    /// <param name="cancellationToken">Cancellation token.</param>
+    /// <returns>True if deleted.</returns>
+    Task<bool> DeleteAsync(string diffId, string format, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Gets statistics about the CAS.
+    /// </summary>
+    /// <param name="cancellationToken">Cancellation token.</param>
+    /// <returns>CAS statistics.</returns>
+    Task<LayerSbomCasStatistics> GetStatisticsAsync(CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Evicts entries older than the specified age.
+    /// </summary>
+    /// <param name="maxAge">Maximum age for entries.</param>
+    /// <param name="cancellationToken">Cancellation token.</param>
+    /// <returns>Number of entries evicted.</returns>
+    Task<int> EvictOlderThanAsync(TimeSpan maxAge, CancellationToken cancellationToken = default);
+}
+
+/// <summary>
+/// Statistics about the Layer SBOM CAS.
+/// </summary>
+public sealed record LayerSbomCasStatistics
+{
+    /// <summary>
+    /// Total number of entries.
+    /// </summary>
+    public long EntryCount { get; init; }
+
+    /// <summary>
+    /// Total size in bytes (compressed).
+    /// </summary>
+    public long TotalSizeBytes { get; init; }
+
+    /// <summary>
+    /// Number of CycloneDX format entries.
+    /// </summary>
+    public long CycloneDxCount { get; init; }
+
+    /// <summary>
+    /// Number of SPDX format entries.
+    /// </summary>
+    public long SpdxCount { get; init; }
+
+    /// <summary>
+    /// Oldest entry timestamp.
+    /// </summary>
+    public DateTimeOffset? OldestEntry { get; init; }
+
+    /// <summary>
+    /// Newest entry timestamp.
+    /// </summary>
+    public DateTimeOffset? NewestEntry { get; init; }
+
+    /// <summary>
+    /// Cache hit count since startup.
+    /// </summary>
+    public long HitCount { get; init; }
+
+    /// <summary>
+    /// Cache miss count since startup.
+    /// </summary>
+    public long MissCount { get; init; }
+
+    /// <summary>
+    /// Hit ratio (0.0 to 1.0).
+    /// </summary>
+    public double HitRatio => (HitCount + MissCount) > 0
+        ? (double)HitCount / (HitCount + MissCount)
+        : 0;
+}
+
+/// <summary>
+/// Options for the Layer SBOM CAS.
+/// </summary>
+public sealed class LayerSbomCasOptions
+{
+    /// <summary>
+    /// Configuration section name.
+    /// </summary>
+    public const string SectionName = "Scanner:LayerSbomCas";
+
+    /// <summary>
+    /// Whether to compress SBOMs before storage.
+    /// Default is true.
+    /// </summary>
+    public bool CompressSboms { get; set; } = true;
+
+    /// <summary>
+    /// TTL for entries in days. Entries older than this are eligible for eviction.
+    /// Default is 90 days.
+    /// </summary>
+    public int TtlDays { get; set; } = 90;
+
+    /// <summary>
+    /// Maximum total storage size in bytes.
+    /// Default is 10GB.
+    /// </summary>
+    public long MaxStorageSizeBytes { get; set; } = 10L * 1024 * 1024 * 1024;
+
+    /// <summary>
+    /// Storage backend type.
+    /// </summary>
+    public StorageBackendType StorageBackend { get; set; } = StorageBackendType.PostgresBlob;
+
+    /// <summary>
+    /// Path for filesystem storage (when using Filesystem backend).
+    /// </summary>
+    public string? FilesystemPath { get; set; }
+}
+
+/// <summary>
+/// Storage backend types for Layer SBOM CAS.
+/// </summary>
+public enum StorageBackendType
+{
+    /// <summary>
+    /// Store SBOMs as compressed blobs in PostgreSQL.
+    /// </summary>
+    PostgresBlob,
+
+    /// <summary>
+    /// Store SBOMs on the filesystem.
+    /// </summary>
+    Filesystem,
+
+    /// <summary>
+    /// Store SBOMs in S3-compatible object storage.
+    /// </summary>
+    S3
+}
--- a/src/Scanner/__Libraries/StellaOps.Scanner.Cache/LayerSbomCas/PostgresLayerSbomCas.cs
+++ b/src/Scanner/__Libraries/StellaOps.Scanner.Cache/LayerSbomCas/PostgresLayerSbomCas.cs
@@ -0,0 +1,292 @@
+using System.IO.Compression;
+using System.Text;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Npgsql;
+
+namespace StellaOps.Scanner.Cache.LayerSbomCas;
+
+/// <summary>
+/// PostgreSQL-backed implementation of ILayerSbomCas.
+/// Stores SBOMs as compressed blobs in the database.
+/// </summary>
+public sealed class PostgresLayerSbomCas : ILayerSbomCas
+{
+    private const string TableName = "scanner.layer_sbom_cas";
+
+    private readonly NpgsqlDataSource _dataSource;
+    private readonly LayerSbomCasOptions _options;
+    private readonly ILogger<PostgresLayerSbomCas> _logger;
+
+    // Metrics counters
+    private long _hitCount;
+    private long _missCount;
+
+    public PostgresLayerSbomCas(
+        NpgsqlDataSource dataSource,
+        IOptions<LayerSbomCasOptions> options,
+        ILogger<PostgresLayerSbomCas> logger)
+    {
+        _dataSource = dataSource ?? throw new ArgumentNullException(nameof(dataSource));
+        _options = options?.Value ?? new LayerSbomCasOptions();
+        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+    }
+
+    public async Task StoreAsync(string diffId, string format, string sbom, CancellationToken cancellationToken = default)
+    {
+        ArgumentException.ThrowIfNullOrWhiteSpace(diffId);
+        ArgumentException.ThrowIfNullOrWhiteSpace(format);
+        ArgumentException.ThrowIfNullOrWhiteSpace(sbom);
+
+        var normalizedFormat = NormalizeFormat(format);
+        var content = _options.CompressSboms ? Compress(sbom) : Encoding.UTF8.GetBytes(sbom);
+
+        await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
+        await using var cmd = conn.CreateCommand();
+
+        cmd.CommandText = $"""
+            INSERT INTO {TableName} (diff_id, format, content, size_bytes, compressed, created_at, last_accessed_at)
+            VALUES (@diffId, @format, @content, @sizeBytes, @compressed, @now, @now)
+            ON CONFLICT (diff_id, format) DO UPDATE SET
+                content = EXCLUDED.content,
+                size_bytes = EXCLUDED.size_bytes,
+                compressed = EXCLUDED.compressed,
+                last_accessed_at = EXCLUDED.last_accessed_at
+            """;
+
+        cmd.Parameters.AddWithValue("diffId", diffId);
+        cmd.Parameters.AddWithValue("format", normalizedFormat);
+        cmd.Parameters.AddWithValue("content", content);
+        cmd.Parameters.AddWithValue("sizeBytes", content.Length);
+        cmd.Parameters.AddWithValue("compressed", _options.CompressSboms);
+        cmd.Parameters.AddWithValue("now", DateTimeOffset.UtcNow);
+
+        try
+        {
+            await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
+            _logger.LogDebug("Stored SBOM for {DiffId} ({Format}), {Size} bytes",
+                diffId, normalizedFormat, content.Length);
+        }
+        catch (PostgresException ex) when (ex.SqlState == "42P01")
+        {
+            _logger.LogWarning("Layer SBOM CAS table does not exist. Run migrations.");
+        }
+    }
+
+    public async Task<string?> GetAsync(string diffId, string format, CancellationToken cancellationToken = default)
+    {
+        ArgumentException.ThrowIfNullOrWhiteSpace(diffId);
+        ArgumentException.ThrowIfNullOrWhiteSpace(format);
+
+        var normalizedFormat = NormalizeFormat(format);
+
+        await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
+
+        // Update last accessed timestamp
+        await using (var updateCmd = conn.CreateCommand())
+        {
+            updateCmd.CommandText = $"""
+                UPDATE {TableName}
+                SET last_accessed_at = @now
+                WHERE diff_id = @diffId AND format = @format
+                """;
+            updateCmd.Parameters.AddWithValue("diffId", diffId);
+            updateCmd.Parameters.AddWithValue("format", normalizedFormat);
+            updateCmd.Parameters.AddWithValue("now", DateTimeOffset.UtcNow);
+
+            await updateCmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
+        }
+
+        await using var cmd = conn.CreateCommand();
+        cmd.CommandText = $"""
+            SELECT content, compressed
+            FROM {TableName}
+            WHERE diff_id = @diffId AND format = @format
+            """;
+        cmd.Parameters.AddWithValue("diffId", diffId);
+        cmd.Parameters.AddWithValue("format", normalizedFormat);
+
+        try
+        {
+            await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
+            if (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
+            {
+                var content = (byte[])reader["content"];
+                var compressed = reader.GetBoolean(reader.GetOrdinal("compressed"));
+
+                Interlocked.Increment(ref _hitCount);
+
+                return compressed ? Decompress(content) : Encoding.UTF8.GetString(content);
+            }
+
+            Interlocked.Increment(ref _missCount);
+            return null;
+        }
+        catch (PostgresException ex) when (ex.SqlState == "42P01")
+        {
+            Interlocked.Increment(ref _missCount);
+            return null;
+        }
+    }
+
+    public async Task<bool> ExistsAsync(string diffId, string format, CancellationToken cancellationToken = default)
+    {
+        ArgumentException.ThrowIfNullOrWhiteSpace(diffId);
+        ArgumentException.ThrowIfNullOrWhiteSpace(format);
+
+        var normalizedFormat = NormalizeFormat(format);
+
+        await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
+        await using var cmd = conn.CreateCommand();
+
+        cmd.CommandText = $"""
+            SELECT 1 FROM {TableName}
+            WHERE diff_id = @diffId AND format = @format
+            LIMIT 1
+            """;
+        cmd.Parameters.AddWithValue("diffId", diffId);
+        cmd.Parameters.AddWithValue("format", normalizedFormat);
+
+        try
+        {
+            var result = await cmd.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
+            return result is not null;
+        }
+        catch (PostgresException ex) when (ex.SqlState == "42P01")
+        {
+            return false;
+        }
+    }
+
+    public async Task<bool> DeleteAsync(string diffId, string format, CancellationToken cancellationToken = default)
+    {
+        ArgumentException.ThrowIfNullOrWhiteSpace(diffId);
+        ArgumentException.ThrowIfNullOrWhiteSpace(format);
+
+        var normalizedFormat = NormalizeFormat(format);
+
+        await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
+        await using var cmd = conn.CreateCommand();
+
+        cmd.CommandText = $"""
+            DELETE FROM {TableName}
+            WHERE diff_id = @diffId AND format = @format
+            """;
+        cmd.Parameters.AddWithValue("diffId", diffId);
+        cmd.Parameters.AddWithValue("format", normalizedFormat);
+
+        try
+        {
+            var rowsAffected = await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
+            return rowsAffected > 0;
+        }
+        catch (PostgresException ex) when (ex.SqlState == "42P01")
+        {
+            return false;
+        }
+    }
+
+    public async Task<LayerSbomCasStatistics> GetStatisticsAsync(CancellationToken cancellationToken = default)
+    {
+        await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
+
+        try
+        {
+            await using var cmd = conn.CreateCommand();
+            cmd.CommandText = $"""
+                SELECT
+                    COUNT(*) as entry_count,
+                    COALESCE(SUM(size_bytes), 0) as total_size,
+                    COUNT(*) FILTER (WHERE format = 'cyclonedx') as cyclonedx_count,
+                    COUNT(*) FILTER (WHERE format = 'spdx') as spdx_count,
+                    MIN(created_at) as oldest,
+                    MAX(created_at) as newest
+                FROM {TableName}
+                """;
+
+            await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
+            if (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
+            {
+                return new LayerSbomCasStatistics
+                {
+                    EntryCount = reader.GetInt64(0),
+                    TotalSizeBytes = reader.GetInt64(1),
+                    CycloneDxCount = reader.GetInt64(2),
+                    SpdxCount = reader.GetInt64(3),
+                    OldestEntry = reader.IsDBNull(4) ? null : reader.GetFieldValue<DateTimeOffset>(4),
+                    NewestEntry = reader.IsDBNull(5) ? null : reader.GetFieldValue<DateTimeOffset>(5),
+                    HitCount = Interlocked.Read(ref _hitCount),
+                    MissCount = Interlocked.Read(ref _missCount)
+                };
+            }
+        }
+        catch (PostgresException ex) when (ex.SqlState == "42P01")
+        {
+            _logger.LogDebug("Statistics table does not exist");
+        }
+
+        return new LayerSbomCasStatistics
+        {
+            HitCount = Interlocked.Read(ref _hitCount),
+            MissCount = Interlocked.Read(ref _missCount)
+        };
+    }
+
+    public async Task<int> EvictOlderThanAsync(TimeSpan maxAge, CancellationToken cancellationToken = default)
+    {
+        var cutoff = DateTimeOffset.UtcNow - maxAge;
+
+        await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
+        await using var cmd = conn.CreateCommand();
+
+        cmd.CommandText = $"""
+            DELETE FROM {TableName}
+            WHERE last_accessed_at < @cutoff
+            """;
+        cmd.Parameters.AddWithValue("cutoff", cutoff);
+
+        try
+        {
+            var rowsDeleted = await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
+            if (rowsDeleted > 0)
+            {
+                _logger.LogInformation("Evicted {Count} layer SBOMs older than {Cutoff}", rowsDeleted, cutoff);
+            }
+            return rowsDeleted;
+        }
+        catch (PostgresException ex) when (ex.SqlState == "42P01")
+        {
+            return 0;
+        }
+    }
+
+    private static string NormalizeFormat(string format)
+    {
+        return format.ToLowerInvariant() switch
+        {
+            "cyclonedx" or "cdx" or "cyclone" => "cyclonedx",
+            "spdx" => "spdx",
+            _ => format.ToLowerInvariant()
+        };
+    }
+
+    private static byte[] Compress(string text)
+    {
+        var bytes = Encoding.UTF8.GetBytes(text);
+        using var output = new MemoryStream();
+        using (var gzip = new GZipStream(output, CompressionLevel.Optimal, leaveOpen: true))
+        {
+            gzip.Write(bytes, 0, bytes.Length);
+        }
+        return output.ToArray();
+    }
+
+    private static string Decompress(byte[] compressed)
+    {
+        using var input = new MemoryStream(compressed);
+        using var gzip = new GZipStream(input, CompressionMode.Decompress);
+        using var output = new MemoryStream();
+        gzip.CopyTo(output);
+        return Encoding.UTF8.GetString(output.ToArray());
+    }
+}
--- a/src/Scanner/__Libraries/StellaOps.Scanner.Cache/ScannerCacheServiceCollectionExtensions.cs
+++ b/src/Scanner/__Libraries/StellaOps.Scanner.Cache/ScannerCacheServiceCollectionExtensions.cs
@@ -6,6 +6,7 @@ using Microsoft.Extensions.Options;
 using StellaOps.Scanner.Cache.Abstractions;
 using StellaOps.Scanner.Cache.FileCas;
 using StellaOps.Scanner.Cache.LayerCache;
+using StellaOps.Scanner.Cache.LayerSbomCas;
 using StellaOps.Scanner.Cache.Maintenance;

 namespace StellaOps.Scanner.Cache;
@@ -48,4 +49,23 @@ public static class ScannerCacheServiceCollectionExtensions

        return services;
    }
+
+    /// <summary>
+    /// Adds Layer SBOM CAS services to the service collection.
+    /// </summary>
+    public static IServiceCollection AddLayerSbomCas(
+        this IServiceCollection services,
+        IConfiguration configuration,
+        string sectionName = LayerSbomCasOptions.SectionName)
+    {
+        ArgumentNullException.ThrowIfNull(services);
+        ArgumentNullException.ThrowIfNull(configuration);
+
+        services.AddOptions<LayerSbomCasOptions>()
+            .Bind(configuration.GetSection(sectionName));
+
+        services.TryAddSingleton<ILayerSbomCas, PostgresLayerSbomCas>();
+
+        return services;
+    }
 }