Refactor code structure and optimize performance across multiple modules

2025-12-26 20:03:22 +02:00
parent c786faae84
commit b4fc66feb6
3353 changed files with 88254 additions and 1590657 deletions
--- a/src/Scanner/StellaOps.Scanner.Worker/Extensions/BinaryIndexServiceExtensions.cs
+++ b/src/Scanner/StellaOps.Scanner.Worker/Extensions/BinaryIndexServiceExtensions.cs
@@ -36,6 +36,7 @@ public static class BinaryIndexServiceExtensions
        services.AddScoped<IBinaryVulnerabilityService, BinaryVulnerabilityService>();
        services.AddScoped<IBinaryFeatureExtractor, ElfFeatureExtractor>();
        services.AddScoped<BinaryVulnerabilityAnalyzer>();
+        services.AddScoped<Processing.BinaryFindingMapper>();

        return services;
    }
@@ -87,4 +88,40 @@ internal sealed class NullBinaryVulnerabilityService : IBinaryVulnerabilityServi
    {
        return Task.FromResult(System.Collections.Immutable.ImmutableDictionary<string, System.Collections.Immutable.ImmutableArray<BinaryVulnMatch>>.Empty);
    }
+
+    public Task<StellaOps.BinaryIndex.FixIndex.Models.FixStatusResult?> GetFixStatusAsync(
+        string distro,
+        string release,
+        string sourcePkg,
+        string cveId,
+        CancellationToken ct = default)
+    {
+        return Task.FromResult<StellaOps.BinaryIndex.FixIndex.Models.FixStatusResult?>(null);
+    }
+
+    public Task<System.Collections.Immutable.ImmutableDictionary<string, StellaOps.BinaryIndex.FixIndex.Models.FixStatusResult>> GetFixStatusBatchAsync(
+        string distro,
+        string release,
+        string sourcePkg,
+        IEnumerable<string> cveIds,
+        CancellationToken ct = default)
+    {
+        return Task.FromResult(System.Collections.Immutable.ImmutableDictionary<string, StellaOps.BinaryIndex.FixIndex.Models.FixStatusResult>.Empty);
+    }
+
+    public Task<System.Collections.Immutable.ImmutableArray<BinaryVulnMatch>> LookupByFingerprintAsync(
+        byte[] fingerprint,
+        FingerprintLookupOptions? options = null,
+        CancellationToken ct = default)
+    {
+        return Task.FromResult(System.Collections.Immutable.ImmutableArray<BinaryVulnMatch>.Empty);
+    }
+
+    public Task<System.Collections.Immutable.ImmutableDictionary<string, System.Collections.Immutable.ImmutableArray<BinaryVulnMatch>>> LookupByFingerprintBatchAsync(
+        IEnumerable<(string Key, byte[] Fingerprint)> fingerprints,
+        FingerprintLookupOptions? options = null,
+        CancellationToken ct = default)
+    {
+        return Task.FromResult(System.Collections.Immutable.ImmutableDictionary<string, System.Collections.Immutable.ImmutableArray<BinaryVulnMatch>>.Empty);
+    }
 }
--- a/src/Scanner/StellaOps.Scanner.Worker/Processing/BinaryFindingMapper.cs
+++ b/src/Scanner/StellaOps.Scanner.Worker/Processing/BinaryFindingMapper.cs
@@ -0,0 +1,288 @@
+// -----------------------------------------------------------------------------
+// BinaryFindingMapper.cs
+// Sprint: SPRINT_20251226_014_BINIDX
+// Task: SCANINT-08 — Create BinaryFindingMapper to convert matches to findings
+// -----------------------------------------------------------------------------
+
+using System.Collections.Immutable;
+using System.Security.Cryptography;
+using System.Text;
+using Microsoft.Extensions.Logging;
+using StellaOps.BinaryIndex.Core.Services;
+using FixStatusResult = StellaOps.BinaryIndex.Core.Services.FixStatusResult;
+
+namespace StellaOps.Scanner.Worker.Processing;
+
+/// <summary>
+/// Maps binary vulnerability findings to the standard scanner finding format.
+/// Enables integration with the Findings Ledger and triage workflow.
+/// </summary>
+public sealed class BinaryFindingMapper
+{
+    private readonly IBinaryVulnerabilityService _binaryVulnService;
+    private readonly ILogger<BinaryFindingMapper> _logger;
+
+    public BinaryFindingMapper(
+        IBinaryVulnerabilityService binaryVulnService,
+        ILogger<BinaryFindingMapper> logger)
+    {
+        _binaryVulnService = binaryVulnService ?? throw new ArgumentNullException(nameof(binaryVulnService));
+        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+    }
+
+    /// <summary>
+    /// Maps a single binary finding to a standard finding.
+    /// </summary>
+    public Finding MapToFinding(BinaryVulnerabilityFinding finding, string? distro = null, string? release = null)
+    {
+        ArgumentNullException.ThrowIfNull(finding);
+
+        var findingId = GenerateFindingId(finding);
+        var severity = GetSeverityFromCve(finding.CveId);
+
+        return new Finding
+        {
+            Id = findingId,
+            Type = FindingType.BinaryVulnerability,
+            Severity = severity,
+            Title = $"Binary contains vulnerable code: {finding.CveId}",
+            Description = GenerateDescription(finding),
+            CveId = finding.CveId,
+            Purl = finding.VulnerablePurl,
+            Evidence = new BinaryFindingEvidence
+            {
+                BinaryKey = finding.BinaryKey,
+                LayerDigest = finding.LayerDigest,
+                MatchMethod = finding.MatchMethod,
+                Confidence = finding.Confidence,
+                Similarity = finding.Evidence?.Similarity,
+                MatchedFunction = finding.Evidence?.MatchedFunction,
+                BuildId = finding.Evidence?.BuildId
+            },
+            Remediation = GenerateRemediation(finding),
+            ScanId = finding.ScanId,
+            DetectedAt = DateTimeOffset.UtcNow
+        };
+    }
+
+    /// <summary>
+    /// Maps multiple binary findings to standard findings with fix status enrichment.
+    /// </summary>
+    public async Task<ImmutableArray<Finding>> MapToFindingsAsync(
+        IEnumerable<BinaryVulnerabilityFinding> findings,
+        string? distro,
+        string? release,
+        CancellationToken ct = default)
+    {
+        var result = new List<Finding>();
+        var findingsList = findings.ToList();
+
+        // Group by source package for batch fix status lookup
+        var groupedByPurl = findingsList
+            .GroupBy(f => ExtractSourcePackage(f.VulnerablePurl))
+            .Where(g => !string.IsNullOrEmpty(g.Key));
+
+        foreach (var group in groupedByPurl)
+        {
+            var sourcePkg = group.Key!;
+            var cveIds = group.Select(f => f.CveId).Distinct().ToList();
+
+            // Batch fix status lookup
+            ImmutableDictionary<string, FixStatusResult>? fixStatuses = null;
+            if (!string.IsNullOrEmpty(distro) && !string.IsNullOrEmpty(release))
+            {
+                try
+                {
+                    fixStatuses = await _binaryVulnService.GetFixStatusBatchAsync(
+                        distro, release, sourcePkg, cveIds, ct).ConfigureAwait(false);
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogWarning(ex, "Failed to get fix status for {SourcePkg}", sourcePkg);
+                }
+            }
+
+            foreach (var finding in group)
+            {
+                var mapped = MapToFinding(finding, distro, release);
+
+                // Enrich with fix status if available
+                if (fixStatuses != null && fixStatuses.TryGetValue(finding.CveId, out var fixStatus))
+                {
+                    mapped = mapped with
+                    {
+                        FixStatus = MapFixStatus(fixStatus),
+                        FixedVersion = fixStatus.FixedVersion
+                    };
+                }
+
+                result.Add(mapped);
+            }
+        }
+
+        // Handle findings without valid PURLs
+        foreach (var finding in findingsList.Where(f => string.IsNullOrEmpty(ExtractSourcePackage(f.VulnerablePurl))))
+        {
+            result.Add(MapToFinding(finding, distro, release));
+        }
+
+        _logger.LogInformation("Mapped {Count} binary findings", result.Count);
+        return result.ToImmutableArray();
+    }
+
+    private static Guid GenerateFindingId(BinaryVulnerabilityFinding finding)
+    {
+        // Generate deterministic ID based on scan, CVE, and binary key
+        var input = $"{finding.ScanId}:{finding.CveId}:{finding.BinaryKey}:{finding.LayerDigest}";
+        var hash = SHA256.HashData(Encoding.UTF8.GetBytes(input));
+        return new Guid(hash.AsSpan()[..16]);
+    }
+
+    private static Severity GetSeverityFromCve(string cveId)
+    {
+        // In production, this would look up CVSS from advisory data
+        // For now, return Unknown and let downstream enrichment set it
+        return Severity.Unknown;
+    }
+
+    private static string GenerateDescription(BinaryVulnerabilityFinding finding)
+    {
+        var sb = new StringBuilder();
+        sb.AppendLine($"A binary file in the container image contains code affected by {finding.CveId}.");
+        sb.AppendLine();
+        sb.AppendLine($"**Detection Method:** {finding.MatchMethod}");
+        sb.AppendLine($"**Confidence:** {finding.Confidence:P0}");
+
+        if (finding.Evidence?.MatchedFunction is not null)
+        {
+            sb.AppendLine($"**Vulnerable Function:** {finding.Evidence.MatchedFunction}");
+        }
+
+        if (finding.Evidence?.BuildId is not null)
+        {
+            sb.AppendLine($"**Build-ID:** {finding.Evidence.BuildId}");
+        }
+
+        return sb.ToString();
+    }
+
+    private static string GenerateRemediation(BinaryVulnerabilityFinding finding)
+    {
+        return $"Update the package containing the binary to a version that includes the fix for {finding.CveId}. " +
+               $"If using a distro package, check if a backported security update is available.";
+    }
+
+    private static string? ExtractSourcePackage(string purl)
+    {
+        // Extract package name from PURL
+        // e.g., "pkg:deb/debian/openssl@1.1.1" -> "openssl"
+        if (string.IsNullOrEmpty(purl))
+            return null;
+
+        var atIndex = purl.IndexOf('@');
+        var slashIndex = purl.LastIndexOf('/', atIndex > 0 ? atIndex : purl.Length);
+
+        if (slashIndex >= 0)
+        {
+            var endIndex = atIndex > slashIndex ? atIndex : purl.Length;
+            return purl[(slashIndex + 1)..endIndex];
+        }
+
+        return null;
+    }
+
+    private static FindingFixStatus MapFixStatus(FixStatusResult status)
+    {
+        return status.State switch
+        {
+            FixState.Fixed => FindingFixStatus.Fixed,
+            FixState.Vulnerable => FindingFixStatus.Vulnerable,
+            FixState.NotAffected => FindingFixStatus.NotAffected,
+            FixState.WontFix => FindingFixStatus.WontFix,
+            _ => FindingFixStatus.Unknown
+        };
+    }
+}
+
+/// <summary>
+/// Standard scanner finding.
+/// </summary>
+public sealed record Finding
+{
+    public required Guid Id { get; init; }
+    public required FindingType Type { get; init; }
+    public required Severity Severity { get; init; }
+    public required string Title { get; init; }
+    public required string Description { get; init; }
+    public string? CveId { get; init; }
+    public string? Purl { get; init; }
+    public required BinaryFindingEvidence Evidence { get; init; }
+    public required string Remediation { get; init; }
+    public Guid ScanId { get; init; }
+    public DateTimeOffset DetectedAt { get; init; }
+    public FindingFixStatus FixStatus { get; init; } = FindingFixStatus.Unknown;
+    public string? FixedVersion { get; init; }
+}
+
+/// <summary>
+/// Evidence specific to binary vulnerability findings.
+/// </summary>
+public sealed record BinaryFindingEvidence
+{
+    public required string BinaryKey { get; init; }
+    public required string LayerDigest { get; init; }
+    public required string MatchMethod { get; init; }
+    public required decimal Confidence { get; init; }
+    public decimal? Similarity { get; init; }
+    public string? MatchedFunction { get; init; }
+    public string? BuildId { get; init; }
+}
+
+/// <summary>
+/// Finding type enumeration.
+/// </summary>
+public enum FindingType
+{
+    PackageVulnerability,
+    BinaryVulnerability,
+    PolicyViolation,
+    SecretExposure,
+    MisconfigurationDebian
+}
+
+/// <summary>
+/// Severity levels for findings.
+/// </summary>
+public enum Severity
+{
+    Unknown,
+    None,
+    Low,
+    Medium,
+    High,
+    Critical
+}
+
+/// <summary>
+/// Fix status for findings.
+/// </summary>
+public enum FindingFixStatus
+{
+    Unknown,
+    Vulnerable,
+    Fixed,
+    NotAffected,
+    WontFix
+}
+
+/// <summary>
+/// Fix state from the binary index.
+/// </summary>
+public enum FixState
+{
+    Unknown,
+    Vulnerable,
+    Fixed,
+    NotAffected,
+    WontFix
+}
--- a/src/Scanner/StellaOps.Scanner.Worker/Processing/BinaryLookupStageExecutor.cs
+++ b/src/Scanner/StellaOps.Scanner.Worker/Processing/BinaryLookupStageExecutor.cs
@@ -0,0 +1,219 @@
+// -----------------------------------------------------------------------------
+// BinaryLookupStageExecutor.cs
+// Sprint: SPRINT_20251226_014_BINIDX
+// Task: SCANINT-02 — Create IBinaryLookupStep in scan pipeline
+// -----------------------------------------------------------------------------
+
+using System.Collections.Immutable;
+using Microsoft.Extensions.Logging;
+using StellaOps.BinaryIndex.Core.Models;
+using StellaOps.BinaryIndex.Core.Services;
+using StellaOps.Scanner.Core.Contracts;
+using StellaOps.Scanner.Worker.Extensions;
+
+namespace StellaOps.Scanner.Worker.Processing;
+
+/// <summary>
+/// Scan pipeline stage that performs binary vulnerability lookups.
+/// Runs after analyzers to correlate binary identities with known vulnerabilities.
+/// </summary>
+public sealed class BinaryLookupStageExecutor : IScanStageExecutor
+{
+    private readonly BinaryVulnerabilityAnalyzer _analyzer;
+    private readonly BinaryIndexOptions _options;
+    private readonly ILogger<BinaryLookupStageExecutor> _logger;
+
+    public BinaryLookupStageExecutor(
+        BinaryVulnerabilityAnalyzer analyzer,
+        BinaryIndexOptions options,
+        ILogger<BinaryLookupStageExecutor> logger)
+    {
+        _analyzer = analyzer ?? throw new ArgumentNullException(nameof(analyzer));
+        _options = options ?? throw new ArgumentNullException(nameof(options));
+        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+    }
+
+    public string StageName => ScanStageNames.BinaryLookup;
+
+    public async ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken)
+    {
+        if (!_options.Enabled)
+        {
+            _logger.LogDebug("Binary vulnerability analysis disabled, skipping");
+            return;
+        }
+
+        _logger.LogInformation(
+            "Starting binary vulnerability lookup for scan {ScanId}",
+            context.ScanId);
+
+        var allFindings = new List<BinaryVulnerabilityFinding>();
+        var layerContexts = BuildLayerContexts(context);
+
+        foreach (var layerContext in layerContexts)
+        {
+            try
+            {
+                var result = await _analyzer.AnalyzeLayerAsync(layerContext, cancellationToken)
+                    .ConfigureAwait(false);
+
+                if (result.Findings.Length > 0)
+                {
+                    allFindings.AddRange(result.Findings);
+                    _logger.LogInformation(
+                        "Found {Count} binary vulnerabilities in layer {Layer}",
+                        result.Findings.Length,
+                        layerContext.LayerDigest);
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning(
+                    ex,
+                    "Failed to analyze layer {Layer} for binary vulnerabilities",
+                    layerContext.LayerDigest);
+            }
+        }
+
+        // Store findings in analysis context for downstream stages
+        context.Analysis.SetBinaryFindings(allFindings.ToImmutableArray());
+
+        _logger.LogInformation(
+            "Binary vulnerability lookup complete for scan {ScanId}: {Count} findings",
+            context.ScanId,
+            allFindings.Count);
+    }
+
+    private IReadOnlyList<BinaryLayerContext> BuildLayerContexts(ScanJobContext context)
+    {
+        var contexts = new List<BinaryLayerContext>();
+
+        // Get layer information from the scan context
+        var layers = context.Analysis.GetLayers();
+        if (layers == null || layers.Count == 0)
+        {
+            _logger.LogDebug("No layers found in scan context");
+            return contexts;
+        }
+
+        var distro = context.Analysis.GetDetectedDistro();
+        var release = context.Analysis.GetDetectedRelease();
+
+        foreach (var layer in layers)
+        {
+            var binaryPaths = context.Analysis.GetBinaryPathsForLayer(layer.Digest);
+            if (binaryPaths == null || binaryPaths.Count == 0)
+            {
+                continue;
+            }
+
+            contexts.Add(new BinaryLayerContext
+            {
+                ScanId = Guid.Parse(context.ScanId),
+                LayerDigest = layer.Digest,
+                BinaryPaths = binaryPaths,
+                DetectedDistro = distro,
+                DetectedRelease = release,
+                OpenFile = path => context.Analysis.OpenLayerFile(layer.Digest, path)
+            });
+        }
+
+        return contexts;
+    }
+}
+
+/// <summary>
+/// Extension methods for ScanAnalysisStore to support binary analysis.
+/// </summary>
+public static class BinaryScanAnalysisStoreExtensions
+{
+    private const string BinaryFindingsKey = "binary_findings";
+    private const string LayersKey = "layers";
+    private const string DistroKey = "detected_distro";
+    private const string ReleaseKey = "detected_release";
+
+    public static void SetBinaryFindings(
+        this ScanAnalysisStore store,
+        ImmutableArray<BinaryVulnerabilityFinding> findings)
+    {
+        ArgumentNullException.ThrowIfNull(store);
+        store.Set(BinaryFindingsKey, findings);
+    }
+
+    public static ImmutableArray<BinaryVulnerabilityFinding> GetBinaryFindings(
+        this ScanAnalysisStore store)
+    {
+        ArgumentNullException.ThrowIfNull(store);
+        if (store.TryGet<ImmutableArray<BinaryVulnerabilityFinding>>(BinaryFindingsKey, out var findings) && !findings.IsDefault)
+        {
+            return findings;
+        }
+        return ImmutableArray<BinaryVulnerabilityFinding>.Empty;
+    }
+
+    public static IReadOnlyList<LayerInfo>? GetLayers(this ScanAnalysisStore store)
+    {
+        ArgumentNullException.ThrowIfNull(store);
+        if (store.TryGet<IReadOnlyList<LayerInfo>>(LayersKey, out var layers))
+        {
+            return layers;
+        }
+        return null;
+    }
+
+    public static string? GetDetectedDistro(this ScanAnalysisStore store)
+    {
+        ArgumentNullException.ThrowIfNull(store);
+        if (store.TryGet<string>(DistroKey, out var distro))
+        {
+            return distro;
+        }
+        return null;
+    }
+
+    public static string? GetDetectedRelease(this ScanAnalysisStore store)
+    {
+        ArgumentNullException.ThrowIfNull(store);
+        if (store.TryGet<string>(ReleaseKey, out var release))
+        {
+            return release;
+        }
+        return null;
+    }
+
+    public static IReadOnlyList<string>? GetBinaryPathsForLayer(
+        this ScanAnalysisStore store,
+        string layerDigest)
+    {
+        ArgumentNullException.ThrowIfNull(store);
+        var key = $"binary_paths_{layerDigest}";
+        if (store.TryGet<IReadOnlyList<string>>(key, out var paths))
+        {
+            return paths;
+        }
+        return null;
+    }
+
+    public static Stream? OpenLayerFile(
+        this ScanAnalysisStore store,
+        string layerDigest,
+        string path)
+    {
+        ArgumentNullException.ThrowIfNull(store);
+        if (store.TryGet<Func<string, string, Stream?>>("layer_file_opener", out var opener))
+        {
+            return opener?.Invoke(layerDigest, path);
+        }
+        return null;
+    }
+}
+
+/// <summary>
+/// Layer metadata for binary analysis.
+/// </summary>
+public sealed record LayerInfo
+{
+    public required string Digest { get; init; }
+    public required string MediaType { get; init; }
+    public long Size { get; init; }
+}
--- a/src/Scanner/StellaOps.Scanner.Worker/Processing/ScanStageNames.cs
+++ b/src/Scanner/StellaOps.Scanner.Worker/Processing/ScanStageNames.cs
@@ -20,6 +20,9 @@ public static class ScanStageNames
    // Sprint: SPRINT_3500_0001_0001 - Proof of Exposure
    public const string GeneratePoE = "generate-poe";

+    // Sprint: SPRINT_20251226_014_BINIDX - Binary Vulnerability Lookup
+    public const string BinaryLookup = "binary-lookup";
+
    public static readonly IReadOnlyList<string> Ordered = new[]
    {
        IngestReplay,
@@ -27,6 +30,7 @@ public static class ScanStageNames
        PullLayers,
        BuildFilesystem,
        ExecuteAnalyzers,
+        BinaryLookup,
        EpssEnrichment,
        ComposeArtifacts,
        Entropy,
--- a/src/Scanner/StellaOps.Scanner.Worker/Program.cs
+++ b/src/Scanner/StellaOps.Scanner.Worker/Program.cs
@@ -27,6 +27,7 @@ using StellaOps.Scanner.Worker.Options;
 using StellaOps.Scanner.Worker.Processing;
 using StellaOps.Scanner.Worker.Processing.Entropy;
 using StellaOps.Scanner.Worker.Determinism;
+using StellaOps.Scanner.Worker.Extensions;
 using StellaOps.Scanner.Worker.Processing.Surface;
 using StellaOps.Scanner.Storage.Extensions;
 using StellaOps.Scanner.Storage;
@@ -93,6 +94,10 @@ builder.Services.AddSingleton<IDelayScheduler, SystemDelayScheduler>();

 builder.Services.AddEntryTraceAnalyzer();
 builder.Services.AddSingleton<IEntryTraceExecutionService, EntryTraceExecutionService>();
+
+// BinaryIndex integration for binary vulnerability detection (Sprint: SPRINT_20251226_014_BINIDX)
+builder.Services.AddBinaryIndexIntegration(builder.Configuration);
+
 builder.Services.AddSingleton<ReachabilityUnionWriter>();
 builder.Services.AddSingleton<ReachabilityUnionPublisher>();
 builder.Services.AddSingleton<IReachabilityUnionPublisherService, ReachabilityUnionPublisherService>();
@@ -156,6 +161,7 @@ builder.Services.AddSingleton<NativeAnalyzerExecutor>();
 builder.Services.AddSingleton<IScanAnalyzerDispatcher, CompositeScanAnalyzerDispatcher>();
 builder.Services.AddSingleton<IScanStageExecutor, RegistrySecretStageExecutor>();
 builder.Services.AddSingleton<IScanStageExecutor, AnalyzerStageExecutor>();
+builder.Services.AddSingleton<IScanStageExecutor, BinaryLookupStageExecutor>();
 builder.Services.AddSingleton<IScanStageExecutor, EpssEnrichmentStageExecutor>();
 builder.Services.AddSingleton<IScanStageExecutor, Reachability.ReachabilityBuildStageExecutor>();
 builder.Services.AddSingleton<IScanStageExecutor, Reachability.ReachabilityPublishStageExecutor>();