save checkpoint

2026-02-12 21:02:43 +02:00
parent 5bca406787
commit 9911b7d73c
593 changed files with 174390 additions and 1376 deletions
--- a/src/Scanner/StellaOps.Scanner.Worker/Processing/BinaryLookupStageExecutor.cs
+++ b/src/Scanner/StellaOps.Scanner.Worker/Processing/BinaryLookupStageExecutor.cs
@@ -6,11 +6,18 @@


 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.DependencyInjection;
 using StellaOps.BinaryIndex.Core.Models;
 using StellaOps.BinaryIndex.Core.Services;
+using StellaOps.Scanner.Analyzers.Native.Index;
 using StellaOps.Scanner.Core.Contracts;
+using StellaOps.Scanner.PatchVerification;
+using StellaOps.Scanner.PatchVerification.Models;
 using StellaOps.Scanner.Worker.Extensions;
 using System.Collections.Immutable;
+using System.Collections.ObjectModel;
+using System.Security.Cryptography;
+using System.Text;

 namespace StellaOps.Scanner.Worker.Processing;

@@ -21,15 +28,24 @@ namespace StellaOps.Scanner.Worker.Processing;
 public sealed class BinaryLookupStageExecutor : IScanStageExecutor
 {
    private readonly BinaryVulnerabilityAnalyzer _analyzer;
+    private readonly BinaryFindingMapper _findingMapper;
+    private readonly IBuildIdIndex _buildIdIndex;
+    private readonly IServiceScopeFactory _scopeFactory;
    private readonly BinaryIndexOptions _options;
    private readonly ILogger<BinaryLookupStageExecutor> _logger;

    public BinaryLookupStageExecutor(
        BinaryVulnerabilityAnalyzer analyzer,
+        BinaryFindingMapper findingMapper,
+        IBuildIdIndex buildIdIndex,
+        IServiceScopeFactory scopeFactory,
        BinaryIndexOptions options,
        ILogger<BinaryLookupStageExecutor> logger)
    {
        _analyzer = analyzer ?? throw new ArgumentNullException(nameof(analyzer));
+        _findingMapper = findingMapper ?? throw new ArgumentNullException(nameof(findingMapper));
+        _buildIdIndex = buildIdIndex ?? throw new ArgumentNullException(nameof(buildIdIndex));
+        _scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory));
        _options = options ?? throw new ArgumentNullException(nameof(options));
        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
    }
@@ -76,8 +92,19 @@ public sealed class BinaryLookupStageExecutor : IScanStageExecutor
            }
        }

-        // Store findings in analysis context for downstream stages
-        context.Analysis.SetBinaryFindings(allFindings.ToImmutableArray());
+        var immutableFindings = allFindings.ToImmutableArray();
+        context.Analysis.SetBinaryFindings(immutableFindings);
+
+        if (!immutableFindings.IsDefaultOrEmpty)
+        {
+            await StoreMappedFindingsAsync(context, immutableFindings, cancellationToken).ConfigureAwait(false);
+            await StoreBuildIdMappingsAsync(context, immutableFindings, cancellationToken).ConfigureAwait(false);
+            await StorePatchVerificationResultAsync(
+                context,
+                immutableFindings,
+                layerContexts,
+                cancellationToken).ConfigureAwait(false);
+        }

        _logger.LogInformation(
            "Binary vulnerability lookup complete for scan {ScanId}: {Count} findings",
@@ -121,6 +148,159 @@ public sealed class BinaryLookupStageExecutor : IScanStageExecutor

        return contexts;
    }
+
+    private async Task StoreMappedFindingsAsync(
+        ScanJobContext context,
+        ImmutableArray<BinaryVulnerabilityFinding> findings,
+        CancellationToken cancellationToken)
+    {
+        var mappedFindings = await _findingMapper.MapToFindingsAsync(
+            findings,
+            context.Analysis.GetDetectedDistro(),
+            context.Analysis.GetDetectedRelease(),
+            cancellationToken).ConfigureAwait(false);
+
+        context.Analysis.Set(ScanAnalysisKeys.BinaryVulnerabilityFindings, mappedFindings.Cast<object>().ToArray());
+    }
+
+    private async Task StoreBuildIdMappingsAsync(
+        ScanJobContext context,
+        ImmutableArray<BinaryVulnerabilityFinding> findings,
+        CancellationToken cancellationToken)
+    {
+        var buildIds = findings
+            .Select(finding => finding.Evidence?.BuildId)
+            .Where(buildId => !string.IsNullOrWhiteSpace(buildId))
+            .Select(buildId => buildId!.Trim())
+            .Distinct(StringComparer.OrdinalIgnoreCase)
+            .ToArray();
+
+        if (buildIds.Length == 0)
+        {
+            return;
+        }
+
+        if (!_buildIdIndex.IsLoaded)
+        {
+            await _buildIdIndex.LoadAsync(cancellationToken).ConfigureAwait(false);
+        }
+
+        var lookupResults = await _buildIdIndex.BatchLookupAsync(buildIds, cancellationToken).ConfigureAwait(false);
+        if (lookupResults.Count == 0)
+        {
+            return;
+        }
+
+        var mapping = lookupResults
+            .GroupBy(result => result.BuildId, StringComparer.OrdinalIgnoreCase)
+            .ToDictionary(group => group.Key, group => group.First(), StringComparer.OrdinalIgnoreCase);
+
+        context.Analysis.Set(
+            ScanAnalysisKeys.BinaryBuildIdMappings,
+            new ReadOnlyDictionary<string, BuildIdLookupResult>(mapping));
+    }
+
+    private async Task StorePatchVerificationResultAsync(
+        ScanJobContext context,
+        ImmutableArray<BinaryVulnerabilityFinding> findings,
+        IReadOnlyList<BinaryLayerContext> layerContexts,
+        CancellationToken cancellationToken)
+    {
+        using var scope = _scopeFactory.CreateScope();
+        var patchVerification = scope.ServiceProvider.GetService<IPatchVerificationOrchestrator>();
+        if (patchVerification is null)
+        {
+            _logger.LogDebug("Patch verification orchestrator not registered; skipping binary patch verification.");
+            return;
+        }
+
+        var cveIds = findings
+            .Select(finding => finding.CveId)
+            .Where(cveId => !string.IsNullOrWhiteSpace(cveId))
+            .Distinct(StringComparer.OrdinalIgnoreCase)
+            .ToArray();
+
+        if (cveIds.Length == 0)
+        {
+            return;
+        }
+
+        var artifactPurl = findings
+            .Select(finding => finding.VulnerablePurl)
+            .FirstOrDefault(purl => !string.IsNullOrWhiteSpace(purl))
+            ?? "pkg:generic/unknown-binary";
+
+        var binaryPaths = BuildPatchBinaryPathMap(context, layerContexts);
+        var patchContext = new PatchVerificationContext
+        {
+            ScanId = context.ScanId,
+            TenantId = ResolveTenant(context),
+            ImageDigest = ResolveImageDigest(context),
+            ArtifactPurl = artifactPurl,
+            CveIds = cveIds,
+            BinaryPaths = binaryPaths,
+            Options = new PatchVerificationOptions
+            {
+                ContinueOnError = true,
+                EmitNoPatchDataEvidence = true
+            }
+        };
+
+        var patchResult = await patchVerification.VerifyAsync(patchContext, cancellationToken).ConfigureAwait(false);
+        context.Analysis.Set(ScanAnalysisKeys.BinaryPatchVerificationResult, patchResult);
+    }
+
+    private static IReadOnlyDictionary<string, string> BuildPatchBinaryPathMap(
+        ScanJobContext context,
+        IReadOnlyList<BinaryLayerContext> layerContexts)
+    {
+        context.Lease.Metadata.TryGetValue(ScanMetadataKeys.RootFilesystemPath, out var rootfsPath);
+
+        var paths = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+        foreach (var layerContext in layerContexts)
+        {
+            foreach (var binaryPath in layerContext.BinaryPaths)
+            {
+                if (paths.ContainsKey(binaryPath))
+                {
+                    continue;
+                }
+
+                var resolved = binaryPath;
+                if (!string.IsNullOrWhiteSpace(rootfsPath))
+                {
+                    resolved = Path.Combine(rootfsPath, binaryPath.TrimStart('/', '\\'));
+                }
+
+                paths[binaryPath] = resolved;
+            }
+        }
+
+        return new ReadOnlyDictionary<string, string>(paths);
+    }
+
+    private static string ResolveTenant(ScanJobContext context)
+    {
+        if (context.Lease.Metadata.TryGetValue("scanner.tenant", out var tenant) &&
+            !string.IsNullOrWhiteSpace(tenant))
+        {
+            return tenant.Trim();
+        }
+
+        return "default";
+    }
+
+    private static string ResolveImageDigest(ScanJobContext context)
+    {
+        if (context.Lease.Metadata.TryGetValue("scanner.image.digest", out var digest) &&
+            !string.IsNullOrWhiteSpace(digest))
+        {
+            return digest.Trim();
+        }
+
+        var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(context.ScanId));
+        return $"sha256:{Convert.ToHexString(bytes).ToLowerInvariant()}";
+    }
 }

 /// <summary>
--- a/src/Scanner/StellaOps.Scanner.Worker/Processing/EntryTraceExecutionService.cs
+++ b/src/Scanner/StellaOps.Scanner.Worker/Processing/EntryTraceExecutionService.cs
@@ -5,6 +5,7 @@ using Microsoft.Extensions.Options;
 using StellaOps.Cryptography;
 using StellaOps.Scanner.Core.Contracts;
 using StellaOps.Scanner.EntryTrace;
+using StellaOps.Scanner.EntryTrace.Binary;
 using StellaOps.Scanner.EntryTrace.FileSystem;
 using StellaOps.Scanner.EntryTrace.Runtime;
 using StellaOps.Scanner.Surface.Env;
@@ -20,8 +21,11 @@ using System.Linq;
 using System.Text;
 using System.Text.Json;
 using System.Text.Json.Serialization;
+using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
+using EntryTraceBinaryArchitecture = StellaOps.Scanner.EntryTrace.Binary.BinaryArchitecture;
+using EntryTraceBinaryFormat = StellaOps.Scanner.EntryTrace.Binary.BinaryFormat;

 namespace StellaOps.Scanner.Worker.Processing;

@@ -38,6 +42,9 @@ public sealed class EntryTraceExecutionService : IEntryTraceExecutionService
    };

    private static readonly UTF8Encoding StrictUtf8 = new(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true);
+    private static readonly Regex CveRegex = new(
+        "CVE-\\d{4}-\\d{4,7}",
+        RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);

    private sealed record FileSystemHandle(
        IRootFileSystem FileSystem,
@@ -196,6 +203,16 @@ public sealed class EntryTraceExecutionService : IEntryTraceExecutionService
        var runtimeGraph = BuildRuntimeGraph(metadata, context.JobId);
        graph = _runtimeReconciler.Reconcile(graph, runtimeGraph);

+        var binaryIntelligence = await BuildBinaryIntelligenceAsync(
+            graph,
+            fileSystemHandle.RootPath,
+            context.TimeProvider,
+            cancellationToken).ConfigureAwait(false);
+        if (binaryIntelligence is not null)
+        {
+            graph = graph with { BinaryIntelligence = binaryIntelligence };
+        }
+
        var generatedAt = context.TimeProvider.GetUtcNow();
        var ndjson = EntryTraceNdjsonWriter.Serialize(
            graph,
@@ -727,6 +744,387 @@ public sealed class EntryTraceExecutionService : IEntryTraceExecutionService
        return builder.ToImmutable();
    }

+    private async Task<EntryTraceBinaryIntelligence?> BuildBinaryIntelligenceAsync(
+        EntryTraceGraph graph,
+        string rootDirectory,
+        TimeProvider timeProvider,
+        CancellationToken cancellationToken)
+    {
+        var terminalPaths = graph.Terminals
+            .Where(terminal => terminal.Type == EntryTraceTerminalType.Native && !string.IsNullOrWhiteSpace(terminal.Path))
+            .Select(terminal => terminal.Path.Trim())
+            .Distinct(StringComparer.Ordinal)
+            .ToImmutableArray();
+
+        if (terminalPaths.IsDefaultOrEmpty)
+        {
+            return null;
+        }
+
+        var targets = ImmutableArray.CreateBuilder<EntryTraceBinaryTarget>();
+
+        foreach (var terminalPath in terminalPaths)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            var fullPath = ResolveTerminalPath(rootDirectory, terminalPath);
+            if (fullPath is null || !File.Exists(fullPath))
+            {
+                continue;
+            }
+
+            byte[] payload;
+            try
+            {
+                payload = await File.ReadAllBytesAsync(fullPath, cancellationToken).ConfigureAwait(false);
+            }
+            catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
+            {
+                _logger.LogDebug(ex, "Unable to read terminal binary '{TerminalPath}' for entry trace binary intelligence.", terminalPath);
+                continue;
+            }
+
+            if (payload.Length == 0)
+            {
+                continue;
+            }
+
+            var functions = ExtractFunctions(payload);
+            if (functions.IsDefaultOrEmpty)
+            {
+                continue;
+            }
+
+            var (architecture, format) = DetectBinaryShape(payload);
+            var binaryHash = "sha256:" + _hash.ComputeHashHex(payload, HashAlgorithms.Sha256);
+            var packagePurl = BuildTerminalPurl(terminalPath);
+            var vulnerabilityIds = DetectVulnerabilityIds(functions);
+
+            var analyzer = new BinaryIntelligenceAnalyzer(timeProvider: timeProvider);
+            await analyzer.IndexPackageAsync(
+                packagePurl,
+                "unknown",
+                functions,
+                vulnerabilityIds,
+                cancellationToken).ConfigureAwait(false);
+
+            var analysis = await analyzer.AnalyzeAsync(
+                terminalPath,
+                binaryHash,
+                functions,
+                architecture,
+                format,
+                cancellationToken).ConfigureAwait(false);
+
+            var matches = analysis.VulnerableMatches
+                .Select(match => new EntryTraceBinaryVulnerability(
+                    match.VulnerabilityId,
+                    match.FunctionName,
+                    match.SourcePackage,
+                    match.VulnerableFunctionName,
+                    match.MatchConfidence,
+                    match.Severity.ToString()))
+                .ToImmutableArray();
+
+            targets.Add(new EntryTraceBinaryTarget(
+                terminalPath,
+                binaryHash,
+                architecture.ToString(),
+                format.ToString(),
+                analysis.Functions.Length,
+                analysis.RecoveredSymbolCount,
+                analysis.SourceCorrelations.Length,
+                analysis.VulnerableMatches.Length,
+                matches));
+        }
+
+        if (targets.Count == 0)
+        {
+            return null;
+        }
+
+        return new EntryTraceBinaryIntelligence(
+            targets.ToImmutable(),
+            terminalPaths.Length,
+            targets.Count,
+            targets.Sum(target => target.VulnerableMatchCount),
+            timeProvider.GetUtcNow());
+    }
+
+    private static string BuildTerminalPurl(string terminalPath)
+    {
+        var fileName = IOPath.GetFileName(terminalPath);
+        if (string.IsNullOrWhiteSpace(fileName))
+        {
+            fileName = "unknown-binary";
+        }
+
+        var normalized = fileName.Trim().ToLowerInvariant();
+        return $"pkg:generic/{normalized}";
+    }
+
+    private static string? ResolveTerminalPath(string rootDirectory, string terminalPath)
+    {
+        if (string.IsNullOrWhiteSpace(rootDirectory) || string.IsNullOrWhiteSpace(terminalPath))
+        {
+            return null;
+        }
+
+        string candidate;
+        try
+        {
+            if (IOPath.IsPathRooted(terminalPath))
+            {
+                candidate = IOPath.GetFullPath(IOPath.Combine(rootDirectory, terminalPath.TrimStart('\\', '/')));
+            }
+            else
+            {
+                candidate = IOPath.GetFullPath(IOPath.Combine(rootDirectory, terminalPath));
+            }
+        }
+        catch
+        {
+            return null;
+        }
+
+        var rootFullPath = IOPath.GetFullPath(rootDirectory);
+        if (!candidate.StartsWith(rootFullPath, StringComparison.OrdinalIgnoreCase))
+        {
+            return null;
+        }
+
+        return candidate;
+    }
+
+    private static (EntryTraceBinaryArchitecture Architecture, EntryTraceBinaryFormat Format) DetectBinaryShape(ReadOnlySpan<byte> payload)
+    {
+        if (payload.Length >= 4 && payload[0] == 0x7F && payload[1] == (byte)'E' && payload[2] == (byte)'L' && payload[3] == (byte)'F')
+        {
+            var architecture = EntryTraceBinaryArchitecture.Unknown;
+            if (payload.Length >= 20)
+            {
+                var machine = BitConverter.ToUInt16(payload.Slice(18, 2));
+                architecture = machine switch
+                {
+                    0x03 => EntryTraceBinaryArchitecture.X86,
+                    0x3E => EntryTraceBinaryArchitecture.X64,
+                    0x28 => EntryTraceBinaryArchitecture.ARM,
+                    0xB7 => EntryTraceBinaryArchitecture.ARM64,
+                    0xF3 => EntryTraceBinaryArchitecture.RISCV64,
+                    _ => EntryTraceBinaryArchitecture.Unknown
+                };
+            }
+
+            return (architecture, EntryTraceBinaryFormat.ELF);
+        }
+
+        if (payload.Length >= 4 && payload[0] == 0x4D && payload[1] == 0x5A)
+        {
+            return (EntryTraceBinaryArchitecture.Unknown, EntryTraceBinaryFormat.PE);
+        }
+
+        if (payload.Length >= 4)
+        {
+            var magic = BitConverter.ToUInt32(payload.Slice(0, 4));
+            if (magic is 0xFEEDFACE or 0xCEFAEDFE or 0xFEEDFACF or 0xCFFAEDFE)
+            {
+                return (EntryTraceBinaryArchitecture.Unknown, EntryTraceBinaryFormat.MachO);
+            }
+        }
+
+        if (payload.Length >= 4 && payload[0] == 0x00 && payload[1] == (byte)'a' && payload[2] == (byte)'s' && payload[3] == (byte)'m')
+        {
+            return (EntryTraceBinaryArchitecture.WASM, EntryTraceBinaryFormat.WASM);
+        }
+
+        return (EntryTraceBinaryArchitecture.Unknown, EntryTraceBinaryFormat.Raw);
+    }
+
+    private static ImmutableArray<FunctionSignature> ExtractFunctions(byte[] payload)
+    {
+        const int minFunctionSize = 16;
+        const int windowSize = 256;
+        const int maxFunctions = 24;
+
+        var functions = ImmutableArray.CreateBuilder<FunctionSignature>();
+        for (var offset = 0; offset < payload.Length && functions.Count < maxFunctions; offset += windowSize)
+        {
+            var length = Math.Min(windowSize, payload.Length - offset);
+            if (length < minFunctionSize)
+            {
+                continue;
+            }
+
+            var window = payload.AsSpan(offset, length);
+            var stringRefs = ExtractAsciiStrings(window);
+            var importRefs = stringRefs
+                .Where(IsLikelyImportReference)
+                .Distinct(StringComparer.OrdinalIgnoreCase)
+                .Take(12)
+                .ToImmutableArray();
+            var basicBlocks = BuildBasicBlocks(window, offset);
+
+            functions.Add(new FunctionSignature(
+                Name: null,
+                Offset: offset,
+                Size: length,
+                CallingConvention: CallingConvention.Unknown,
+                ParameterCount: null,
+                ReturnType: null,
+                Fingerprint: CodeFingerprint.Empty,
+                BasicBlocks: basicBlocks,
+                StringReferences: stringRefs,
+                ImportReferences: importRefs));
+        }
+
+        return functions.ToImmutable();
+    }
+
+    private static ImmutableArray<BasicBlock> BuildBasicBlocks(ReadOnlySpan<byte> window, int baseOffset)
+    {
+        const int blockSize = 16;
+        var blocks = ImmutableArray.CreateBuilder<BasicBlock>();
+        var blockCount = (int)Math.Ceiling(window.Length / (double)blockSize);
+
+        for (var i = 0; i < blockCount; i++)
+        {
+            var offset = i * blockSize;
+            var size = Math.Min(blockSize, window.Length - offset);
+            if (size <= 0)
+            {
+                continue;
+            }
+
+            var normalizedBytes = window.Slice(offset, size).ToArray().ToImmutableArray();
+            var successors = i < blockCount - 1
+                ? ImmutableArray.Create(i + 1)
+                : ImmutableArray<int>.Empty;
+            var predecessors = i > 0
+                ? ImmutableArray.Create(i - 1)
+                : ImmutableArray<int>.Empty;
+
+            blocks.Add(new BasicBlock(
+                Id: i,
+                Offset: baseOffset + offset,
+                Size: size,
+                InstructionCount: Math.Max(1, size / 4),
+                Successors: successors,
+                Predecessors: predecessors,
+                NormalizedBytes: normalizedBytes));
+        }
+
+        return blocks.ToImmutable();
+    }
+
+    private static ImmutableArray<string> ExtractAsciiStrings(ReadOnlySpan<byte> payload)
+    {
+        const int minLength = 4;
+        const int maxStrings = 24;
+
+        var result = new List<string>(maxStrings);
+        var current = new List<char>(64);
+
+        static bool IsPrintable(byte value) => value >= 32 && value <= 126;
+
+        void Flush()
+        {
+            if (current.Count >= minLength && result.Count < maxStrings)
+            {
+                result.Add(new string(current.ToArray()));
+            }
+            current.Clear();
+        }
+
+        foreach (var value in payload)
+        {
+            if (IsPrintable(value))
+            {
+                current.Add((char)value);
+            }
+            else
+            {
+                Flush();
+                if (result.Count >= maxStrings)
+                {
+                    break;
+                }
+            }
+        }
+
+        if (result.Count < maxStrings)
+        {
+            Flush();
+        }
+
+        return result
+            .Distinct(StringComparer.OrdinalIgnoreCase)
+            .ToImmutableArray();
+    }
+
+    private static bool IsLikelyImportReference(string candidate)
+    {
+        if (string.IsNullOrWhiteSpace(candidate) || candidate.Length > 96)
+        {
+            return false;
+        }
+
+        if (candidate.Contains(' ') || candidate.Contains('/') || candidate.Contains('\\'))
+        {
+            return false;
+        }
+
+        var likelyPrefixes = new[]
+        {
+            "SSL_",
+            "EVP_",
+            "BIO_",
+            "inflate",
+            "deflate",
+            "mem",
+            "str",
+            "free",
+            "malloc",
+            "open",
+            "read",
+            "write"
+        };
+
+        return likelyPrefixes.Any(prefix => candidate.StartsWith(prefix, StringComparison.OrdinalIgnoreCase));
+    }
+
+    private static ImmutableArray<string> DetectVulnerabilityIds(ImmutableArray<FunctionSignature> functions)
+    {
+        var ids = ImmutableHashSet.CreateBuilder<string>(StringComparer.OrdinalIgnoreCase);
+
+        foreach (var function in functions)
+        {
+            foreach (var reference in function.StringReferences)
+            {
+                foreach (Match match in CveRegex.Matches(reference))
+                {
+                    ids.Add(match.Value.ToUpperInvariant());
+                }
+
+                if (reference.Contains("HEARTBLEED", StringComparison.OrdinalIgnoreCase))
+                {
+                    ids.Add("CVE-2014-0160");
+                }
+
+                if (reference.Contains("SHELLSHOCK", StringComparison.OrdinalIgnoreCase))
+                {
+                    ids.Add("CVE-2014-6271");
+                }
+
+                if (reference.Contains("LOG4J", StringComparison.OrdinalIgnoreCase))
+                {
+                    ids.Add("CVE-2021-44228");
+                }
+            }
+        }
+
+        return ids.ToImmutableArray();
+    }
+
    private static IEnumerable<string> SplitLayerString(string raw)
        => raw.Split(new[] { '\n', '\r', ';' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);