Add unit tests for AST parsing and security sink detection

- Created `StellaOps.AuditPack.Tests.csproj` for unit testing the AuditPack library. - Implemented comprehensive unit tests in `index.test.js` for AST parsing, covering various JavaScript and TypeScript constructs including functions, classes, decorators, and JSX. - Added `sink-detect.test.js` to test security sink detection patterns, validating command injection, SQL injection, file write, deserialization, SSRF, NoSQL injection, and more. - Included tests for taint source detection in various contexts such as Express, Koa, and AWS Lambda.
2025-12-23 09:23:42 +02:00
parent 7e384ab610
commit 56e2dc01ee
96 changed files with 8555 additions and 1455 deletions
--- a/src/Scanner/StellaOps.Scanner.Worker/Extensions/BinaryIndexServiceExtensions.cs
+++ b/src/Scanner/StellaOps.Scanner.Worker/Extensions/BinaryIndexServiceExtensions.cs
@@ -0,0 +1,90 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// Sprint: SPRINT_6000_0004_0001 - Scanner Worker Integration
+// Task: T5 - Add Configuration and DI Registration
+
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using StellaOps.BinaryIndex.Core.Services;
+using StellaOps.Scanner.Worker.Processing;
+
+namespace StellaOps.Scanner.Worker.Extensions;
+
+/// <summary>
+/// Extension methods for registering BinaryIndex integration services.
+/// </summary>
+public static class BinaryIndexServiceExtensions
+{
+    /// <summary>
+    /// Adds BinaryIndex integration services to the service collection.
+    /// </summary>
+    public static IServiceCollection AddBinaryIndexIntegration(
+        this IServiceCollection services,
+        IConfiguration configuration)
+    {
+        var options = configuration
+            .GetSection("BinaryIndex")
+            .Get<BinaryIndexOptions>() ?? new BinaryIndexOptions();
+
+        if (!options.Enabled)
+        {
+            services.AddSingleton<IBinaryVulnerabilityService, NullBinaryVulnerabilityService>();
+            return services;
+        }
+
+        services.AddSingleton(options);
+        services.AddScoped<IBinaryVulnerabilityService, BinaryVulnerabilityService>();
+        services.AddScoped<IBinaryFeatureExtractor, ElfFeatureExtractor>();
+        services.AddScoped<BinaryVulnerabilityAnalyzer>();
+
+        return services;
+    }
+}
+
+/// <summary>
+/// Configuration options for BinaryIndex integration.
+/// </summary>
+public sealed class BinaryIndexOptions
+{
+    /// <summary>
+    /// Whether binary vulnerability analysis is enabled.
+    /// </summary>
+    public bool Enabled { get; init; } = true;
+
+    /// <summary>
+    /// Batch size for binary lookups.
+    /// </summary>
+    public int BatchSize { get; init; } = 100;
+
+    /// <summary>
+    /// Timeout in milliseconds for binary lookups.
+    /// </summary>
+    public int TimeoutMs { get; init; } = 5000;
+
+    /// <summary>
+    /// Minimum confidence threshold for reporting matches.
+    /// </summary>
+    public decimal MinConfidence { get; init; } = 0.7m;
+}
+
+/// <summary>
+/// Null implementation of IBinaryVulnerabilityService for when binary analysis is disabled.
+/// </summary>
+internal sealed class NullBinaryVulnerabilityService : IBinaryVulnerabilityService
+{
+    public Task<System.Collections.Immutable.ImmutableArray<BinaryVulnMatch>> LookupByIdentityAsync(
+        StellaOps.BinaryIndex.Core.Models.BinaryIdentity identity,
+        LookupOptions? options = null,
+        CancellationToken ct = default)
+    {
+        return Task.FromResult(System.Collections.Immutable.ImmutableArray<BinaryVulnMatch>.Empty);
+    }
+
+    public Task<System.Collections.Immutable.ImmutableDictionary<string, System.Collections.Immutable.ImmutableArray<BinaryVulnMatch>>> LookupBatchAsync(
+        IEnumerable<StellaOps.BinaryIndex.Core.Models.BinaryIdentity> identities,
+        LookupOptions? options = null,
+        CancellationToken ct = default)
+    {
+        return Task.FromResult(System.Collections.Immutable.ImmutableDictionary<string, System.Collections.Immutable.ImmutableArray<BinaryVulnMatch>>.Empty);
+    }
+}
--- a/src/Scanner/StellaOps.Scanner.Worker/Processing/BinaryVulnerabilityAnalyzer.cs
+++ b/src/Scanner/StellaOps.Scanner.Worker/Processing/BinaryVulnerabilityAnalyzer.cs
@@ -0,0 +1,216 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// Sprint: SPRINT_6000_0004_0001 - Scanner Worker Integration
+// Task: T3 - Create Scanner.Worker Integration Point
+
+using System.Collections.Immutable;
+using Microsoft.Extensions.Logging;
+using StellaOps.BinaryIndex.Core.Models;
+using StellaOps.BinaryIndex.Core.Services;
+
+namespace StellaOps.Scanner.Worker.Processing;
+
+/// <summary>
+/// Analyzer that queries BinaryIndex for vulnerable binaries during scan.
+/// Integrates with the Scanner.Worker pipeline to detect binary vulnerabilities.
+/// </summary>
+public sealed class BinaryVulnerabilityAnalyzer
+{
+    private readonly IBinaryVulnerabilityService _binaryVulnService;
+    private readonly IBinaryFeatureExtractor _featureExtractor;
+    private readonly ILogger<BinaryVulnerabilityAnalyzer> _logger;
+
+    public BinaryVulnerabilityAnalyzer(
+        IBinaryVulnerabilityService binaryVulnService,
+        IBinaryFeatureExtractor featureExtractor,
+        ILogger<BinaryVulnerabilityAnalyzer> logger)
+    {
+        _binaryVulnService = binaryVulnService ?? throw new ArgumentNullException(nameof(binaryVulnService));
+        _featureExtractor = featureExtractor ?? throw new ArgumentNullException(nameof(featureExtractor));
+        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+    }
+
+    public string AnalyzerId => "binary-vulnerability";
+    public int Priority => 100; // Run after package analyzers
+
+    /// <summary>
+    /// Analyzes a layer for binary vulnerabilities.
+    /// </summary>
+    public async Task<BinaryAnalysisResult> AnalyzeLayerAsync(
+        BinaryLayerContext context,
+        CancellationToken ct = default)
+    {
+        var findings = new List<BinaryVulnerabilityFinding>();
+        var identities = new List<BinaryIdentity>();
+        var extractionErrors = new List<string>();
+
+        _logger.LogDebug("Scanning layer {LayerDigest} for binary vulnerabilities", context.LayerDigest);
+
+        // Extract identities from all binaries in layer
+        foreach (var filePath in context.BinaryPaths)
+        {
+            if (!IsBinaryFile(filePath))
+                continue;
+
+            try
+            {
+                using var stream = context.OpenFile(filePath);
+                if (stream == null)
+                {
+                    _logger.LogDebug("Could not open file {Path}", filePath);
+                    continue;
+                }
+
+                var identity = await _featureExtractor.ExtractIdentityAsync(stream, ct).ConfigureAwait(false);
+                if (identity != null)
+                {
+                    identities.Add(identity);
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogDebug(ex, "Failed to extract identity from {Path}", filePath);
+                extractionErrors.Add($"{filePath}: {ex.Message}");
+            }
+        }
+
+        if (identities.Count == 0)
+        {
+            _logger.LogDebug("No binary identities extracted from layer {LayerDigest}", context.LayerDigest);
+            return BinaryAnalysisResult.Empty(context.ScanId, context.LayerDigest);
+        }
+
+        _logger.LogDebug("Extracted {Count} binary identities from layer {LayerDigest}",
+            identities.Count, context.LayerDigest);
+
+        // Batch lookup
+        var options = new LookupOptions
+        {
+            DistroHint = context.DetectedDistro,
+            ReleaseHint = context.DetectedRelease,
+            CheckFixIndex = true
+        };
+
+        var matches = await _binaryVulnService.LookupBatchAsync(identities, options, ct).ConfigureAwait(false);
+
+        foreach (var (binaryKey, vulnMatches) in matches)
+        {
+            foreach (var match in vulnMatches)
+            {
+                findings.Add(new BinaryVulnerabilityFinding
+                {
+                    ScanId = context.ScanId,
+                    LayerDigest = context.LayerDigest,
+                    BinaryKey = binaryKey,
+                    CveId = match.CveId,
+                    VulnerablePurl = match.VulnerablePurl,
+                    MatchMethod = match.Method.ToString(),
+                    Confidence = match.Confidence,
+                    Evidence = match.Evidence
+                });
+            }
+        }
+
+        _logger.LogInformation(
+            "Found {FindingCount} binary vulnerability findings in layer {LayerDigest}",
+            findings.Count, context.LayerDigest);
+
+        return new BinaryAnalysisResult
+        {
+            ScanId = context.ScanId,
+            LayerDigest = context.LayerDigest,
+            AnalyzerId = AnalyzerId,
+            Findings = findings.ToImmutableArray(),
+            ExtractedBinaryCount = identities.Count,
+            ExtractionErrors = extractionErrors.ToImmutableArray()
+        };
+    }
+
+    /// <summary>
+    /// Checks if a file path indicates a binary file.
+    /// </summary>
+    private static bool IsBinaryFile(string path)
+    {
+        // Check common binary paths
+        if (path.StartsWith("/usr/lib/", StringComparison.OrdinalIgnoreCase) ||
+            path.StartsWith("/lib/", StringComparison.OrdinalIgnoreCase) ||
+            path.StartsWith("/lib64/", StringComparison.OrdinalIgnoreCase) ||
+            path.StartsWith("/usr/lib64/", StringComparison.OrdinalIgnoreCase) ||
+            path.StartsWith("/usr/bin/", StringComparison.OrdinalIgnoreCase) ||
+            path.StartsWith("/bin/", StringComparison.OrdinalIgnoreCase) ||
+            path.StartsWith("/usr/sbin/", StringComparison.OrdinalIgnoreCase) ||
+            path.StartsWith("/sbin/", StringComparison.OrdinalIgnoreCase))
+        {
+            return true;
+        }
+
+        // Check common binary extensions
+        if (path.EndsWith(".so", StringComparison.OrdinalIgnoreCase) ||
+            path.Contains(".so.", StringComparison.OrdinalIgnoreCase) ||
+            path.EndsWith(".a", StringComparison.OrdinalIgnoreCase))
+        {
+            return true;
+        }
+
+        return false;
+    }
+}
+
+/// <summary>
+/// Context for binary layer analysis.
+/// </summary>
+public sealed class BinaryLayerContext
+{
+    public required Guid ScanId { get; init; }
+    public required string LayerDigest { get; init; }
+    public required IReadOnlyList<string> BinaryPaths { get; init; }
+    public string? DetectedDistro { get; init; }
+    public string? DetectedRelease { get; init; }
+
+    /// <summary>
+    /// Function to open a file for reading.
+    /// </summary>
+    public required Func<string, Stream?> OpenFile { get; init; }
+}
+
+/// <summary>
+/// Result of binary vulnerability analysis.
+/// </summary>
+public sealed record BinaryAnalysisResult
+{
+    public required Guid ScanId { get; init; }
+    public required string LayerDigest { get; init; }
+    public required string AnalyzerId { get; init; }
+    public required ImmutableArray<BinaryVulnerabilityFinding> Findings { get; init; }
+    public int ExtractedBinaryCount { get; init; }
+    public ImmutableArray<string> ExtractionErrors { get; init; } = [];
+
+    public static BinaryAnalysisResult Empty(Guid scanId, string layerDigest) => new()
+    {
+        ScanId = scanId,
+        LayerDigest = layerDigest,
+        AnalyzerId = "binary-vulnerability",
+        Findings = [],
+        ExtractedBinaryCount = 0,
+        ExtractionErrors = []
+    };
+}
+
+/// <summary>
+/// A binary vulnerability finding.
+/// </summary>
+public sealed record BinaryVulnerabilityFinding
+{
+    public Guid ScanId { get; init; }
+    public required string LayerDigest { get; init; }
+    public required string BinaryKey { get; init; }
+    public required string CveId { get; init; }
+    public required string VulnerablePurl { get; init; }
+    public required string MatchMethod { get; init; }
+    public required decimal Confidence { get; init; }
+    public MatchEvidence? Evidence { get; init; }
+
+    public string FindingType => "binary-vulnerability";
+
+    public string GetSummary() =>
+        $"{CveId} in {VulnerablePurl} (via {MatchMethod}, confidence {Confidence:P0})";
+}
--- a/src/Scanner/StellaOps.Scanner.Worker/Processing/CompositeScanAnalyzerDispatcher.cs
+++ b/src/Scanner/StellaOps.Scanner.Worker/Processing/CompositeScanAnalyzerDispatcher.cs
@@ -22,6 +22,7 @@ using StellaOps.Scanner.Surface.Env;
 using StellaOps.Scanner.Surface.FS;
 using StellaOps.Scanner.Surface.Validation;
 using StellaOps.Scanner.Worker.Options;
+using StellaOps.Scanner.Worker.Extensions;
 using StellaOps.Scanner.Worker.Diagnostics;
 using StellaOps.Cryptography;

@@ -96,6 +97,9 @@ internal sealed class CompositeScanAnalyzerDispatcher : IScanAnalyzerDispatcher
        {
            await ExecuteNativeAnalyzerAsync(context, services, rootfsPath, cancellationToken).ConfigureAwait(false);
        }
+
+        // Binary vulnerability analysis (SPRINT_6000_0004_0001)
+        await ExecuteBinaryAnalyzerAsync(context, services, rootfsPath, cancellationToken).ConfigureAwait(false);
    }

    private async Task ExecuteOsAnalyzersAsync(
@@ -382,6 +386,133 @@ internal sealed class CompositeScanAnalyzerDispatcher : IScanAnalyzerDispatcher
        context.Analysis.AppendLayerFragments(ImmutableArray.Create(fragment));
    }

+    private async Task ExecuteBinaryAnalyzerAsync(
+        ScanJobContext context,
+        IServiceProvider services,
+        string? rootfsPath,
+        CancellationToken cancellationToken)
+    {
+        // Check if binary analysis is enabled via options
+        var binaryOptions = services.GetService<BinaryIndexOptions>();
+        if (binaryOptions is null || !binaryOptions.Enabled)
+        {
+            _logger.LogDebug("Binary vulnerability analysis is disabled for job {JobId}.", context.JobId);
+            return;
+        }
+
+        if (rootfsPath is null)
+        {
+            _logger.LogDebug(
+                "Root filesystem path not available for job {JobId}; skipping binary vulnerability analysis.",
+                context.JobId);
+            return;
+        }
+
+        try
+        {
+            var analyzer = services.GetService<BinaryVulnerabilityAnalyzer>();
+            if (analyzer is null)
+            {
+                _logger.LogDebug("BinaryVulnerabilityAnalyzer not registered; skipping binary analysis.");
+                return;
+            }
+
+            // Build list of binary paths from rootfs
+            var binaryPaths = DiscoverBinaryPaths(rootfsPath);
+            if (binaryPaths.Count == 0)
+            {
+                _logger.LogDebug("No binary files found in rootfs for job {JobId}.", context.JobId);
+                return;
+            }
+
+            var layerDigest = ComputeLayerDigest("binary");
+            var scanIdGuid = Guid.TryParse(context.ScanId, out var parsedGuid) ? parsedGuid : Guid.Empty;
+            var layerContext = new BinaryLayerContext
+            {
+                ScanId = scanIdGuid,
+                LayerDigest = layerDigest,
+                BinaryPaths = binaryPaths,
+                DetectedDistro = null, // Could be enriched from OS analyzer results
+                DetectedRelease = null,
+                OpenFile = path =>
+                {
+                    var fullPath = Path.Combine(rootfsPath, path.TrimStart('/'));
+                    return File.Exists(fullPath) ? File.OpenRead(fullPath) : null;
+                }
+            };
+
+            var result = await analyzer.AnalyzeLayerAsync(layerContext, cancellationToken).ConfigureAwait(false);
+
+            if (result.Findings.Length > 0)
+            {
+                context.Analysis.Set(ScanAnalysisKeys.BinaryVulnerabilityFindings, result.Findings);
+                _logger.LogInformation(
+                    "Binary vulnerability analysis found {Count} findings for job {JobId}.",
+                    result.Findings.Length, context.JobId);
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Binary vulnerability analysis failed for job {JobId}.", context.JobId);
+        }
+    }
+
+    private static IReadOnlyList<string> DiscoverBinaryPaths(string rootfsPath)
+    {
+        var binaryPaths = new List<string>();
+        var searchDirs = new[]
+        {
+            "usr/lib", "usr/lib64", "lib", "lib64",
+            "usr/bin", "usr/sbin", "bin", "sbin"
+        };
+
+        foreach (var dir in searchDirs)
+        {
+            var fullDir = Path.Combine(rootfsPath, dir);
+            if (!Directory.Exists(fullDir))
+                continue;
+
+            try
+            {
+                var files = Directory.EnumerateFiles(fullDir, "*", SearchOption.AllDirectories);
+                foreach (var file in files)
+                {
+                    var relativePath = "/" + Path.GetRelativePath(rootfsPath, file).Replace('\\', '/');
+                    if (IsPotentialBinary(file))
+                    {
+                        binaryPaths.Add(relativePath);
+                    }
+                }
+            }
+            catch (Exception)
+            {
+                // Directory access issues are expected in some scenarios
+            }
+        }
+
+        return binaryPaths;
+    }
+
+    private static bool IsPotentialBinary(string filePath)
+    {
+        // Quick heuristic: check for .so files and executables
+        var name = Path.GetFileName(filePath);
+        if (name.EndsWith(".so", StringComparison.OrdinalIgnoreCase) ||
+            name.Contains(".so.", StringComparison.OrdinalIgnoreCase))
+        {
+            return true;
+        }
+
+        // Check if file is executable by looking at extension (no extension often = binary)
+        var ext = Path.GetExtension(filePath);
+        if (string.IsNullOrEmpty(ext))
+        {
+            return true;
+        }
+
+        return false;
+    }
+
    private static string ComputeLayerDigest(string kind)
    {
        var normalized = $"stellaops:{kind.Trim().ToLowerInvariant()}";
--- a/src/Scanner/StellaOps.Scanner.Worker/Program.cs
+++ b/src/Scanner/StellaOps.Scanner.Worker/Program.cs
@@ -185,7 +185,7 @@ if (workerOptions.VerdictPush.Enabled)
        client.Timeout = workerOptions.VerdictPush.Timeout;
    });
    builder.Services.AddSingleton<StellaOps.Scanner.Storage.Oci.VerdictOciPublisher>();
-    builder.Services.AddSingleton<IScanStageExecutor, Processing.VerdictPushStageExecutor>();
+    builder.Services.AddSingleton<IScanStageExecutor, VerdictPushStageExecutor>();
 }

 builder.Services.AddSingleton<ScannerWorkerHostedService>();
--- a/src/Scanner/StellaOps.Scanner.Worker/StellaOps.Scanner.Worker.csproj
+++ b/src/Scanner/StellaOps.Scanner.Worker/StellaOps.Scanner.Worker.csproj
@@ -32,5 +32,6 @@
    <ProjectReference Include="../__Libraries/StellaOps.Scanner.Emit/StellaOps.Scanner.Emit.csproj" />
    <ProjectReference Include="../StellaOps.Scanner.Analyzers.Native/StellaOps.Scanner.Analyzers.Native.csproj" />
    <ProjectReference Include="../../Unknowns/__Libraries/StellaOps.Unknowns.Core/StellaOps.Unknowns.Core.csproj" />
+    <ProjectReference Include="../../BinaryIndex/__Libraries/StellaOps.BinaryIndex.Core/StellaOps.BinaryIndex.Core.csproj" />
  </ItemGroup>
 </Project>