feat: Add initial implementation of Vulnerability Resolver Jobs
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Created project for StellaOps.Scanner.Analyzers.Native.Tests with necessary dependencies.
- Documented roles and guidelines in AGENTS.md for Scheduler module.
- Implemented IResolverJobService interface and InMemoryResolverJobService for handling resolver jobs.
- Added ResolverBacklogNotifier and ResolverBacklogService for monitoring job metrics.
- Developed API endpoints for managing resolver jobs and retrieving metrics.
- Defined models for resolver job requests and responses.
- Integrated dependency injection for resolver job services.
- Implemented ImpactIndexSnapshot for persisting impact index data.
- Introduced SignalsScoringOptions for configurable scoring weights in reachability scoring.
- Added unit tests for ReachabilityScoringService and RuntimeFactsIngestionService.
- Created dotnet-filter.sh script to handle command-line arguments for dotnet.
- Established nuget-prime project for managing package downloads.
This commit is contained in:
master
2025-11-18 07:52:15 +02:00
parent e69b57d467
commit 8355e2ff75
299 changed files with 13293 additions and 2444 deletions

View File

@@ -1,9 +1,13 @@
global using System;
global using System.Collections.Generic;
global using System.IO;
global using System.Linq;
global using System.Text.Json;
global using System.Threading;
global using System.Threading.Tasks;
global using StellaOps.Scanner.Analyzers.Lang;
global using System.Collections.Generic;
global using System.IO;
global using System.IO.Compression;
global using System.Linq;
global using System.Formats.Tar;
global using System.Security.Cryptography;
global using System.Text;
global using System.Text.Json;
global using System.Threading;
global using System.Threading.Tasks;
global using StellaOps.Scanner.Analyzers.Lang;

View File

@@ -9,15 +9,17 @@ internal sealed class NodePackage
string packageJsonLocator,
bool? isPrivate,
NodeLockEntry? lockEntry,
bool isWorkspaceMember,
string? workspaceRoot,
IReadOnlyList<string> workspaceTargets,
string? workspaceLink,
bool isWorkspaceMember,
string? workspaceRoot,
IReadOnlyList<string> workspaceTargets,
string? workspaceLink,
IReadOnlyList<NodeLifecycleScript> lifecycleScripts,
IReadOnlyList<NodeVersionTarget> nodeVersions,
bool usedByEntrypoint,
bool declaredOnly = false,
string? lockSource = null,
string? lockLocator = null)
string? lockLocator = null,
string? packageSha256 = null)
{
Name = name;
Version = version;
@@ -26,14 +28,16 @@ internal sealed class NodePackage
IsPrivate = isPrivate;
LockEntry = lockEntry;
IsWorkspaceMember = isWorkspaceMember;
WorkspaceRoot = workspaceRoot;
WorkspaceTargets = workspaceTargets;
WorkspaceRoot = workspaceRoot;
WorkspaceTargets = workspaceTargets;
WorkspaceLink = workspaceLink;
LifecycleScripts = lifecycleScripts ?? Array.Empty<NodeLifecycleScript>();
NodeVersions = nodeVersions ?? Array.Empty<NodeVersionTarget>();
IsUsedByEntrypoint = usedByEntrypoint;
DeclaredOnly = declaredOnly;
LockSource = lockSource;
LockLocator = lockLocator;
PackageSha256 = packageSha256;
}
public string Name { get; }
@@ -58,6 +62,8 @@ internal sealed class NodePackage
public IReadOnlyList<NodeLifecycleScript> LifecycleScripts { get; }
public IReadOnlyList<NodeVersionTarget> NodeVersions { get; }
public bool HasInstallScripts => LifecycleScripts.Count > 0;
public bool IsUsedByEntrypoint { get; }
@@ -67,6 +73,8 @@ internal sealed class NodePackage
public string? LockSource { get; }
public string? LockLocator { get; }
public string? PackageSha256 { get; }
public string RelativePathNormalized => string.IsNullOrEmpty(RelativePath) ? string.Empty : RelativePath.Replace(Path.DirectorySeparatorChar, '/');
@@ -80,13 +88,23 @@ internal sealed class NodePackage
{
CreateRootEvidence()
};
foreach (var script in LifecycleScripts)
{
var locator = string.IsNullOrEmpty(PackageJsonLocator)
? $"package.json#scripts.{script.Name}"
: $"{PackageJsonLocator}#scripts.{script.Name}";
foreach (var version in NodeVersions.OrderBy(static v => v.Kind, StringComparer.Ordinal))
{
evidence.Add(new LanguageComponentEvidence(
LanguageEvidenceKind.File,
$"node-version:{version.Kind}",
version.Locator,
version.Version,
version.Sha256));
}
foreach (var script in LifecycleScripts)
{
var locator = string.IsNullOrEmpty(PackageJsonLocator)
? $"package.json#scripts.{script.Name}"
: $"{PackageJsonLocator}#scripts.{script.Name}";
evidence.Add(new LanguageComponentEvidence(
LanguageEvidenceKind.Metadata,
"package.json:scripts",
@@ -95,7 +113,9 @@ internal sealed class NodePackage
script.Sha256));
}
return evidence;
return evidence
.OrderBy(static e => e.ComparisonKey, StringComparer.Ordinal)
.ToArray();
}
public IReadOnlyCollection<KeyValuePair<string, string?>> CreateMetadata()
@@ -137,16 +157,36 @@ internal sealed class NodePackage
entries.Add(new KeyValuePair<string, string?>("workspaceLink", WorkspaceLink));
}
if (WorkspaceTargets.Count > 0)
{
entries.Add(new KeyValuePair<string, string?>("workspaceTargets", string.Join(';', WorkspaceTargets)));
}
if (HasInstallScripts)
{
entries.Add(new KeyValuePair<string, string?>("installScripts", "true"));
var lifecycleNames = LifecycleScripts
.Select(static script => script.Name)
if (WorkspaceTargets.Count > 0)
{
entries.Add(new KeyValuePair<string, string?>("workspaceTargets", string.Join(';', WorkspaceTargets)));
}
if (NodeVersions.Count > 0)
{
var distinctVersions = NodeVersions
.Select(static v => v.Version)
.Where(static v => !string.IsNullOrWhiteSpace(v))
.Distinct(StringComparer.Ordinal)
.OrderBy(static v => v, StringComparer.Ordinal)
.ToArray();
if (distinctVersions.Length > 0)
{
entries.Add(new KeyValuePair<string, string?>("nodeVersion", string.Join(';', distinctVersions)));
}
foreach (var versionTarget in NodeVersions.OrderBy(static v => v.Kind, StringComparer.Ordinal))
{
entries.Add(new KeyValuePair<string, string?>($"nodeVersionSource.{versionTarget.Kind}", versionTarget.Version));
}
}
if (HasInstallScripts)
{
entries.Add(new KeyValuePair<string, string?>("installScripts", "true"));
var lifecycleNames = LifecycleScripts
.Select(static script => script.Name)
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static name => name, StringComparer.OrdinalIgnoreCase)
.ToArray();
@@ -216,6 +256,6 @@ internal sealed class NodePackage
var kind = DeclaredOnly ? LanguageEvidenceKind.Metadata : LanguageEvidenceKind.File;
return new LanguageComponentEvidence(kind, evidenceSource, locator, Value: null, Sha256: null);
return new LanguageComponentEvidence(kind, evidenceSource, locator, Value: null, Sha256: PackageSha256);
}
}

View File

@@ -48,14 +48,16 @@ internal static class NodePackageCollector
}
}
var nodeModules = Path.Combine(context.RootPath, "node_modules");
TraverseDirectory(context, nodeModules, lockData, workspaceIndex, packages, visited, cancellationToken);
foreach (var pendingRoot in pendingNodeModuleRoots.OrderBy(static path => path, StringComparer.Ordinal))
{
TraverseDirectory(context, pendingRoot, lockData, workspaceIndex, packages, visited, cancellationToken);
}
var nodeModules = Path.Combine(context.RootPath, "node_modules");
TraverseDirectory(context, nodeModules, lockData, workspaceIndex, packages, visited, cancellationToken);
foreach (var pendingRoot in pendingNodeModuleRoots.OrderBy(static path => path, StringComparer.Ordinal))
{
TraverseDirectory(context, pendingRoot, lockData, workspaceIndex, packages, visited, cancellationToken);
}
TraverseTarballs(context, lockData, workspaceIndex, packages, visited, cancellationToken);
AppendDeclaredPackages(packages, lockData);
return packages;
@@ -181,6 +183,108 @@ internal static class NodePackageCollector
TraverseDirectory(context, nestedNodeModules, lockData, workspaceIndex, packages, visited, cancellationToken);
}
private static void TraverseTarballs(
LanguageAnalyzerContext context,
NodeLockData lockData,
NodeWorkspaceIndex workspaceIndex,
List<NodePackage> packages,
HashSet<string> visited,
CancellationToken cancellationToken)
{
var enumerationOptions = new EnumerationOptions
{
RecurseSubdirectories = true,
IgnoreInaccessible = true,
AttributesToSkip = FileAttributes.ReparsePoint | FileAttributes.Device
};
foreach (var tgzPath in Directory.EnumerateFiles(context.RootPath, "*.tgz", enumerationOptions))
{
cancellationToken.ThrowIfCancellationRequested();
TryProcessTarball(context, tgzPath, packages, visited, cancellationToken);
}
}
private static void TryProcessTarball(
LanguageAnalyzerContext context,
string tgzPath,
List<NodePackage> packages,
HashSet<string> visited,
CancellationToken cancellationToken)
{
try
{
using var fileStream = File.OpenRead(tgzPath);
using var gzipStream = new GZipStream(fileStream, CompressionMode.Decompress, leaveOpen: false);
using var tarReader = new TarReader(gzipStream);
TarEntry? entry;
while ((entry = tarReader.GetNextEntry()) is not null)
{
cancellationToken.ThrowIfCancellationRequested();
if (entry.EntryType != TarEntryType.RegularFile)
{
continue;
}
var normalizedName = entry.Name.Replace('\\', '/');
if (!normalizedName.EndsWith("package.json", StringComparison.OrdinalIgnoreCase))
{
continue;
}
using var buffer = new MemoryStream();
entry.DataStream?.CopyTo(buffer);
buffer.Position = 0;
var sha256 = SHA256.HashData(buffer.ToArray());
var sha256Hex = Convert.ToHexString(sha256).ToLowerInvariant();
buffer.Position = 0;
using var document = JsonDocument.Parse(buffer);
var root = document.RootElement;
var relativeDirectory = NormalizeRelativeDirectoryTar(context, tgzPath);
var locator = BuildTarLocator(context, tgzPath, normalizedName);
var usedByEntrypoint = context.UsageHints.IsPathUsed(tgzPath);
var package = TryCreatePackageFromJson(
context,
root,
relativeDirectory,
locator,
usedByEntrypoint,
cancellationToken,
packageSha256: sha256Hex);
if (package is null)
{
continue;
}
if (visited.Add($"tar::{locator}"))
{
packages.Add(package);
}
break;
}
}
catch (IOException)
{
// ignore unreadable tarballs
}
catch (InvalidDataException)
{
// ignore invalid gzip/tar payloads
}
catch (JsonException)
{
// ignore malformed package definitions in tarballs
}
}
private static void AppendDeclaredPackages(List<NodePackage> packages, NodeLockData lockData)
{
if (lockData.DeclaredPackages.Count == 0)
@@ -223,6 +327,7 @@ internal static class NodePackageCollector
workspaceTargets: Array.Empty<string>(),
workspaceLink: null,
lifecycleScripts: Array.Empty<NodeLifecycleScript>(),
nodeVersions: Array.Empty<NodeVersionTarget>(),
usedByEntrypoint: false,
declaredOnly: true,
lockSource: entry.Source,
@@ -257,87 +362,113 @@ internal static class NodePackageCollector
return $"{entry.Source}:{entry.Locator}";
}
private static NodePackage? TryCreatePackage(
LanguageAnalyzerContext context,
string packageJsonPath,
string relativeDirectory,
NodeLockData lockData,
NodeWorkspaceIndex workspaceIndex,
CancellationToken cancellationToken)
{
try
{
using var stream = File.OpenRead(packageJsonPath);
using var document = JsonDocument.Parse(stream);
var root = document.RootElement;
if (!root.TryGetProperty("name", out var nameElement))
{
return null;
}
var name = nameElement.GetString();
if (string.IsNullOrWhiteSpace(name))
{
return null;
}
if (!root.TryGetProperty("version", out var versionElement))
{
return null;
}
var version = versionElement.GetString();
if (string.IsNullOrWhiteSpace(version))
{
return null;
}
bool? isPrivate = null;
if (root.TryGetProperty("private", out var privateElement) && privateElement.ValueKind is JsonValueKind.True or JsonValueKind.False)
{
isPrivate = privateElement.GetBoolean();
}
var lockEntry = lockData.TryGet(relativeDirectory, name, out var entry) ? entry : null;
var locator = BuildLocator(relativeDirectory);
var usedByEntrypoint = context.UsageHints.IsPathUsed(packageJsonPath);
var lockLocator = BuildLockLocator(lockEntry);
var lockSource = lockEntry?.Source;
var isWorkspaceMember = workspaceIndex.TryGetMember(relativeDirectory, out var workspaceRoot);
var workspaceTargets = ExtractWorkspaceTargets(relativeDirectory, root, workspaceIndex);
var workspaceLink = !isWorkspaceMember && workspaceIndex.TryGetWorkspacePathByName(name, out var workspacePathByName)
? NormalizeRelativeDirectory(context, Path.Combine(context.RootPath, relativeDirectory))
: null;
var lifecycleScripts = ExtractLifecycleScripts(root);
return new NodePackage(
name: name.Trim(),
version: version.Trim(),
relativePath: relativeDirectory,
packageJsonLocator: locator,
isPrivate: isPrivate,
lockEntry: lockEntry,
isWorkspaceMember: isWorkspaceMember,
workspaceRoot: workspaceRoot,
workspaceTargets: workspaceTargets,
workspaceLink: workspaceLink,
lifecycleScripts: lifecycleScripts,
usedByEntrypoint: usedByEntrypoint,
declaredOnly: false,
lockSource: lockSource,
lockLocator: lockLocator);
}
catch (IOException)
{
return null;
}
private static NodePackage? TryCreatePackage(
LanguageAnalyzerContext context,
string packageJsonPath,
string relativeDirectory,
NodeLockData lockData,
NodeWorkspaceIndex workspaceIndex,
CancellationToken cancellationToken)
{
try
{
using var stream = File.OpenRead(packageJsonPath);
using var document = JsonDocument.Parse(stream);
var root = document.RootElement;
return TryCreatePackageFromJson(
context,
root,
relativeDirectory,
BuildLocator(relativeDirectory),
context.UsageHints.IsPathUsed(packageJsonPath),
cancellationToken,
lockData,
workspaceIndex,
packageJsonPath);
}
catch (IOException)
{
return null;
}
catch (JsonException)
{
return null;
}
}
}
private static NodePackage? TryCreatePackageFromJson(
LanguageAnalyzerContext context,
JsonElement root,
string relativeDirectory,
string packageJsonLocator,
bool usedByEntrypoint,
CancellationToken cancellationToken,
NodeLockData? lockData = null,
NodeWorkspaceIndex? workspaceIndex = null,
string? packageJsonPath = null,
string? packageSha256 = null)
{
if (!root.TryGetProperty("name", out var nameElement))
{
return null;
}
var name = nameElement.GetString();
if (string.IsNullOrWhiteSpace(name))
{
return null;
}
if (!root.TryGetProperty("version", out var versionElement))
{
return null;
}
var version = versionElement.GetString();
if (string.IsNullOrWhiteSpace(version))
{
return null;
}
bool? isPrivate = null;
if (root.TryGetProperty("private", out var privateElement) && privateElement.ValueKind is JsonValueKind.True or JsonValueKind.False)
{
isPrivate = privateElement.GetBoolean();
}
var lockEntry = lockData?.TryGet(relativeDirectory, name, out var entry) == true ? entry : null;
var lockLocator = BuildLockLocator(lockEntry);
var lockSource = lockEntry?.Source;
var isWorkspaceMember = workspaceIndex?.TryGetMember(relativeDirectory, out var workspaceRoot) == true;
var workspaceRootValue = isWorkspaceMember && workspaceIndex is not null ? workspaceRoot : null;
var workspaceTargets = workspaceIndex is null ? Array.Empty<string>() : ExtractWorkspaceTargets(relativeDirectory, root, workspaceIndex);
var workspaceLink = workspaceIndex is not null && !isWorkspaceMember && workspaceIndex.TryGetWorkspacePathByName(name, out var workspacePathByName)
? NormalizeRelativeDirectory(context, Path.Combine(context.RootPath, relativeDirectory))
: null;
var lifecycleScripts = ExtractLifecycleScripts(root);
var nodeVersions = NodeVersionDetector.Detect(context, relativeDirectory, cancellationToken);
return new NodePackage(
name: name.Trim(),
version: version.Trim(),
relativePath: relativeDirectory,
packageJsonLocator: packageJsonLocator,
isPrivate: isPrivate,
lockEntry: lockEntry,
isWorkspaceMember: isWorkspaceMember,
workspaceRoot: workspaceRootValue,
workspaceTargets: workspaceTargets,
workspaceLink: workspaceLink,
lifecycleScripts: lifecycleScripts,
nodeVersions: nodeVersions,
usedByEntrypoint: usedByEntrypoint,
declaredOnly: false,
lockSource: lockSource,
lockLocator: lockLocator,
packageSha256: packageSha256);
}
private static string NormalizeRelativeDirectory(LanguageAnalyzerContext context, string directory)
{
@@ -350,15 +481,37 @@ internal static class NodePackageCollector
return relative.Replace(Path.DirectorySeparatorChar, '/');
}
private static string BuildLocator(string relativeDirectory)
{
if (string.IsNullOrEmpty(relativeDirectory))
{
return "package.json";
}
return relativeDirectory + "/package.json";
}
private static string BuildLocator(string relativeDirectory)
{
if (string.IsNullOrEmpty(relativeDirectory))
{
return "package.json";
}
return relativeDirectory + "/package.json";
}
private static string BuildTarLocator(LanguageAnalyzerContext context, string tgzPath, string entryName)
{
var relative = context.GetRelativePath(tgzPath);
var normalizedArchive = string.IsNullOrWhiteSpace(relative) || relative == "."
? Path.GetFileName(tgzPath)
: relative.Replace(Path.DirectorySeparatorChar, '/');
var normalizedEntry = entryName.Replace('\\', '/');
return $"{normalizedArchive}!{normalizedEntry}";
}
private static string NormalizeRelativeDirectoryTar(LanguageAnalyzerContext context, string tgzPath)
{
var relative = context.GetRelativePath(Path.GetDirectoryName(tgzPath)!);
if (string.IsNullOrEmpty(relative) || relative == ".")
{
return "tgz";
}
return relative.Replace(Path.DirectorySeparatorChar, '/');
}
private static bool ShouldSkipDirectory(string name)
{

View File

@@ -0,0 +1,145 @@
using System.Globalization;
namespace StellaOps.Scanner.Analyzers.Lang.Node.Internal;
internal static class NodeVersionDetector
{
private static readonly string[] VersionFiles = { ".nvmrc", ".node-version" };
public static IReadOnlyList<NodeVersionTarget> Detect(LanguageAnalyzerContext context, string relativeDirectory, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
var targets = new List<NodeVersionTarget>();
var baseDirectory = ResolveAbsolutePath(context.RootPath, relativeDirectory);
foreach (var versionFile in VersionFiles)
{
var path = Path.Combine(baseDirectory, versionFile);
if (!File.Exists(path))
{
continue;
}
var version = ReadFirstNonEmptyLine(path, cancellationToken);
if (string.IsNullOrWhiteSpace(version))
{
continue;
}
targets.Add(CreateTarget(context, relativeDirectory, path, version.Trim(), GetSha256(path), versionFile.TrimStart('.')));
}
var dockerfilePath = Path.Combine(baseDirectory, "Dockerfile");
if (File.Exists(dockerfilePath))
{
var dockerVersion = ExtractNodeTagFromDockerfile(dockerfilePath, cancellationToken);
if (!string.IsNullOrWhiteSpace(dockerVersion))
{
targets.Add(CreateTarget(context, relativeDirectory, dockerfilePath, dockerVersion!, GetSha256(dockerfilePath), "dockerfile"));
}
}
return targets
.OrderBy(static t => t.Kind, StringComparer.Ordinal)
.ThenBy(static t => t.Version, StringComparer.Ordinal)
.ThenBy(static t => t.Locator, StringComparer.Ordinal)
.ToArray();
}
private static NodeVersionTarget CreateTarget(LanguageAnalyzerContext context, string relativeDirectory, string absolutePath, string version, string sha256, string kind)
{
var locator = BuildLocator(context, absolutePath);
return new NodeVersionTarget(kind, version, locator, sha256);
}
private static string BuildLocator(LanguageAnalyzerContext context, string absolutePath)
{
var relative = context.GetRelativePath(absolutePath);
if (string.IsNullOrWhiteSpace(relative) || relative == ".")
{
return Path.GetFileName(absolutePath);
}
return relative.Replace(Path.DirectorySeparatorChar, '/');
}
private static string ResolveAbsolutePath(string rootPath, string relativeDirectory)
{
if (string.IsNullOrWhiteSpace(relativeDirectory))
{
return rootPath;
}
return Path.Combine(rootPath, relativeDirectory.Replace('/', Path.DirectorySeparatorChar));
}
private static string GetSha256(string path)
{
var bytes = File.ReadAllBytes(path);
var hash = SHA256.HashData(bytes);
return Convert.ToHexString(hash).ToLowerInvariant();
}
private static string? ReadFirstNonEmptyLine(string path, CancellationToken cancellationToken)
{
using var stream = File.OpenText(path);
while (!stream.EndOfStream)
{
cancellationToken.ThrowIfCancellationRequested();
var line = stream.ReadLine();
if (string.IsNullOrWhiteSpace(line))
{
continue;
}
return line.Trim();
}
return null;
}
private static string? ExtractNodeTagFromDockerfile(string path, CancellationToken cancellationToken)
{
using var reader = File.OpenText(path);
var linesChecked = 0;
while (!reader.EndOfStream && linesChecked < 200)
{
cancellationToken.ThrowIfCancellationRequested();
var line = reader.ReadLine();
if (line is null)
{
break;
}
linesChecked++;
var trimmed = line.Trim();
if (!trimmed.StartsWith("FROM", true, CultureInfo.InvariantCulture))
{
continue;
}
var tokens = trimmed.Split(' ', StringSplitOptions.RemoveEmptyEntries);
var tag = tokens
.Skip(1)
.FirstOrDefault(static token => token.StartsWith("node:", StringComparison.OrdinalIgnoreCase));
if (string.IsNullOrWhiteSpace(tag))
{
continue;
}
var versionPart = tag["node:".Length..];
var atIndex = versionPart.IndexOf('@');
if (atIndex > 0)
{
versionPart = versionPart[..atIndex];
}
return versionPart;
}
return null;
}
}

View File

@@ -0,0 +1,7 @@
namespace StellaOps.Scanner.Analyzers.Lang.Node.Internal;
internal sealed record NodeVersionTarget(
string Kind,
string Version,
string Locator,
string? Sha256);