nuget reorganization

This commit is contained in:
master
2025-11-18 23:45:25 +02:00
parent 77cee6a209
commit d3ecd7f8e6
7712 changed files with 13963 additions and 10007504 deletions

View File

@@ -19,7 +19,8 @@ internal sealed class NodePackage
bool declaredOnly = false,
string? lockSource = null,
string? lockLocator = null,
string? packageSha256 = null)
string? packageSha256 = null,
bool isYarnPnp = false)
{
Name = name;
Version = version;
@@ -38,6 +39,7 @@ internal sealed class NodePackage
LockSource = lockSource;
LockLocator = lockLocator;
PackageSha256 = packageSha256;
IsYarnPnp = isYarnPnp;
}
public string Name { get; }
@@ -75,8 +77,10 @@ internal sealed class NodePackage
public string? LockLocator { get; }
public string? PackageSha256 { get; }
public string RelativePathNormalized => string.IsNullOrEmpty(RelativePath) ? string.Empty : RelativePath.Replace(Path.DirectorySeparatorChar, '/');
public bool IsYarnPnp { get; }
public string RelativePathNormalized => string.IsNullOrEmpty(RelativePath) ? string.Empty : RelativePath.Replace(Path.DirectorySeparatorChar, '/');
public string ComponentKey => $"purl::{Purl}";
@@ -217,6 +221,11 @@ internal sealed class NodePackage
entries.Add(new KeyValuePair<string, string?>("lockLocator", LockLocator));
}
if (IsYarnPnp)
{
entries.Add(new KeyValuePair<string, string?>("yarnPnp", "true"));
}
return entries
.OrderBy(static pair => pair.Key, StringComparer.Ordinal)
.ToArray();

View File

@@ -1,175 +1,179 @@
using System.Text.Json;
namespace StellaOps.Scanner.Analyzers.Lang.Node.Internal;
internal static class NodePackageCollector
{
private static readonly string[] IgnoredDirectories =
{
".bin",
".cache",
".store",
"__pycache__"
};
public static IReadOnlyList<NodePackage> CollectPackages(LanguageAnalyzerContext context, NodeLockData lockData, CancellationToken cancellationToken)
{
var packages = new List<NodePackage>();
var visited = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var pendingNodeModuleRoots = new List<string>();
var rootPackageJson = Path.Combine(context.RootPath, "package.json");
var workspaceIndex = NodeWorkspaceIndex.Create(context.RootPath);
if (File.Exists(rootPackageJson))
{
var rootPackage = TryCreatePackage(context, rootPackageJson, string.Empty, lockData, workspaceIndex, cancellationToken);
if (rootPackage is not null)
{
packages.Add(rootPackage);
visited.Add(rootPackage.RelativePathNormalized);
}
}
foreach (var workspaceRelative in workspaceIndex.GetMembers())
{
var workspaceAbsolute = Path.Combine(context.RootPath, workspaceRelative.Replace('/', Path.DirectorySeparatorChar));
if (!Directory.Exists(workspaceAbsolute))
{
continue;
}
ProcessPackageDirectory(context, workspaceAbsolute, lockData, workspaceIndex, includeNestedNodeModules: false, packages, visited, cancellationToken);
var workspaceNodeModules = Path.Combine(workspaceAbsolute, "node_modules");
if (Directory.Exists(workspaceNodeModules))
{
pendingNodeModuleRoots.Add(workspaceNodeModules);
}
}
using System.Text.Json;
namespace StellaOps.Scanner.Analyzers.Lang.Node.Internal;
internal static class NodePackageCollector
{
private static readonly string[] IgnoredDirectories =
{
".bin",
".cache",
".store",
"__pycache__"
};
public static IReadOnlyList<NodePackage> CollectPackages(LanguageAnalyzerContext context, NodeLockData lockData, CancellationToken cancellationToken)
{
var packages = new List<NodePackage>();
var visited = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var pendingNodeModuleRoots = new List<string>();
var rootPackageJson = Path.Combine(context.RootPath, "package.json");
var workspaceIndex = NodeWorkspaceIndex.Create(context.RootPath);
var yarnPnpPresent = HasYarnPnp(context.RootPath);
if (File.Exists(rootPackageJson))
{
var rootPackage = TryCreatePackage(context, rootPackageJson, string.Empty, lockData, workspaceIndex, yarnPnpPresent, cancellationToken);
if (rootPackage is not null)
{
packages.Add(rootPackage);
visited.Add(rootPackage.RelativePathNormalized);
}
}
foreach (var workspaceRelative in workspaceIndex.GetMembers())
{
var workspaceAbsolute = Path.Combine(context.RootPath, workspaceRelative.Replace('/', Path.DirectorySeparatorChar));
if (!Directory.Exists(workspaceAbsolute))
{
continue;
}
ProcessPackageDirectory(context, workspaceAbsolute, lockData, workspaceIndex, includeNestedNodeModules: false, packages, visited, yarnPnpPresent, cancellationToken);
var workspaceNodeModules = Path.Combine(workspaceAbsolute, "node_modules");
if (Directory.Exists(workspaceNodeModules))
{
pendingNodeModuleRoots.Add(workspaceNodeModules);
}
}
var nodeModules = Path.Combine(context.RootPath, "node_modules");
TraverseDirectory(context, nodeModules, lockData, workspaceIndex, packages, visited, cancellationToken);
TraverseDirectory(context, nodeModules, lockData, workspaceIndex, packages, visited, yarnPnpPresent, cancellationToken);
foreach (var pendingRoot in pendingNodeModuleRoots.OrderBy(static path => path, StringComparer.Ordinal))
{
TraverseDirectory(context, pendingRoot, lockData, workspaceIndex, packages, visited, cancellationToken);
TraverseDirectory(context, pendingRoot, lockData, workspaceIndex, packages, visited, yarnPnpPresent, cancellationToken);
}
TraverseTarballs(context, lockData, workspaceIndex, packages, visited, cancellationToken);
TraverseTarballs(context, lockData, workspaceIndex, packages, visited, yarnPnpPresent, cancellationToken);
AppendDeclaredPackages(packages, lockData);
return packages;
}
private static void TraverseDirectory(
LanguageAnalyzerContext context,
string directory,
NodeLockData lockData,
NodeWorkspaceIndex workspaceIndex,
List<NodePackage> packages,
HashSet<string> visited,
CancellationToken cancellationToken)
{
if (!Directory.Exists(directory))
{
return;
}
foreach (var child in Directory.EnumerateDirectories(directory))
{
cancellationToken.ThrowIfCancellationRequested();
var name = Path.GetFileName(child);
if (string.IsNullOrEmpty(name))
{
continue;
}
if (ShouldSkipDirectory(name))
{
continue;
}
if (string.Equals(name, ".pnpm", StringComparison.OrdinalIgnoreCase))
{
TraversePnpmStore(context, child, lockData, workspaceIndex, packages, visited, cancellationToken);
continue;
}
if (name.StartsWith('@'))
{
foreach (var scoped in Directory.EnumerateDirectories(child))
{
ProcessPackageDirectory(context, scoped, lockData, workspaceIndex, includeNestedNodeModules: true, packages, visited, cancellationToken);
}
continue;
}
ProcessPackageDirectory(context, child, lockData, workspaceIndex, includeNestedNodeModules: true, packages, visited, cancellationToken);
}
}
private static void TraversePnpmStore(
LanguageAnalyzerContext context,
string pnpmDirectory,
NodeLockData lockData,
NodeWorkspaceIndex workspaceIndex,
List<NodePackage> packages,
HashSet<string> visited,
CancellationToken cancellationToken)
{
foreach (var storeEntry in Directory.EnumerateDirectories(pnpmDirectory))
{
cancellationToken.ThrowIfCancellationRequested();
var nestedNodeModules = Path.Combine(storeEntry, "node_modules");
if (Directory.Exists(nestedNodeModules))
{
TraverseDirectory(context, nestedNodeModules, lockData, workspaceIndex, packages, visited, cancellationToken);
}
}
}
private static void ProcessPackageDirectory(
LanguageAnalyzerContext context,
string directory,
NodeLockData lockData,
NodeWorkspaceIndex workspaceIndex,
bool includeNestedNodeModules,
List<NodePackage> packages,
HashSet<string> visited,
CancellationToken cancellationToken)
{
var packageJsonPath = Path.Combine(directory, "package.json");
var relativeDirectory = NormalizeRelativeDirectory(context, directory);
if (!visited.Add(relativeDirectory))
{
// Already processed this path.
if (includeNestedNodeModules)
{
TraverseNestedNodeModules(context, directory, lockData, workspaceIndex, packages, visited, cancellationToken);
}
return;
}
if (File.Exists(packageJsonPath))
{
var package = TryCreatePackage(context, packageJsonPath, relativeDirectory, lockData, workspaceIndex, cancellationToken);
if (package is not null)
{
packages.Add(package);
}
}
if (includeNestedNodeModules)
{
TraverseNestedNodeModules(context, directory, lockData, workspaceIndex, packages, visited, cancellationToken);
}
}
private static void TraverseDirectory(
LanguageAnalyzerContext context,
string directory,
NodeLockData lockData,
NodeWorkspaceIndex workspaceIndex,
List<NodePackage> packages,
HashSet<string> visited,
bool yarnPnpPresent,
CancellationToken cancellationToken)
{
if (!Directory.Exists(directory))
{
return;
}
foreach (var child in Directory.EnumerateDirectories(directory))
{
cancellationToken.ThrowIfCancellationRequested();
var name = Path.GetFileName(child);
if (string.IsNullOrEmpty(name))
{
continue;
}
if (ShouldSkipDirectory(name))
{
continue;
}
if (string.Equals(name, ".pnpm", StringComparison.OrdinalIgnoreCase))
{
TraversePnpmStore(context, child, lockData, workspaceIndex, packages, visited, yarnPnpPresent, cancellationToken);
continue;
}
if (name.StartsWith('@'))
{
foreach (var scoped in Directory.EnumerateDirectories(child))
{
ProcessPackageDirectory(context, scoped, lockData, workspaceIndex, includeNestedNodeModules: true, packages, visited, yarnPnpPresent, cancellationToken);
}
continue;
}
ProcessPackageDirectory(context, child, lockData, workspaceIndex, includeNestedNodeModules: true, packages, visited, yarnPnpPresent, cancellationToken);
}
}
private static void TraversePnpmStore(
LanguageAnalyzerContext context,
string pnpmDirectory,
NodeLockData lockData,
NodeWorkspaceIndex workspaceIndex,
List<NodePackage> packages,
HashSet<string> visited,
bool yarnPnpPresent,
CancellationToken cancellationToken)
{
foreach (var storeEntry in Directory.EnumerateDirectories(pnpmDirectory))
{
cancellationToken.ThrowIfCancellationRequested();
var nestedNodeModules = Path.Combine(storeEntry, "node_modules");
if (Directory.Exists(nestedNodeModules))
{
TraverseDirectory(context, nestedNodeModules, lockData, workspaceIndex, packages, visited, yarnPnpPresent, cancellationToken);
}
}
}
private static void ProcessPackageDirectory(
LanguageAnalyzerContext context,
string directory,
NodeLockData lockData,
NodeWorkspaceIndex workspaceIndex,
bool includeNestedNodeModules,
List<NodePackage> packages,
HashSet<string> visited,
bool yarnPnpPresent,
CancellationToken cancellationToken)
{
var packageJsonPath = Path.Combine(directory, "package.json");
var relativeDirectory = NormalizeRelativeDirectory(context, directory);
if (!visited.Add(relativeDirectory))
{
// Already processed this path.
if (includeNestedNodeModules)
{
TraverseNestedNodeModules(context, directory, lockData, workspaceIndex, packages, visited, yarnPnpPresent, cancellationToken);
}
return;
}
if (File.Exists(packageJsonPath))
{
var package = TryCreatePackage(context, packageJsonPath, relativeDirectory, lockData, workspaceIndex, yarnPnpPresent, cancellationToken);
if (package is not null)
{
packages.Add(package);
}
}
if (includeNestedNodeModules)
{
TraverseNestedNodeModules(context, directory, lockData, workspaceIndex, packages, visited, yarnPnpPresent, cancellationToken);
}
}
private static void TraverseNestedNodeModules(
LanguageAnalyzerContext context,
string directory,
@@ -177,10 +181,11 @@ internal static class NodePackageCollector
NodeWorkspaceIndex workspaceIndex,
List<NodePackage> packages,
HashSet<string> visited,
bool yarnPnpPresent,
CancellationToken cancellationToken)
{
var nestedNodeModules = Path.Combine(directory, "node_modules");
TraverseDirectory(context, nestedNodeModules, lockData, workspaceIndex, packages, visited, cancellationToken);
TraverseDirectory(context, nestedNodeModules, lockData, workspaceIndex, packages, visited, yarnPnpPresent, cancellationToken);
}
private static void TraverseTarballs(
@@ -189,6 +194,7 @@ internal static class NodePackageCollector
NodeWorkspaceIndex workspaceIndex,
List<NodePackage> packages,
HashSet<string> visited,
bool yarnPnpPresent,
CancellationToken cancellationToken)
{
var enumerationOptions = new EnumerationOptions
@@ -201,7 +207,7 @@ internal static class NodePackageCollector
foreach (var tgzPath in Directory.EnumerateFiles(context.RootPath, "*.tgz", enumerationOptions))
{
cancellationToken.ThrowIfCancellationRequested();
TryProcessTarball(context, tgzPath, packages, visited, cancellationToken);
TryProcessTarball(context, tgzPath, packages, visited, yarnPnpPresent, cancellationToken);
}
}
@@ -210,6 +216,7 @@ internal static class NodePackageCollector
string tgzPath,
List<NodePackage> packages,
HashSet<string> visited,
bool yarnPnpPresent,
CancellationToken cancellationToken)
{
try
@@ -256,7 +263,8 @@ internal static class NodePackageCollector
locator,
usedByEntrypoint,
cancellationToken,
packageSha256: sha256Hex);
packageSha256: sha256Hex,
yarnPnpPresent: yarnPnpPresent);
if (package is null)
{
@@ -361,13 +369,20 @@ internal static class NodePackageCollector
return $"{entry.Source}:{entry.Locator}";
}
private static bool HasYarnPnp(string rootPath)
{
return File.Exists(Path.Combine(rootPath, ".pnp.cjs"))
|| File.Exists(Path.Combine(rootPath, ".pnp.data.cjs"));
}
private static NodePackage? TryCreatePackage(
LanguageAnalyzerContext context,
string packageJsonPath,
string relativeDirectory,
NodeLockData lockData,
NodeWorkspaceIndex workspaceIndex,
bool yarnPnpPresent,
CancellationToken cancellationToken)
{
try
@@ -385,16 +400,18 @@ internal static class NodePackageCollector
cancellationToken,
lockData,
workspaceIndex,
packageJsonPath);
packageJsonPath,
packageSha256: null,
yarnPnpPresent: yarnPnpPresent);
}
catch (IOException)
{
return null;
}
catch (JsonException)
{
return null;
}
catch (JsonException)
{
return null;
}
}
private static NodePackage? TryCreatePackageFromJson(
@@ -407,7 +424,8 @@ internal static class NodePackageCollector
NodeLockData? lockData = null,
NodeWorkspaceIndex? workspaceIndex = null,
string? packageJsonPath = null,
string? packageSha256 = null)
string? packageSha256 = null,
bool yarnPnpPresent = false)
{
if (!root.TryGetProperty("name", out var nameElement))
{
@@ -467,20 +485,21 @@ internal static class NodePackageCollector
declaredOnly: false,
lockSource: lockSource,
lockLocator: lockLocator,
packageSha256: packageSha256);
packageSha256: packageSha256,
isYarnPnp: yarnPnpPresent);
}
private static string NormalizeRelativeDirectory(LanguageAnalyzerContext context, string directory)
{
var relative = context.GetRelativePath(directory);
if (string.IsNullOrEmpty(relative) || relative == ".")
{
return string.Empty;
}
return relative.Replace(Path.DirectorySeparatorChar, '/');
}
private static string NormalizeRelativeDirectory(LanguageAnalyzerContext context, string directory)
{
var relative = context.GetRelativePath(directory);
if (string.IsNullOrEmpty(relative) || relative == ".")
{
return string.Empty;
}
return relative.Replace(Path.DirectorySeparatorChar, '/');
}
private static string BuildLocator(string relativeDirectory)
{
if (string.IsNullOrEmpty(relativeDirectory))
@@ -512,103 +531,119 @@ internal static class NodePackageCollector
return relative.Replace(Path.DirectorySeparatorChar, '/');
}
private static bool ShouldSkipDirectory(string name)
{
if (name.Length == 0)
{
return true;
}
if (name[0] == '.')
{
return !string.Equals(name, ".pnpm", StringComparison.OrdinalIgnoreCase);
}
return IgnoredDirectories.Any(ignored => string.Equals(name, ignored, StringComparison.OrdinalIgnoreCase));
}
private static IReadOnlyList<string> ExtractWorkspaceTargets(string relativeDirectory, JsonElement root, NodeWorkspaceIndex workspaceIndex)
{
var dependencies = workspaceIndex.ResolveWorkspaceTargets(relativeDirectory, TryGetProperty(root, "dependencies"));
var devDependencies = workspaceIndex.ResolveWorkspaceTargets(relativeDirectory, TryGetProperty(root, "devDependencies"));
var peerDependencies = workspaceIndex.ResolveWorkspaceTargets(relativeDirectory, TryGetProperty(root, "peerDependencies"));
if (dependencies.Count == 0 && devDependencies.Count == 0 && peerDependencies.Count == 0)
{
return Array.Empty<string>();
}
var combined = new HashSet<string>(StringComparer.Ordinal);
foreach (var item in dependencies)
{
combined.Add(item);
}
foreach (var item in devDependencies)
{
combined.Add(item);
}
foreach (var item in peerDependencies)
{
combined.Add(item);
}
return combined.OrderBy(static x => x, StringComparer.Ordinal).ToArray();
}
private static JsonElement? TryGetProperty(JsonElement element, string propertyName)
=> element.TryGetProperty(propertyName, out var property) ? property : null;
private static IReadOnlyList<NodeLifecycleScript> ExtractLifecycleScripts(JsonElement root)
{
if (!root.TryGetProperty("scripts", out var scriptsElement) || scriptsElement.ValueKind != JsonValueKind.Object)
{
return Array.Empty<NodeLifecycleScript>();
}
var lifecycleScripts = new Dictionary<string, NodeLifecycleScript>(StringComparer.OrdinalIgnoreCase);
foreach (var script in scriptsElement.EnumerateObject())
{
if (!IsLifecycleScriptName(script.Name))
{
continue;
}
if (script.Value.ValueKind != JsonValueKind.String)
{
continue;
}
var command = script.Value.GetString();
if (string.IsNullOrWhiteSpace(command))
{
continue;
}
var canonicalName = script.Name.Trim().ToLowerInvariant();
var lifecycleScript = new NodeLifecycleScript(canonicalName, command);
if (!lifecycleScripts.ContainsKey(canonicalName))
{
NodeAnalyzerMetrics.RecordLifecycleScript(canonicalName);
}
lifecycleScripts[canonicalName] = lifecycleScript;
}
if (lifecycleScripts.Count == 0)
{
return Array.Empty<NodeLifecycleScript>();
}
return lifecycleScripts.Values
.OrderBy(static script => script.Name, StringComparer.Ordinal)
.ToArray();
}
private static bool IsLifecycleScriptName(string name)
=> name.Equals("preinstall", StringComparison.OrdinalIgnoreCase)
|| name.Equals("install", StringComparison.OrdinalIgnoreCase)
|| name.Equals("postinstall", StringComparison.OrdinalIgnoreCase);
}
private static bool ShouldSkipDirectory(string name)
{
if (name.Length == 0)
{
return true;
}
if (name[0] == '.')
{
return !string.Equals(name, ".pnpm", StringComparison.OrdinalIgnoreCase);
}
return IgnoredDirectories.Any(ignored => string.Equals(name, ignored, StringComparison.OrdinalIgnoreCase));
}
private static bool HasYarnPnp(string rootPath)
{
if (string.IsNullOrWhiteSpace(rootPath))
{
return false;
}
var pnpCjs = Path.Combine(rootPath, ".pnp.cjs");
var pnpData = Path.Combine(rootPath, ".pnp.data.json");
var yarnCache = Path.Combine(rootPath, ".yarn", "cache");
return File.Exists(pnpCjs)
|| File.Exists(pnpData)
|| Directory.Exists(yarnCache);
}
private static IReadOnlyList<string> ExtractWorkspaceTargets(string relativeDirectory, JsonElement root, NodeWorkspaceIndex workspaceIndex)
{
var dependencies = workspaceIndex.ResolveWorkspaceTargets(relativeDirectory, TryGetProperty(root, "dependencies"));
var devDependencies = workspaceIndex.ResolveWorkspaceTargets(relativeDirectory, TryGetProperty(root, "devDependencies"));
var peerDependencies = workspaceIndex.ResolveWorkspaceTargets(relativeDirectory, TryGetProperty(root, "peerDependencies"));
if (dependencies.Count == 0 && devDependencies.Count == 0 && peerDependencies.Count == 0)
{
return Array.Empty<string>();
}
var combined = new HashSet<string>(StringComparer.Ordinal);
foreach (var item in dependencies)
{
combined.Add(item);
}
foreach (var item in devDependencies)
{
combined.Add(item);
}
foreach (var item in peerDependencies)
{
combined.Add(item);
}
return combined.OrderBy(static x => x, StringComparer.Ordinal).ToArray();
}
private static JsonElement? TryGetProperty(JsonElement element, string propertyName)
=> element.TryGetProperty(propertyName, out var property) ? property : null;
private static IReadOnlyList<NodeLifecycleScript> ExtractLifecycleScripts(JsonElement root)
{
if (!root.TryGetProperty("scripts", out var scriptsElement) || scriptsElement.ValueKind != JsonValueKind.Object)
{
return Array.Empty<NodeLifecycleScript>();
}
var lifecycleScripts = new Dictionary<string, NodeLifecycleScript>(StringComparer.OrdinalIgnoreCase);
foreach (var script in scriptsElement.EnumerateObject())
{
if (!IsLifecycleScriptName(script.Name))
{
continue;
}
if (script.Value.ValueKind != JsonValueKind.String)
{
continue;
}
var command = script.Value.GetString();
if (string.IsNullOrWhiteSpace(command))
{
continue;
}
var canonicalName = script.Name.Trim().ToLowerInvariant();
var lifecycleScript = new NodeLifecycleScript(canonicalName, command);
if (!lifecycleScripts.ContainsKey(canonicalName))
{
NodeAnalyzerMetrics.RecordLifecycleScript(canonicalName);
}
lifecycleScripts[canonicalName] = lifecycleScript;
}
if (lifecycleScripts.Count == 0)
{
return Array.Empty<NodeLifecycleScript>();
}
return lifecycleScripts.Values
.OrderBy(static script => script.Name, StringComparer.Ordinal)
.ToArray();
}
private static bool IsLifecycleScriptName(string name)
=> name.Equals("preinstall", StringComparison.OrdinalIgnoreCase)
|| name.Equals("install", StringComparison.OrdinalIgnoreCase)
|| name.Equals("postinstall", StringComparison.OrdinalIgnoreCase);
}