Add unit tests for VexLens normalizer, CPE parser, product mapper, and PURL parser

- Implemented comprehensive tests for VexLensNormalizer including format detection and normalization scenarios.
- Added tests for CpeParser covering CPE 2.3 and 2.2 formats, invalid inputs, and canonical key generation.
- Created tests for ProductMapper to validate parsing and matching logic across different strictness levels.
- Developed tests for PurlParser to ensure correct parsing of various PURL formats and validation of identifiers.
- Introduced stubs for Monaco editor and worker to facilitate testing in the web application.
- Updated project file for the test project to include necessary dependencies.
This commit is contained in:
StellaOps Bot
2025-12-06 16:28:12 +02:00
parent 2b892ad1b2
commit efd6850c38
132 changed files with 16675 additions and 5428 deletions

View File

@@ -38,6 +38,12 @@ public sealed class BunLanguageAnalyzer : ILanguageAnalyzer
continue;
}
// Parse workspace info for direct dependency detection
var workspaceInfo = BunWorkspaceHelper.ParseWorkspaceInfo(projectRoot);
// Parse bunfig.toml for custom registry info
var bunConfig = BunConfigHelper.ParseConfig(projectRoot);
// Stage 3: Collect packages based on classification
IReadOnlyList<BunPackage> packages;
if (classification.Kind == BunInputKind.InstalledModules)
@@ -61,6 +67,35 @@ public sealed class BunLanguageAnalyzer : ILanguageAnalyzer
continue;
}
// Mark direct, patched dependencies and custom registries
foreach (var package in packages)
{
package.IsDirect = workspaceInfo.DirectDependencies.ContainsKey(package.Name);
if (workspaceInfo.PatchedDependencies.TryGetValue(package.Name, out var patchFile))
{
package.IsPatched = true;
package.PatchFile = patchFile;
}
// Check for custom registry (scoped or default)
if (bunConfig.HasCustomRegistry)
{
// Check scoped registry first (e.g., @company/pkg uses company's registry)
if (package.Name.StartsWith('@'))
{
var scope = package.Name.Split('/')[0];
if (bunConfig.ScopeRegistries.TryGetValue(scope, out var scopeRegistry))
{
package.CustomRegistry = scopeRegistry;
}
}
// Fall back to default custom registry if no scope match
package.CustomRegistry ??= bunConfig.DefaultRegistry;
}
}
// Stage 4: Normalize and emit
var normalized = BunPackageNormalizer.Normalize(packages);
foreach (var package in normalized.OrderBy(static p => p.ComponentKey, StringComparer.Ordinal))

View File

@@ -0,0 +1,166 @@
using System.Collections.Immutable;
using System.Text.RegularExpressions;
namespace StellaOps.Scanner.Analyzers.Lang.Bun.Internal;
/// <summary>
/// Helper for parsing bunfig.toml configuration files.
/// Provides registry and scope information for dependency source tracking.
/// </summary>
internal static partial class BunConfigHelper
{
/// <summary>
/// Configuration information from bunfig.toml.
/// </summary>
public sealed record BunConfig
{
public static readonly BunConfig Empty = new(
null,
ImmutableDictionary<string, string>.Empty);
public BunConfig(
string? defaultRegistry,
IReadOnlyDictionary<string, string> scopeRegistries)
{
DefaultRegistry = defaultRegistry;
ScopeRegistries = scopeRegistries;
}
/// <summary>
/// Default registry URL for packages (from install.registry).
/// </summary>
public string? DefaultRegistry { get; }
/// <summary>
/// Scoped registries mapping scope name to registry URL.
/// </summary>
public IReadOnlyDictionary<string, string> ScopeRegistries { get; }
/// <summary>
/// Returns true if any custom registry configuration exists.
/// </summary>
public bool HasCustomRegistry => DefaultRegistry is not null || ScopeRegistries.Count > 0;
}
/// <summary>
/// Parses bunfig.toml from the project root.
/// </summary>
public static BunConfig ParseConfig(string projectRoot)
{
ArgumentException.ThrowIfNullOrWhiteSpace(projectRoot);
var bunfigPath = Path.Combine(projectRoot, "bunfig.toml");
if (!File.Exists(bunfigPath))
{
return BunConfig.Empty;
}
try
{
var content = File.ReadAllText(bunfigPath);
return ParseToml(content);
}
catch (IOException)
{
return BunConfig.Empty;
}
}
/// <summary>
/// Simple TOML parser for bunfig.toml registry configuration.
/// Extracts [install] registry and [install.scopes] sections.
/// </summary>
private static BunConfig ParseToml(string content)
{
if (string.IsNullOrWhiteSpace(content))
{
return BunConfig.Empty;
}
string? defaultRegistry = null;
var scopeRegistries = new Dictionary<string, string>(StringComparer.Ordinal);
var lines = content.Split('\n');
var currentSection = string.Empty;
foreach (var rawLine in lines)
{
var line = rawLine.Trim();
// Skip comments and empty lines
if (string.IsNullOrEmpty(line) || line.StartsWith('#'))
{
continue;
}
// Section header
if (line.StartsWith('[') && line.EndsWith(']'))
{
currentSection = line[1..^1].Trim();
continue;
}
// Key-value pair
var equalsIndex = line.IndexOf('=');
if (equalsIndex > 0)
{
var key = line[..equalsIndex].Trim();
var value = line[(equalsIndex + 1)..].Trim();
// Remove quotes from value
value = StripQuotes(value);
// [install] registry = "..."
if (currentSection.Equals("install", StringComparison.OrdinalIgnoreCase) &&
key.Equals("registry", StringComparison.OrdinalIgnoreCase))
{
defaultRegistry = value;
}
// [install.scopes] "@scope" = { url = "..." } or "@scope" = "..."
else if (currentSection.Equals("install.scopes", StringComparison.OrdinalIgnoreCase))
{
var scopeName = StripQuotes(key);
var registryUrl = ExtractRegistryUrl(value);
if (!string.IsNullOrEmpty(scopeName) && !string.IsNullOrEmpty(registryUrl))
{
scopeRegistries[scopeName] = registryUrl;
}
}
}
}
return new BunConfig(
defaultRegistry,
scopeRegistries.ToImmutableDictionary(StringComparer.Ordinal));
}
private static string StripQuotes(string value)
{
if (value.Length >= 2)
{
if ((value.StartsWith('"') && value.EndsWith('"')) ||
(value.StartsWith('\'') && value.EndsWith('\'')))
{
return value[1..^1];
}
}
return value;
}
private static string? ExtractRegistryUrl(string value)
{
// Simple case: just a URL string
if (value.StartsWith("http", StringComparison.OrdinalIgnoreCase))
{
return value;
}
// Inline table: { url = "..." }
var urlMatch = UrlPattern().Match(value);
return urlMatch.Success ? urlMatch.Groups[1].Value : null;
}
[GeneratedRegex(@"url\s*=\s*[""']([^""']+)[""']", RegexOptions.IgnoreCase)]
private static partial Regex UrlPattern();
}

View File

@@ -27,6 +27,48 @@ internal sealed class BunPackage
public string? Source { get; private init; }
public bool IsPrivate { get; private init; }
public bool IsDev { get; private init; }
public bool IsOptional { get; private init; }
public bool IsPeer { get; private init; }
/// <summary>
/// Source type: npm, git, tarball, file, link, workspace.
/// </summary>
public string SourceType { get; private init; } = "npm";
/// <summary>
/// Git commit hash for git dependencies.
/// </summary>
public string? GitCommit { get; private init; }
/// <summary>
/// Original specifier (e.g., "github:user/repo#tag").
/// </summary>
public string? Specifier { get; private init; }
/// <summary>
/// Direct dependencies of this package (for transitive analysis).
/// </summary>
public IReadOnlyList<string> Dependencies { get; private init; } = Array.Empty<string>();
/// <summary>
/// Whether this is a direct dependency (in root package.json) or transitive.
/// </summary>
public bool IsDirect { get; set; }
/// <summary>
/// Whether this package has been patched (via patchedDependencies or .patches directory).
/// </summary>
public bool IsPatched { get; set; }
/// <summary>
/// Path to the patch file if this package is patched.
/// </summary>
public string? PatchFile { get; set; }
/// <summary>
/// Custom registry URL if this package comes from a non-default registry.
/// </summary>
public string? CustomRegistry { get; set; }
/// <summary>
/// Logical path where this package was found (may be symlink).
@@ -67,7 +109,13 @@ internal sealed class BunPackage
Source = "node_modules",
Resolved = lockEntry?.Resolved,
Integrity = lockEntry?.Integrity,
IsDev = lockEntry?.IsDev ?? false
IsDev = lockEntry?.IsDev ?? false,
IsOptional = lockEntry?.IsOptional ?? false,
IsPeer = lockEntry?.IsPeer ?? false,
SourceType = lockEntry?.SourceType ?? "npm",
GitCommit = lockEntry?.GitCommit,
Specifier = lockEntry?.Specifier,
Dependencies = lockEntry?.Dependencies ?? Array.Empty<string>()
};
}
@@ -80,7 +128,13 @@ internal sealed class BunPackage
Source = source,
Resolved = entry.Resolved,
Integrity = entry.Integrity,
IsDev = entry.IsDev
IsDev = entry.IsDev,
IsOptional = entry.IsOptional,
IsPeer = entry.IsPeer,
SourceType = entry.SourceType,
GitCommit = entry.GitCommit,
Specifier = entry.Specifier,
Dependencies = entry.Dependencies
};
}
@@ -118,13 +172,58 @@ internal sealed class BunPackage
metadata["private"] = "true";
}
if (!string.IsNullOrEmpty(CustomRegistry))
{
metadata["customRegistry"] = CustomRegistry;
}
if (IsDev)
{
metadata["dev"] = "true";
}
if (IsDirect)
{
metadata["direct"] = "true";
}
if (!string.IsNullOrEmpty(GitCommit))
{
metadata["gitCommit"] = GitCommit;
}
if (IsOptional)
{
metadata["optional"] = "true";
}
metadata["packageManager"] = "bun";
if (IsPatched)
{
metadata["patched"] = "true";
}
if (!string.IsNullOrEmpty(PatchFile))
{
metadata["patchFile"] = NormalizePath(PatchFile);
}
if (IsPeer)
{
metadata["peer"] = "true";
}
if (SourceType != "npm")
{
metadata["sourceType"] = SourceType;
}
if (!string.IsNullOrEmpty(Specifier))
{
metadata["specifier"] = Specifier;
}
if (_occurrencePaths.Count > 1)
{
metadata["occurrences"] = string.Join(";", _occurrencePaths.Select(NormalizePath).Order(StringComparer.Ordinal));

View File

@@ -0,0 +1,414 @@
using System.Collections.Immutable;
using System.Text.Json;
namespace StellaOps.Scanner.Analyzers.Lang.Bun.Internal;
/// <summary>
/// Helper for parsing workspace configuration and direct dependencies from package.json files.
/// </summary>
internal static class BunWorkspaceHelper
{
/// <summary>
/// Information about workspaces and direct dependencies in a Bun project.
/// </summary>
public sealed record WorkspaceInfo
{
public static readonly WorkspaceInfo Empty = new(
ImmutableHashSet<string>.Empty,
ImmutableHashSet<string>.Empty,
ImmutableDictionary<string, DependencyType>.Empty,
ImmutableDictionary<string, string>.Empty);
public WorkspaceInfo(
IReadOnlySet<string> workspacePatterns,
IReadOnlySet<string> workspacePaths,
IReadOnlyDictionary<string, DependencyType> directDependencies,
IReadOnlyDictionary<string, string> patchedDependencies)
{
WorkspacePatterns = workspacePatterns;
WorkspacePaths = workspacePaths;
DirectDependencies = directDependencies;
PatchedDependencies = patchedDependencies;
}
/// <summary>
/// Glob patterns for workspace members from root package.json.
/// </summary>
public IReadOnlySet<string> WorkspacePatterns { get; }
/// <summary>
/// Resolved paths to workspace member directories.
/// </summary>
public IReadOnlySet<string> WorkspacePaths { get; }
/// <summary>
/// Direct dependencies declared in root and workspace package.json files.
/// Key is package name, value is dependency type.
/// </summary>
public IReadOnlyDictionary<string, DependencyType> DirectDependencies { get; }
/// <summary>
/// Patched dependencies. Key is package name (or name@version), value is patch file path.
/// </summary>
public IReadOnlyDictionary<string, string> PatchedDependencies { get; }
}
[Flags]
public enum DependencyType
{
None = 0,
Production = 1,
Dev = 2,
Optional = 4,
Peer = 8
}
/// <summary>
/// Parses workspace configuration and direct dependencies from project root.
/// </summary>
public static WorkspaceInfo ParseWorkspaceInfo(string projectRoot)
{
ArgumentException.ThrowIfNullOrWhiteSpace(projectRoot);
var rootPackageJsonPath = Path.Combine(projectRoot, "package.json");
if (!File.Exists(rootPackageJsonPath))
{
return WorkspaceInfo.Empty;
}
try
{
var content = File.ReadAllText(rootPackageJsonPath);
using var document = JsonDocument.Parse(content);
var root = document.RootElement;
// Parse workspace patterns
var workspacePatterns = ParseWorkspacePatterns(root);
// Resolve workspace paths
var workspacePaths = ResolveWorkspacePaths(projectRoot, workspacePatterns);
// Parse direct dependencies from root
var directDependencies = new Dictionary<string, DependencyType>(StringComparer.Ordinal);
ParseDependencies(root, directDependencies);
// Parse direct dependencies from each workspace
foreach (var wsPath in workspacePaths)
{
var wsPackageJsonPath = Path.Combine(projectRoot, wsPath, "package.json");
if (File.Exists(wsPackageJsonPath))
{
try
{
var wsContent = File.ReadAllText(wsPackageJsonPath);
using var wsDocument = JsonDocument.Parse(wsContent);
ParseDependencies(wsDocument.RootElement, directDependencies);
}
catch (JsonException)
{
// Skip malformed workspace package.json
}
}
}
// Parse patched dependencies
var patchedDependencies = ParsePatchedDependencies(root, projectRoot);
return new WorkspaceInfo(
workspacePatterns.ToImmutableHashSet(StringComparer.Ordinal),
workspacePaths.ToImmutableHashSet(StringComparer.Ordinal),
directDependencies.ToImmutableDictionary(StringComparer.Ordinal),
patchedDependencies.ToImmutableDictionary(StringComparer.Ordinal));
}
catch (JsonException)
{
return WorkspaceInfo.Empty;
}
catch (IOException)
{
return WorkspaceInfo.Empty;
}
}
/// <summary>
/// Checks if a package name is a direct dependency.
/// </summary>
public static bool IsDirect(string packageName, IReadOnlyDictionary<string, DependencyType> directDependencies)
{
return directDependencies.ContainsKey(packageName);
}
private static HashSet<string> ParseWorkspacePatterns(JsonElement root)
{
var patterns = new HashSet<string>(StringComparer.Ordinal);
if (!root.TryGetProperty("workspaces", out var workspaces))
{
return patterns;
}
// workspaces can be an array of patterns
if (workspaces.ValueKind == JsonValueKind.Array)
{
foreach (var pattern in workspaces.EnumerateArray())
{
var patternStr = pattern.GetString();
if (!string.IsNullOrWhiteSpace(patternStr))
{
patterns.Add(patternStr);
}
}
}
// Or an object with "packages" array (npm/yarn format)
else if (workspaces.ValueKind == JsonValueKind.Object &&
workspaces.TryGetProperty("packages", out var packages) &&
packages.ValueKind == JsonValueKind.Array)
{
foreach (var pattern in packages.EnumerateArray())
{
var patternStr = pattern.GetString();
if (!string.IsNullOrWhiteSpace(patternStr))
{
patterns.Add(patternStr);
}
}
}
return patterns;
}
private static HashSet<string> ResolveWorkspacePaths(string projectRoot, IEnumerable<string> patterns)
{
var paths = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var pattern in patterns)
{
// Handle glob patterns like "packages/*" or "apps/**"
if (pattern.Contains('*'))
{
var resolvedPaths = ExpandGlobPattern(projectRoot, pattern);
foreach (var path in resolvedPaths)
{
paths.Add(path);
}
}
else
{
// Direct path
var fullPath = Path.Combine(projectRoot, pattern);
if (Directory.Exists(fullPath) && File.Exists(Path.Combine(fullPath, "package.json")))
{
paths.Add(pattern);
}
}
}
return paths;
}
private static IEnumerable<string> ExpandGlobPattern(string projectRoot, string pattern)
{
// Simple glob expansion for common patterns
// Handles: "packages/*", "apps/*", "libs/**", etc.
var parts = pattern.Split('/', '\\');
var baseParts = new List<string>();
var hasGlob = false;
foreach (var part in parts)
{
if (part.Contains('*'))
{
hasGlob = true;
break;
}
baseParts.Add(part);
}
var baseDir = baseParts.Count > 0
? Path.Combine(projectRoot, string.Join(Path.DirectorySeparatorChar.ToString(), baseParts))
: projectRoot;
if (!Directory.Exists(baseDir))
{
yield break;
}
// For simple patterns like "packages/*", enumerate immediate subdirectories
if (hasGlob)
{
var isRecursive = pattern.Contains("**");
foreach (var dir in Directory.EnumerateDirectories(baseDir))
{
var dirPath = Path.Combine(string.Join("/", baseParts), Path.GetFileName(dir));
// Check if this is a package (has package.json)
if (File.Exists(Path.Combine(dir, "package.json")))
{
yield return dirPath;
}
// For recursive patterns, search subdirectories
if (isRecursive)
{
foreach (var subResult in EnumeratePackagesRecursively(dir, dirPath))
{
yield return subResult;
}
}
}
}
}
private static List<string> EnumeratePackagesRecursively(string directory, string relativePath)
{
var results = new List<string>();
try
{
foreach (var subdir in Directory.EnumerateDirectories(directory))
{
var subdirName = Path.GetFileName(subdir);
// Skip node_modules and hidden directories
if (subdirName == "node_modules" || subdirName.StartsWith('.'))
{
continue;
}
var subdirRelative = $"{relativePath}/{subdirName}";
if (File.Exists(Path.Combine(subdir, "package.json")))
{
results.Add(subdirRelative);
}
results.AddRange(EnumeratePackagesRecursively(subdir, subdirRelative));
}
}
catch (UnauthorizedAccessException)
{
// Skip inaccessible directories
}
return results;
}
private static void ParseDependencies(JsonElement root, Dictionary<string, DependencyType> result)
{
AddDependencies(root, "dependencies", DependencyType.Production, result);
AddDependencies(root, "devDependencies", DependencyType.Dev, result);
AddDependencies(root, "optionalDependencies", DependencyType.Optional, result);
AddDependencies(root, "peerDependencies", DependencyType.Peer, result);
}
private static Dictionary<string, string> ParsePatchedDependencies(JsonElement root, string projectRoot)
{
var result = new Dictionary<string, string>(StringComparer.Ordinal);
// Check for patchedDependencies in package.json (Bun/pnpm style)
// Format: { "patchedDependencies": { "package-name@version": "patches/package-name@version.patch" } }
if (root.TryGetProperty("patchedDependencies", out var patchedDeps) &&
patchedDeps.ValueKind == JsonValueKind.Object)
{
foreach (var entry in patchedDeps.EnumerateObject())
{
var patchFile = entry.Value.GetString();
if (!string.IsNullOrEmpty(patchFile))
{
// Parse package name from key (could be "pkg@version" or just "pkg")
var packageName = ExtractPackageName(entry.Name);
result[packageName] = patchFile;
}
}
}
// Also check for patches directory
var patchesDir = Path.Combine(projectRoot, "patches");
if (Directory.Exists(patchesDir))
{
ScanPatchesDirectory(patchesDir, result);
}
// Bun uses .patches directory
var bunPatchesDir = Path.Combine(projectRoot, ".patches");
if (Directory.Exists(bunPatchesDir))
{
ScanPatchesDirectory(bunPatchesDir, result);
}
return result;
}
private static void ScanPatchesDirectory(string patchesDir, Dictionary<string, string> result)
{
try
{
foreach (var patchFile in Directory.EnumerateFiles(patchesDir, "*.patch"))
{
// Patch file name format: package-name@version.patch
var fileName = Path.GetFileNameWithoutExtension(patchFile);
var packageName = ExtractPackageName(fileName);
if (!string.IsNullOrEmpty(packageName) && !result.ContainsKey(packageName))
{
result[packageName] = patchFile;
}
}
}
catch (UnauthorizedAccessException)
{
// Skip inaccessible directory
}
}
private static string ExtractPackageName(string nameWithVersion)
{
// Format: package-name@version or @scope/package-name@version
if (string.IsNullOrEmpty(nameWithVersion))
{
return string.Empty;
}
// For scoped packages, find @ after the scope
if (nameWithVersion.StartsWith('@'))
{
var slashIndex = nameWithVersion.IndexOf('/');
if (slashIndex > 0)
{
var atIndex = nameWithVersion.IndexOf('@', slashIndex);
return atIndex > slashIndex ? nameWithVersion[..atIndex] : nameWithVersion;
}
}
// For regular packages
var lastAtIndex = nameWithVersion.LastIndexOf('@');
return lastAtIndex > 0 ? nameWithVersion[..lastAtIndex] : nameWithVersion;
}
private static void AddDependencies(
JsonElement root,
string propertyName,
DependencyType type,
Dictionary<string, DependencyType> result)
{
if (!root.TryGetProperty(propertyName, out var deps) ||
deps.ValueKind != JsonValueKind.Object)
{
return;
}
foreach (var dep in deps.EnumerateObject())
{
var name = dep.Name;
if (result.TryGetValue(name, out var existingType))
{
result[name] = existingType | type;
}
else
{
result[name] = type;
}
}
}
}