This commit is contained in:
StellaOps Bot
2025-12-09 00:20:52 +02:00
parent 3d01bf9edc
commit bc0762e97d
261 changed files with 14033 additions and 4427 deletions

View File

@@ -0,0 +1,57 @@
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Scanner.Analyzers.Lang.DotNet.Internal;
internal sealed class DotNetAnalyzerOptions
{
private const string DefaultConfigFileName = "dotnet-il.config.json";
[JsonPropertyName("emitDependencyEdges")]
public bool EmitDependencyEdges { get; init; } = false;
[JsonPropertyName("includeEntrypoints")]
public bool IncludeEntrypoints { get; init; } = false;
[JsonPropertyName("runtimeEvidencePath")]
public string? RuntimeEvidencePath { get; init; }
[JsonPropertyName("runtimeEvidenceConfidence")]
public string? RuntimeEvidenceConfidence { get; init; }
public static DotNetAnalyzerOptions Load(LanguageAnalyzerContext context)
{
ArgumentNullException.ThrowIfNull(context);
var path = Path.Combine(context.RootPath, DefaultConfigFileName);
if (!File.Exists(path))
{
return new DotNetAnalyzerOptions();
}
try
{
var json = File.ReadAllText(path);
var options = new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true,
ReadCommentHandling = JsonCommentHandling.Skip,
AllowTrailingCommas = true
};
return JsonSerializer.Deserialize<DotNetAnalyzerOptions>(json, options) ?? new DotNetAnalyzerOptions();
}
catch (IOException)
{
return new DotNetAnalyzerOptions();
}
catch (JsonException)
{
return new DotNetAnalyzerOptions();
}
catch (UnauthorizedAccessException)
{
return new DotNetAnalyzerOptions();
}
}
}

View File

@@ -0,0 +1,7 @@
namespace StellaOps.Scanner.Analyzers.Lang.DotNet.Internal;
internal sealed record DotNetDependencyEdge(
string Target,
string Reason,
string Confidence,
string Source);

View File

@@ -0,0 +1,110 @@
using System.Text.Json;
namespace StellaOps.Scanner.Analyzers.Lang.DotNet.Internal;
internal static class DotNetRuntimeEvidenceLoader
{
public static IReadOnlyDictionary<string, IReadOnlyList<DotNetDependencyEdge>> Load(
LanguageAnalyzerContext context,
DotNetAnalyzerOptions options,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
ArgumentNullException.ThrowIfNull(options);
if (string.IsNullOrWhiteSpace(options.RuntimeEvidencePath))
{
return new Dictionary<string, IReadOnlyList<DotNetDependencyEdge>>(StringComparer.OrdinalIgnoreCase);
}
var absolute = context.ResolvePath(options.RuntimeEvidencePath);
if (!File.Exists(absolute))
{
return new Dictionary<string, IReadOnlyList<DotNetDependencyEdge>>(StringComparer.OrdinalIgnoreCase);
}
var edges = new Dictionary<string, List<DotNetDependencyEdge>>(StringComparer.OrdinalIgnoreCase);
var confidence = string.IsNullOrWhiteSpace(options.RuntimeEvidenceConfidence) ? "medium" : options.RuntimeEvidenceConfidence!.Trim();
foreach (var line in File.ReadLines(absolute))
{
cancellationToken.ThrowIfCancellationRequested();
if (string.IsNullOrWhiteSpace(line))
{
continue;
}
try
{
using var document = JsonDocument.Parse(line);
var root = document.RootElement;
if (!root.TryGetProperty("package", out var packageElement) || packageElement.ValueKind != JsonValueKind.String)
{
continue;
}
var packageId = packageElement.GetString();
if (string.IsNullOrWhiteSpace(packageId))
{
continue;
}
if (!root.TryGetProperty("target", out var targetElement) || targetElement.ValueKind != JsonValueKind.String)
{
continue;
}
var target = targetElement.GetString();
if (string.IsNullOrWhiteSpace(target))
{
continue;
}
var reason = root.TryGetProperty("reason", out var reasonElement) && reasonElement.ValueKind == JsonValueKind.String
? reasonElement.GetString()
: "runtime";
var conf = root.TryGetProperty("confidence", out var confidenceElement) && confidenceElement.ValueKind == JsonValueKind.String
? confidenceElement.GetString()
: confidence;
var source = root.TryGetProperty("source", out var sourceElement) && sourceElement.ValueKind == JsonValueKind.String
? sourceElement.GetString()
: "runtime-evidence";
var edge = new DotNetDependencyEdge(
Target: target!.Trim(),
Reason: string.IsNullOrWhiteSpace(reason) ? "runtime" : reason!.Trim(),
Confidence: string.IsNullOrWhiteSpace(conf) ? confidence : conf!.Trim(),
Source: string.IsNullOrWhiteSpace(source) ? "runtime-evidence" : source!.Trim());
if (!edges.TryGetValue(packageId.Trim().ToLowerInvariant(), out var list))
{
list = new List<DotNetDependencyEdge>();
edges[packageId.Trim().ToLowerInvariant()] = list;
}
list.Add(edge);
}
catch (JsonException)
{
continue;
}
catch (IOException)
{
continue;
}
catch (UnauthorizedAccessException)
{
continue;
}
}
return edges.ToDictionary(
kvp => kvp.Key,
kvp => (IReadOnlyList<DotNetDependencyEdge>)kvp.Value.OrderBy(edge => edge.Target, StringComparer.OrdinalIgnoreCase).ToArray(),
StringComparer.OrdinalIgnoreCase);
}
}

View File

@@ -0,0 +1,15 @@
namespace StellaOps.Scanner.Analyzers.Lang.Node.Internal;
internal static class NodeDeclarationKeyBuilder
{
public static string Build(string name, string? version)
{
if (string.IsNullOrWhiteSpace(name) || string.IsNullOrWhiteSpace(version))
{
return string.Empty;
}
return $"{name.Trim().ToLowerInvariant()}@{version.Trim()}";
}
}

View File

@@ -162,25 +162,81 @@ internal static class NodePackageCollector
return;
}
var componentIndex = packages.ToDictionary(static p => p.ComponentKey, StringComparer.Ordinal);
foreach (var package in pnpPackages)
{
cancellationToken.ThrowIfCancellationRequested();
if (componentIndex.TryGetValue(package.ComponentKey, out var existing))
{
var merged = MergePackageMetadata(existing, package);
var existingIndex = packages.FindIndex(p => string.Equals(p.ComponentKey, existing.ComponentKey, StringComparison.Ordinal));
if (existingIndex >= 0)
{
packages[existingIndex] = merged;
}
componentIndex[merged.ComponentKey] = merged;
if (!string.IsNullOrEmpty(merged.RelativePathNormalized))
{
visited.Add(merged.RelativePathNormalized);
}
continue;
}
var key = package.RelativePathNormalized;
if (!string.IsNullOrEmpty(key) && !visited.Add(key))
{
continue;
}
if (packages.Any(p => string.Equals(p.ComponentKey, package.ComponentKey, StringComparison.Ordinal)))
{
continue;
}
packages.Add(package);
componentIndex[package.ComponentKey] = package;
}
}
private static NodePackage MergePackageMetadata(NodePackage existing, NodePackage pnpPackage)
{
var merged = new NodePackage(
name: existing.Name,
version: existing.Version,
relativePath: string.IsNullOrWhiteSpace(pnpPackage.RelativePathNormalized) ? existing.RelativePath : pnpPackage.RelativePathNormalized,
packageJsonLocator: string.IsNullOrWhiteSpace(pnpPackage.PackageJsonLocator) ? existing.PackageJsonLocator : pnpPackage.PackageJsonLocator,
isPrivate: pnpPackage.IsPrivate ?? existing.IsPrivate,
lockEntry: existing.LockEntry,
isWorkspaceMember: existing.IsWorkspaceMember,
workspaceRoot: existing.WorkspaceRoot,
workspaceTargets: existing.WorkspaceTargets,
workspaceLink: existing.WorkspaceLink,
lifecycleScripts: existing.LifecycleScripts,
nodeVersions: existing.NodeVersions,
usedByEntrypoint: existing.IsUsedByEntrypoint,
declaredOnly: existing.DeclaredOnly,
lockSource: string.IsNullOrWhiteSpace(pnpPackage.LockSource) ? existing.LockSource : pnpPackage.LockSource,
lockLocator: string.IsNullOrWhiteSpace(pnpPackage.LockLocator) ? existing.LockLocator : pnpPackage.LockLocator,
packageSha256: pnpPackage.PackageSha256 ?? existing.PackageSha256,
isYarnPnp: existing.IsYarnPnp || pnpPackage.IsYarnPnp,
scope: existing.Scope,
isOptional: existing.IsOptional,
license: string.IsNullOrWhiteSpace(existing.License) ? pnpPackage.License : existing.License);
foreach (var entrypoint in existing.Entrypoints)
{
merged.AddEntrypoint(entrypoint.Path, entrypoint.ConditionSet, entrypoint.BinName, entrypoint.MainField, entrypoint.ModuleField);
}
foreach (var importEdge in existing.Imports)
{
merged.AddImport(importEdge);
}
merged.SetResolvedImports(existing.ResolvedImports);
return merged;
}
private static IEnumerable<string> EnumerateSourceFiles(string root)
{
foreach (var extension in new[] { ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx" })
@@ -525,7 +581,7 @@ internal static class NodePackageCollector
}
}
private static string? BuildLockLocator(NodeLockEntry? entry)
internal static string? BuildLockLocator(NodeLockEntry? entry)
{
if (entry is null)
{

View File

@@ -255,6 +255,20 @@ internal static class NodePnpDataLoader
private static string NormalizeRelativePath(LanguageAnalyzerContext context, string packageLocation)
{
if (!Path.IsPathRooted(packageLocation))
{
var normalized = packageLocation
.Replace('\\', '/')
.TrimEnd('/');
while (normalized.StartsWith("./", StringComparison.Ordinal))
{
normalized = normalized[2..];
}
return string.IsNullOrWhiteSpace(normalized) ? "." : normalized;
}
var relative = context.GetRelativePath(packageLocation);
if (string.IsNullOrWhiteSpace(relative) || relative == ".")
{

View File

@@ -201,27 +201,19 @@ internal sealed class PythonImportGraph
/// </summary>
public IReadOnlyList<string>? GetTopologicalOrder()
{
var inDegree = new Dictionary<string, int>(StringComparer.Ordinal);
var dependencyCounts = new Dictionary<string, int>(StringComparer.Ordinal);
foreach (var module in _modules.Keys)
{
inDegree[module] = 0;
}
foreach (var edges in _edges.Values)
{
foreach (var edge in edges)
{
if (inDegree.ContainsKey(edge.To))
{
inDegree[edge.To]++;
}
}
var count = _edges.TryGetValue(module, out var edges)
? edges.Count
: 0;
dependencyCounts[module] = count;
}
var queue = new Queue<string>();
foreach (var (module, degree) in inDegree)
foreach (var (module, count) in dependencyCounts)
{
if (degree == 0)
if (count == 0)
{
queue.Enqueue(module);
}
@@ -233,23 +225,27 @@ internal sealed class PythonImportGraph
var module = queue.Dequeue();
result.Add(module);
if (_edges.TryGetValue(module, out var edges))
if (!_reverseEdges.TryGetValue(module, out var dependents))
{
foreach (var edge in edges)
continue;
}
foreach (var edge in dependents)
{
if (!dependencyCounts.TryGetValue(edge.From, out var remaining))
{
if (inDegree.ContainsKey(edge.To))
{
inDegree[edge.To]--;
if (inDegree[edge.To] == 0)
{
queue.Enqueue(edge.To);
}
}
continue;
}
remaining--;
dependencyCounts[edge.From] = remaining;
if (remaining == 0)
{
queue.Enqueue(edge.From);
}
}
}
// If not all modules are in result, there's a cycle
return result.Count == _modules.Count ? result : null;
}
@@ -437,27 +433,23 @@ internal sealed class PythonImportGraph
{
var parts = sourceModulePath.Split('.');
// Calculate the package to start from
// Level 1 (.) = current package
// Level 2 (..) = parent package
var levelsUp = import.RelativeLevel;
// If source is not a package (__init__.py), we need to go one more level up
var sourceVirtualPath = _modules.TryGetValue(sourceModulePath, out var node) ? node.VirtualPath : null;
var isSourcePackage = sourceVirtualPath?.EndsWith("__init__.py", StringComparison.Ordinal) == true;
if (!isSourcePackage)
// Base package is the containing package of the source module
var packageParts = isSourcePackage ? parts : parts[..^1];
// RelativeLevel counts from the package boundary; level 1 = current package
var levelsUp = Math.Max(import.RelativeLevel - 1, 0);
if (levelsUp > packageParts.Length)
{
levelsUp++;
return null;
}
if (levelsUp > parts.Length)
{
return null; // Invalid relative import (goes beyond top-level package)
}
var baseParts = parts[..^(levelsUp)];
var basePackage = string.Join('.', baseParts);
var basePartsLength = packageParts.Length - levelsUp;
var basePackage = basePartsLength <= 0
? string.Empty
: string.Join('.', packageParts.Take(basePartsLength));
if (string.IsNullOrEmpty(import.Module))
{

View File

@@ -11,8 +11,9 @@ internal sealed partial class PythonSourceImportExtractor
private readonly List<PythonImport> _imports = new();
private bool _inTryBlock;
private bool _inTypeCheckingBlock;
private int _functionDepth;
private int _classDepth;
private int? _typeCheckingIndent;
private readonly Stack<int> _functionIndentStack = new();
private readonly Stack<int> _classIndentStack = new();
public PythonSourceImportExtractor(string sourceFile)
{
@@ -37,11 +38,15 @@ internal sealed partial class PythonSourceImportExtractor
var lines = content.Split('\n');
var lineNumber = 0;
var continuedLine = string.Empty;
var parenBuffer = string.Empty;
var inParenContinuation = false;
foreach (var rawLine in lines)
foreach (var rawLineOriginal in lines)
{
lineNumber++;
var line = rawLine.TrimEnd('\r');
var rawLine = rawLineOriginal.TrimEnd('\r');
var indent = CountIndentation(rawLine);
var line = rawLine;
// Handle line continuations
if (line.EndsWith('\\'))
@@ -54,7 +59,7 @@ internal sealed partial class PythonSourceImportExtractor
continuedLine = string.Empty;
// Track context
UpdateContext(fullLine.TrimStart());
UpdateContext(fullLine.TrimStart(), indent);
// Skip comments and empty lines
var trimmed = fullLine.TrimStart();
@@ -75,6 +80,29 @@ internal sealed partial class PythonSourceImportExtractor
continue;
}
// Handle parenthesized from-imports spanning multiple lines
if (inParenContinuation)
{
parenBuffer += " " + trimmed;
if (trimmed.Contains(')'))
{
inParenContinuation = false;
ExtractImports(parenBuffer, lineNumber);
parenBuffer = string.Empty;
}
continue;
}
if (trimmed.StartsWith("from ", StringComparison.Ordinal) &&
trimmed.Contains("import", StringComparison.Ordinal) &&
trimmed.Contains('(') &&
!trimmed.Contains(')'))
{
inParenContinuation = true;
parenBuffer = trimmed;
continue;
}
// Try to extract imports
ExtractImports(trimmed, lineNumber);
}
@@ -82,8 +110,24 @@ internal sealed partial class PythonSourceImportExtractor
return this;
}
private void UpdateContext(string line)
private void UpdateContext(string line, int indent)
{
// unwind function/class stacks based on indentation
while (_functionIndentStack.Count > 0 && indent <= _functionIndentStack.Peek())
{
_functionIndentStack.Pop();
}
while (_classIndentStack.Count > 0 && indent <= _classIndentStack.Peek())
{
_classIndentStack.Pop();
}
if (_typeCheckingIndent is not null && indent <= _typeCheckingIndent.Value)
{
_typeCheckingIndent = null;
}
// Track try blocks
if (line.StartsWith("try:") || line == "try")
{
@@ -94,31 +138,30 @@ internal sealed partial class PythonSourceImportExtractor
_inTryBlock = false;
}
// Track TYPE_CHECKING blocks
if (line.Contains("TYPE_CHECKING") && line.Contains("if"))
{
_inTypeCheckingBlock = true;
}
// Track function depth
if (line.StartsWith("def ") || line.StartsWith("async def "))
{
_functionDepth++;
_functionIndentStack.Push(indent);
}
// Track class depth (for nested classes)
if (line.StartsWith("class "))
{
_classDepth++;
_classIndentStack.Push(indent);
}
// Reset context at module level definitions
if ((line.StartsWith("def ") || line.StartsWith("class ") || line.StartsWith("async def ")) &&
!line.StartsWith(" ") && !line.StartsWith("\t"))
// Track TYPE_CHECKING blocks
if (line.StartsWith("if", StringComparison.Ordinal) && line.Contains("TYPE_CHECKING", StringComparison.Ordinal))
{
_typeCheckingIndent = indent;
}
_inTypeCheckingBlock = _typeCheckingIndent is not null && indent > _typeCheckingIndent.Value;
// Reset TYPE_CHECKING when hitting new top-level if
if (_typeCheckingIndent is null && indent == 0 && line.StartsWith("if ", StringComparison.Ordinal))
{
_inTypeCheckingBlock = false;
_functionDepth = 0;
_classDepth = 0;
}
}
@@ -212,7 +255,7 @@ internal sealed partial class PythonSourceImportExtractor
LineNumber: lineNumber,
Confidence: PythonImportConfidence.Definitive,
IsConditional: _inTryBlock,
IsLazy: _functionDepth > 0,
IsLazy: _functionIndentStack.Count > 0,
IsTypeCheckingOnly: _inTypeCheckingBlock));
}
}
@@ -241,7 +284,7 @@ internal sealed partial class PythonSourceImportExtractor
LineNumber: lineNumber,
Confidence: PythonImportConfidence.Definitive,
IsConditional: _inTryBlock,
IsLazy: _functionDepth > 0,
IsLazy: _functionIndentStack.Count > 0,
IsTypeCheckingOnly: _inTypeCheckingBlock));
return;
}
@@ -294,7 +337,7 @@ internal sealed partial class PythonSourceImportExtractor
LineNumber: lineNumber,
Confidence: PythonImportConfidence.Definitive,
IsConditional: _inTryBlock,
IsLazy: _functionDepth > 0,
IsLazy: _functionIndentStack.Count > 0,
IsTypeCheckingOnly: _inTypeCheckingBlock));
}
@@ -324,7 +367,7 @@ internal sealed partial class PythonSourceImportExtractor
LineNumber: lineNumber,
Confidence: PythonImportConfidence.High,
IsConditional: _inTryBlock,
IsLazy: _functionDepth > 0,
IsLazy: _functionIndentStack.Count > 0,
IsTypeCheckingOnly: _inTypeCheckingBlock));
}
@@ -342,7 +385,7 @@ internal sealed partial class PythonSourceImportExtractor
LineNumber: lineNumber,
Confidence: PythonImportConfidence.High,
IsConditional: _inTryBlock,
IsLazy: _functionDepth > 0,
IsLazy: _functionIndentStack.Count > 0,
IsTypeCheckingOnly: _inTypeCheckingBlock));
}
@@ -359,10 +402,32 @@ internal sealed partial class PythonSourceImportExtractor
LineNumber: lineNumber,
Confidence: PythonImportConfidence.Medium,
IsConditional: _inTryBlock,
IsLazy: _functionDepth > 0,
IsLazy: _functionIndentStack.Count > 0,
IsTypeCheckingOnly: _inTypeCheckingBlock));
}
private static int CountIndentation(string line)
{
var count = 0;
foreach (var c in line)
{
if (c == ' ')
{
count++;
}
else if (c == '\t')
{
count += 4;
}
else
{
break;
}
}
return count;
}
private static int FindCommentStart(string line)
{
var inSingleQuote = false;

View File

@@ -1,5 +1,6 @@
using System.Text.Json;
using System.Text.RegularExpressions;
using System.Linq;
namespace StellaOps.Scanner.Analyzers.Lang.Python.Internal.VirtualFileSystem;
@@ -276,10 +277,10 @@ internal sealed partial class PythonInputNormalizer
var version = parts[1].Trim();
if (!string.IsNullOrEmpty(version))
{
_versionTargets.Add(new PythonVersionTarget(
AddVersionTarget(
version,
"pyvenv.cfg",
PythonVersionConfidence.Definitive));
PythonVersionConfidence.Definitive);
}
}
}
@@ -304,8 +305,7 @@ internal sealed partial class PythonInputNormalizer
var content = await File.ReadAllTextAsync(path, cancellationToken).ConfigureAwait(false);
// Look for requires-python in [project] section
var requiresPythonMatch = RequiresPythonPattern().Match(content);
if (requiresPythonMatch.Success)
foreach (Match requiresPythonMatch in RequiresPythonPattern().Matches(content))
{
var version = requiresPythonMatch.Groups["version"].Value.Trim().Trim('"', '\'');
if (!string.IsNullOrEmpty(version))
@@ -314,17 +314,16 @@ internal sealed partial class PythonInputNormalizer
version.StartsWith(">", StringComparison.Ordinal);
version = Regex.Replace(version, @"^[><=!~]+", string.Empty).Trim();
_versionTargets.Add(new PythonVersionTarget(
AddVersionTarget(
version,
"pyproject.toml",
PythonVersionConfidence.High,
isMinimum));
isMinimum);
}
}
// Look for python_requires in [tool.poetry] or similar
var pythonMatch = PythonVersionTomlPattern().Match(content);
if (pythonMatch.Success)
foreach (Match pythonMatch in PythonVersionTomlPattern().Matches(content))
{
var version = pythonMatch.Groups["version"].Value.Trim().Trim('"', '\'');
if (!string.IsNullOrEmpty(version))
@@ -333,11 +332,11 @@ internal sealed partial class PythonInputNormalizer
version.StartsWith(">=", StringComparison.Ordinal);
version = Regex.Replace(version, @"^[\^><=!~]+", string.Empty).Trim();
_versionTargets.Add(new PythonVersionTarget(
AddVersionTarget(
version,
"pyproject.toml",
PythonVersionConfidence.High,
isMinimum));
isMinimum);
}
}
}
@@ -367,11 +366,11 @@ internal sealed partial class PythonInputNormalizer
var isMinimum = version.StartsWith(">=", StringComparison.Ordinal);
version = Regex.Replace(version, @"^[><=!~]+", string.Empty).Trim();
_versionTargets.Add(new PythonVersionTarget(
AddVersionTarget(
version,
"setup.py",
PythonVersionConfidence.High,
isMinimum));
isMinimum);
}
}
}
@@ -401,11 +400,11 @@ internal sealed partial class PythonInputNormalizer
var isMinimum = version.StartsWith(">=", StringComparison.Ordinal);
version = Regex.Replace(version, @"^[><=!~]+", string.Empty).Trim();
_versionTargets.Add(new PythonVersionTarget(
AddVersionTarget(
version,
"setup.cfg",
PythonVersionConfidence.High,
isMinimum));
isMinimum);
}
}
}
@@ -433,10 +432,10 @@ internal sealed partial class PythonInputNormalizer
if (match.Success)
{
var version = match.Groups["version"].Value;
_versionTargets.Add(new PythonVersionTarget(
AddVersionTarget(
version,
"runtime.txt",
PythonVersionConfidence.Definitive));
PythonVersionConfidence.Definitive);
}
}
catch (IOException)
@@ -462,10 +461,10 @@ internal sealed partial class PythonInputNormalizer
if (fromMatch.Success)
{
var version = fromMatch.Groups["version"].Value;
_versionTargets.Add(new PythonVersionTarget(
AddVersionTarget(
version,
"Dockerfile",
PythonVersionConfidence.High));
PythonVersionConfidence.High);
return;
}
@@ -474,10 +473,10 @@ internal sealed partial class PythonInputNormalizer
if (envMatch.Success)
{
var version = envMatch.Groups["version"].Value;
_versionTargets.Add(new PythonVersionTarget(
AddVersionTarget(
version,
"Dockerfile",
PythonVersionConfidence.Medium));
PythonVersionConfidence.Medium);
}
}
catch (IOException)
@@ -509,10 +508,10 @@ internal sealed partial class PythonInputNormalizer
{
// Convert py311 to 3.11
var formatted = $"{version[0]}.{version[1..]}";
_versionTargets.Add(new PythonVersionTarget(
AddVersionTarget(
formatted,
"tox.ini",
PythonVersionConfidence.Medium));
PythonVersionConfidence.Medium);
}
}
}
@@ -553,10 +552,10 @@ internal sealed partial class PythonInputNormalizer
if (match.Success)
{
var version = match.Groups["version"].Value;
_versionTargets.Add(new PythonVersionTarget(
AddVersionTarget(
version,
$"lib/{dirName}",
PythonVersionConfidence.Medium));
PythonVersionConfidence.Medium);
}
}
}
@@ -659,6 +658,46 @@ internal sealed partial class PythonInputNormalizer
}
}
private void AddVersionTarget(string version, string source, PythonVersionConfidence confidence, bool isMinimum = false)
{
if (string.IsNullOrWhiteSpace(version) || string.IsNullOrWhiteSpace(source))
{
return;
}
var normalizedVersion = version.Trim();
var normalizedSource = source.Trim();
var existingIndex = _versionTargets.FindIndex(target =>
string.Equals(target.Version, normalizedVersion, StringComparison.OrdinalIgnoreCase) &&
string.Equals(target.Source, normalizedSource, StringComparison.OrdinalIgnoreCase));
if (existingIndex >= 0)
{
var existing = _versionTargets[existingIndex];
var effectiveConfidence = (PythonVersionConfidence)Math.Max((int)existing.Confidence, (int)confidence);
var effectiveIsMinimum = existing.IsMinimum || isMinimum;
if (existing.Confidence == effectiveConfidence && existing.IsMinimum == effectiveIsMinimum)
{
return;
}
_versionTargets[existingIndex] = new PythonVersionTarget(
normalizedVersion,
normalizedSource,
effectiveConfidence,
effectiveIsMinimum);
return;
}
_versionTargets.Add(new PythonVersionTarget(
normalizedVersion,
normalizedSource,
confidence,
isMinimum));
}
private void DetectZipapps()
{
if (!Directory.Exists(_rootPath))
@@ -779,7 +818,7 @@ internal sealed partial class PythonInputNormalizer
[GeneratedRegex(@"requires-python\s*=\s*[""']?(?<version>[^""'\n]+)", RegexOptions.IgnoreCase)]
private static partial Regex RequiresPythonPattern();
[GeneratedRegex(@"python\s*=\s*[""'](?<version>[^""']+)[""']", RegexOptions.IgnoreCase)]
[GeneratedRegex(@"^\s*python\s*=\s*[""'](?<version>[^""']+)[""']", RegexOptions.IgnoreCase | RegexOptions.Multiline)]
private static partial Regex PythonVersionTomlPattern();
[GeneratedRegex(@"python_requires\s*=\s*[""'](?<version>[^""']+)[""']", RegexOptions.IgnoreCase)]

View File

@@ -38,7 +38,7 @@ internal sealed class ApkPackageAnalyzer : OsPackageAnalyzerBase
using var stream = File.OpenRead(installedPath);
var entries = _parser.Parse(stream, cancellationToken);
context.Metadata.TryGetValue(ScanMetadataKeys.CurrentLayerDigest, out var layerDigest);
var evidenceFactory = OsFileEvidenceFactory.Create(context.RootPath, context.Metadata);
var records = new List<OSPackageRecord>(entries.Count);
foreach (var entry in entries)
@@ -68,16 +68,17 @@ internal sealed class ApkPackageAnalyzer : OsPackageAnalyzerBase
vendorMetadata[$"apk:{pair.Key}"] = pair.Value;
}
var files = new List<OSPackageFileEvidence>(entry.Files.Count);
foreach (var file in entry.Files)
{
files.Add(new OSPackageFileEvidence(
var files = entry.Files
.Select(file => evidenceFactory.Create(
file.Path,
layerDigest: layerDigest,
sha256: file.Digest,
sizeBytes: null,
isConfigFile: file.IsConfig));
}
file.IsConfig,
string.IsNullOrWhiteSpace(file.Digest)
? null
: new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["sha256"] = file.Digest
}))
.ToList();
var cveHints = CveHintExtractor.Extract(
string.Join(' ', entry.Depends),

View File

@@ -40,10 +40,9 @@ internal sealed class DpkgPackageAnalyzer : OsPackageAnalyzerBase
using var stream = File.OpenRead(statusPath);
var entries = _parser.Parse(stream, cancellationToken);
context.Metadata.TryGetValue(ScanMetadataKeys.CurrentLayerDigest, out var layerDigest);
var infoDirectory = Path.Combine(context.RootPath, "var", "lib", "dpkg", "info");
var records = new List<OSPackageRecord>();
var evidenceFactory = OsFileEvidenceFactory.Create(context.RootPath, context.Metadata);
foreach (var entry in entries)
{
@@ -86,7 +85,7 @@ internal sealed class DpkgPackageAnalyzer : OsPackageAnalyzerBase
var dependencies = entry.Depends.Concat(entry.PreDepends).ToArray();
var provides = entry.Provides.ToArray();
var fileEvidence = BuildFileEvidence(infoDirectory, entry, layerDigest, cancellationToken);
var fileEvidence = BuildFileEvidence(infoDirectory, entry, evidenceFactory, cancellationToken);
var cveHints = CveHintExtractor.Extract(entry.Description, string.Join(' ', dependencies), string.Join(' ', provides));
@@ -128,7 +127,11 @@ internal sealed class DpkgPackageAnalyzer : OsPackageAnalyzerBase
return parts.Length == 0 ? null : parts[0];
}
private static IReadOnlyList<OSPackageFileEvidence> BuildFileEvidence(string infoDirectory, DpkgPackageEntry entry, string? layerDigest, CancellationToken cancellationToken)
private static IReadOnlyList<OSPackageFileEvidence> BuildFileEvidence(
string infoDirectory,
DpkgPackageEntry entry,
OsFileEvidenceFactory evidenceFactory,
CancellationToken cancellationToken)
{
if (!Directory.Exists(infoDirectory))
{
@@ -140,7 +143,7 @@ internal sealed class DpkgPackageAnalyzer : OsPackageAnalyzerBase
{
if (!files.TryGetValue(path, out _))
{
files[path] = new FileEvidenceBuilder(path, layerDigest);
files[path] = new FileEvidenceBuilder(path);
}
}
@@ -236,7 +239,7 @@ internal sealed class DpkgPackageAnalyzer : OsPackageAnalyzerBase
}
var evidence = files.Values
.Select(builder => builder.ToEvidence())
.Select(builder => evidenceFactory.Create(builder.Path, builder.IsConfig, builder.Digests))
.OrderBy(e => e)
.ToArray();
@@ -251,23 +254,15 @@ internal sealed class DpkgPackageAnalyzer : OsPackageAnalyzerBase
private sealed class FileEvidenceBuilder
{
public FileEvidenceBuilder(string path, string? layerDigest)
public FileEvidenceBuilder(string path)
{
Path = path;
LayerDigest = layerDigest;
}
public string Path { get; }
public string? LayerDigest { get; }
public bool IsConfig { get; set; }
public Dictionary<string, string> Digests { get; } = new(StringComparer.OrdinalIgnoreCase);
public OSPackageFileEvidence ToEvidence()
{
return new OSPackageFileEvidence(Path, layerDigest: LayerDigest, isConfigFile: IsConfig, digests: Digests);
}
}
}

View File

@@ -43,7 +43,7 @@ internal sealed class RpmPackageAnalyzer : OsPackageAnalyzerBase
return ValueTask.FromResult<IReadOnlyList<OSPackageRecord>>(EmptyPackages);
}
context.Metadata.TryGetValue(ScanMetadataKeys.CurrentLayerDigest, out var layerDigest);
var evidenceFactory = OsFileEvidenceFactory.Create(context.RootPath, context.Metadata);
var records = new List<OSPackageRecord>(headers.Count);
foreach (var header in headers)
@@ -80,7 +80,7 @@ internal sealed class RpmPackageAnalyzer : OsPackageAnalyzerBase
digests = new Dictionary<string, string>(file.Digests, StringComparer.OrdinalIgnoreCase);
}
files.Add(new OSPackageFileEvidence(file.Path, layerDigest: layerDigest, isConfigFile: file.IsConfig, digests: digests));
files.Add(evidenceFactory.Create(file.Path, file.IsConfig, digests));
}
var cveHints = CveHintExtractor.Extract(

View File

@@ -0,0 +1,222 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using StellaOps.Scanner.Core.Contracts;
namespace StellaOps.Scanner.Analyzers.OS.Helpers;
/// <summary>
/// Enriches OS package file evidence with layer attribution and stable hashes/sizes.
/// </summary>
public sealed class OsFileEvidenceFactory
{
private readonly string _rootPath;
private readonly ImmutableArray<(string? Digest, string Path)> _layerDirectories;
private readonly string? _defaultLayerDigest;
private OsFileEvidenceFactory(string rootPath, ImmutableArray<(string? Digest, string Path)> layerDirectories, string? defaultLayerDigest)
{
_rootPath = rootPath;
_layerDirectories = layerDirectories;
_defaultLayerDigest = NormalizeDigest(defaultLayerDigest);
}
public static OsFileEvidenceFactory Create(string rootPath, IReadOnlyDictionary<string, string> metadata)
{
ArgumentException.ThrowIfNullOrWhiteSpace(rootPath);
ArgumentNullException.ThrowIfNull(metadata);
var layerDirectories = ParseLayerEntries(metadata, ScanMetadataKeys.LayerDirectories);
metadata.TryGetValue(ScanMetadataKeys.CurrentLayerDigest, out var defaultLayerDigest);
return new OsFileEvidenceFactory(rootPath, layerDirectories, defaultLayerDigest);
}
public OSPackageFileEvidence Create(string path, bool isConfigFile, IDictionary<string, string>? digests = null)
{
var digestMap = digests is null
? new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
: new Dictionary<string, string>(digests, StringComparer.OrdinalIgnoreCase);
var layerDigest = ResolveLayerDigest(path) ?? _defaultLayerDigest;
string? sha256 = null;
long? size = null;
var fullPath = CombineWithRoot(path);
if (fullPath is not null && File.Exists(fullPath))
{
try
{
var info = new FileInfo(fullPath);
size = info.Length;
if (info.Length > 0 && !digestMap.TryGetValue("sha256", out sha256))
{
sha256 = ComputeSha256(fullPath);
digestMap["sha256"] = sha256;
}
}
catch (IOException)
{
// Best-effort: ignore IO failures and fall back to existing metadata
}
catch (UnauthorizedAccessException)
{
// Ignore permission issues
}
}
return new OSPackageFileEvidence(
path,
layerDigest: layerDigest,
sha256: sha256,
sizeBytes: size,
isConfigFile: isConfigFile,
digests: digestMap);
}
private string? CombineWithRoot(string path)
{
if (string.IsNullOrWhiteSpace(path))
{
return null;
}
var trimmed = path.TrimStart('/', '\\');
var combined = Path.Combine(_rootPath, trimmed);
return Path.GetFullPath(combined);
}
private string? ResolveLayerDigest(string path)
{
if (_layerDirectories.IsDefaultOrEmpty)
{
return null;
}
var relative = path.TrimStart('/', '\\');
foreach (var (digest, layerPath) in _layerDirectories)
{
string? layerDigest = NormalizeDigest(digest);
string candidate;
try
{
candidate = Path.GetFullPath(Path.Combine(layerPath, relative));
}
catch
{
continue;
}
if (!File.Exists(candidate))
{
continue;
}
return layerDigest ?? ComputeDirectoryDigest(layerPath);
}
return null;
}
private static string ComputeSha256(string path)
{
using var stream = File.OpenRead(path);
return Convert.ToHexString(SHA256.HashData(stream)).ToLowerInvariant();
}
private static string ComputeDirectoryDigest(string path)
{
var payload = Encoding.UTF8.GetBytes(Path.GetFullPath(path));
var hash = SHA256.HashData(payload);
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
}
private static ImmutableArray<(string? Digest, string Path)> ParseLayerEntries(
IReadOnlyDictionary<string, string> metadata,
string metadataKey)
{
if (string.IsNullOrWhiteSpace(metadataKey) ||
!metadata.TryGetValue(metadataKey, out var rawValue) ||
string.IsNullOrWhiteSpace(rawValue))
{
return ImmutableArray<(string?, string)>.Empty;
}
rawValue = rawValue.Trim();
IEnumerable<string> tokens;
if (rawValue.StartsWith("[", StringComparison.Ordinal))
{
try
{
var parsed = JsonSerializer.Deserialize<string[]>(rawValue);
tokens = parsed ?? Array.Empty<string>();
}
catch
{
tokens = SplitLayerString(rawValue);
}
}
else
{
tokens = SplitLayerString(rawValue);
}
var builder = ImmutableArray.CreateBuilder<(string?, string)>();
foreach (var token in tokens)
{
var entry = token.Trim();
if (entry.Length == 0)
{
continue;
}
var separator = entry.IndexOf('=');
string? digest = null;
var pathPart = entry;
if (separator >= 0)
{
digest = entry[..separator].Trim();
pathPart = entry[(separator + 1)..].Trim();
}
if (pathPart.Length == 0)
{
continue;
}
builder.Add((NormalizeDigest(digest), pathPart));
}
return builder.ToImmutable();
}
private static IEnumerable<string> SplitLayerString(string raw)
=> raw.Split(new[] { '\n', '\r', ';' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
private static string? NormalizeDigest(string? digest)
{
if (string.IsNullOrWhiteSpace(digest))
{
return null;
}
var trimmed = digest.Trim();
if (!trimmed.Contains(':', StringComparison.Ordinal))
{
return trimmed.ToLowerInvariant();
}
var parts = trimmed.Split(':', 2, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
return parts.Length == 2
? $"{parts[0].ToLowerInvariant()}:{parts[1].ToLowerInvariant()}"
: trimmed.ToLowerInvariant();
}
}

View File

@@ -4,6 +4,7 @@ using System.Collections.Immutable;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Globalization;
using StellaOps.Scanner.Core.Contracts;
namespace StellaOps.Scanner.Analyzers.OS.Mapping;
@@ -144,6 +145,11 @@ public static class OsComponentMapper
properties[$"digest.{digest.Key}.{NormalizePathKey(file.Path)}"] = digest.Value.Trim();
}
if (file.SizeBytes.HasValue)
{
properties[$"size.{NormalizePathKey(file.Path)}"] = file.SizeBytes.Value.ToString(CultureInfo.InvariantCulture);
}
}
IReadOnlyList<string>? licenses = null;