Add unit tests for RancherHubConnector and various exporters
- Implemented tests for RancherHubConnector to validate fetching documents, handling errors, and managing state. - Added tests for CsafExporter to ensure deterministic serialization of CSAF documents. - Created tests for CycloneDX exporters and reconciler to verify correct handling of VEX claims and output structure. - Developed OpenVEX exporter tests to confirm the generation of canonical OpenVEX documents and statement merging logic. - Introduced Rust file caching and license scanning functionality, including a cache key structure and hash computation. - Added sample Cargo.toml and LICENSE files for testing Rust license scanning functionality.
This commit is contained in:
@@ -241,25 +241,40 @@ internal static class JavaReflectionAnalyzer
|
||||
instructionOffset,
|
||||
null));
|
||||
}
|
||||
else if (normalizedOwner == "java/lang/ClassLoader" && (name == "getResource" || name == "getResourceAsStream" || name == "getResources"))
|
||||
{
|
||||
var target = pendingString;
|
||||
var confidence = pendingString is null ? JavaReflectionConfidence.Low : JavaReflectionConfidence.High;
|
||||
edges.Add(new JavaReflectionEdge(
|
||||
normalizedSource,
|
||||
segmentIdentifier,
|
||||
target,
|
||||
JavaReflectionReason.ResourceLookup,
|
||||
confidence,
|
||||
method.Name,
|
||||
method.Descriptor,
|
||||
instructionOffset,
|
||||
null));
|
||||
}
|
||||
else if (normalizedOwner == "java/lang/Thread" && name == "currentThread")
|
||||
{
|
||||
sawCurrentThread = true;
|
||||
}
|
||||
else if (normalizedOwner == "java/lang/ClassLoader" && (name == "getResource" || name == "getResourceAsStream" || name == "getResources"))
|
||||
{
|
||||
var target = pendingString;
|
||||
var confidence = pendingString is null ? JavaReflectionConfidence.Low : JavaReflectionConfidence.High;
|
||||
edges.Add(new JavaReflectionEdge(
|
||||
normalizedSource,
|
||||
segmentIdentifier,
|
||||
target,
|
||||
JavaReflectionReason.ResourceLookup,
|
||||
confidence,
|
||||
method.Name,
|
||||
method.Descriptor,
|
||||
instructionOffset,
|
||||
null));
|
||||
}
|
||||
else if (normalizedOwner == "java/lang/Class" && (name == "getResource" || name == "getResourceAsStream"))
|
||||
{
|
||||
var target = pendingString;
|
||||
var confidence = pendingString is null ? JavaReflectionConfidence.Low : JavaReflectionConfidence.High;
|
||||
edges.Add(new JavaReflectionEdge(
|
||||
normalizedSource,
|
||||
segmentIdentifier,
|
||||
target,
|
||||
JavaReflectionReason.ResourceLookup,
|
||||
confidence,
|
||||
method.Name,
|
||||
method.Descriptor,
|
||||
instructionOffset,
|
||||
null));
|
||||
}
|
||||
else if (normalizedOwner == "java/lang/Thread" && name == "currentThread")
|
||||
{
|
||||
sawCurrentThread = true;
|
||||
}
|
||||
else if (normalizedOwner == "java/lang/Thread" && name == "getContextClassLoader")
|
||||
{
|
||||
if (sawCurrentThread && !emittedTcclWarning)
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
| SCANNER-ANALYZERS-JAVA-21-001 | DONE (2025-10-27) | Java Analyzer Guild | SCANNER-CORE-09-501 | Build input normalizer and virtual file system for JAR/WAR/EAR/fat-jar/JMOD/jimage/container roots. Detect packaging type, layered dirs (BOOT-INF/WEB-INF), multi-release overlays, and jlink runtime metadata. | Normalizer walks fixtures without extraction, classifies packaging, selects MR overlays deterministically, records java version + vendor from runtime images. |
|
||||
| SCANNER-ANALYZERS-JAVA-21-002 | DONE (2025-10-27) | Java Analyzer Guild | SCANNER-ANALYZERS-JAVA-21-001 | Implement module/classpath builder: JPMS graph parser (`module-info.class`), classpath order rules (fat jar, war, ear), duplicate & split-package detection, package fingerprinting. | Classpath order reproduced for fixtures; module graph serialized; duplicate provider + split-package warnings emitted deterministically. |
|
||||
| SCANNER-ANALYZERS-JAVA-21-003 | DONE (2025-10-27) | Java Analyzer Guild | SCANNER-ANALYZERS-JAVA-21-002 | SPI scanner covering META-INF/services, provider selection, and warning generation. Include configurable SPI corpus (JDK, Spring, logging, Jackson, MicroProfile). | SPI tables produced with selected provider + candidates; fixtures show first-wins behaviour; warnings recorded for duplicate providers. |
|
||||
| SCANNER-ANALYZERS-JAVA-21-004 | DOING (2025-10-27) | Java Analyzer Guild | SCANNER-ANALYZERS-JAVA-21-002 | Reflection/dynamic loader heuristics: scan constant pools, bytecode sites (Class.forName, loadClass, TCCL usage), resource-based plugin hints, manifest loader hints. Emit edges with reason codes + confidence. | Reflection edges generated for fixtures (classpath, boot, war); includes call site metadata and confidence scoring; TCCL warning emitted where detected. |
|
||||
| SCANNER-ANALYZERS-JAVA-21-004 | DONE (2025-10-29) | Java Analyzer Guild | SCANNER-ANALYZERS-JAVA-21-002 | Reflection/dynamic loader heuristics: scan constant pools, bytecode sites (Class.forName, loadClass, TCCL usage), resource-based plugin hints, manifest loader hints. Emit edges with reason codes + confidence. | Reflection edges generated for fixtures (classpath, boot, war); includes call site metadata and confidence scoring; TCCL warning emitted where detected. |
|
||||
| SCANNER-ANALYZERS-JAVA-21-005 | TODO | Java Analyzer Guild | SCANNER-ANALYZERS-JAVA-21-002 | Framework config extraction: Spring Boot imports, spring.factories, application properties/yaml, Jakarta web.xml & fragments, JAX-RS/JPA/CDI/JAXB configs, logging files, Graal native-image configs. | Framework fixtures parsed; relevant class FQCNs surfaced with reasons (`config-spring`, `config-jaxrs`, etc.); non-class config ignored; determinism guard passes. |
|
||||
| SCANNER-ANALYZERS-JAVA-21-006 | TODO | Java Analyzer Guild | SCANNER-ANALYZERS-JAVA-21-002 | JNI/native hint scanner: detect native methods, System.load/Library literals, bundled native libs, Graal JNI configs; emit `jni-load` edges for native analyzer correlation. | JNI fixtures produce hint edges pointing at embedded libs; metadata includes candidate paths and reason `jni`. |
|
||||
| SCANNER-ANALYZERS-JAVA-21-007 | TODO | Java Analyzer Guild | SCANNER-ANALYZERS-JAVA-21-003 | Signature and manifest metadata collector: verify JAR signature structure, capture signers, manifest loader attributes (Main-Class, Agent-Class, Start-Class, Class-Path). | Signed jar fixture reports signer info and structural validation result; manifest metadata attached to entrypoints. |
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Rust.Internal;
|
||||
|
||||
@@ -30,6 +29,7 @@ internal static class RustAnalyzerCollector
|
||||
private readonly Dictionary<string, List<RustCrateBuilder>> _cratesByName = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, RustHeuristicBuilder> _heuristics = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, RustBinaryRecord> _binaries = new(StringComparer.Ordinal);
|
||||
private RustLicenseIndex _licenseIndex = RustLicenseIndex.Empty;
|
||||
|
||||
public Collector(LanguageAnalyzerContext context)
|
||||
{
|
||||
@@ -38,6 +38,7 @@ internal static class RustAnalyzerCollector
|
||||
|
||||
public void Execute(CancellationToken cancellationToken)
|
||||
{
|
||||
_licenseIndex = RustLicenseScanner.GetOrCreate(_context.RootPath, cancellationToken);
|
||||
CollectCargoLocks(cancellationToken);
|
||||
CollectFingerprints(cancellationToken);
|
||||
CollectBinaries(cancellationToken);
|
||||
@@ -81,6 +82,7 @@ internal static class RustAnalyzerCollector
|
||||
{
|
||||
var builder = GetOrCreateCrate(package.Name, package.Version);
|
||||
builder.ApplyCargoPackage(package, relativePath);
|
||||
TryApplyLicense(builder);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -95,6 +97,7 @@ internal static class RustAnalyzerCollector
|
||||
var builder = GetOrCreateCrate(record.Name, record.Version);
|
||||
var relative = NormalizeRelative(_context.GetRelativePath(record.AbsolutePath));
|
||||
builder.ApplyFingerprint(record, relative);
|
||||
TryApplyLicense(builder);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -149,25 +152,30 @@ internal static class RustAnalyzerCollector
|
||||
|
||||
private RustCrateBuilder GetOrCreateCrate(string name, string? version)
|
||||
{
|
||||
var key = new RustCrateKey(name, version);
|
||||
if (_crates.TryGetValue(key, out var existing))
|
||||
{
|
||||
existing.EnsureVersion(version);
|
||||
return existing;
|
||||
}
|
||||
var key = new RustCrateKey(name, version);
|
||||
if (_crates.TryGetValue(key, out var existing))
|
||||
{
|
||||
existing.EnsureVersion(version);
|
||||
if (!existing.HasLicenseMetadata)
|
||||
{
|
||||
TryApplyLicense(existing);
|
||||
}
|
||||
return existing;
|
||||
}
|
||||
|
||||
var builder = new RustCrateBuilder(name, version);
|
||||
_crates[key] = builder;
|
||||
var builder = new RustCrateBuilder(name, version);
|
||||
_crates[key] = builder;
|
||||
|
||||
if (!_cratesByName.TryGetValue(builder.Name, out var list))
|
||||
{
|
||||
list = new List<RustCrateBuilder>();
|
||||
_cratesByName[builder.Name] = list;
|
||||
}
|
||||
{
|
||||
list = new List<RustCrateBuilder>();
|
||||
_cratesByName[builder.Name] = list;
|
||||
}
|
||||
|
||||
list.Add(builder);
|
||||
return builder;
|
||||
}
|
||||
list.Add(builder);
|
||||
TryApplyLicense(builder);
|
||||
return builder;
|
||||
}
|
||||
|
||||
private RustCrateBuilder? FindCrateByName(string candidate)
|
||||
{
|
||||
@@ -250,6 +258,15 @@ internal static class RustAnalyzerCollector
|
||||
|
||||
return relativePath.Replace('\\', '/');
|
||||
}
|
||||
|
||||
private void TryApplyLicense(RustCrateBuilder builder)
|
||||
{
|
||||
var info = _licenseIndex.Find(builder.Name, builder.Version);
|
||||
if (info is not null)
|
||||
{
|
||||
builder.ApplyLicense(info);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -274,6 +291,8 @@ internal sealed class RustCrateBuilder
|
||||
private readonly HashSet<LanguageComponentEvidence> _evidence = new(new LanguageComponentEvidenceComparer());
|
||||
private readonly SortedSet<string> _binaryPaths = new(StringComparer.Ordinal);
|
||||
private readonly SortedSet<string> _binaryHashes = new(StringComparer.Ordinal);
|
||||
private readonly SortedSet<string> _licenseExpressions = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly SortedDictionary<string, string?> _licenseFiles = new(StringComparer.Ordinal);
|
||||
|
||||
private string? _version;
|
||||
private string? _source;
|
||||
@@ -290,6 +309,8 @@ internal sealed class RustCrateBuilder
|
||||
|
||||
public string? Version => _version;
|
||||
|
||||
public bool HasLicenseMetadata => _licenseExpressions.Count > 0 || _licenseFiles.Count > 0;
|
||||
|
||||
public static string NormalizeName(string value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
@@ -399,9 +420,40 @@ internal sealed class RustCrateBuilder
|
||||
|
||||
var metadata = _metadata
|
||||
.Select(static pair => new KeyValuePair<string, string?>(pair.Key, pair.Value))
|
||||
.OrderBy(static pair => pair.Key, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
|
||||
if (_licenseExpressions.Count > 0)
|
||||
{
|
||||
var index = 0;
|
||||
foreach (var expression in _licenseExpressions)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(expression))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
metadata.Add(new KeyValuePair<string, string?>($"license.expression[{index}]", expression));
|
||||
index++;
|
||||
}
|
||||
}
|
||||
|
||||
if (_licenseFiles.Count > 0)
|
||||
{
|
||||
var index = 0;
|
||||
foreach (var pair in _licenseFiles)
|
||||
{
|
||||
metadata.Add(new KeyValuePair<string, string?>($"license.file[{index}]", pair.Key));
|
||||
if (!string.IsNullOrWhiteSpace(pair.Value))
|
||||
{
|
||||
metadata.Add(new KeyValuePair<string, string?>($"license.file.sha256[{index}]", pair.Value));
|
||||
}
|
||||
|
||||
index++;
|
||||
}
|
||||
}
|
||||
|
||||
metadata.Sort(static (left, right) => string.CompareOrdinal(left.Key, right.Key));
|
||||
|
||||
var evidence = _evidence
|
||||
.OrderBy(static item => item.ComparisonKey, StringComparer.Ordinal)
|
||||
.ToImmutableArray();
|
||||
@@ -422,6 +474,45 @@ internal sealed class RustCrateBuilder
|
||||
UsedByEntrypoint: _usedByEntrypoint);
|
||||
}
|
||||
|
||||
public void ApplyLicense(RustLicenseInfo info)
|
||||
{
|
||||
if (info is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var expression in info.Expressions)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(expression))
|
||||
{
|
||||
_licenseExpressions.Add(expression.Trim());
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var file in info.Files)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(file.RelativePath))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var normalized = file.RelativePath.Replace('\\', '/');
|
||||
if (_licenseFiles.ContainsKey(normalized))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(file.Sha256))
|
||||
{
|
||||
_licenseFiles[normalized] = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
_licenseFiles[normalized] = file.Sha256!.Trim();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void AddMetadataIfEmpty(string key, string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(key) || string.IsNullOrWhiteSpace(value))
|
||||
@@ -577,28 +668,14 @@ internal sealed class RustBinaryRecord
|
||||
_hash ??= hash;
|
||||
}
|
||||
|
||||
if (_hash is null)
|
||||
if (!string.IsNullOrEmpty(_hash))
|
||||
{
|
||||
_hash = ComputeHashSafely();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
private string? ComputeHashSafely()
|
||||
{
|
||||
try
|
||||
if (RustFileHashCache.TryGetSha256(AbsolutePath, out var computed) && !string.IsNullOrEmpty(computed))
|
||||
{
|
||||
using var stream = new FileStream(AbsolutePath, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
using var sha = SHA256.Create();
|
||||
var hash = sha.ComputeHash(stream);
|
||||
return Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
catch (UnauthorizedAccessException)
|
||||
{
|
||||
return null;
|
||||
_hash = computed;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
using System.Buffers;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Rust.Internal;
|
||||
@@ -32,6 +32,8 @@ internal static class RustBinaryClassifier
|
||||
AttributesToSkip = FileAttributes.Device | FileAttributes.ReparsePoint,
|
||||
};
|
||||
|
||||
private static readonly ConcurrentDictionary<RustFileCacheKey, ImmutableArray<string>> CandidateCache = new();
|
||||
|
||||
public static IReadOnlyList<RustBinaryInfo> Scan(string rootPath, CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(rootPath))
|
||||
@@ -49,7 +51,16 @@ internal static class RustBinaryClassifier
|
||||
continue;
|
||||
}
|
||||
|
||||
var candidates = ExtractCrateNames(path, cancellationToken);
|
||||
if (!RustFileCacheKey.TryCreate(path, out var key))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var candidates = CandidateCache.GetOrAdd(
|
||||
key,
|
||||
static (_, state) => ExtractCrateNames(state.Path, state.CancellationToken),
|
||||
(Path: path, CancellationToken: cancellationToken));
|
||||
|
||||
binaries.Add(new RustBinaryInfo(path, candidates));
|
||||
}
|
||||
|
||||
@@ -220,31 +231,13 @@ internal static class RustBinaryClassifier
|
||||
|
||||
internal sealed record RustBinaryInfo(string AbsolutePath, ImmutableArray<string> CrateCandidates)
|
||||
{
|
||||
private string? _sha256;
|
||||
|
||||
public string ComputeSha256()
|
||||
{
|
||||
if (_sha256 is not null)
|
||||
if (RustFileHashCache.TryGetSha256(AbsolutePath, out var sha256) && !string.IsNullOrEmpty(sha256))
|
||||
{
|
||||
return _sha256;
|
||||
return sha256;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using var stream = new FileStream(AbsolutePath, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
using var sha = SHA256.Create();
|
||||
var hash = sha.ComputeHash(stream);
|
||||
_sha256 = Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
_sha256 = string.Empty;
|
||||
}
|
||||
catch (UnauthorizedAccessException)
|
||||
{
|
||||
_sha256 = string.Empty;
|
||||
}
|
||||
|
||||
return _sha256 ?? string.Empty;
|
||||
return string.Empty;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Immutable;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Rust.Internal;
|
||||
|
||||
internal static class RustCargoLockParser
|
||||
{
|
||||
private static readonly ConcurrentDictionary<RustFileCacheKey, ImmutableArray<RustCargoPackage>> Cache = new();
|
||||
|
||||
public static IReadOnlyList<RustCargoPackage> Parse(string path, CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(path))
|
||||
@@ -9,17 +14,26 @@ internal static class RustCargoLockParser
|
||||
throw new ArgumentException("Lock path is required", nameof(path));
|
||||
}
|
||||
|
||||
var info = new FileInfo(path);
|
||||
if (!info.Exists)
|
||||
if (!RustFileCacheKey.TryCreate(path, out var key))
|
||||
{
|
||||
return Array.Empty<RustCargoPackage>();
|
||||
}
|
||||
|
||||
var packages = new List<RustCargoPackage>();
|
||||
var packages = Cache.GetOrAdd(
|
||||
key,
|
||||
static (_, state) => ParseInternal(state.Path, state.CancellationToken),
|
||||
(Path: path, CancellationToken: cancellationToken));
|
||||
|
||||
return packages.IsDefaultOrEmpty ? Array.Empty<RustCargoPackage>() : packages;
|
||||
}
|
||||
|
||||
private static ImmutableArray<RustCargoPackage> ParseInternal(string path, CancellationToken cancellationToken)
|
||||
{
|
||||
var resultBuilder = ImmutableArray.CreateBuilder<RustCargoPackage>();
|
||||
|
||||
using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
using var reader = new StreamReader(stream);
|
||||
|
||||
RustCargoPackageBuilder? builder = null;
|
||||
RustCargoPackageBuilder? packageBuilder = null;
|
||||
string? currentArrayKey = null;
|
||||
var arrayValues = new List<string>();
|
||||
|
||||
@@ -41,14 +55,14 @@ internal static class RustCargoLockParser
|
||||
|
||||
if (IsPackageHeader(trimmed))
|
||||
{
|
||||
FlushCurrent(builder, packages);
|
||||
builder = new RustCargoPackageBuilder();
|
||||
FlushCurrent(packageBuilder, resultBuilder);
|
||||
packageBuilder = new RustCargoPackageBuilder();
|
||||
currentArrayKey = null;
|
||||
arrayValues.Clear();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (builder is null)
|
||||
if (packageBuilder is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
@@ -94,12 +108,12 @@ internal static class RustCargoLockParser
|
||||
}
|
||||
|
||||
if (valuePart[0] == '[')
|
||||
{
|
||||
currentArrayKey = key.ToString();
|
||||
arrayValues.Clear();
|
||||
|
||||
if (valuePart.Length > 1 && valuePart[^1] == ']')
|
||||
{
|
||||
currentArrayKey = key.ToString();
|
||||
arrayValues.Clear();
|
||||
|
||||
if (valuePart.Length > 1 && valuePart[^1] == ']')
|
||||
{
|
||||
var inline = valuePart[1..^1].Trim();
|
||||
if (inline.Length > 0)
|
||||
{
|
||||
@@ -113,7 +127,7 @@ internal static class RustCargoLockParser
|
||||
}
|
||||
}
|
||||
|
||||
builder.SetArray(currentArrayKey, arrayValues);
|
||||
packageBuilder.SetArray(currentArrayKey, arrayValues);
|
||||
currentArrayKey = null;
|
||||
arrayValues.Clear();
|
||||
}
|
||||
@@ -124,17 +138,17 @@ internal static class RustCargoLockParser
|
||||
var parsed = ExtractString(valuePart);
|
||||
if (parsed is not null)
|
||||
{
|
||||
builder.SetField(key, parsed);
|
||||
packageBuilder.SetField(key, parsed);
|
||||
}
|
||||
}
|
||||
|
||||
if (currentArrayKey is not null && arrayValues.Count > 0)
|
||||
{
|
||||
builder?.SetArray(currentArrayKey, arrayValues);
|
||||
packageBuilder?.SetArray(currentArrayKey, arrayValues);
|
||||
}
|
||||
|
||||
FlushCurrent(builder, packages);
|
||||
return packages;
|
||||
FlushCurrent(packageBuilder, resultBuilder);
|
||||
return resultBuilder.ToImmutable();
|
||||
}
|
||||
|
||||
private static ReadOnlySpan<char> TrimComments(ReadOnlySpan<char> line)
|
||||
@@ -204,14 +218,14 @@ internal static class RustCargoLockParser
|
||||
return trimmed.Length == 0 ? null : trimmed.ToString();
|
||||
}
|
||||
|
||||
private static void FlushCurrent(RustCargoPackageBuilder? builder, List<RustCargoPackage> packages)
|
||||
private static void FlushCurrent(RustCargoPackageBuilder? packageBuilder, ImmutableArray<RustCargoPackage>.Builder packages)
|
||||
{
|
||||
if (builder is null || !builder.HasData)
|
||||
if (packageBuilder is null || !packageBuilder.HasData)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (builder.TryBuild(out var package))
|
||||
if (packageBuilder.TryBuild(out var package))
|
||||
{
|
||||
packages.Add(package);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
using System.Security;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Rust.Internal;
|
||||
|
||||
internal readonly struct RustFileCacheKey : IEquatable<RustFileCacheKey>
|
||||
{
|
||||
private readonly string _normalizedPath;
|
||||
private readonly long _length;
|
||||
private readonly long _lastWriteTicks;
|
||||
|
||||
private RustFileCacheKey(string normalizedPath, long length, long lastWriteTicks)
|
||||
{
|
||||
_normalizedPath = normalizedPath;
|
||||
_length = length;
|
||||
_lastWriteTicks = lastWriteTicks;
|
||||
}
|
||||
|
||||
public static bool TryCreate(string path, out RustFileCacheKey key)
|
||||
{
|
||||
key = default;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(path))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var info = new FileInfo(path);
|
||||
if (!info.Exists)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var normalizedPath = OperatingSystem.IsWindows()
|
||||
? info.FullName.ToLowerInvariant()
|
||||
: info.FullName;
|
||||
|
||||
key = new RustFileCacheKey(normalizedPath, info.Length, info.LastWriteTimeUtc.Ticks);
|
||||
return true;
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
catch (UnauthorizedAccessException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
catch (SecurityException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
catch (NotSupportedException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public bool Equals(RustFileCacheKey other)
|
||||
=> _length == other._length
|
||||
&& _lastWriteTicks == other._lastWriteTicks
|
||||
&& string.Equals(_normalizedPath, other._normalizedPath, StringComparison.Ordinal);
|
||||
|
||||
public override bool Equals(object? obj)
|
||||
=> obj is RustFileCacheKey other && Equals(other);
|
||||
|
||||
public override int GetHashCode()
|
||||
=> HashCode.Combine(_normalizedPath, _length, _lastWriteTicks);
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Security;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Rust.Internal;
|
||||
|
||||
internal static class RustFileHashCache
|
||||
{
|
||||
private static readonly ConcurrentDictionary<RustFileCacheKey, string> Sha256Cache = new();
|
||||
|
||||
public static bool TryGetSha256(string path, out string? sha256)
|
||||
{
|
||||
sha256 = null;
|
||||
|
||||
if (!RustFileCacheKey.TryCreate(path, out var key))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
sha256 = Sha256Cache.GetOrAdd(key, static (_, state) => ComputeSha256(state), path);
|
||||
return !string.IsNullOrEmpty(sha256);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
catch (UnauthorizedAccessException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
catch (SecurityException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static string ComputeSha256(string path)
|
||||
{
|
||||
using var stream = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
using var sha = System.Security.Cryptography.SHA256.Create();
|
||||
var hash = sha.ComputeHash(stream);
|
||||
return Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Rust.Internal;
|
||||
@@ -13,6 +14,7 @@ internal static class RustFingerprintScanner
|
||||
};
|
||||
|
||||
private static readonly string FingerprintSegment = $"{Path.DirectorySeparatorChar}.fingerprint{Path.DirectorySeparatorChar}";
|
||||
private static readonly ConcurrentDictionary<RustFileCacheKey, RustFingerprintRecord?> Cache = new();
|
||||
|
||||
public static IReadOnlyList<RustFingerprintRecord> Scan(string rootPath, CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -31,7 +33,17 @@ internal static class RustFingerprintScanner
|
||||
continue;
|
||||
}
|
||||
|
||||
if (TryParse(path, out var record))
|
||||
if (!RustFileCacheKey.TryCreate(path, out var key))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var record = Cache.GetOrAdd(
|
||||
key,
|
||||
static (_, state) => ParseFingerprint(state),
|
||||
path);
|
||||
|
||||
if (record is not null)
|
||||
{
|
||||
results.Add(record);
|
||||
}
|
||||
@@ -40,10 +52,8 @@ internal static class RustFingerprintScanner
|
||||
return results;
|
||||
}
|
||||
|
||||
private static bool TryParse(string path, out RustFingerprintRecord record)
|
||||
private static RustFingerprintRecord? ParseFingerprint(string path)
|
||||
{
|
||||
record = default!;
|
||||
|
||||
try
|
||||
{
|
||||
using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
@@ -57,33 +67,31 @@ internal static class RustFingerprintScanner
|
||||
var (name, version, source) = ParseIdentity(pkgId, path);
|
||||
if (string.IsNullOrWhiteSpace(name))
|
||||
{
|
||||
return false;
|
||||
return null;
|
||||
}
|
||||
|
||||
var profile = TryGetString(root, "profile");
|
||||
var targetKind = TryGetKind(root);
|
||||
|
||||
record = new RustFingerprintRecord(
|
||||
return new RustFingerprintRecord(
|
||||
Name: name!,
|
||||
Version: version,
|
||||
Source: source,
|
||||
TargetKind: targetKind,
|
||||
Profile: profile,
|
||||
AbsolutePath: path);
|
||||
|
||||
return true;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return false;
|
||||
return null;
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
return false;
|
||||
return null;
|
||||
}
|
||||
catch (UnauthorizedAccessException)
|
||||
{
|
||||
return false;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,298 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Immutable;
|
||||
using System.Security;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Rust.Internal;
|
||||
|
||||
internal static class RustLicenseScanner
|
||||
{
|
||||
private static readonly ConcurrentDictionary<string, RustLicenseIndex> IndexCache = new(StringComparer.Ordinal);
|
||||
|
||||
public static RustLicenseIndex GetOrCreate(string rootPath, CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(rootPath) || !Directory.Exists(rootPath))
|
||||
{
|
||||
return RustLicenseIndex.Empty;
|
||||
}
|
||||
|
||||
var normalizedRoot = NormalizeRoot(rootPath);
|
||||
return IndexCache.GetOrAdd(
|
||||
normalizedRoot,
|
||||
static (_, state) => BuildIndex(state.RootPath, state.CancellationToken),
|
||||
(RootPath: rootPath, CancellationToken: cancellationToken));
|
||||
}
|
||||
|
||||
private static RustLicenseIndex BuildIndex(string rootPath, CancellationToken cancellationToken)
|
||||
{
|
||||
var byName = new Dictionary<string, List<RustLicenseInfo>>(StringComparer.Ordinal);
|
||||
var enumeration = new EnumerationOptions
|
||||
{
|
||||
MatchCasing = MatchCasing.CaseSensitive,
|
||||
IgnoreInaccessible = true,
|
||||
RecurseSubdirectories = true,
|
||||
AttributesToSkip = FileAttributes.Device | FileAttributes.ReparsePoint,
|
||||
};
|
||||
|
||||
foreach (var cargoTomlPath in Directory.EnumerateFiles(rootPath, "Cargo.toml", enumeration))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (IsUnderTargetDirectory(cargoTomlPath))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!TryParseCargoToml(rootPath, cargoTomlPath, out var info))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var normalizedName = RustCrateBuilder.NormalizeName(info.Name);
|
||||
if (!byName.TryGetValue(normalizedName, out var entries))
|
||||
{
|
||||
entries = new List<RustLicenseInfo>();
|
||||
byName[normalizedName] = entries;
|
||||
}
|
||||
|
||||
entries.Add(info);
|
||||
}
|
||||
|
||||
foreach (var entry in byName.Values)
|
||||
{
|
||||
entry.Sort(static (left, right) =>
|
||||
{
|
||||
var versionCompare = string.Compare(left.Version, right.Version, StringComparison.OrdinalIgnoreCase);
|
||||
if (versionCompare != 0)
|
||||
{
|
||||
return versionCompare;
|
||||
}
|
||||
|
||||
return string.Compare(left.CargoTomlRelativePath, right.CargoTomlRelativePath, StringComparison.Ordinal);
|
||||
});
|
||||
}
|
||||
|
||||
return new RustLicenseIndex(byName);
|
||||
}
|
||||
|
||||
private static bool TryParseCargoToml(string rootPath, string cargoTomlPath, out RustLicenseInfo info)
|
||||
{
|
||||
info = default!;
|
||||
|
||||
try
|
||||
{
|
||||
using var stream = new FileStream(cargoTomlPath, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
using var reader = new StreamReader(stream, leaveOpen: false);
|
||||
|
||||
string? name = null;
|
||||
string? version = null;
|
||||
string? licenseExpression = null;
|
||||
string? licenseFile = null;
|
||||
var inPackageSection = false;
|
||||
|
||||
while (reader.ReadLine() is { } line)
|
||||
{
|
||||
line = StripComment(line).Trim();
|
||||
if (line.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (line.StartsWith("[", StringComparison.Ordinal))
|
||||
{
|
||||
inPackageSection = string.Equals(line, "[package]", StringComparison.OrdinalIgnoreCase);
|
||||
if (!inPackageSection && line.StartsWith("[dependency", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
// Exiting package section.
|
||||
break;
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!inPackageSection)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (TryParseStringAssignment(line, "name", out var parsedName))
|
||||
{
|
||||
name ??= parsedName;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (TryParseStringAssignment(line, "version", out var parsedVersion))
|
||||
{
|
||||
version ??= parsedVersion;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (TryParseStringAssignment(line, "license", out var parsedLicense))
|
||||
{
|
||||
licenseExpression ??= parsedLicense;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (TryParseStringAssignment(line, "license-file", out var parsedLicenseFile))
|
||||
{
|
||||
licenseFile ??= parsedLicenseFile;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(name))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var expressions = ImmutableArray<string>.Empty;
|
||||
if (!string.IsNullOrWhiteSpace(licenseExpression))
|
||||
{
|
||||
expressions = ImmutableArray.Create(licenseExpression!);
|
||||
}
|
||||
|
||||
var files = ImmutableArray<RustLicenseFileReference>.Empty;
|
||||
if (!string.IsNullOrWhiteSpace(licenseFile))
|
||||
{
|
||||
var directory = Path.GetDirectoryName(cargoTomlPath) ?? string.Empty;
|
||||
var absolute = Path.GetFullPath(Path.Combine(directory, licenseFile!));
|
||||
if (File.Exists(absolute))
|
||||
{
|
||||
var relative = NormalizeRelativePath(rootPath, absolute);
|
||||
if (RustFileHashCache.TryGetSha256(absolute, out var sha256))
|
||||
{
|
||||
files = ImmutableArray.Create(new RustLicenseFileReference(relative, sha256));
|
||||
}
|
||||
else
|
||||
{
|
||||
files = ImmutableArray.Create(new RustLicenseFileReference(relative, null));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var cargoRelative = NormalizeRelativePath(rootPath, cargoTomlPath);
|
||||
|
||||
info = new RustLicenseInfo(
|
||||
name!.Trim(),
|
||||
string.IsNullOrWhiteSpace(version) ? null : version!.Trim(),
|
||||
expressions,
|
||||
files,
|
||||
cargoRelative);
|
||||
|
||||
return true;
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
catch (UnauthorizedAccessException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
catch (SecurityException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static string NormalizeRoot(string rootPath)
|
||||
{
|
||||
var full = Path.GetFullPath(rootPath);
|
||||
return OperatingSystem.IsWindows()
|
||||
? full.ToLowerInvariant()
|
||||
: full;
|
||||
}
|
||||
|
||||
private static bool TryParseStringAssignment(string line, string key, out string? value)
|
||||
{
|
||||
value = null;
|
||||
|
||||
if (!line.StartsWith(key, StringComparison.Ordinal))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var remaining = line[key.Length..].TrimStart();
|
||||
if (remaining.Length == 0 || remaining[0] != '=')
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
remaining = remaining[1..].TrimStart();
|
||||
if (remaining.Length < 2 || remaining[0] != '"' || remaining[^1] != '"')
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
value = remaining[1..^1];
|
||||
return true;
|
||||
}
|
||||
|
||||
private static string StripComment(string line)
|
||||
{
|
||||
var index = line.IndexOf('#');
|
||||
return index < 0 ? line : line[..index];
|
||||
}
|
||||
|
||||
private static bool IsUnderTargetDirectory(string path)
|
||||
{
|
||||
var segment = $"{Path.DirectorySeparatorChar}target{Path.DirectorySeparatorChar}";
|
||||
return path.Contains(segment, OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
private static string NormalizeRelativePath(string rootPath, string absolutePath)
|
||||
{
|
||||
var relative = Path.GetRelativePath(rootPath, absolutePath);
|
||||
if (string.IsNullOrWhiteSpace(relative) || relative == ".")
|
||||
{
|
||||
return ".";
|
||||
}
|
||||
|
||||
return relative.Replace('\\', '/');
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class RustLicenseIndex
|
||||
{
|
||||
private readonly Dictionary<string, List<RustLicenseInfo>> _byName;
|
||||
|
||||
public static readonly RustLicenseIndex Empty = new(new Dictionary<string, List<RustLicenseInfo>>(StringComparer.Ordinal));
|
||||
|
||||
public RustLicenseIndex(Dictionary<string, List<RustLicenseInfo>> byName)
|
||||
{
|
||||
_byName = byName ?? throw new ArgumentNullException(nameof(byName));
|
||||
}
|
||||
|
||||
public RustLicenseInfo? Find(string crateName, string? version)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(crateName))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var normalized = RustCrateBuilder.NormalizeName(crateName);
|
||||
if (!_byName.TryGetValue(normalized, out var list) || list.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(version))
|
||||
{
|
||||
var match = list.FirstOrDefault(entry => string.Equals(entry.Version, version, StringComparison.OrdinalIgnoreCase));
|
||||
if (match is not null)
|
||||
{
|
||||
return match;
|
||||
}
|
||||
}
|
||||
|
||||
return list[0];
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record RustLicenseInfo(
|
||||
string Name,
|
||||
string? Version,
|
||||
ImmutableArray<string> Expressions,
|
||||
ImmutableArray<RustLicenseFileReference> Files,
|
||||
string CargoTomlRelativePath);
|
||||
|
||||
internal sealed record RustLicenseFileReference(string RelativePath, string? Sha256);
|
||||
@@ -5,6 +5,6 @@
|
||||
| 1 | SCANNER-ANALYZERS-LANG-10-306A | DONE (2025-10-22) | SCANNER-ANALYZERS-LANG-10-307 | Parse Cargo metadata (`Cargo.lock`, `.fingerprint`, `.metadata`) and map crates to components with evidence. | Fixtures confirm crate attribution ≥85 % coverage; metadata normalized; evidence includes path + hash. |
|
||||
| 2 | SCANNER-ANALYZERS-LANG-10-306B | DONE (2025-10-22) | SCANNER-ANALYZERS-LANG-10-306A | Implement heuristic classifier using ELF section names, symbol mangling, and `.comment` data for stripped binaries. | Heuristic output flagged as `heuristic`; regression tests ensure no false “observed” classifications. |
|
||||
| 3 | SCANNER-ANALYZERS-LANG-10-306C | DONE (2025-10-22) | SCANNER-ANALYZERS-LANG-10-306B | Integrate binary hash fallback (`bin:{sha256}`) and tie into shared quiet provenance helpers. | Fallback path deterministic; shared helpers reused; tests verify consistent hashing. |
|
||||
| 4 | SCANNER-ANALYZERS-LANG-10-307R | DOING (2025-10-23) | SCANNER-ANALYZERS-LANG-10-306C | Finalize shared helper usage (license, usage flags) and concurrency-safe caches. | Analyzer uses shared utilities; concurrency tests pass; no race conditions. |
|
||||
| 4 | SCANNER-ANALYZERS-LANG-10-307R | DONE (2025-10-29) | SCANNER-ANALYZERS-LANG-10-306C | Finalize shared helper usage (license, usage flags) and concurrency-safe caches. | Analyzer uses shared utilities; concurrency tests pass; no race conditions. |
|
||||
| 5 | SCANNER-ANALYZERS-LANG-10-308R | TODO | SCANNER-ANALYZERS-LANG-10-307R | Determinism fixtures + performance benchmarks; compare against competitor heuristic coverage. | Fixtures `Fixtures/lang/rust/` committed; determinism guard; benchmark shows ≥15 % better coverage vs competitor. |
|
||||
| 6 | SCANNER-ANALYZERS-LANG-10-309R | TODO | SCANNER-ANALYZERS-LANG-10-308R | Package plug-in manifest + Offline Kit documentation; ensure Worker integration. | Manifest copied; Worker loads analyzer; Offline Kit doc updated. |
|
||||
|
||||
Reference in New Issue
Block a user