Add unit tests for SBOM ingestion and transformation
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Implement `SbomIngestServiceCollectionExtensionsTests` to verify the SBOM ingestion pipeline exports snapshots correctly.
- Create `SbomIngestTransformerTests` to ensure the transformation produces expected nodes and edges, including deduplication of license nodes and normalization of timestamps.
- Add `SbomSnapshotExporterTests` to test the export functionality for manifest, adjacency, nodes, and edges.
- Introduce `VexOverlayTransformerTests` to validate the transformation of VEX nodes and edges.
- Set up project file for the test project with necessary dependencies and configurations.
- Include JSON fixture files for testing purposes.
This commit is contained in:
master
2025-11-04 07:49:39 +02:00
parent f72c5c513a
commit 2eb6852d34
491 changed files with 39445 additions and 3917 deletions

View File

@@ -0,0 +1,6 @@
namespace StellaOps.Scanner.Analyzers.Lang.Ruby.Internal;
internal sealed record RubyCapabilities(
bool UsesExec,
bool UsesNetwork,
bool UsesSerialization);

View File

@@ -38,7 +38,7 @@ internal sealed class RubyPackage
public string ComponentKey => $"purl::{Purl}";
public IReadOnlyCollection<KeyValuePair<string, string?>> CreateMetadata(RubyCapabilities? capabilities)
public IReadOnlyCollection<KeyValuePair<string, string?>> CreateMetadata(RubyCapabilities? capabilities = null)
{
var metadata = new List<KeyValuePair<string, string?>>
{

View File

@@ -10,17 +10,17 @@ internal static class RubyPackageCollector
if (!lockData.IsEmpty)
{
var relativeLockPath = lockData.LockFilePath is null
? Gemfile.lock
? "Gemfile.lock"
: context.GetRelativePath(lockData.LockFilePath);
if (string.IsNullOrWhiteSpace(relativeLockPath))
{
relativeLockPath = Gemfile.lock;
relativeLockPath = "Gemfile.lock";
}
foreach (var entry in lockData.Entries)
{
var key = ${entry.Name}@{entry.Version};
var key = $"{entry.Name}@{entry.Version}";
if (!seen.Add(key))
{
continue;
@@ -37,27 +37,27 @@ internal static class RubyPackageCollector
private static void CollectVendorCachePackages(LanguageAnalyzerContext context, List<RubyPackage> packages, HashSet<string> seen)
{
var vendorCache = Path.Combine(context.RootPath, vendor, cache);
var vendorCache = Path.Combine(context.RootPath, "vendor", "cache");
if (!Directory.Exists(vendorCache))
{
return;
}
foreach (var gemPath in Directory.EnumerateFiles(vendorCache, *.gem, SearchOption.AllDirectories))
foreach (var gemPath in Directory.EnumerateFiles(vendorCache, "*.gem", SearchOption.AllDirectories))
{
if (!TryParseGemArchive(gemPath, out var name, out var version, out var platform))
{
continue;
}
var key = ${name}@{version};
var key = $"{name}@{version}";
if (!seen.Add(key))
{
continue;
}
var locator = context.GetRelativePath(gemPath);
packages.Add(RubyPackage.FromVendor(name, version, source: vendor-cache, platform, locator));
packages.Add(RubyPackage.FromVendor(name, version, source: "vendor-cache", platform, locator));
}
}

View File

@@ -6,7 +6,7 @@ namespace StellaOps.Scanner.Analyzers.Lang.Ruby;
public sealed class RubyAnalyzerPlugin : ILanguageAnalyzerPlugin
{
public string Name => StellaOps.Scanner.Analyzers.Lang.Ruby;
public string Name => "ruby";
public bool IsAvailable(IServiceProvider services) => services is not null;

View File

@@ -0,0 +1,108 @@
namespace StellaOps.Scanner.Analyzers.Lang.Internal;
using System.Text;
using Microsoft.Extensions.Logging;
using StellaOps.Scanner.Surface.FS;
public sealed class LanguageAnalyzerSurfaceCache
{
private const string CacheNamespace = "scanner/lang/analyzers";
private static readonly JsonSerializerOptions JsonOptions = LanguageAnalyzerJson.CreateDefault(indent: false);
private readonly ISurfaceCache _cache;
private readonly string _tenant;
public LanguageAnalyzerSurfaceCache(ISurfaceCache cache, string tenant)
{
_cache = cache ?? throw new ArgumentNullException(nameof(cache));
_tenant = string.IsNullOrWhiteSpace(tenant) ? "default" : tenant.Trim();
}
public async ValueTask<LanguageAnalyzerResult> GetOrCreateAsync(
ILogger logger,
string analyzerId,
string fingerprint,
Func<CancellationToken, ValueTask<LanguageAnalyzerResult>> factory,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(logger);
ArgumentNullException.ThrowIfNull(factory);
if (string.IsNullOrWhiteSpace(analyzerId))
{
throw new ArgumentException("Analyzer identifier is required.", nameof(analyzerId));
}
var contentKey = $"{fingerprint}:{analyzerId}";
var key = new SurfaceCacheKey(CacheNamespace, _tenant, contentKey);
var cacheHit = true;
LanguageAnalyzerResult result;
try
{
result = await _cache.GetOrCreateAsync(
key,
async token =>
{
cacheHit = false;
return await factory(token).ConfigureAwait(false);
},
Serialize,
Deserialize,
cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or JsonException)
{
cacheHit = false;
logger.LogWarning(
ex,
"Surface cache lookup failed for analyzer {AnalyzerId} (tenant {Tenant}, fingerprint {Fingerprint}); running analyzer without cache.",
analyzerId,
_tenant,
fingerprint);
result = await factory(cancellationToken).ConfigureAwait(false);
return result;
}
if (cacheHit)
{
logger.LogDebug(
"Surface cache hit for analyzer {AnalyzerId} (tenant {Tenant}, fingerprint {Fingerprint}).",
analyzerId,
_tenant,
fingerprint);
}
else
{
logger.LogDebug(
"Surface cache miss for analyzer {AnalyzerId} (tenant {Tenant}, fingerprint {Fingerprint}); stored result.",
analyzerId,
_tenant,
fingerprint);
}
return result;
}
private static ReadOnlyMemory<byte> Serialize(LanguageAnalyzerResult result)
{
ArgumentNullException.ThrowIfNull(result);
var json = result.ToJson(indent: false);
var bytes = Encoding.UTF8.GetBytes(json);
return bytes;
}
private static LanguageAnalyzerResult Deserialize(ReadOnlyMemory<byte> payload)
{
if (payload.IsEmpty)
{
return LanguageAnalyzerResult.FromSnapshots(Array.Empty<LanguageComponentSnapshot>());
}
var snapshots = JsonSerializer.Deserialize<IReadOnlyList<LanguageComponentSnapshot>>(payload.Span, JsonOptions)
?? Array.Empty<LanguageComponentSnapshot>();
return LanguageAnalyzerResult.FromSnapshots(snapshots);
}
}

View File

@@ -0,0 +1,112 @@
namespace StellaOps.Scanner.Analyzers.Lang.Internal;
using System.Buffers;
using System.Security.Cryptography;
using System.Text;
public static class LanguageWorkspaceFingerprint
{
private static readonly EnumerationOptions Enumeration = new()
{
RecurseSubdirectories = true,
IgnoreInaccessible = true,
AttributesToSkip = FileAttributes.ReparsePoint,
ReturnSpecialDirectories = false
};
public static string Compute(string rootPath, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(rootPath))
{
throw new ArgumentException("Workspace root path is required.", nameof(rootPath));
}
var fullRoot = Path.GetFullPath(rootPath);
if (!Directory.Exists(fullRoot))
{
return HashPrimitive(fullRoot);
}
var entries = Directory
.EnumerateFileSystemEntries(fullRoot, "*", Enumeration)
.Select(Path.GetFullPath)
.ToList();
entries.Sort(StringComparer.Ordinal);
using var aggregate = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
Append(aggregate, $"ROOT|{NormalizeRelative(fullRoot, fullRoot)}");
foreach (var entry in entries)
{
cancellationToken.ThrowIfCancellationRequested();
if (Directory.Exists(entry))
{
Append(aggregate, $"D|{NormalizeRelative(fullRoot, entry)}");
continue;
}
var relative = NormalizeRelative(fullRoot, entry);
try
{
var info = new FileInfo(entry);
var timestamp = new DateTimeOffset(info.LastWriteTimeUtc).ToUnixTimeMilliseconds();
Append(aggregate, $"F|{relative}|{info.Length}|{timestamp}");
Append(aggregate, $"H|{ComputeFileHash(entry, cancellationToken)}");
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Append(aggregate, $"E|{relative}|{ex.GetType().Name}");
}
}
return Convert.ToHexString(aggregate.GetHashAndReset()).ToLowerInvariant();
}
private static string ComputeFileHash(string path, CancellationToken cancellationToken)
{
using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read);
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
var buffer = ArrayPool<byte>.Shared.Rent(64 * 1024);
try
{
int read;
while ((read = stream.Read(buffer, 0, buffer.Length)) > 0)
{
cancellationToken.ThrowIfCancellationRequested();
hash.AppendData(buffer, 0, read);
}
}
finally
{
ArrayPool<byte>.Shared.Return(buffer);
}
return Convert.ToHexString(hash.GetHashAndReset()).ToLowerInvariant();
}
private static void Append(IncrementalHash hash, string value)
{
var bytes = Encoding.UTF8.GetBytes(value + "\n");
hash.AppendData(bytes);
}
private static string NormalizeRelative(string root, string path)
{
if (string.Equals(root, path, StringComparison.Ordinal))
{
return ".";
}
var relative = Path.GetRelativePath(root, path);
return relative.Replace('\\', '/');
}
private static string HashPrimitive(string value)
{
var bytes = Encoding.UTF8.GetBytes(value);
return Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant();
}
}

View File

@@ -1,9 +1,14 @@
namespace StellaOps.Scanner.Analyzers.Lang;
public sealed class LanguageAnalyzerContext
{
public LanguageAnalyzerContext(string rootPath, TimeProvider timeProvider, LanguageUsageHints? usageHints = null, IServiceProvider? services = null)
{
using StellaOps.Scanner.Surface.Env;
using StellaOps.Scanner.Surface.Secrets;
namespace StellaOps.Scanner.Analyzers.Lang;
public sealed class LanguageAnalyzerContext
{
private const string SecretsComponentName = "ScannerWorkerLanguageAnalyzers";
public LanguageAnalyzerContext(string rootPath, TimeProvider timeProvider, LanguageUsageHints? usageHints = null, IServiceProvider? services = null)
{
if (string.IsNullOrWhiteSpace(rootPath))
{
throw new ArgumentException("Root path is required", nameof(rootPath));
@@ -15,24 +20,27 @@ public sealed class LanguageAnalyzerContext
throw new DirectoryNotFoundException($"Root path '{RootPath}' does not exist.");
}
TimeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
UsageHints = usageHints ?? LanguageUsageHints.Empty;
Services = services;
}
public string RootPath { get; }
public TimeProvider TimeProvider { get; }
public LanguageUsageHints UsageHints { get; }
public IServiceProvider? Services { get; }
public bool TryGetService<T>([NotNullWhen(true)] out T? service) where T : class
{
if (Services is null)
{
service = null;
TimeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
UsageHints = usageHints ?? LanguageUsageHints.Empty;
Services = services;
Secrets = CreateSecrets(services);
}
public string RootPath { get; }
public TimeProvider TimeProvider { get; }
public LanguageUsageHints UsageHints { get; }
public IServiceProvider? Services { get; }
public LanguageAnalyzerSecrets Secrets { get; }
public bool TryGetService<T>([NotNullWhen(true)] out T? service) where T : class
{
if (Services is null)
{
service = null;
return false;
}
@@ -48,11 +56,11 @@ public sealed class LanguageAnalyzerContext
}
var relativeString = new string(relative);
var combined = Path.Combine(RootPath, relativeString);
return Path.GetFullPath(combined);
}
public string GetRelativePath(string absolutePath)
var combined = Path.Combine(RootPath, relativeString);
return Path.GetFullPath(combined);
}
public string GetRelativePath(string absolutePath)
{
if (string.IsNullOrWhiteSpace(absolutePath))
{
@@ -62,6 +70,23 @@ public sealed class LanguageAnalyzerContext
var relative = Path.GetRelativePath(RootPath, absolutePath);
return OperatingSystem.IsWindows()
? relative.Replace('\\', '/')
: relative;
}
}
: relative;
}
private static LanguageAnalyzerSecrets CreateSecrets(IServiceProvider? services)
{
if (services is null)
{
return LanguageAnalyzerSecrets.Empty;
}
var environment = services.GetService(typeof(ISurfaceEnvironment)) as ISurfaceEnvironment;
if (environment is null)
{
return LanguageAnalyzerSecrets.Empty;
}
var provider = services.GetService(typeof(ISurfaceSecretProvider)) as ISurfaceSecretProvider;
return new LanguageAnalyzerSecrets(provider, environment.Settings.Tenant, SecretsComponentName);
}
}

View File

@@ -22,14 +22,25 @@ public sealed class LanguageAnalyzerResult
public LayerComponentFragment ToLayerFragment(string analyzerId, string? layerDigest = null)
=> LanguageComponentMapper.ToLayerFragment(analyzerId, _components, layerDigest);
public IReadOnlyList<LanguageComponentSnapshot> ToSnapshots()
=> _components.Select(static component => component.ToSnapshot()).ToImmutableArray();
public string ToJson(bool indent = true)
{
var snapshots = ToSnapshots();
var options = Internal.LanguageAnalyzerJson.CreateDefault(indent);
return JsonSerializer.Serialize(snapshots, options);
public IReadOnlyList<LanguageComponentSnapshot> ToSnapshots()
=> _components.Select(static component => component.ToSnapshot()).ToImmutableArray();
internal static LanguageAnalyzerResult FromSnapshots(IEnumerable<LanguageComponentSnapshot> snapshots)
{
if (snapshots is null)
{
throw new ArgumentNullException(nameof(snapshots));
}
var records = snapshots.Select(LanguageComponentRecord.FromSnapshot).ToArray();
return new LanguageAnalyzerResult(records);
}
public string ToJson(bool indent = true)
{
var snapshots = ToSnapshots();
var options = Internal.LanguageAnalyzerJson.CreateDefault(indent);
return JsonSerializer.Serialize(snapshots, options);
}
}

View File

@@ -0,0 +1,80 @@
namespace StellaOps.Scanner.Analyzers.Lang;
using StellaOps.Scanner.Surface.Env;
using StellaOps.Scanner.Surface.Secrets;
public sealed class LanguageAnalyzerSecrets
{
private const string DefaultComponent = "ScannerWorkerLanguageAnalyzers";
public static LanguageAnalyzerSecrets Empty { get; } = new(null, "default", DefaultComponent);
private readonly ISurfaceSecretProvider? _provider;
private readonly string _tenant;
private readonly string _component;
internal LanguageAnalyzerSecrets(ISurfaceSecretProvider? provider, string tenant, string component)
{
_provider = provider;
_tenant = string.IsNullOrWhiteSpace(tenant) ? "default" : tenant.Trim();
_component = NormalizeComponentName(component);
}
public bool IsAvailable => _provider is not null;
public async ValueTask<SurfaceSecretHandle> GetAsync(
string secretType,
string? name = null,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(secretType))
{
throw new ArgumentException("Secret type is required.", nameof(secretType));
}
if (_provider is null)
{
throw new SurfaceSecretNotFoundException(CreateRequest(secretType, name));
}
return await _provider.GetAsync(CreateRequest(secretType, name), cancellationToken).ConfigureAwait(false);
}
public async ValueTask<SurfaceSecretHandle?> TryGetAsync(
string secretType,
string? name = null,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(secretType) || _provider is null)
{
return null;
}
try
{
return await _provider.GetAsync(CreateRequest(secretType, name), cancellationToken).ConfigureAwait(false);
}
catch (SurfaceSecretNotFoundException)
{
return null;
}
}
private SurfaceSecretRequest CreateRequest(string secretType, string? name)
=> new(_tenant, _component, secretType, name);
private static string NormalizeComponentName(string component)
{
if (string.IsNullOrWhiteSpace(component))
{
return DefaultComponent;
}
var normalized = component.Trim()
.Replace(".", string.Empty, StringComparison.Ordinal)
.Replace("-", string.Empty, StringComparison.Ordinal)
.Replace(" ", string.Empty, StringComparison.Ordinal);
return string.IsNullOrWhiteSpace(normalized) ? DefaultComponent : normalized;
}
}

View File

@@ -109,23 +109,71 @@ public sealed class LanguageComponentRecord
throw new ArgumentException("Component key is required", nameof(componentKey));
}
return new LanguageComponentRecord(
analyzerId,
componentKey.Trim(),
purl,
name,
version,
type,
metadata ?? Array.Empty<KeyValuePair<string, string?>>(),
evidence ?? Array.Empty<LanguageComponentEvidence>(),
usedByEntrypoint);
}
internal void Merge(LanguageComponentRecord other)
{
ArgumentNullException.ThrowIfNull(other);
if (!ComponentKey.Equals(other.ComponentKey, StringComparison.Ordinal))
return new LanguageComponentRecord(
analyzerId,
componentKey.Trim(),
purl,
name,
version,
type,
metadata ?? Array.Empty<KeyValuePair<string, string?>>(),
evidence ?? Array.Empty<LanguageComponentEvidence>(),
usedByEntrypoint);
}
internal static LanguageComponentRecord FromSnapshot(LanguageComponentSnapshot snapshot)
{
if (snapshot is null)
{
throw new ArgumentNullException(nameof(snapshot));
}
var metadata = snapshot.Metadata is null
? Array.Empty<KeyValuePair<string, string?>>()
: snapshot.Metadata.Select(static entry => new KeyValuePair<string, string?>(entry.Key, entry.Value));
var evidence = snapshot.Evidence is null or { Count: 0 }
? Array.Empty<LanguageComponentEvidence>()
: snapshot.Evidence
.Where(static item => item is not null)
.Select(static item => new LanguageComponentEvidence(
item.Kind,
item.Source ?? string.Empty,
item.Locator ?? string.Empty,
item.Value,
item.Sha256))
.ToArray();
if (!string.IsNullOrWhiteSpace(snapshot.Purl))
{
return FromPurl(
snapshot.AnalyzerId,
snapshot.Purl!,
snapshot.Name,
snapshot.Version,
snapshot.Type,
metadata,
evidence,
snapshot.UsedByEntrypoint);
}
return FromExplicitKey(
snapshot.AnalyzerId,
snapshot.ComponentKey,
snapshot.Purl,
snapshot.Name,
snapshot.Version,
snapshot.Type,
metadata,
evidence,
snapshot.UsedByEntrypoint);
}
internal void Merge(LanguageComponentRecord other)
{
ArgumentNullException.ThrowIfNull(other);
if (!ComponentKey.Equals(other.ComponentKey, StringComparison.Ordinal))
{
throw new InvalidOperationException($"Cannot merge component '{ComponentKey}' with '{other.ComponentKey}'.");
}

View File

@@ -18,6 +18,9 @@
<ItemGroup>
<ProjectReference Include="../StellaOps.Scanner.Core/StellaOps.Scanner.Core.csproj" />
<ProjectReference Include="../../../__Libraries/StellaOps.Plugin/StellaOps.Plugin.csproj" />
<ProjectReference Include="../StellaOps.Scanner.Surface.Env/StellaOps.Scanner.Surface.Env.csproj" />
<ProjectReference Include="../StellaOps.Scanner.Surface.FS/StellaOps.Scanner.Surface.FS.csproj" />
<ProjectReference Include="../StellaOps.Scanner.Surface.Secrets/StellaOps.Scanner.Surface.Secrets.csproj" />
</ItemGroup>
</Project>
</Project>

View File

@@ -2,6 +2,6 @@
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|----|--------|----------|------------|-------------|---------------|
| LANG-SURFACE-01 | TODO | Language Analyzer Guild | SURFACE-VAL-02, SURFACE-FS-02 | Invoke Surface.Validation checks (env/cache/secrets) before analyzer execution to ensure consistent prerequisites. | Validation pipeline integrated; regression tests updated; failures bubble with actionable errors. |
| LANG-SURFACE-02 | TODO | Language Analyzer Guild | SURFACE-FS-02 | Consume Surface.FS APIs for layer/source caching (instead of bespoke caches) to improve determinism. | Analyzer outputs match baseline; performance benchmarks recorded; docs updated. |
| LANG-SURFACE-03 | TODO | Language Analyzer Guild | SURFACE-SECRETS-02 | Replace direct secret/env reads with Surface.Secrets references when fetching package feeds or registry creds. | Analyzer uses shared provider; tests cover rotation/failure; config docs updated. |
| LANG-SURFACE-01 | DONE | Language Analyzer Guild | SURFACE-VAL-02, SURFACE-FS-02 | Invoke Surface.Validation checks (env/cache/secrets) before analyzer execution to ensure consistent prerequisites. | Validation pipeline integrated; regression tests updated; failures bubble with actionable errors. |
| LANG-SURFACE-02 | DONE | Language Analyzer Guild | SURFACE-FS-02 | Consume Surface.FS APIs for layer/source caching (instead of bespoke caches) to improve determinism. | Analyzer outputs match baseline; performance benchmarks recorded; docs updated. |
| LANG-SURFACE-03 | DONE | Language Analyzer Guild | SURFACE-SECRETS-02 | Replace direct secret/env reads with Surface.Secrets references when fetching package feeds or registry creds. | Analyzer uses shared provider; tests cover rotation/failure; config docs updated. |