feat: Implement Runtime Facts ingestion service and NDJSON reader
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Added RuntimeFactsNdjsonReader for reading NDJSON formatted runtime facts.
- Introduced IRuntimeFactsIngestionService interface and its implementation.
- Enhanced Program.cs to register new services and endpoints for runtime facts.
- Updated CallgraphIngestionService to include CAS URI in stored artifacts.
- Created RuntimeFactsValidationException for validation errors during ingestion.
- Added tests for RuntimeFactsIngestionService and RuntimeFactsNdjsonReader.
- Implemented SignalsSealedModeMonitor for compliance checks in sealed mode.
- Updated project dependencies for testing utilities.
This commit is contained in:
master
2025-11-10 07:56:15 +02:00
parent 9df52d84aa
commit 69c59defdc
132 changed files with 19718 additions and 9334 deletions

View File

@@ -0,0 +1,184 @@
using System.Collections.Generic;
using System.Linq;
using System.Xml.Linq;
namespace StellaOps.Scanner.Analyzers.Lang.Java.Internal;
internal static class JavaLockFileCollector
{
private static readonly string[] GradleLockPatterns = { "gradle.lockfile" };
public static async Task<JavaLockData> LoadAsync(LanguageAnalyzerContext context, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
var entries = new Dictionary<string, JavaLockEntry>(StringComparer.OrdinalIgnoreCase);
var root = context.RootPath;
foreach (var pattern in GradleLockPatterns)
{
var lockPath = Path.Combine(root, pattern);
if (File.Exists(lockPath))
{
await ParseGradleLockFileAsync(context, lockPath, entries, cancellationToken).ConfigureAwait(false);
}
}
var dependencyLocksDir = Path.Combine(root, "gradle", "dependency-locks");
if (Directory.Exists(dependencyLocksDir))
{
foreach (var file in Directory.EnumerateFiles(dependencyLocksDir, "*.lockfile", SearchOption.AllDirectories))
{
await ParseGradleLockFileAsync(context, file, entries, cancellationToken).ConfigureAwait(false);
}
}
foreach (var pomPath in Directory.EnumerateFiles(root, "pom.xml", SearchOption.AllDirectories))
{
await ParsePomAsync(context, pomPath, entries, cancellationToken).ConfigureAwait(false);
}
return entries.Count == 0 ? JavaLockData.Empty : new JavaLockData(entries);
}
private static async Task ParseGradleLockFileAsync(LanguageAnalyzerContext context, string path, IDictionary<string, JavaLockEntry> entries, CancellationToken cancellationToken)
{
await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read);
using var reader = new StreamReader(stream);
string? line;
while ((line = await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false)) is not null)
{
cancellationToken.ThrowIfCancellationRequested();
line = line.Trim();
if (string.IsNullOrWhiteSpace(line) || line.StartsWith("#", StringComparison.Ordinal))
{
continue;
}
var parts = line.Split('=', 2, StringSplitOptions.TrimEntries);
var coordinates = parts[0];
var configuration = parts.Length > 1 ? parts[1] : null;
var coordinateParts = coordinates.Split(':');
if (coordinateParts.Length < 3)
{
continue;
}
var groupId = coordinateParts[0];
var artifactId = coordinateParts[1];
var version = coordinateParts[^1];
if (string.IsNullOrWhiteSpace(groupId) || string.IsNullOrWhiteSpace(artifactId) || string.IsNullOrWhiteSpace(version))
{
continue;
}
var entry = new JavaLockEntry(
groupId.Trim(),
artifactId.Trim(),
version.Trim(),
Path.GetFileName(path),
NormalizeLocator(context, path),
configuration,
null,
null);
entries[entry.Key] = entry;
}
}
private static async Task ParsePomAsync(LanguageAnalyzerContext context, string path, IDictionary<string, JavaLockEntry> entries, CancellationToken cancellationToken)
{
await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read);
var document = await XDocument.LoadAsync(stream, LoadOptions.None, cancellationToken).ConfigureAwait(false);
var dependencies = document
.Descendants()
.Where(static element => element.Name.LocalName.Equals("dependency", StringComparison.OrdinalIgnoreCase));
foreach (var dependency in dependencies)
{
cancellationToken.ThrowIfCancellationRequested();
var groupId = dependency.Elements().FirstOrDefault(static e => e.Name.LocalName.Equals("groupId", StringComparison.OrdinalIgnoreCase))?.Value?.Trim();
var artifactId = dependency.Elements().FirstOrDefault(static e => e.Name.LocalName.Equals("artifactId", StringComparison.OrdinalIgnoreCase))?.Value?.Trim();
var version = dependency.Elements().FirstOrDefault(static e => e.Name.LocalName.Equals("version", StringComparison.OrdinalIgnoreCase))?.Value?.Trim();
var scope = dependency.Elements().FirstOrDefault(static e => e.Name.LocalName.Equals("scope", StringComparison.OrdinalIgnoreCase))?.Value?.Trim();
var repository = dependency.Elements().FirstOrDefault(static e => e.Name.LocalName.Equals("repository", StringComparison.OrdinalIgnoreCase))?.Value?.Trim();
if (string.IsNullOrWhiteSpace(groupId) ||
string.IsNullOrWhiteSpace(artifactId) ||
string.IsNullOrWhiteSpace(version) ||
version.Contains("${", StringComparison.Ordinal))
{
continue;
}
var entry = new JavaLockEntry(
groupId,
artifactId,
version,
"pom.xml",
NormalizeLocator(context, path),
scope,
repository,
null);
entries[entry.Key] = entry;
}
}
private static string NormalizeLocator(LanguageAnalyzerContext context, string path)
=> context.GetRelativePath(path).Replace('\\', '/');
}
internal sealed record JavaLockEntry(
string GroupId,
string ArtifactId,
string Version,
string Source,
string Locator,
string? Configuration,
string? Repository,
string? ResolvedUrl)
{
public string Key => BuildKey(GroupId, ArtifactId, Version);
private static string BuildKey(string groupId, string artifactId, string version)
=> $"{groupId}:{artifactId}:{version}".ToLowerInvariant();
}
internal sealed class JavaLockData
{
public static readonly JavaLockData Empty = new(new Dictionary<string, JavaLockEntry>(StringComparer.OrdinalIgnoreCase));
private readonly Dictionary<string, JavaLockEntry> _entriesByKey;
private readonly IReadOnlyList<JavaLockEntry> _orderedEntries;
public JavaLockData(Dictionary<string, JavaLockEntry> entries)
{
_entriesByKey = entries ?? throw new ArgumentNullException(nameof(entries));
_orderedEntries = entries.Count == 0
? Array.Empty<JavaLockEntry>()
: entries.Values
.OrderBy(static entry => entry.GroupId, StringComparer.OrdinalIgnoreCase)
.ThenBy(static entry => entry.ArtifactId, StringComparer.OrdinalIgnoreCase)
.ThenBy(static entry => entry.Version, StringComparer.OrdinalIgnoreCase)
.ThenBy(static entry => entry.Source, StringComparer.OrdinalIgnoreCase)
.ThenBy(static entry => entry.Locator, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
public IReadOnlyList<JavaLockEntry> Entries => _orderedEntries;
public bool HasEntries => _entriesByKey.Count > 0;
public bool TryGet(string groupId, string artifactId, string version, out JavaLockEntry? entry)
{
var key = $"{groupId}:{artifactId}:{version}".ToLowerInvariant();
return _entriesByKey.TryGetValue(key, out entry);
}
}

View File

@@ -1,138 +1,167 @@
using StellaOps.Scanner.Analyzers.Lang.Java.Internal;
namespace StellaOps.Scanner.Analyzers.Lang.Java;
public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer
{
public string Id => "java";
using System.Collections.Generic;
using System.IO;
using System.Text;
using StellaOps.Scanner.Analyzers.Lang.Java.Internal;
namespace StellaOps.Scanner.Analyzers.Lang.Java;
public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer
{
public string Id => "java";
public string DisplayName => "Java/Maven Analyzer";
public async ValueTask AnalyzeAsync(LanguageAnalyzerContext context, LanguageComponentWriter writer, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
ArgumentNullException.ThrowIfNull(writer);
var workspace = JavaWorkspaceNormalizer.Normalize(context, cancellationToken);
foreach (var archive in workspace.Archives)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
await ProcessArchiveAsync(archive, context, writer, cancellationToken).ConfigureAwait(false);
}
catch (IOException)
{
// Corrupt archives should not abort the scan.
}
catch (InvalidDataException)
{
// Skip non-zip payloads despite supported extensions.
}
}
}
private async ValueTask ProcessArchiveAsync(JavaArchive archive, LanguageAnalyzerContext context, LanguageComponentWriter writer, CancellationToken cancellationToken)
{
ManifestMetadata? manifestMetadata = null;
if (archive.TryGetEntry("META-INF/MANIFEST.MF", out var manifestEntry))
{
manifestMetadata = await ParseManifestAsync(archive, manifestEntry, cancellationToken).ConfigureAwait(false);
}
foreach (var entry in archive.Entries)
{
cancellationToken.ThrowIfCancellationRequested();
if (IsManifestEntry(entry.EffectivePath))
{
continue;
}
if (!IsPomPropertiesEntry(entry.EffectivePath))
{
continue;
}
var artifact = await ParsePomPropertiesAsync(archive, entry, cancellationToken).ConfigureAwait(false);
if (artifact is null)
{
continue;
}
var metadata = new Dictionary<string, string?>(StringComparer.Ordinal)
{
["groupId"] = artifact.GroupId,
["artifactId"] = artifact.ArtifactId,
["jarPath"] = NormalizeArchivePath(archive.RelativePath),
};
if (!string.IsNullOrEmpty(artifact.Packaging))
{
metadata["packaging"] = artifact.Packaging;
}
if (!string.IsNullOrEmpty(artifact.Name))
{
metadata["displayName"] = artifact.Name;
}
if (manifestMetadata is not null)
{
manifestMetadata.ApplyMetadata(metadata);
}
var evidence = new List<LanguageComponentEvidence>
{
new(LanguageEvidenceKind.File, "pom.properties", BuildLocator(archive, entry.OriginalPath), null, artifact.PomSha256),
};
if (manifestMetadata is not null)
{
evidence.Add(manifestMetadata.CreateEvidence(archive));
}
var usedByEntrypoint = context.UsageHints.IsPathUsed(archive.AbsolutePath);
writer.AddFromPurl(
analyzerId: Id,
purl: artifact.Purl,
name: artifact.ArtifactId,
version: artifact.Version,
type: "maven",
metadata: metadata,
evidence: evidence,
usedByEntrypoint: usedByEntrypoint);
}
}
public async ValueTask AnalyzeAsync(LanguageAnalyzerContext context, LanguageComponentWriter writer, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
ArgumentNullException.ThrowIfNull(writer);
private static string BuildLocator(JavaArchive archive, string entryPath)
{
var relativeArchive = NormalizeArchivePath(archive.RelativePath);
var normalizedEntry = NormalizeEntry(entryPath);
if (string.Equals(relativeArchive, ".", StringComparison.Ordinal) || string.IsNullOrEmpty(relativeArchive))
{
return normalizedEntry;
}
return string.Concat(relativeArchive, "!", normalizedEntry);
}
private static string NormalizeEntry(string entryPath)
=> entryPath.Replace('\\', '/');
private static string NormalizeArchivePath(string relativePath)
{
if (string.IsNullOrEmpty(relativePath) || string.Equals(relativePath, ".", StringComparison.Ordinal))
{
return ".";
}
return relativePath.Replace('\\', '/');
}
var workspace = JavaWorkspaceNormalizer.Normalize(context, cancellationToken);
var lockData = await JavaLockFileCollector.LoadAsync(context, cancellationToken).ConfigureAwait(false);
var matchedLocks = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var hasLockEntries = lockData.HasEntries;
foreach (var archive in workspace.Archives)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
await ProcessArchiveAsync(archive, context, writer, lockData, matchedLocks, hasLockEntries, cancellationToken).ConfigureAwait(false);
}
catch (IOException)
{
// Corrupt archives should not abort the scan.
}
catch (InvalidDataException)
{
// Skip non-zip payloads despite supported extensions.
}
}
if (lockData.Entries.Count > 0)
{
foreach (var entry in lockData.Entries)
{
if (matchedLocks.Contains(entry.Key))
{
continue;
}
var metadata = CreateDeclaredMetadata(entry);
var evidence = new[] { CreateDeclaredEvidence(entry) };
var purl = BuildPurl(entry.GroupId, entry.ArtifactId, entry.Version, packaging: null);
writer.AddFromPurl(
analyzerId: Id,
purl: purl,
name: entry.ArtifactId,
version: entry.Version,
type: "maven",
metadata: metadata,
evidence: evidence,
usedByEntrypoint: false);
}
}
}
private async ValueTask ProcessArchiveAsync(
JavaArchive archive,
LanguageAnalyzerContext context,
LanguageComponentWriter writer,
JavaLockData lockData,
HashSet<string> matchedLocks,
bool hasLockEntries,
CancellationToken cancellationToken)
{
ManifestMetadata? manifestMetadata = null;
if (archive.TryGetEntry("META-INF/MANIFEST.MF", out var manifestEntry))
{
manifestMetadata = await ParseManifestAsync(archive, manifestEntry, cancellationToken).ConfigureAwait(false);
}
foreach (var entry in archive.Entries)
{
cancellationToken.ThrowIfCancellationRequested();
if (IsManifestEntry(entry.EffectivePath))
{
continue;
}
if (!IsPomPropertiesEntry(entry.EffectivePath))
{
continue;
}
var artifact = await ParsePomPropertiesAsync(archive, entry, cancellationToken).ConfigureAwait(false);
if (artifact is null)
{
continue;
}
var metadata = CreateInstalledMetadata(artifact, archive, manifestMetadata);
if (lockData.TryGet(artifact.GroupId, artifact.ArtifactId, artifact.Version, out var lockEntry))
{
matchedLocks.Add(lockEntry!.Key);
AppendLockMetadata(metadata, lockEntry);
}
else if (hasLockEntries)
{
AddMetadata(metadata, "lockMissing", "true");
}
var evidence = new List<LanguageComponentEvidence>
{
new(LanguageEvidenceKind.File, "pom.properties", BuildLocator(archive, entry.OriginalPath), null, artifact.PomSha256),
};
if (manifestMetadata is not null)
{
evidence.Add(manifestMetadata.CreateEvidence(archive));
}
var usedByEntrypoint = context.UsageHints.IsPathUsed(archive.AbsolutePath);
writer.AddFromPurl(
analyzerId: Id,
purl: artifact.Purl,
name: artifact.ArtifactId,
version: artifact.Version,
type: "maven",
metadata: SortMetadata(metadata),
evidence: evidence,
usedByEntrypoint: usedByEntrypoint);
}
}
private static string BuildLocator(JavaArchive archive, string entryPath)
{
var relativeArchive = NormalizeArchivePath(archive.RelativePath);
var normalizedEntry = NormalizeEntry(entryPath);
if (string.Equals(relativeArchive, ".", StringComparison.Ordinal) || string.IsNullOrEmpty(relativeArchive))
{
return normalizedEntry;
}
return string.Concat(relativeArchive, "!", normalizedEntry);
}
private static string NormalizeEntry(string entryPath)
=> entryPath.Replace('\\', '/');
private static string NormalizeArchivePath(string relativePath)
{
if (string.IsNullOrEmpty(relativePath) || string.Equals(relativePath, ".", StringComparison.Ordinal))
{
return ".";
}
return relativePath.Replace('\\', '/');
}
private static bool IsPomPropertiesEntry(string entryName)
=> entryName.StartsWith("META-INF/maven/", StringComparison.OrdinalIgnoreCase)
@@ -141,14 +170,21 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer
private static bool IsManifestEntry(string entryName)
=> string.Equals(entryName, "META-INF/MANIFEST.MF", StringComparison.OrdinalIgnoreCase);
private static async ValueTask<MavenArtifact?> ParsePomPropertiesAsync(JavaArchive archive, JavaArchiveEntry entry, CancellationToken cancellationToken)
{
await using var entryStream = archive.OpenEntry(entry);
using var buffer = new MemoryStream();
await entryStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
buffer.Position = 0;
using var reader = new StreamReader(buffer, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, leaveOpen: true);
private static void AppendLockMetadata(ICollection<KeyValuePair<string, string?>> metadata, JavaLockEntry entry)
{
AddMetadata(metadata, "lockConfiguration", entry.Configuration);
AddMetadata(metadata, "lockRepository", entry.Repository);
AddMetadata(metadata, "lockResolved", entry.ResolvedUrl);
}
private static async ValueTask<MavenArtifact?> ParsePomPropertiesAsync(JavaArchive archive, JavaArchiveEntry entry, CancellationToken cancellationToken)
{
await using var entryStream = archive.OpenEntry(entry);
using var buffer = new MemoryStream();
await entryStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
buffer.Position = 0;
using var reader = new StreamReader(buffer, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, leaveOpen: true);
var properties = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
while (await reader.ReadLineAsync().ConfigureAwait(false) is { } line)
@@ -209,14 +245,14 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer
PomSha256: pomSha);
}
private static async ValueTask<ManifestMetadata?> ParseManifestAsync(JavaArchive archive, JavaArchiveEntry entry, CancellationToken cancellationToken)
{
await using var entryStream = archive.OpenEntry(entry);
using var reader = new StreamReader(entryStream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, leaveOpen: false);
string? title = null;
string? version = null;
string? vendor = null;
private static async ValueTask<ManifestMetadata?> ParseManifestAsync(JavaArchive archive, JavaArchiveEntry entry, CancellationToken cancellationToken)
{
await using var entryStream = archive.OpenEntry(entry);
using var reader = new StreamReader(entryStream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, leaveOpen: false);
string? title = null;
string? version = null;
string? vendor = null;
while (await reader.ReadLineAsync().ConfigureAwait(false) is { } line)
{
@@ -289,32 +325,21 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer
private sealed record ManifestMetadata(string? ImplementationTitle, string? ImplementationVersion, string? ImplementationVendor)
{
public void ApplyMetadata(IDictionary<string, string?> target)
public void ApplyMetadata(ICollection<KeyValuePair<string, string?>> target)
{
if (!string.IsNullOrWhiteSpace(ImplementationTitle))
{
target["manifestTitle"] = ImplementationTitle;
}
if (!string.IsNullOrWhiteSpace(ImplementationVersion))
{
target["manifestVersion"] = ImplementationVersion;
}
if (!string.IsNullOrWhiteSpace(ImplementationVendor))
{
target["manifestVendor"] = ImplementationVendor;
}
AddMetadata(target, "manifestTitle", ImplementationTitle);
AddMetadata(target, "manifestVersion", ImplementationVersion);
AddMetadata(target, "manifestVendor", ImplementationVendor);
}
public LanguageComponentEvidence CreateEvidence(JavaArchive archive)
{
var locator = BuildLocator(archive, "META-INF/MANIFEST.MF");
var valueBuilder = new StringBuilder();
if (!string.IsNullOrWhiteSpace(ImplementationTitle))
{
valueBuilder.Append("title=").Append(ImplementationTitle);
public LanguageComponentEvidence CreateEvidence(JavaArchive archive)
{
var locator = BuildLocator(archive, "META-INF/MANIFEST.MF");
var valueBuilder = new StringBuilder();
if (!string.IsNullOrWhiteSpace(ImplementationTitle))
{
valueBuilder.Append("title=").Append(ImplementationTitle);
}
if (!string.IsNullOrWhiteSpace(ImplementationVersion))
@@ -340,5 +365,89 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer
var value = valueBuilder.Length > 0 ? valueBuilder.ToString() : null;
return new LanguageComponentEvidence(LanguageEvidenceKind.File, "MANIFEST.MF", locator, value, null);
}
}
private static List<KeyValuePair<string, string?>> CreateInstalledMetadata(
MavenArtifact artifact,
JavaArchive archive,
ManifestMetadata? manifestMetadata)
{
var metadata = new List<KeyValuePair<string, string?>>(8);
AddMetadata(metadata, "groupId", artifact.GroupId);
AddMetadata(metadata, "artifactId", artifact.ArtifactId);
AddMetadata(metadata, "jarPath", NormalizeArchivePath(archive.RelativePath), allowEmpty: true);
AddMetadata(metadata, "packaging", artifact.Packaging);
AddMetadata(metadata, "displayName", artifact.Name);
manifestMetadata?.ApplyMetadata(metadata);
return metadata;
}
private static IReadOnlyList<KeyValuePair<string, string?>> CreateDeclaredMetadata(JavaLockEntry entry)
{
var metadata = new List<KeyValuePair<string, string?>>(6);
var lockSource = NormalizeLockSource(entry.Source);
var lockLocator = string.IsNullOrWhiteSpace(entry.Locator) ? lockSource : entry.Locator;
AddMetadata(metadata, "declaredOnly", "true");
AddMetadata(metadata, "lockSource", lockSource);
AddMetadata(metadata, "lockLocator", lockLocator, allowEmpty: true);
AppendLockMetadata(metadata, entry);
return SortMetadata(metadata);
}
private static LanguageComponentEvidence CreateDeclaredEvidence(JavaLockEntry entry)
{
var lockSource = NormalizeLockSource(entry.Source);
var lockLocator = string.IsNullOrWhiteSpace(entry.Locator) ? lockSource : entry.Locator;
return new LanguageComponentEvidence(
LanguageEvidenceKind.Metadata,
lockSource,
lockLocator,
entry.ResolvedUrl,
Sha256: null);
}
private static IReadOnlyList<KeyValuePair<string, string?>> SortMetadata(List<KeyValuePair<string, string?>> metadata)
{
metadata.Sort(static (left, right) =>
{
var keyComparison = string.CompareOrdinal(left.Key, right.Key);
if (keyComparison != 0)
{
return keyComparison;
}
return string.CompareOrdinal(left.Value ?? string.Empty, right.Value ?? string.Empty);
});
return metadata;
}
private static void AddMetadata(
ICollection<KeyValuePair<string, string?>> metadata,
string key,
string? value,
bool allowEmpty = false)
{
if (string.IsNullOrWhiteSpace(key))
{
return;
}
if (!allowEmpty && string.IsNullOrWhiteSpace(value))
{
return;
}
metadata.Add(new KeyValuePair<string, string?>(key, value));
}
private static string NormalizeLockSource(string? source)
=> string.IsNullOrWhiteSpace(source) ? "lockfile" : source;
}