feat: Implement Runtime Facts ingestion service and NDJSON reader
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
- Added RuntimeFactsNdjsonReader for reading NDJSON formatted runtime facts. - Introduced IRuntimeFactsIngestionService interface and its implementation. - Enhanced Program.cs to register new services and endpoints for runtime facts. - Updated CallgraphIngestionService to include CAS URI in stored artifacts. - Created RuntimeFactsValidationException for validation errors during ingestion. - Added tests for RuntimeFactsIngestionService and RuntimeFactsNdjsonReader. - Implemented SignalsSealedModeMonitor for compliance checks in sealed mode. - Updated project dependencies for testing utilities.
This commit is contained in:
master
@@ -1,138 +1,167 @@
|
||||
using StellaOps.Scanner.Analyzers.Lang.Java.Internal;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Java;
|
||||
|
||||
public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer
|
||||
{
|
||||
public string Id => "java";
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Java.Internal;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Java;
|
||||
|
||||
public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer
|
||||
{
|
||||
public string Id => "java";
|
||||
|
||||
public string DisplayName => "Java/Maven Analyzer";
|
||||
|
||||
public async ValueTask AnalyzeAsync(LanguageAnalyzerContext context, LanguageComponentWriter writer, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
ArgumentNullException.ThrowIfNull(writer);
|
||||
|
||||
var workspace = JavaWorkspaceNormalizer.Normalize(context, cancellationToken);
|
||||
|
||||
foreach (var archive in workspace.Archives)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
try
|
||||
{
|
||||
await ProcessArchiveAsync(archive, context, writer, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
// Corrupt archives should not abort the scan.
|
||||
}
|
||||
catch (InvalidDataException)
|
||||
{
|
||||
// Skip non-zip payloads despite supported extensions.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async ValueTask ProcessArchiveAsync(JavaArchive archive, LanguageAnalyzerContext context, LanguageComponentWriter writer, CancellationToken cancellationToken)
|
||||
{
|
||||
ManifestMetadata? manifestMetadata = null;
|
||||
if (archive.TryGetEntry("META-INF/MANIFEST.MF", out var manifestEntry))
|
||||
{
|
||||
manifestMetadata = await ParseManifestAsync(archive, manifestEntry, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
foreach (var entry in archive.Entries)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (IsManifestEntry(entry.EffectivePath))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!IsPomPropertiesEntry(entry.EffectivePath))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var artifact = await ParsePomPropertiesAsync(archive, entry, cancellationToken).ConfigureAwait(false);
|
||||
if (artifact is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var metadata = new Dictionary<string, string?>(StringComparer.Ordinal)
|
||||
{
|
||||
["groupId"] = artifact.GroupId,
|
||||
["artifactId"] = artifact.ArtifactId,
|
||||
["jarPath"] = NormalizeArchivePath(archive.RelativePath),
|
||||
};
|
||||
|
||||
if (!string.IsNullOrEmpty(artifact.Packaging))
|
||||
{
|
||||
metadata["packaging"] = artifact.Packaging;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(artifact.Name))
|
||||
{
|
||||
metadata["displayName"] = artifact.Name;
|
||||
}
|
||||
|
||||
if (manifestMetadata is not null)
|
||||
{
|
||||
manifestMetadata.ApplyMetadata(metadata);
|
||||
}
|
||||
|
||||
var evidence = new List<LanguageComponentEvidence>
|
||||
{
|
||||
new(LanguageEvidenceKind.File, "pom.properties", BuildLocator(archive, entry.OriginalPath), null, artifact.PomSha256),
|
||||
};
|
||||
|
||||
if (manifestMetadata is not null)
|
||||
{
|
||||
evidence.Add(manifestMetadata.CreateEvidence(archive));
|
||||
}
|
||||
|
||||
var usedByEntrypoint = context.UsageHints.IsPathUsed(archive.AbsolutePath);
|
||||
|
||||
writer.AddFromPurl(
|
||||
analyzerId: Id,
|
||||
purl: artifact.Purl,
|
||||
name: artifact.ArtifactId,
|
||||
version: artifact.Version,
|
||||
type: "maven",
|
||||
metadata: metadata,
|
||||
evidence: evidence,
|
||||
usedByEntrypoint: usedByEntrypoint);
|
||||
}
|
||||
}
|
||||
public async ValueTask AnalyzeAsync(LanguageAnalyzerContext context, LanguageComponentWriter writer, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
ArgumentNullException.ThrowIfNull(writer);
|
||||
|
||||
private static string BuildLocator(JavaArchive archive, string entryPath)
|
||||
{
|
||||
var relativeArchive = NormalizeArchivePath(archive.RelativePath);
|
||||
var normalizedEntry = NormalizeEntry(entryPath);
|
||||
|
||||
if (string.Equals(relativeArchive, ".", StringComparison.Ordinal) || string.IsNullOrEmpty(relativeArchive))
|
||||
{
|
||||
return normalizedEntry;
|
||||
}
|
||||
|
||||
return string.Concat(relativeArchive, "!", normalizedEntry);
|
||||
}
|
||||
|
||||
private static string NormalizeEntry(string entryPath)
|
||||
=> entryPath.Replace('\\', '/');
|
||||
|
||||
private static string NormalizeArchivePath(string relativePath)
|
||||
{
|
||||
if (string.IsNullOrEmpty(relativePath) || string.Equals(relativePath, ".", StringComparison.Ordinal))
|
||||
{
|
||||
return ".";
|
||||
}
|
||||
|
||||
return relativePath.Replace('\\', '/');
|
||||
}
|
||||
var workspace = JavaWorkspaceNormalizer.Normalize(context, cancellationToken);
|
||||
var lockData = await JavaLockFileCollector.LoadAsync(context, cancellationToken).ConfigureAwait(false);
|
||||
var matchedLocks = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var hasLockEntries = lockData.HasEntries;
|
||||
|
||||
foreach (var archive in workspace.Archives)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
try
|
||||
{
|
||||
await ProcessArchiveAsync(archive, context, writer, lockData, matchedLocks, hasLockEntries, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
// Corrupt archives should not abort the scan.
|
||||
}
|
||||
catch (InvalidDataException)
|
||||
{
|
||||
// Skip non-zip payloads despite supported extensions.
|
||||
}
|
||||
}
|
||||
|
||||
if (lockData.Entries.Count > 0)
|
||||
{
|
||||
foreach (var entry in lockData.Entries)
|
||||
{
|
||||
if (matchedLocks.Contains(entry.Key))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var metadata = CreateDeclaredMetadata(entry);
|
||||
var evidence = new[] { CreateDeclaredEvidence(entry) };
|
||||
|
||||
var purl = BuildPurl(entry.GroupId, entry.ArtifactId, entry.Version, packaging: null);
|
||||
|
||||
writer.AddFromPurl(
|
||||
analyzerId: Id,
|
||||
purl: purl,
|
||||
name: entry.ArtifactId,
|
||||
version: entry.Version,
|
||||
type: "maven",
|
||||
metadata: metadata,
|
||||
evidence: evidence,
|
||||
usedByEntrypoint: false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async ValueTask ProcessArchiveAsync(
|
||||
JavaArchive archive,
|
||||
LanguageAnalyzerContext context,
|
||||
LanguageComponentWriter writer,
|
||||
JavaLockData lockData,
|
||||
HashSet<string> matchedLocks,
|
||||
bool hasLockEntries,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ManifestMetadata? manifestMetadata = null;
|
||||
if (archive.TryGetEntry("META-INF/MANIFEST.MF", out var manifestEntry))
|
||||
{
|
||||
manifestMetadata = await ParseManifestAsync(archive, manifestEntry, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
foreach (var entry in archive.Entries)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (IsManifestEntry(entry.EffectivePath))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!IsPomPropertiesEntry(entry.EffectivePath))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var artifact = await ParsePomPropertiesAsync(archive, entry, cancellationToken).ConfigureAwait(false);
|
||||
if (artifact is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var metadata = CreateInstalledMetadata(artifact, archive, manifestMetadata);
|
||||
|
||||
if (lockData.TryGet(artifact.GroupId, artifact.ArtifactId, artifact.Version, out var lockEntry))
|
||||
{
|
||||
matchedLocks.Add(lockEntry!.Key);
|
||||
AppendLockMetadata(metadata, lockEntry);
|
||||
}
|
||||
else if (hasLockEntries)
|
||||
{
|
||||
AddMetadata(metadata, "lockMissing", "true");
|
||||
}
|
||||
|
||||
var evidence = new List<LanguageComponentEvidence>
|
||||
{
|
||||
new(LanguageEvidenceKind.File, "pom.properties", BuildLocator(archive, entry.OriginalPath), null, artifact.PomSha256),
|
||||
};
|
||||
|
||||
if (manifestMetadata is not null)
|
||||
{
|
||||
evidence.Add(manifestMetadata.CreateEvidence(archive));
|
||||
}
|
||||
|
||||
var usedByEntrypoint = context.UsageHints.IsPathUsed(archive.AbsolutePath);
|
||||
|
||||
writer.AddFromPurl(
|
||||
analyzerId: Id,
|
||||
purl: artifact.Purl,
|
||||
name: artifact.ArtifactId,
|
||||
version: artifact.Version,
|
||||
type: "maven",
|
||||
metadata: SortMetadata(metadata),
|
||||
evidence: evidence,
|
||||
usedByEntrypoint: usedByEntrypoint);
|
||||
}
|
||||
}
|
||||
|
||||
private static string BuildLocator(JavaArchive archive, string entryPath)
|
||||
{
|
||||
var relativeArchive = NormalizeArchivePath(archive.RelativePath);
|
||||
var normalizedEntry = NormalizeEntry(entryPath);
|
||||
|
||||
if (string.Equals(relativeArchive, ".", StringComparison.Ordinal) || string.IsNullOrEmpty(relativeArchive))
|
||||
{
|
||||
return normalizedEntry;
|
||||
}
|
||||
|
||||
return string.Concat(relativeArchive, "!", normalizedEntry);
|
||||
}
|
||||
|
||||
private static string NormalizeEntry(string entryPath)
|
||||
=> entryPath.Replace('\\', '/');
|
||||
|
||||
private static string NormalizeArchivePath(string relativePath)
|
||||
{
|
||||
if (string.IsNullOrEmpty(relativePath) || string.Equals(relativePath, ".", StringComparison.Ordinal))
|
||||
{
|
||||
return ".";
|
||||
}
|
||||
|
||||
return relativePath.Replace('\\', '/');
|
||||
}
|
||||
|
||||
private static bool IsPomPropertiesEntry(string entryName)
|
||||
=> entryName.StartsWith("META-INF/maven/", StringComparison.OrdinalIgnoreCase)
|
||||
@@ -141,14 +170,21 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer
|
||||
private static bool IsManifestEntry(string entryName)
|
||||
=> string.Equals(entryName, "META-INF/MANIFEST.MF", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private static async ValueTask<MavenArtifact?> ParsePomPropertiesAsync(JavaArchive archive, JavaArchiveEntry entry, CancellationToken cancellationToken)
|
||||
{
|
||||
await using var entryStream = archive.OpenEntry(entry);
|
||||
using var buffer = new MemoryStream();
|
||||
await entryStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
|
||||
buffer.Position = 0;
|
||||
|
||||
using var reader = new StreamReader(buffer, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, leaveOpen: true);
|
||||
private static void AppendLockMetadata(ICollection<KeyValuePair<string, string?>> metadata, JavaLockEntry entry)
|
||||
{
|
||||
AddMetadata(metadata, "lockConfiguration", entry.Configuration);
|
||||
AddMetadata(metadata, "lockRepository", entry.Repository);
|
||||
AddMetadata(metadata, "lockResolved", entry.ResolvedUrl);
|
||||
}
|
||||
|
||||
private static async ValueTask<MavenArtifact?> ParsePomPropertiesAsync(JavaArchive archive, JavaArchiveEntry entry, CancellationToken cancellationToken)
|
||||
{
|
||||
await using var entryStream = archive.OpenEntry(entry);
|
||||
using var buffer = new MemoryStream();
|
||||
await entryStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
|
||||
buffer.Position = 0;
|
||||
|
||||
using var reader = new StreamReader(buffer, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, leaveOpen: true);
|
||||
var properties = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
while (await reader.ReadLineAsync().ConfigureAwait(false) is { } line)
|
||||
@@ -209,14 +245,14 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer
|
||||
PomSha256: pomSha);
|
||||
}
|
||||
|
||||
private static async ValueTask<ManifestMetadata?> ParseManifestAsync(JavaArchive archive, JavaArchiveEntry entry, CancellationToken cancellationToken)
|
||||
{
|
||||
await using var entryStream = archive.OpenEntry(entry);
|
||||
using var reader = new StreamReader(entryStream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, leaveOpen: false);
|
||||
|
||||
string? title = null;
|
||||
string? version = null;
|
||||
string? vendor = null;
|
||||
private static async ValueTask<ManifestMetadata?> ParseManifestAsync(JavaArchive archive, JavaArchiveEntry entry, CancellationToken cancellationToken)
|
||||
{
|
||||
await using var entryStream = archive.OpenEntry(entry);
|
||||
using var reader = new StreamReader(entryStream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, leaveOpen: false);
|
||||
|
||||
string? title = null;
|
||||
string? version = null;
|
||||
string? vendor = null;
|
||||
|
||||
while (await reader.ReadLineAsync().ConfigureAwait(false) is { } line)
|
||||
{
|
||||
@@ -289,32 +325,21 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer
|
||||
|
||||
private sealed record ManifestMetadata(string? ImplementationTitle, string? ImplementationVersion, string? ImplementationVendor)
|
||||
{
|
||||
public void ApplyMetadata(IDictionary<string, string?> target)
|
||||
public void ApplyMetadata(ICollection<KeyValuePair<string, string?>> target)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(ImplementationTitle))
|
||||
{
|
||||
target["manifestTitle"] = ImplementationTitle;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(ImplementationVersion))
|
||||
{
|
||||
target["manifestVersion"] = ImplementationVersion;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(ImplementationVendor))
|
||||
{
|
||||
target["manifestVendor"] = ImplementationVendor;
|
||||
}
|
||||
AddMetadata(target, "manifestTitle", ImplementationTitle);
|
||||
AddMetadata(target, "manifestVersion", ImplementationVersion);
|
||||
AddMetadata(target, "manifestVendor", ImplementationVendor);
|
||||
}
|
||||
|
||||
public LanguageComponentEvidence CreateEvidence(JavaArchive archive)
|
||||
{
|
||||
var locator = BuildLocator(archive, "META-INF/MANIFEST.MF");
|
||||
var valueBuilder = new StringBuilder();
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(ImplementationTitle))
|
||||
{
|
||||
valueBuilder.Append("title=").Append(ImplementationTitle);
|
||||
public LanguageComponentEvidence CreateEvidence(JavaArchive archive)
|
||||
{
|
||||
var locator = BuildLocator(archive, "META-INF/MANIFEST.MF");
|
||||
var valueBuilder = new StringBuilder();
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(ImplementationTitle))
|
||||
{
|
||||
valueBuilder.Append("title=").Append(ImplementationTitle);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(ImplementationVersion))
|
||||
@@ -340,5 +365,89 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer
|
||||
var value = valueBuilder.Length > 0 ? valueBuilder.ToString() : null;
|
||||
return new LanguageComponentEvidence(LanguageEvidenceKind.File, "MANIFEST.MF", locator, value, null);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private static List<KeyValuePair<string, string?>> CreateInstalledMetadata(
|
||||
MavenArtifact artifact,
|
||||
JavaArchive archive,
|
||||
ManifestMetadata? manifestMetadata)
|
||||
{
|
||||
var metadata = new List<KeyValuePair<string, string?>>(8);
|
||||
|
||||
AddMetadata(metadata, "groupId", artifact.GroupId);
|
||||
AddMetadata(metadata, "artifactId", artifact.ArtifactId);
|
||||
AddMetadata(metadata, "jarPath", NormalizeArchivePath(archive.RelativePath), allowEmpty: true);
|
||||
AddMetadata(metadata, "packaging", artifact.Packaging);
|
||||
AddMetadata(metadata, "displayName", artifact.Name);
|
||||
|
||||
manifestMetadata?.ApplyMetadata(metadata);
|
||||
|
||||
return metadata;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<KeyValuePair<string, string?>> CreateDeclaredMetadata(JavaLockEntry entry)
|
||||
{
|
||||
var metadata = new List<KeyValuePair<string, string?>>(6);
|
||||
var lockSource = NormalizeLockSource(entry.Source);
|
||||
var lockLocator = string.IsNullOrWhiteSpace(entry.Locator) ? lockSource : entry.Locator;
|
||||
|
||||
AddMetadata(metadata, "declaredOnly", "true");
|
||||
AddMetadata(metadata, "lockSource", lockSource);
|
||||
AddMetadata(metadata, "lockLocator", lockLocator, allowEmpty: true);
|
||||
AppendLockMetadata(metadata, entry);
|
||||
|
||||
return SortMetadata(metadata);
|
||||
}
|
||||
|
||||
private static LanguageComponentEvidence CreateDeclaredEvidence(JavaLockEntry entry)
|
||||
{
|
||||
var lockSource = NormalizeLockSource(entry.Source);
|
||||
var lockLocator = string.IsNullOrWhiteSpace(entry.Locator) ? lockSource : entry.Locator;
|
||||
|
||||
return new LanguageComponentEvidence(
|
||||
LanguageEvidenceKind.Metadata,
|
||||
lockSource,
|
||||
lockLocator,
|
||||
entry.ResolvedUrl,
|
||||
Sha256: null);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<KeyValuePair<string, string?>> SortMetadata(List<KeyValuePair<string, string?>> metadata)
|
||||
{
|
||||
metadata.Sort(static (left, right) =>
|
||||
{
|
||||
var keyComparison = string.CompareOrdinal(left.Key, right.Key);
|
||||
if (keyComparison != 0)
|
||||
{
|
||||
return keyComparison;
|
||||
}
|
||||
|
||||
return string.CompareOrdinal(left.Value ?? string.Empty, right.Value ?? string.Empty);
|
||||
});
|
||||
|
||||
return metadata;
|
||||
}
|
||||
|
||||
private static void AddMetadata(
|
||||
ICollection<KeyValuePair<string, string?>> metadata,
|
||||
string key,
|
||||
string? value,
|
||||
bool allowEmpty = false)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(key))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!allowEmpty && string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
metadata.Add(new KeyValuePair<string, string?>(key, value));
|
||||
}
|
||||
|
||||
private static string NormalizeLockSource(string? source)
|
||||
=> string.IsNullOrWhiteSpace(source) ? "lockfile" : source;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user