feat: Implement Runtime Facts ingestion service and NDJSON reader
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
- Added RuntimeFactsNdjsonReader for reading NDJSON formatted runtime facts. - Introduced IRuntimeFactsIngestionService interface and its implementation. - Enhanced Program.cs to register new services and endpoints for runtime facts. - Updated CallgraphIngestionService to include CAS URI in stored artifacts. - Created RuntimeFactsValidationException for validation errors during ingestion. - Added tests for RuntimeFactsIngestionService and RuntimeFactsNdjsonReader. - Implemented SignalsSealedModeMonitor for compliance checks in sealed mode. - Updated project dependencies for testing utilities.
This commit is contained in:
master
@@ -36,7 +36,7 @@ internal static class PythonDistributionLoader
|
||||
|
||||
var trimmedName = name.Trim();
|
||||
var trimmedVersion = version.Trim();
|
||||
var normalizedName = NormalizePackageName(trimmedName);
|
||||
var normalizedName = PythonPathHelper.NormalizePackageName(trimmedName);
|
||||
var purl = $"pkg:pypi/{normalizedName}@{trimmedVersion}";
|
||||
|
||||
var metadataEntries = new List<KeyValuePair<string, string?>>();
|
||||
@@ -321,28 +321,6 @@ internal static class PythonDistributionLoader
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string NormalizePackageName(string name)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(name))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var builder = new StringBuilder(name.Length);
|
||||
foreach (var ch in name.Trim().ToLowerInvariant())
|
||||
{
|
||||
builder.Append(ch switch
|
||||
{
|
||||
'_' => '-',
|
||||
'.' => '-',
|
||||
' ' => '-',
|
||||
_ => ch
|
||||
});
|
||||
}
|
||||
|
||||
return builder.ToString();
|
||||
}
|
||||
|
||||
private static string ResolvePackageRoot(string distInfoPath)
|
||||
{
|
||||
var parent = Directory.GetParent(distInfoPath);
|
||||
@@ -1063,21 +1041,45 @@ internal sealed class PythonDirectUrlInfo
|
||||
}
|
||||
}
|
||||
|
||||
internal static class PythonPathHelper
|
||||
{
|
||||
public static string NormalizeRelative(LanguageAnalyzerContext context, string path)
|
||||
{
|
||||
var relative = context.GetRelativePath(path);
|
||||
if (string.IsNullOrEmpty(relative) || relative == ".")
|
||||
{
|
||||
return ".";
|
||||
}
|
||||
|
||||
return relative;
|
||||
}
|
||||
}
|
||||
|
||||
internal static class PythonEncoding
|
||||
internal static class PythonPathHelper
|
||||
{
|
||||
public static string NormalizeRelative(LanguageAnalyzerContext context, string path)
|
||||
{
|
||||
var relative = context.GetRelativePath(path);
|
||||
if (string.IsNullOrEmpty(relative) || relative == ".")
|
||||
{
|
||||
return ".";
|
||||
}
|
||||
|
||||
return relative;
|
||||
}
|
||||
|
||||
public static string NormalizePackageName(string name)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(name))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var trimmed = name.Trim();
|
||||
var builder = new StringBuilder(trimmed.Length);
|
||||
foreach (var ch in trimmed)
|
||||
{
|
||||
var lower = char.ToLowerInvariant(ch);
|
||||
builder.Append(lower switch
|
||||
{
|
||||
'_' => '-',
|
||||
'.' => '-',
|
||||
' ' => '-',
|
||||
_ => lower
|
||||
});
|
||||
}
|
||||
|
||||
return builder.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
internal static class PythonEncoding
|
||||
{
|
||||
public static readonly UTF8Encoding Utf8 = new(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,283 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Python.Internal;
|
||||
|
||||
internal static class PythonLockFileCollector
|
||||
{
|
||||
private static readonly string[] RequirementPatterns =
|
||||
{
|
||||
"requirements.txt",
|
||||
"requirements-dev.txt",
|
||||
"requirements.prod.txt"
|
||||
};
|
||||
|
||||
private static readonly Regex RequirementLinePattern = new(@"^\s*(?<name>[A-Za-z0-9_.\-]+)(?<extras>\[[^\]]+\])?\s*(?<op>==|===)\s*(?<version>[^\s;#]+)", RegexOptions.Compiled);
|
||||
private static readonly Regex EditablePattern = new(@"^-{1,2}editable\s*=?\s*(?<path>.+)$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
|
||||
|
||||
public static async Task<PythonLockData> LoadAsync(LanguageAnalyzerContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
|
||||
var entries = new Dictionary<string, PythonLockEntry>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var pattern in RequirementPatterns)
|
||||
{
|
||||
var candidate = Path.Combine(context.RootPath, pattern);
|
||||
if (File.Exists(candidate))
|
||||
{
|
||||
await ParseRequirementsFileAsync(context, candidate, entries, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
var pipfileLock = Path.Combine(context.RootPath, "Pipfile.lock");
|
||||
if (File.Exists(pipfileLock))
|
||||
{
|
||||
await ParsePipfileLockAsync(context, pipfileLock, entries, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var poetryLock = Path.Combine(context.RootPath, "poetry.lock");
|
||||
if (File.Exists(poetryLock))
|
||||
{
|
||||
await ParsePoetryLockAsync(context, poetryLock, entries, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return entries.Count == 0 ? PythonLockData.Empty : new PythonLockData(entries);
|
||||
}
|
||||
|
||||
private static async Task ParseRequirementsFileAsync(LanguageAnalyzerContext context, string path, IDictionary<string, PythonLockEntry> entries, CancellationToken cancellationToken)
|
||||
{
|
||||
await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
using var reader = new StreamReader(stream);
|
||||
string? line;
|
||||
var locator = PythonPathHelper.NormalizeRelative(context, path);
|
||||
|
||||
while ((line = await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false)) is not null)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
line = line.Trim();
|
||||
if (string.IsNullOrWhiteSpace(line) || line.StartsWith("#", StringComparison.Ordinal) || line.StartsWith("-r ", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var editableMatch = EditablePattern.Match(line);
|
||||
if (editableMatch.Success)
|
||||
{
|
||||
var editablePath = editableMatch.Groups["path"].Value.Trim().Trim('"', '\'');
|
||||
var packageName = Path.GetFileName(editablePath.TrimEnd(Path.DirectorySeparatorChar, '/'));
|
||||
if (string.IsNullOrWhiteSpace(packageName))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var entry = new PythonLockEntry(
|
||||
Name: packageName,
|
||||
Version: null,
|
||||
Source: Path.GetFileName(path),
|
||||
Locator: locator,
|
||||
Extras: Array.Empty<string>(),
|
||||
Resolved: null,
|
||||
Index: null,
|
||||
EditablePath: editablePath);
|
||||
|
||||
entries[entry.DeclarationKey] = entry;
|
||||
continue;
|
||||
}
|
||||
|
||||
var match = RequirementLinePattern.Match(line);
|
||||
if (!match.Success)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var name = match.Groups["name"].Value;
|
||||
var version = match.Groups["version"].Value;
|
||||
var extras = match.Groups["extras"].Success
|
||||
? match.Groups["extras"].Value.Trim('[', ']').Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
|
||||
: Array.Empty<string>();
|
||||
|
||||
var requirementEntry = new PythonLockEntry(
|
||||
Name: name,
|
||||
Version: version,
|
||||
Source: Path.GetFileName(path),
|
||||
Locator: locator,
|
||||
Extras: extras,
|
||||
Resolved: null,
|
||||
Index: null,
|
||||
EditablePath: null);
|
||||
|
||||
entries[requirementEntry.DeclarationKey] = requirementEntry;
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task ParsePipfileLockAsync(LanguageAnalyzerContext context, string path, IDictionary<string, PythonLockEntry> entries, CancellationToken cancellationToken)
|
||||
{
|
||||
await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var root = document.RootElement;
|
||||
if (!root.TryGetProperty("default", out var defaultDeps))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var property in defaultDeps.EnumerateObject())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (!property.Value.TryGetProperty("version", out var versionElement))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var version = versionElement.GetString();
|
||||
if (string.IsNullOrWhiteSpace(version))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
version = version.TrimStart('=', ' ');
|
||||
var entry = new PythonLockEntry(
|
||||
Name: property.Name,
|
||||
Version: version,
|
||||
Source: "Pipfile.lock",
|
||||
Locator: PythonPathHelper.NormalizeRelative(context, path),
|
||||
Extras: Array.Empty<string>(),
|
||||
Resolved: property.Value.TryGetProperty("file", out var fileElement) ? fileElement.GetString() : null,
|
||||
Index: property.Value.TryGetProperty("index", out var indexElement) ? indexElement.GetString() : null,
|
||||
EditablePath: null);
|
||||
|
||||
entries[entry.DeclarationKey] = entry;
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task ParsePoetryLockAsync(LanguageAnalyzerContext context, string path, IDictionary<string, PythonLockEntry> entries, CancellationToken cancellationToken)
|
||||
{
|
||||
using var reader = new StreamReader(path);
|
||||
string? line;
|
||||
string? currentName = null;
|
||||
string? currentVersion = null;
|
||||
var extras = new List<string>();
|
||||
|
||||
void Flush()
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(currentName) || string.IsNullOrWhiteSpace(currentVersion))
|
||||
{
|
||||
currentName = null;
|
||||
currentVersion = null;
|
||||
extras.Clear();
|
||||
return;
|
||||
}
|
||||
|
||||
var entry = new PythonLockEntry(
|
||||
Name: currentName!,
|
||||
Version: currentVersion!,
|
||||
Source: "poetry.lock",
|
||||
Locator: PythonPathHelper.NormalizeRelative(context, path),
|
||||
Extras: extras.ToArray(),
|
||||
Resolved: null,
|
||||
Index: null,
|
||||
EditablePath: null);
|
||||
|
||||
entries[entry.DeclarationKey] = entry;
|
||||
currentName = null;
|
||||
currentVersion = null;
|
||||
extras.Clear();
|
||||
}
|
||||
|
||||
while ((line = await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false)) is not null)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
line = line.Trim();
|
||||
if (line.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (line.StartsWith("[[package]]", StringComparison.Ordinal))
|
||||
{
|
||||
Flush();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (line.StartsWith("name = ", StringComparison.Ordinal))
|
||||
{
|
||||
currentName = TrimQuoted(line);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (line.StartsWith("version = ", StringComparison.Ordinal))
|
||||
{
|
||||
currentVersion = TrimQuoted(line);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (line.StartsWith("extras = [", StringComparison.Ordinal))
|
||||
{
|
||||
|
||||
var extrasValue = line["extras = ".Length..].Trim();
|
||||
extrasValue = extrasValue.Trim('[', ']');
|
||||
extras.AddRange(extrasValue.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).Select(static x => x.Trim('"')));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
Flush();
|
||||
}
|
||||
|
||||
private static string TrimQuoted(string line)
|
||||
{
|
||||
var index = line.IndexOf('=', StringComparison.Ordinal);
|
||||
if (index < 0)
|
||||
{
|
||||
return line;
|
||||
}
|
||||
|
||||
var value = line[(index + 1)..].Trim();
|
||||
return value.Trim('"');
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record PythonLockEntry(
|
||||
string Name,
|
||||
string? Version,
|
||||
string Source,
|
||||
string Locator,
|
||||
IReadOnlyCollection<string> Extras,
|
||||
string? Resolved,
|
||||
string? Index,
|
||||
string? EditablePath)
|
||||
{
|
||||
public string DeclarationKey => BuildKey(Name, Version);
|
||||
|
||||
private static string BuildKey(string name, string? version)
|
||||
{
|
||||
var normalized = PythonPathHelper.NormalizePackageName(name);
|
||||
return version is null
|
||||
? normalized
|
||||
: $"{normalized}@{version}".ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class PythonLockData
|
||||
{
|
||||
public static readonly PythonLockData Empty = new(new Dictionary<string, PythonLockEntry>(StringComparer.OrdinalIgnoreCase));
|
||||
|
||||
private readonly Dictionary<string, PythonLockEntry> _entries;
|
||||
|
||||
public PythonLockData(Dictionary<string, PythonLockEntry> entries)
|
||||
{
|
||||
_entries = entries;
|
||||
}
|
||||
|
||||
public IReadOnlyCollection<PythonLockEntry> Entries => _entries.Values;
|
||||
|
||||
public bool TryGet(string name, string version, out PythonLockEntry? entry)
|
||||
{
|
||||
var key = $"{PythonPathHelper.NormalizePackageName(name)}@{version}".ToLowerInvariant();
|
||||
return _entries.TryGetValue(key, out entry);
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
using System.Text.Json;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Python.Internal;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Python;
|
||||
@@ -24,18 +25,22 @@ public sealed class PythonLanguageAnalyzer : ILanguageAnalyzer
|
||||
return AnalyzeInternalAsync(context, writer, cancellationToken);
|
||||
}
|
||||
|
||||
private static async ValueTask AnalyzeInternalAsync(LanguageAnalyzerContext context, LanguageComponentWriter writer, CancellationToken cancellationToken)
|
||||
{
|
||||
var distInfoDirectories = Directory
|
||||
.EnumerateDirectories(context.RootPath, "*.dist-info", Enumeration)
|
||||
.OrderBy(static path => path, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
|
||||
foreach (var distInfoPath in distInfoDirectories)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
PythonDistribution? distribution;
|
||||
private static async ValueTask AnalyzeInternalAsync(LanguageAnalyzerContext context, LanguageComponentWriter writer, CancellationToken cancellationToken)
|
||||
{
|
||||
var lockData = await PythonLockFileCollector.LoadAsync(context, cancellationToken).ConfigureAwait(false);
|
||||
var matchedLocks = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var hasLockEntries = lockData.Entries.Count > 0;
|
||||
|
||||
var distInfoDirectories = Directory
|
||||
.EnumerateDirectories(context.RootPath, "*.dist-info", Enumeration)
|
||||
.OrderBy(static path => path, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
|
||||
foreach (var distInfoPath in distInfoDirectories)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
PythonDistribution? distribution;
|
||||
try
|
||||
{
|
||||
distribution = await PythonDistributionLoader.LoadAsync(context, distInfoPath, cancellationToken).ConfigureAwait(false);
|
||||
@@ -54,19 +59,104 @@ public sealed class PythonLanguageAnalyzer : ILanguageAnalyzer
|
||||
}
|
||||
|
||||
if (distribution is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
writer.AddFromPurl(
|
||||
analyzerId: "python",
|
||||
purl: distribution.Purl,
|
||||
name: distribution.Name,
|
||||
version: distribution.Version,
|
||||
type: "pypi",
|
||||
metadata: distribution.SortedMetadata,
|
||||
evidence: distribution.SortedEvidence,
|
||||
usedByEntrypoint: distribution.UsedByEntrypoint);
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var metadata = distribution.SortedMetadata.ToList();
|
||||
|
||||
if (lockData.TryGet(distribution.Name, distribution.Version, out var lockEntry))
|
||||
{
|
||||
matchedLocks.Add(lockEntry!.DeclarationKey);
|
||||
AppendLockMetadata(metadata, lockEntry);
|
||||
}
|
||||
else if (hasLockEntries)
|
||||
{
|
||||
metadata.Add(new KeyValuePair<string, string?>("lockMissing", "true"));
|
||||
}
|
||||
|
||||
writer.AddFromPurl(
|
||||
analyzerId: "python",
|
||||
purl: distribution.Purl,
|
||||
name: distribution.Name,
|
||||
version: distribution.Version,
|
||||
type: "pypi",
|
||||
metadata: metadata,
|
||||
evidence: distribution.SortedEvidence,
|
||||
usedByEntrypoint: distribution.UsedByEntrypoint);
|
||||
}
|
||||
|
||||
if (lockData.Entries.Count > 0)
|
||||
{
|
||||
foreach (var entry in lockData.Entries)
|
||||
{
|
||||
if (matchedLocks.Contains(entry.DeclarationKey))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var declaredMetadata = new List<KeyValuePair<string, string?>>
|
||||
{
|
||||
new("declaredOnly", "true"),
|
||||
new("lockSource", entry.Source),
|
||||
new("lockLocator", entry.Locator)
|
||||
};
|
||||
|
||||
AppendCommonLockFields(declaredMetadata, entry);
|
||||
|
||||
var version = string.IsNullOrWhiteSpace(entry.Version) ? "editable" : entry.Version!;
|
||||
var purl = $"pkg:pypi/{PythonPathHelper.NormalizePackageName(entry.Name)}@{version}";
|
||||
|
||||
var evidence = new[]
|
||||
{
|
||||
new LanguageComponentEvidence(
|
||||
LanguageEvidenceKind.Metadata,
|
||||
entry.Source,
|
||||
entry.Locator,
|
||||
entry.Resolved,
|
||||
Sha256: null)
|
||||
};
|
||||
|
||||
writer.AddFromPurl(
|
||||
analyzerId: "python",
|
||||
purl: purl,
|
||||
name: entry.Name,
|
||||
version: version,
|
||||
type: "pypi",
|
||||
metadata: declaredMetadata,
|
||||
evidence: evidence,
|
||||
usedByEntrypoint: false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void AppendLockMetadata(List<KeyValuePair<string, string?>> metadata, PythonLockEntry entry)
|
||||
{
|
||||
metadata.Add(new KeyValuePair<string, string?>("lockSource", entry.Source));
|
||||
metadata.Add(new KeyValuePair<string, string?>("lockLocator", entry.Locator));
|
||||
AppendCommonLockFields(metadata, entry);
|
||||
}
|
||||
|
||||
private static void AppendCommonLockFields(List<KeyValuePair<string, string?>> metadata, PythonLockEntry entry)
|
||||
{
|
||||
if (entry.Extras.Count > 0)
|
||||
{
|
||||
metadata.Add(new KeyValuePair<string, string?>("lockExtras", string.Join(';', entry.Extras)));
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(entry.Resolved))
|
||||
{
|
||||
metadata.Add(new KeyValuePair<string, string?>("lockResolved", entry.Resolved));
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(entry.Index))
|
||||
{
|
||||
metadata.Add(new KeyValuePair<string, string?>("lockIndex", entry.Index));
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(entry.EditablePath))
|
||||
{
|
||||
metadata.Add(new KeyValuePair<string, string?>("lockEditablePath", entry.EditablePath));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user