feat(ruby): Implement RubyManifestParser for parsing gem groups and dependencies
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

feat(ruby): Add RubyVendorArtifactCollector to collect vendor artifacts

test(deno): Add golden tests for Deno analyzer with various fixtures

test(deno): Create Deno module and package files for testing

test(deno): Implement Deno lock and import map for dependency management

test(deno): Add FFI and worker scripts for Deno testing

feat(ruby): Set up Ruby workspace with Gemfile and dependencies

feat(ruby): Add expected output for Ruby workspace tests

feat(signals): Introduce CallgraphManifest model for signal processing
This commit is contained in:
master
2025-11-10 09:27:03 +02:00
parent 69c59defdc
commit 56c687253f
87 changed files with 2462 additions and 542 deletions

View File

@@ -1,8 +1,10 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Security.Cryptography;
using System.Text;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
@@ -26,10 +28,11 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
private readonly ICallgraphParserResolver parserResolver;
private readonly ICallgraphArtifactStore artifactStore;
private readonly ICallgraphRepository repository;
private readonly ILogger<CallgraphIngestionService> logger;
private readonly SignalsOptions options;
private readonly TimeProvider timeProvider;
private readonly ICallgraphRepository repository;
private readonly ILogger<CallgraphIngestionService> logger;
private readonly SignalsOptions options;
private readonly TimeProvider timeProvider;
private static readonly JsonSerializerOptions ManifestSerializerOptions = new(JsonSerializerDefaults.Web);
public CallgraphIngestionService(
ICallgraphParserResolver parserResolver,
@@ -53,23 +56,42 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
var parser = parserResolver.Resolve(request.Language);
var artifactBytes = Convert.FromBase64String(request.ArtifactContentBase64);
await using var parseStream = new MemoryStream(artifactBytes, writable: false);
var parseResult = await parser.ParseAsync(parseStream, cancellationToken).ConfigureAwait(false);
parseStream.Position = 0;
var hash = ComputeSha256(artifactBytes);
var artifactMetadata = await artifactStore.SaveAsync(
new CallgraphArtifactSaveRequest(
request.Language,
request.Component,
request.Version,
request.ArtifactFileName,
request.ArtifactContentType,
hash),
parseStream,
cancellationToken).ConfigureAwait(false);
var artifactBytes = Convert.FromBase64String(request.ArtifactContentBase64);
await using var parseStream = new MemoryStream(artifactBytes, writable: false);
var parseResult = await parser.ParseAsync(parseStream, cancellationToken).ConfigureAwait(false);
parseStream.Position = 0;
var artifactHash = ComputeSha256(artifactBytes);
var graphHash = ComputeGraphHash(parseResult);
var manifest = new CallgraphManifest
{
Language = request.Language,
Component = request.Component,
Version = request.Version,
ArtifactHash = artifactHash,
GraphHash = graphHash,
NodeCount = parseResult.Nodes.Count,
EdgeCount = parseResult.Edges.Count,
CreatedAt = timeProvider.GetUtcNow()
};
await using var manifestStream = new MemoryStream();
await JsonSerializer.SerializeAsync(manifestStream, manifest, ManifestSerializerOptions, cancellationToken).ConfigureAwait(false);
manifestStream.Position = 0;
parseStream.Position = 0;
var artifactMetadata = await artifactStore.SaveAsync(
new CallgraphArtifactSaveRequest(
request.Language,
request.Component,
request.Version,
request.ArtifactFileName,
request.ArtifactContentType,
artifactHash,
manifestStream),
parseStream,
cancellationToken).ConfigureAwait(false);
var document = new CallgraphDocument
{
@@ -81,21 +103,25 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
Metadata = request.Metadata is null
? null
: new Dictionary<string, string?>(request.Metadata, StringComparer.OrdinalIgnoreCase),
Artifact = new CallgraphArtifactMetadata
{
Path = artifactMetadata.Path,
Hash = artifactMetadata.Hash,
CasUri = artifactMetadata.CasUri,
ContentType = artifactMetadata.ContentType,
Length = artifactMetadata.Length
},
IngestedAt = timeProvider.GetUtcNow()
};
document.Metadata ??= new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
document.Metadata["formatVersion"] = parseResult.FormatVersion;
document = await repository.UpsertAsync(document, cancellationToken).ConfigureAwait(false);
Artifact = new CallgraphArtifactMetadata
{
Path = artifactMetadata.Path,
Hash = artifactMetadata.Hash,
CasUri = artifactMetadata.CasUri,
ManifestPath = artifactMetadata.ManifestPath,
ManifestCasUri = artifactMetadata.ManifestCasUri,
GraphHash = graphHash,
ContentType = artifactMetadata.ContentType,
Length = artifactMetadata.Length
},
IngestedAt = timeProvider.GetUtcNow()
};
document.Metadata ??= new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
document.Metadata["formatVersion"] = parseResult.FormatVersion;
document.GraphHash = graphHash;
document = await repository.UpsertAsync(document, cancellationToken).ConfigureAwait(false);
logger.LogInformation(
"Ingested callgraph {Language}:{Component}:{Version} (id={Id}) with {NodeCount} nodes and {EdgeCount} edges.",
@@ -106,7 +132,13 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
document.Nodes.Count,
document.Edges.Count);
return new CallgraphIngestResponse(document.Id, document.Artifact.Path, document.Artifact.Hash, document.Artifact.CasUri);
return new CallgraphIngestResponse(
document.Id,
document.Artifact.Path,
document.Artifact.Hash,
document.Artifact.CasUri,
graphHash,
document.Artifact.ManifestCasUri);
}
private static void ValidateRequest(CallgraphIngestRequest request)
@@ -144,13 +176,29 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
}
}
private static string ComputeSha256(ReadOnlySpan<byte> buffer)
{
Span<byte> hash = stackalloc byte[SHA256.HashSizeInBytes];
SHA256.HashData(buffer, hash);
return Convert.ToHexString(hash);
}
}
private static string ComputeSha256(ReadOnlySpan<byte> buffer)
{
Span<byte> hash = stackalloc byte[SHA256.HashSizeInBytes];
SHA256.HashData(buffer, hash);
return Convert.ToHexString(hash);
}
private static string ComputeGraphHash(CallgraphParseResult result)
{
var builder = new StringBuilder();
foreach (var node in result.Nodes.OrderBy(n => n.Id, StringComparer.Ordinal))
{
builder.Append(node.Id).Append('|').Append(node.Name).AppendLine();
}
foreach (var edge in result.Edges.OrderBy(e => e.SourceId, StringComparer.Ordinal).ThenBy(e => e.TargetId, StringComparer.Ordinal))
{
builder.Append(edge.SourceId).Append("->").Append(edge.TargetId).AppendLine();
}
return ComputeSha256(Encoding.UTF8.GetBytes(builder.ToString()));
}
}
/// <summary>
/// Exception thrown when the ingestion request is invalid.

View File

@@ -40,6 +40,14 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
ValidateRequest(request);
var subjectKey = request.Subject?.ToSubjectKey();
if (string.IsNullOrWhiteSpace(subjectKey))
{
throw new ReachabilityScoringValidationException("Subject must include scanId, imageDigest, or component/version.");
}
var existingFact = await factRepository.GetBySubjectAsync(subjectKey, cancellationToken).ConfigureAwait(false);
var callgraph = await callgraphRepository.GetByIdAsync(request.CallgraphId, cancellationToken).ConfigureAwait(false);
if (callgraph is null)
{
@@ -57,10 +65,24 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
throw new ReachabilityScoringValidationException("At least one target symbol is required.");
}
var runtimeHits = request.RuntimeHits?.Where(hit => !string.IsNullOrWhiteSpace(hit))
.Select(hit => hit.Trim())
.Distinct(StringComparer.Ordinal)
.ToList() ?? new List<string>();
var runtimeHitSet = new HashSet<string>(StringComparer.Ordinal);
if (existingFact?.RuntimeFacts is { Count: > 0 })
{
foreach (var fact in existingFact.RuntimeFacts.Where(f => !string.IsNullOrWhiteSpace(f.SymbolId)))
{
runtimeHitSet.Add(fact.SymbolId);
}
}
if (request.RuntimeHits is { Count: > 0 })
{
foreach (var hit in request.RuntimeHits.Where(h => !string.IsNullOrWhiteSpace(h)))
{
runtimeHitSet.Add(hit.Trim());
}
}
var runtimeHits = runtimeHitSet.ToList();
var states = new List<ReachabilityStateDocument>(targets.Count);
foreach (var target in targets)
@@ -95,12 +117,13 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
var document = new ReachabilityFactDocument
{
CallgraphId = request.CallgraphId,
Subject = request.Subject,
Subject = request.Subject!,
EntryPoints = entryPoints,
States = states,
Metadata = request.Metadata,
ComputedAt = timeProvider.GetUtcNow(),
SubjectKey = request.Subject.ToSubjectKey()
SubjectKey = subjectKey,
RuntimeFacts = existingFact?.RuntimeFacts
};
logger.LogInformation("Computed reachability fact for subject {SubjectKey} with {StateCount} targets.", document.SubjectKey, states.Count);

View File

@@ -115,6 +115,11 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
SymbolId = key.SymbolId,
CodeId = key.CodeId,
LoaderBase = key.LoaderBase,
ProcessId = evt.ProcessId,
ProcessName = Normalize(evt.ProcessName),
SocketAddress = Normalize(evt.SocketAddress),
ContainerId = Normalize(evt.ContainerId),
EvidenceUri = Normalize(evt.EvidenceUri),
Metadata = evt.Metadata != null
? new Dictionary<string, string?>(evt.Metadata, StringComparer.Ordinal)
: null
@@ -152,9 +157,9 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
if (incoming != null)
{
foreach (var (key, value) in incoming)
foreach (var (metaKey, metaValue) in incoming)
{
merged[key] = value;
merged[metaKey] = metaValue;
}
}
@@ -177,6 +182,11 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
SymbolId = fact.SymbolId,
CodeId = fact.CodeId,
LoaderBase = fact.LoaderBase,
ProcessId = fact.ProcessId,
ProcessName = fact.ProcessName,
SocketAddress = fact.SocketAddress,
ContainerId = fact.ContainerId,
EvidenceUri = fact.EvidenceUri,
HitCount = fact.HitCount,
Metadata = fact.Metadata is null
? null
@@ -197,6 +207,11 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
SymbolId = fact.SymbolId,
CodeId = fact.CodeId,
LoaderBase = fact.LoaderBase,
ProcessId = fact.ProcessId,
ProcessName = fact.ProcessName,
SocketAddress = fact.SocketAddress,
ContainerId = fact.ContainerId,
EvidenceUri = fact.EvidenceUri,
HitCount = fact.HitCount,
Metadata = fact.Metadata is null
? null
@@ -206,12 +221,17 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
}
existingFact.HitCount = Math.Clamp(existingFact.HitCount + fact.HitCount, 1, int.MaxValue);
existingFact.ProcessId ??= fact.ProcessId;
existingFact.ProcessName ??= fact.ProcessName;
existingFact.SocketAddress ??= fact.SocketAddress;
existingFact.ContainerId ??= fact.ContainerId;
existingFact.EvidenceUri ??= fact.EvidenceUri;
if (fact.Metadata != null && fact.Metadata.Count > 0)
{
existingFact.Metadata ??= new Dictionary<string, string?>(StringComparer.Ordinal);
foreach (var (key, value) in fact.Metadata)
foreach (var (metaKey, metaValue) in fact.Metadata)
{
existingFact.Metadata[key] = value;
existingFact.Metadata[metaKey] = metaValue;
}
}
}
@@ -224,6 +244,9 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
.ToList();
}
private static string? Normalize(string? value) =>
string.IsNullOrWhiteSpace(value) ? null : value.Trim();
private readonly record struct RuntimeFactKey(string SymbolId, string? CodeId, string? LoaderBase);
private sealed class RuntimeFactKeyComparer : IEqualityComparer<RuntimeFactKey>