feat(ruby): Implement RubyManifestParser for parsing gem groups and dependencies
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
feat(ruby): Add RubyVendorArtifactCollector to collect vendor artifacts test(deno): Add golden tests for Deno analyzer with various fixtures test(deno): Create Deno module and package files for testing test(deno): Implement Deno lock and import map for dependency management test(deno): Add FFI and worker scripts for Deno testing feat(ruby): Set up Ruby workspace with Gemfile and dependencies feat(ruby): Add expected output for Ruby workspace tests feat(signals): Introduce CallgraphManifest model for signal processing
This commit is contained in:
@@ -1,8 +1,10 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -26,10 +28,11 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
|
||||
private readonly ICallgraphParserResolver parserResolver;
|
||||
private readonly ICallgraphArtifactStore artifactStore;
|
||||
private readonly ICallgraphRepository repository;
|
||||
private readonly ILogger<CallgraphIngestionService> logger;
|
||||
private readonly SignalsOptions options;
|
||||
private readonly TimeProvider timeProvider;
|
||||
private readonly ICallgraphRepository repository;
|
||||
private readonly ILogger<CallgraphIngestionService> logger;
|
||||
private readonly SignalsOptions options;
|
||||
private readonly TimeProvider timeProvider;
|
||||
private static readonly JsonSerializerOptions ManifestSerializerOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
public CallgraphIngestionService(
|
||||
ICallgraphParserResolver parserResolver,
|
||||
@@ -53,23 +56,42 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
|
||||
var parser = parserResolver.Resolve(request.Language);
|
||||
|
||||
var artifactBytes = Convert.FromBase64String(request.ArtifactContentBase64);
|
||||
await using var parseStream = new MemoryStream(artifactBytes, writable: false);
|
||||
var parseResult = await parser.ParseAsync(parseStream, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
parseStream.Position = 0;
|
||||
var hash = ComputeSha256(artifactBytes);
|
||||
|
||||
var artifactMetadata = await artifactStore.SaveAsync(
|
||||
new CallgraphArtifactSaveRequest(
|
||||
request.Language,
|
||||
request.Component,
|
||||
request.Version,
|
||||
request.ArtifactFileName,
|
||||
request.ArtifactContentType,
|
||||
hash),
|
||||
parseStream,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
var artifactBytes = Convert.FromBase64String(request.ArtifactContentBase64);
|
||||
await using var parseStream = new MemoryStream(artifactBytes, writable: false);
|
||||
var parseResult = await parser.ParseAsync(parseStream, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
parseStream.Position = 0;
|
||||
var artifactHash = ComputeSha256(artifactBytes);
|
||||
var graphHash = ComputeGraphHash(parseResult);
|
||||
|
||||
var manifest = new CallgraphManifest
|
||||
{
|
||||
Language = request.Language,
|
||||
Component = request.Component,
|
||||
Version = request.Version,
|
||||
ArtifactHash = artifactHash,
|
||||
GraphHash = graphHash,
|
||||
NodeCount = parseResult.Nodes.Count,
|
||||
EdgeCount = parseResult.Edges.Count,
|
||||
CreatedAt = timeProvider.GetUtcNow()
|
||||
};
|
||||
|
||||
await using var manifestStream = new MemoryStream();
|
||||
await JsonSerializer.SerializeAsync(manifestStream, manifest, ManifestSerializerOptions, cancellationToken).ConfigureAwait(false);
|
||||
manifestStream.Position = 0;
|
||||
|
||||
parseStream.Position = 0;
|
||||
var artifactMetadata = await artifactStore.SaveAsync(
|
||||
new CallgraphArtifactSaveRequest(
|
||||
request.Language,
|
||||
request.Component,
|
||||
request.Version,
|
||||
request.ArtifactFileName,
|
||||
request.ArtifactContentType,
|
||||
artifactHash,
|
||||
manifestStream),
|
||||
parseStream,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var document = new CallgraphDocument
|
||||
{
|
||||
@@ -81,21 +103,25 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
Metadata = request.Metadata is null
|
||||
? null
|
||||
: new Dictionary<string, string?>(request.Metadata, StringComparer.OrdinalIgnoreCase),
|
||||
Artifact = new CallgraphArtifactMetadata
|
||||
{
|
||||
Path = artifactMetadata.Path,
|
||||
Hash = artifactMetadata.Hash,
|
||||
CasUri = artifactMetadata.CasUri,
|
||||
ContentType = artifactMetadata.ContentType,
|
||||
Length = artifactMetadata.Length
|
||||
},
|
||||
IngestedAt = timeProvider.GetUtcNow()
|
||||
};
|
||||
|
||||
document.Metadata ??= new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
|
||||
document.Metadata["formatVersion"] = parseResult.FormatVersion;
|
||||
|
||||
document = await repository.UpsertAsync(document, cancellationToken).ConfigureAwait(false);
|
||||
Artifact = new CallgraphArtifactMetadata
|
||||
{
|
||||
Path = artifactMetadata.Path,
|
||||
Hash = artifactMetadata.Hash,
|
||||
CasUri = artifactMetadata.CasUri,
|
||||
ManifestPath = artifactMetadata.ManifestPath,
|
||||
ManifestCasUri = artifactMetadata.ManifestCasUri,
|
||||
GraphHash = graphHash,
|
||||
ContentType = artifactMetadata.ContentType,
|
||||
Length = artifactMetadata.Length
|
||||
},
|
||||
IngestedAt = timeProvider.GetUtcNow()
|
||||
};
|
||||
|
||||
document.Metadata ??= new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
|
||||
document.Metadata["formatVersion"] = parseResult.FormatVersion;
|
||||
document.GraphHash = graphHash;
|
||||
|
||||
document = await repository.UpsertAsync(document, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
logger.LogInformation(
|
||||
"Ingested callgraph {Language}:{Component}:{Version} (id={Id}) with {NodeCount} nodes and {EdgeCount} edges.",
|
||||
@@ -106,7 +132,13 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
document.Nodes.Count,
|
||||
document.Edges.Count);
|
||||
|
||||
return new CallgraphIngestResponse(document.Id, document.Artifact.Path, document.Artifact.Hash, document.Artifact.CasUri);
|
||||
return new CallgraphIngestResponse(
|
||||
document.Id,
|
||||
document.Artifact.Path,
|
||||
document.Artifact.Hash,
|
||||
document.Artifact.CasUri,
|
||||
graphHash,
|
||||
document.Artifact.ManifestCasUri);
|
||||
}
|
||||
|
||||
private static void ValidateRequest(CallgraphIngestRequest request)
|
||||
@@ -144,13 +176,29 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
}
|
||||
}
|
||||
|
||||
private static string ComputeSha256(ReadOnlySpan<byte> buffer)
|
||||
{
|
||||
Span<byte> hash = stackalloc byte[SHA256.HashSizeInBytes];
|
||||
SHA256.HashData(buffer, hash);
|
||||
return Convert.ToHexString(hash);
|
||||
}
|
||||
}
|
||||
private static string ComputeSha256(ReadOnlySpan<byte> buffer)
|
||||
{
|
||||
Span<byte> hash = stackalloc byte[SHA256.HashSizeInBytes];
|
||||
SHA256.HashData(buffer, hash);
|
||||
return Convert.ToHexString(hash);
|
||||
}
|
||||
|
||||
private static string ComputeGraphHash(CallgraphParseResult result)
|
||||
{
|
||||
var builder = new StringBuilder();
|
||||
foreach (var node in result.Nodes.OrderBy(n => n.Id, StringComparer.Ordinal))
|
||||
{
|
||||
builder.Append(node.Id).Append('|').Append(node.Name).AppendLine();
|
||||
}
|
||||
|
||||
foreach (var edge in result.Edges.OrderBy(e => e.SourceId, StringComparer.Ordinal).ThenBy(e => e.TargetId, StringComparer.Ordinal))
|
||||
{
|
||||
builder.Append(edge.SourceId).Append("->").Append(edge.TargetId).AppendLine();
|
||||
}
|
||||
|
||||
return ComputeSha256(Encoding.UTF8.GetBytes(builder.ToString()));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Exception thrown when the ingestion request is invalid.
|
||||
|
||||
@@ -40,6 +40,14 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
|
||||
|
||||
ValidateRequest(request);
|
||||
|
||||
var subjectKey = request.Subject?.ToSubjectKey();
|
||||
if (string.IsNullOrWhiteSpace(subjectKey))
|
||||
{
|
||||
throw new ReachabilityScoringValidationException("Subject must include scanId, imageDigest, or component/version.");
|
||||
}
|
||||
|
||||
var existingFact = await factRepository.GetBySubjectAsync(subjectKey, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var callgraph = await callgraphRepository.GetByIdAsync(request.CallgraphId, cancellationToken).ConfigureAwait(false);
|
||||
if (callgraph is null)
|
||||
{
|
||||
@@ -57,10 +65,24 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
|
||||
throw new ReachabilityScoringValidationException("At least one target symbol is required.");
|
||||
}
|
||||
|
||||
var runtimeHits = request.RuntimeHits?.Where(hit => !string.IsNullOrWhiteSpace(hit))
|
||||
.Select(hit => hit.Trim())
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.ToList() ?? new List<string>();
|
||||
var runtimeHitSet = new HashSet<string>(StringComparer.Ordinal);
|
||||
if (existingFact?.RuntimeFacts is { Count: > 0 })
|
||||
{
|
||||
foreach (var fact in existingFact.RuntimeFacts.Where(f => !string.IsNullOrWhiteSpace(f.SymbolId)))
|
||||
{
|
||||
runtimeHitSet.Add(fact.SymbolId);
|
||||
}
|
||||
}
|
||||
|
||||
if (request.RuntimeHits is { Count: > 0 })
|
||||
{
|
||||
foreach (var hit in request.RuntimeHits.Where(h => !string.IsNullOrWhiteSpace(h)))
|
||||
{
|
||||
runtimeHitSet.Add(hit.Trim());
|
||||
}
|
||||
}
|
||||
|
||||
var runtimeHits = runtimeHitSet.ToList();
|
||||
|
||||
var states = new List<ReachabilityStateDocument>(targets.Count);
|
||||
foreach (var target in targets)
|
||||
@@ -95,12 +117,13 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
|
||||
var document = new ReachabilityFactDocument
|
||||
{
|
||||
CallgraphId = request.CallgraphId,
|
||||
Subject = request.Subject,
|
||||
Subject = request.Subject!,
|
||||
EntryPoints = entryPoints,
|
||||
States = states,
|
||||
Metadata = request.Metadata,
|
||||
ComputedAt = timeProvider.GetUtcNow(),
|
||||
SubjectKey = request.Subject.ToSubjectKey()
|
||||
SubjectKey = subjectKey,
|
||||
RuntimeFacts = existingFact?.RuntimeFacts
|
||||
};
|
||||
|
||||
logger.LogInformation("Computed reachability fact for subject {SubjectKey} with {StateCount} targets.", document.SubjectKey, states.Count);
|
||||
|
||||
@@ -115,6 +115,11 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
SymbolId = key.SymbolId,
|
||||
CodeId = key.CodeId,
|
||||
LoaderBase = key.LoaderBase,
|
||||
ProcessId = evt.ProcessId,
|
||||
ProcessName = Normalize(evt.ProcessName),
|
||||
SocketAddress = Normalize(evt.SocketAddress),
|
||||
ContainerId = Normalize(evt.ContainerId),
|
||||
EvidenceUri = Normalize(evt.EvidenceUri),
|
||||
Metadata = evt.Metadata != null
|
||||
? new Dictionary<string, string?>(evt.Metadata, StringComparer.Ordinal)
|
||||
: null
|
||||
@@ -152,9 +157,9 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
|
||||
if (incoming != null)
|
||||
{
|
||||
foreach (var (key, value) in incoming)
|
||||
foreach (var (metaKey, metaValue) in incoming)
|
||||
{
|
||||
merged[key] = value;
|
||||
merged[metaKey] = metaValue;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -177,6 +182,11 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
SymbolId = fact.SymbolId,
|
||||
CodeId = fact.CodeId,
|
||||
LoaderBase = fact.LoaderBase,
|
||||
ProcessId = fact.ProcessId,
|
||||
ProcessName = fact.ProcessName,
|
||||
SocketAddress = fact.SocketAddress,
|
||||
ContainerId = fact.ContainerId,
|
||||
EvidenceUri = fact.EvidenceUri,
|
||||
HitCount = fact.HitCount,
|
||||
Metadata = fact.Metadata is null
|
||||
? null
|
||||
@@ -197,6 +207,11 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
SymbolId = fact.SymbolId,
|
||||
CodeId = fact.CodeId,
|
||||
LoaderBase = fact.LoaderBase,
|
||||
ProcessId = fact.ProcessId,
|
||||
ProcessName = fact.ProcessName,
|
||||
SocketAddress = fact.SocketAddress,
|
||||
ContainerId = fact.ContainerId,
|
||||
EvidenceUri = fact.EvidenceUri,
|
||||
HitCount = fact.HitCount,
|
||||
Metadata = fact.Metadata is null
|
||||
? null
|
||||
@@ -206,12 +221,17 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
}
|
||||
|
||||
existingFact.HitCount = Math.Clamp(existingFact.HitCount + fact.HitCount, 1, int.MaxValue);
|
||||
existingFact.ProcessId ??= fact.ProcessId;
|
||||
existingFact.ProcessName ??= fact.ProcessName;
|
||||
existingFact.SocketAddress ??= fact.SocketAddress;
|
||||
existingFact.ContainerId ??= fact.ContainerId;
|
||||
existingFact.EvidenceUri ??= fact.EvidenceUri;
|
||||
if (fact.Metadata != null && fact.Metadata.Count > 0)
|
||||
{
|
||||
existingFact.Metadata ??= new Dictionary<string, string?>(StringComparer.Ordinal);
|
||||
foreach (var (key, value) in fact.Metadata)
|
||||
foreach (var (metaKey, metaValue) in fact.Metadata)
|
||||
{
|
||||
existingFact.Metadata[key] = value;
|
||||
existingFact.Metadata[metaKey] = metaValue;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -224,6 +244,9 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static string? Normalize(string? value) =>
|
||||
string.IsNullOrWhiteSpace(value) ? null : value.Trim();
|
||||
|
||||
private readonly record struct RuntimeFactKey(string SymbolId, string? CodeId, string? LoaderBase);
|
||||
|
||||
private sealed class RuntimeFactKeyComparer : IEqualityComparer<RuntimeFactKey>
|
||||
|
||||
Reference in New Issue
Block a user