feat(ruby): Implement RubyManifestParser for parsing gem groups and dependencies
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
feat(ruby): Add RubyVendorArtifactCollector to collect vendor artifacts test(deno): Add golden tests for Deno analyzer with various fixtures test(deno): Create Deno module and package files for testing test(deno): Implement Deno lock and import map for dependency management test(deno): Add FFI and worker scripts for Deno testing feat(ruby): Set up Ruby workspace with Gemfile and dependencies feat(ruby): Add expected output for Ruby workspace tests feat(signals): Introduce CallgraphManifest model for signal processing
This commit is contained in:
@@ -16,6 +16,15 @@ public sealed class CallgraphArtifactMetadata
|
||||
[BsonElement("casUri")]
|
||||
public string CasUri { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("manifestPath")]
|
||||
public string ManifestPath { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("manifestCasUri")]
|
||||
public string ManifestCasUri { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("graphHash")]
|
||||
public string GraphHash { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("contentType")]
|
||||
public string ContentType { get; set; } = string.Empty;
|
||||
|
||||
|
||||
@@ -35,7 +35,10 @@ public sealed class CallgraphDocument
|
||||
[BsonElement("edges")]
|
||||
public List<CallgraphEdge> Edges { get; set; } = new();
|
||||
|
||||
[BsonElement("metadata")]
|
||||
[BsonIgnoreIfNull]
|
||||
public Dictionary<string, string?>? Metadata { get; set; }
|
||||
}
|
||||
[BsonElement("metadata")]
|
||||
[BsonIgnoreIfNull]
|
||||
public Dictionary<string, string?>? Metadata { get; set; }
|
||||
|
||||
[BsonElement("graphHash")]
|
||||
public string GraphHash { get; set; } = string.Empty;
|
||||
}
|
||||
|
||||
@@ -7,4 +7,6 @@ public sealed record CallgraphIngestResponse(
|
||||
string CallgraphId,
|
||||
string ArtifactPath,
|
||||
string ArtifactHash,
|
||||
string CasUri);
|
||||
string CasUri,
|
||||
string GraphHash,
|
||||
string ManifestCasUri);
|
||||
|
||||
31
src/Signals/StellaOps.Signals/Models/CallgraphManifest.cs
Normal file
31
src/Signals/StellaOps.Signals/Models/CallgraphManifest.cs
Normal file
@@ -0,0 +1,31 @@
|
||||
using System;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Signals.Models;
|
||||
|
||||
public sealed class CallgraphManifest
|
||||
{
|
||||
[JsonPropertyName("language")]
|
||||
public string Language { get; set; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("component")]
|
||||
public string Component { get; set; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("version")]
|
||||
public string Version { get; set; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("graphHash")]
|
||||
public string GraphHash { get; set; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("artifactHash")]
|
||||
public string ArtifactHash { get; set; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("nodeCount")]
|
||||
public int NodeCount { get; set; }
|
||||
|
||||
[JsonPropertyName("edgeCount")]
|
||||
public int EdgeCount { get; set; }
|
||||
|
||||
[JsonPropertyName("createdAt")]
|
||||
public DateTimeOffset CreatedAt { get; set; }
|
||||
}
|
||||
@@ -114,6 +114,26 @@ public sealed class RuntimeFactDocument
|
||||
[BsonIgnoreIfNull]
|
||||
public string? LoaderBase { get; set; }
|
||||
|
||||
[BsonElement("processId")]
|
||||
[BsonIgnoreIfNull]
|
||||
public int? ProcessId { get; set; }
|
||||
|
||||
[BsonElement("processName")]
|
||||
[BsonIgnoreIfNull]
|
||||
public string? ProcessName { get; set; }
|
||||
|
||||
[BsonElement("socketAddress")]
|
||||
[BsonIgnoreIfNull]
|
||||
public string? SocketAddress { get; set; }
|
||||
|
||||
[BsonElement("containerId")]
|
||||
[BsonIgnoreIfNull]
|
||||
public string? ContainerId { get; set; }
|
||||
|
||||
[BsonElement("evidenceUri")]
|
||||
[BsonIgnoreIfNull]
|
||||
public string? EvidenceUri { get; set; }
|
||||
|
||||
[BsonElement("hitCount")]
|
||||
public int HitCount { get; set; }
|
||||
|
||||
|
||||
@@ -26,6 +26,16 @@ public sealed class RuntimeFactEvent
|
||||
|
||||
public string? LoaderBase { get; set; }
|
||||
|
||||
public int? ProcessId { get; set; }
|
||||
|
||||
public string? ProcessName { get; set; }
|
||||
|
||||
public string? SocketAddress { get; set; }
|
||||
|
||||
public string? ContainerId { get; set; }
|
||||
|
||||
public string? EvidenceUri { get; set; }
|
||||
|
||||
public int HitCount { get; set; } = 1;
|
||||
|
||||
public Dictionary<string, string?>? Metadata { get; set; }
|
||||
|
||||
@@ -1,14 +1,29 @@
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
|
||||
namespace StellaOps.Signals.Models;
|
||||
|
||||
public sealed class RuntimeFactsStreamMetadata
|
||||
{
|
||||
[FromQuery(Name = "callgraphId")]
|
||||
public string CallgraphId { get; set; } = string.Empty;
|
||||
|
||||
[FromQuery(Name = "scanId")]
|
||||
public string? ScanId { get; set; }
|
||||
|
||||
[FromQuery(Name = "imageDigest")]
|
||||
public string? ImageDigest { get; set; }
|
||||
|
||||
[FromQuery(Name = "component")]
|
||||
public string? Component { get; set; }
|
||||
|
||||
[FromQuery(Name = "version")]
|
||||
public string? Version { get; set; }
|
||||
|
||||
public ReachabilitySubject ToSubject() => new()
|
||||
{
|
||||
ScanId = ScanId,
|
||||
ImageDigest = ImageDigest,
|
||||
Component = Component,
|
||||
Version = Version
|
||||
};
|
||||
}
|
||||
|
||||
@@ -206,15 +206,22 @@ app.MapGet("/readyz", (SignalsStartupState state, SignalsSealedModeMonitor seale
|
||||
: Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
|
||||
}).AllowAnonymous();
|
||||
|
||||
var fallbackAllowed = !bootstrap.Authority.Enabled || bootstrap.Authority.AllowAnonymousFallback;
|
||||
|
||||
var signalsGroup = app.MapGroup("/signals");
|
||||
|
||||
signalsGroup.MapGet("/ping", (HttpContext context, SignalsOptions options, SignalsSealedModeMonitor sealedModeMonitor) =>
|
||||
Program.TryAuthorize(context, requiredScope: SignalsPolicies.Read, fallbackAllowed: options.Authority.AllowAnonymousFallback, out var authFailure) &&
|
||||
Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure)
|
||||
? Results.NoContent()
|
||||
: authFailure ?? sealedFailure ?? Results.Unauthorized()).WithName("SignalsPing");
|
||||
{
|
||||
if (!Program.TryAuthorize(context, requiredScope: SignalsPolicies.Read, fallbackAllowed: options.Authority.AllowAnonymousFallback, out var authFailure))
|
||||
{
|
||||
return authFailure ?? Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
{
|
||||
return sealedFailure ?? Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
|
||||
}
|
||||
|
||||
return Results.NoContent();
|
||||
}).WithName("SignalsPing");
|
||||
|
||||
signalsGroup.MapGet("/status", (HttpContext context, SignalsOptions options, SignalsSealedModeMonitor sealedModeMonitor) =>
|
||||
{
|
||||
@@ -245,33 +252,37 @@ signalsGroup.MapPost("/callgraphs", async Task<IResult> (
|
||||
SignalsSealedModeMonitor sealedModeMonitor,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Write, options.Authority.AllowAnonymousFallback, out var authFailure) ||
|
||||
!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Write, options.Authority.AllowAnonymousFallback, out var authFailure))
|
||||
{
|
||||
return authFailure ?? sealedFailure ?? Results.Unauthorized();
|
||||
return authFailure ?? Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
{
|
||||
return sealedFailure ?? Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var result = await ingestionService.IngestAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Accepted($"/signals/callgraphs/{result.CallgraphId}", result);
|
||||
}
|
||||
catch (CallgraphIngestionValidationException ex)
|
||||
{
|
||||
return Results.BadRequest(new { error = ex.Message });
|
||||
}
|
||||
catch (CallgraphParserNotFoundException ex)
|
||||
{
|
||||
return Results.BadRequest(new { error = ex.Message });
|
||||
}
|
||||
catch (CallgraphParserValidationException ex)
|
||||
{
|
||||
return Results.UnprocessableEntity(new { error = ex.Message });
|
||||
}
|
||||
catch (FormatException ex)
|
||||
{
|
||||
return Results.BadRequest(new { error = ex.Message });
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var result = await ingestionService.IngestAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Accepted($"/signals/callgraphs/{result.CallgraphId}", result);
|
||||
}
|
||||
catch (CallgraphIngestionValidationException ex)
|
||||
{
|
||||
return Results.BadRequest(new { error = ex.Message });
|
||||
}
|
||||
catch (CallgraphParserNotFoundException ex)
|
||||
{
|
||||
return Results.BadRequest(new { error = ex.Message });
|
||||
}
|
||||
catch (CallgraphParserValidationException ex)
|
||||
{
|
||||
return Results.UnprocessableEntity(new { error = ex.Message });
|
||||
}
|
||||
catch (FormatException ex)
|
||||
{
|
||||
return Results.BadRequest(new { error = ex.Message });
|
||||
}
|
||||
}).WithName("SignalsCallgraphIngest");
|
||||
|
||||
signalsGroup.MapGet("/callgraphs/{callgraphId}", async Task<IResult> (
|
||||
@@ -282,10 +293,14 @@ signalsGroup.MapGet("/callgraphs/{callgraphId}", async Task<IResult> (
|
||||
SignalsSealedModeMonitor sealedModeMonitor,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Read, options.Authority.AllowAnonymousFallback, out var authFailure) ||
|
||||
!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Read, options.Authority.AllowAnonymousFallback, out var authFailure))
|
||||
{
|
||||
return authFailure ?? sealedFailure ?? Results.Unauthorized();
|
||||
return authFailure ?? Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
{
|
||||
return sealedFailure ?? Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(callgraphId))
|
||||
@@ -296,7 +311,46 @@ signalsGroup.MapGet("/callgraphs/{callgraphId}", async Task<IResult> (
|
||||
var document = await callgraphRepository.GetByIdAsync(callgraphId.Trim(), cancellationToken).ConfigureAwait(false);
|
||||
return document is null ? Results.NotFound() : Results.Ok(document);
|
||||
}).WithName("SignalsCallgraphGet");
|
||||
|
||||
|
||||
signalsGroup.MapGet("/callgraphs/{callgraphId}/manifest", async Task<IResult> (
|
||||
HttpContext context,
|
||||
SignalsOptions options,
|
||||
string callgraphId,
|
||||
ICallgraphRepository callgraphRepository,
|
||||
SignalsSealedModeMonitor sealedModeMonitor,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Read, options.Authority.AllowAnonymousFallback, out var authFailure))
|
||||
{
|
||||
return authFailure ?? Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
{
|
||||
return sealedFailure ?? Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(callgraphId))
|
||||
{
|
||||
return Results.BadRequest(new { error = "callgraphId is required." });
|
||||
}
|
||||
|
||||
var document = await callgraphRepository.GetByIdAsync(callgraphId.Trim(), cancellationToken).ConfigureAwait(false);
|
||||
if (document is null || string.IsNullOrWhiteSpace(document.Artifact.ManifestPath))
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
var manifestPath = Path.Combine(options.Storage.RootPath, document.Artifact.ManifestPath);
|
||||
if (!File.Exists(manifestPath))
|
||||
{
|
||||
return Results.NotFound(new { error = "manifest not found" });
|
||||
}
|
||||
|
||||
var bytes = await File.ReadAllBytesAsync(manifestPath, cancellationToken).ConfigureAwait(false);
|
||||
return Results.File(bytes, "application/json");
|
||||
}).WithName("SignalsCallgraphManifestGet");
|
||||
|
||||
signalsGroup.MapPost("/runtime-facts", async Task<IResult> (
|
||||
HttpContext context,
|
||||
SignalsOptions options,
|
||||
@@ -305,10 +359,14 @@ signalsGroup.MapPost("/runtime-facts", async Task<IResult> (
|
||||
SignalsSealedModeMonitor sealedModeMonitor,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Write, options.Authority.AllowAnonymousFallback, out var authFailure) ||
|
||||
!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Write, options.Authority.AllowAnonymousFallback, out var authFailure))
|
||||
{
|
||||
return authFailure ?? sealedFailure ?? Results.Unauthorized();
|
||||
return authFailure ?? Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
{
|
||||
return sealedFailure ?? Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
|
||||
}
|
||||
|
||||
try
|
||||
@@ -325,15 +383,19 @@ signalsGroup.MapPost("/runtime-facts", async Task<IResult> (
|
||||
signalsGroup.MapPost("/runtime-facts/ndjson", async Task<IResult> (
|
||||
HttpContext context,
|
||||
SignalsOptions options,
|
||||
RuntimeFactsStreamMetadata metadata,
|
||||
[AsParameters] RuntimeFactsStreamMetadata metadata,
|
||||
IRuntimeFactsIngestionService ingestionService,
|
||||
SignalsSealedModeMonitor sealedModeMonitor,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Write, options.Authority.AllowAnonymousFallback, out var authFailure) ||
|
||||
!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Write, options.Authority.AllowAnonymousFallback, out var authFailure))
|
||||
{
|
||||
return authFailure ?? sealedFailure ?? Results.Unauthorized();
|
||||
return authFailure ?? Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
{
|
||||
return sealedFailure ?? Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
|
||||
}
|
||||
|
||||
if (metadata is null || string.IsNullOrWhiteSpace(metadata.CallgraphId))
|
||||
@@ -341,13 +403,7 @@ signalsGroup.MapPost("/runtime-facts/ndjson", async Task<IResult> (
|
||||
return Results.BadRequest(new { error = "callgraphId is required." });
|
||||
}
|
||||
|
||||
var subject = new ReachabilitySubject
|
||||
{
|
||||
ScanId = metadata.ScanId,
|
||||
ImageDigest = metadata.ImageDigest,
|
||||
Component = metadata.Component,
|
||||
Version = metadata.Version
|
||||
};
|
||||
var subject = metadata.ToSubject();
|
||||
|
||||
var isGzip = string.Equals(context.Request.Headers.ContentEncoding, "gzip", StringComparison.OrdinalIgnoreCase);
|
||||
var events = await RuntimeFactsNdjsonReader.ReadAsync(context.Request.Body, isGzip, cancellationToken).ConfigureAwait(false);
|
||||
@@ -382,10 +438,14 @@ signalsGroup.MapGet("/facts/{subjectKey}", async Task<IResult> (
|
||||
SignalsSealedModeMonitor sealedModeMonitor,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Read, options.Authority.AllowAnonymousFallback, out var authFailure) ||
|
||||
!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Read, options.Authority.AllowAnonymousFallback, out var authFailure))
|
||||
{
|
||||
return authFailure ?? sealedFailure ?? Results.Unauthorized();
|
||||
return authFailure ?? Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
{
|
||||
return sealedFailure ?? Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(subjectKey))
|
||||
@@ -405,10 +465,14 @@ signalsGroup.MapPost("/reachability/recompute", async Task<IResult> (
|
||||
SignalsSealedModeMonitor sealedModeMonitor,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Admin, options.Authority.AllowAnonymousFallback, out var authFailure) ||
|
||||
!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Admin, options.Authority.AllowAnonymousFallback, out var authFailure))
|
||||
{
|
||||
return authFailure ?? sealedFailure ?? Results.Unauthorized();
|
||||
return authFailure ?? Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
{
|
||||
return sealedFailure ?? Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
|
||||
}
|
||||
|
||||
try
|
||||
@@ -434,26 +498,6 @@ signalsGroup.MapPost("/reachability/recompute", async Task<IResult> (
|
||||
}
|
||||
}).WithName("SignalsReachabilityRecompute");
|
||||
|
||||
signalsGroup.MapGet("/facts/{subjectKey}", async Task<IResult> (
|
||||
HttpContext context,
|
||||
SignalsOptions options,
|
||||
string subjectKey,
|
||||
IReachabilityFactRepository factRepository,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Read, options.Authority.AllowAnonymousFallback, out var failure))
|
||||
{
|
||||
return failure ?? Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(subjectKey))
|
||||
{
|
||||
return Results.BadRequest(new { error = "subjectKey is required." });
|
||||
}
|
||||
|
||||
var fact = await factRepository.GetBySubjectAsync(subjectKey.Trim(), cancellationToken).ConfigureAwait(false);
|
||||
return fact is null ? Results.NotFound() : Results.Ok(fact);
|
||||
}).WithName("SignalsFactsGet");
|
||||
|
||||
app.Run();
|
||||
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -26,10 +28,11 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
|
||||
private readonly ICallgraphParserResolver parserResolver;
|
||||
private readonly ICallgraphArtifactStore artifactStore;
|
||||
private readonly ICallgraphRepository repository;
|
||||
private readonly ILogger<CallgraphIngestionService> logger;
|
||||
private readonly SignalsOptions options;
|
||||
private readonly TimeProvider timeProvider;
|
||||
private readonly ICallgraphRepository repository;
|
||||
private readonly ILogger<CallgraphIngestionService> logger;
|
||||
private readonly SignalsOptions options;
|
||||
private readonly TimeProvider timeProvider;
|
||||
private static readonly JsonSerializerOptions ManifestSerializerOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
public CallgraphIngestionService(
|
||||
ICallgraphParserResolver parserResolver,
|
||||
@@ -53,23 +56,42 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
|
||||
var parser = parserResolver.Resolve(request.Language);
|
||||
|
||||
var artifactBytes = Convert.FromBase64String(request.ArtifactContentBase64);
|
||||
await using var parseStream = new MemoryStream(artifactBytes, writable: false);
|
||||
var parseResult = await parser.ParseAsync(parseStream, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
parseStream.Position = 0;
|
||||
var hash = ComputeSha256(artifactBytes);
|
||||
|
||||
var artifactMetadata = await artifactStore.SaveAsync(
|
||||
new CallgraphArtifactSaveRequest(
|
||||
request.Language,
|
||||
request.Component,
|
||||
request.Version,
|
||||
request.ArtifactFileName,
|
||||
request.ArtifactContentType,
|
||||
hash),
|
||||
parseStream,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
var artifactBytes = Convert.FromBase64String(request.ArtifactContentBase64);
|
||||
await using var parseStream = new MemoryStream(artifactBytes, writable: false);
|
||||
var parseResult = await parser.ParseAsync(parseStream, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
parseStream.Position = 0;
|
||||
var artifactHash = ComputeSha256(artifactBytes);
|
||||
var graphHash = ComputeGraphHash(parseResult);
|
||||
|
||||
var manifest = new CallgraphManifest
|
||||
{
|
||||
Language = request.Language,
|
||||
Component = request.Component,
|
||||
Version = request.Version,
|
||||
ArtifactHash = artifactHash,
|
||||
GraphHash = graphHash,
|
||||
NodeCount = parseResult.Nodes.Count,
|
||||
EdgeCount = parseResult.Edges.Count,
|
||||
CreatedAt = timeProvider.GetUtcNow()
|
||||
};
|
||||
|
||||
await using var manifestStream = new MemoryStream();
|
||||
await JsonSerializer.SerializeAsync(manifestStream, manifest, ManifestSerializerOptions, cancellationToken).ConfigureAwait(false);
|
||||
manifestStream.Position = 0;
|
||||
|
||||
parseStream.Position = 0;
|
||||
var artifactMetadata = await artifactStore.SaveAsync(
|
||||
new CallgraphArtifactSaveRequest(
|
||||
request.Language,
|
||||
request.Component,
|
||||
request.Version,
|
||||
request.ArtifactFileName,
|
||||
request.ArtifactContentType,
|
||||
artifactHash,
|
||||
manifestStream),
|
||||
parseStream,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var document = new CallgraphDocument
|
||||
{
|
||||
@@ -81,21 +103,25 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
Metadata = request.Metadata is null
|
||||
? null
|
||||
: new Dictionary<string, string?>(request.Metadata, StringComparer.OrdinalIgnoreCase),
|
||||
Artifact = new CallgraphArtifactMetadata
|
||||
{
|
||||
Path = artifactMetadata.Path,
|
||||
Hash = artifactMetadata.Hash,
|
||||
CasUri = artifactMetadata.CasUri,
|
||||
ContentType = artifactMetadata.ContentType,
|
||||
Length = artifactMetadata.Length
|
||||
},
|
||||
IngestedAt = timeProvider.GetUtcNow()
|
||||
};
|
||||
|
||||
document.Metadata ??= new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
|
||||
document.Metadata["formatVersion"] = parseResult.FormatVersion;
|
||||
|
||||
document = await repository.UpsertAsync(document, cancellationToken).ConfigureAwait(false);
|
||||
Artifact = new CallgraphArtifactMetadata
|
||||
{
|
||||
Path = artifactMetadata.Path,
|
||||
Hash = artifactMetadata.Hash,
|
||||
CasUri = artifactMetadata.CasUri,
|
||||
ManifestPath = artifactMetadata.ManifestPath,
|
||||
ManifestCasUri = artifactMetadata.ManifestCasUri,
|
||||
GraphHash = graphHash,
|
||||
ContentType = artifactMetadata.ContentType,
|
||||
Length = artifactMetadata.Length
|
||||
},
|
||||
IngestedAt = timeProvider.GetUtcNow()
|
||||
};
|
||||
|
||||
document.Metadata ??= new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
|
||||
document.Metadata["formatVersion"] = parseResult.FormatVersion;
|
||||
document.GraphHash = graphHash;
|
||||
|
||||
document = await repository.UpsertAsync(document, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
logger.LogInformation(
|
||||
"Ingested callgraph {Language}:{Component}:{Version} (id={Id}) with {NodeCount} nodes and {EdgeCount} edges.",
|
||||
@@ -106,7 +132,13 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
document.Nodes.Count,
|
||||
document.Edges.Count);
|
||||
|
||||
return new CallgraphIngestResponse(document.Id, document.Artifact.Path, document.Artifact.Hash, document.Artifact.CasUri);
|
||||
return new CallgraphIngestResponse(
|
||||
document.Id,
|
||||
document.Artifact.Path,
|
||||
document.Artifact.Hash,
|
||||
document.Artifact.CasUri,
|
||||
graphHash,
|
||||
document.Artifact.ManifestCasUri);
|
||||
}
|
||||
|
||||
private static void ValidateRequest(CallgraphIngestRequest request)
|
||||
@@ -144,13 +176,29 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
}
|
||||
}
|
||||
|
||||
private static string ComputeSha256(ReadOnlySpan<byte> buffer)
|
||||
{
|
||||
Span<byte> hash = stackalloc byte[SHA256.HashSizeInBytes];
|
||||
SHA256.HashData(buffer, hash);
|
||||
return Convert.ToHexString(hash);
|
||||
}
|
||||
}
|
||||
private static string ComputeSha256(ReadOnlySpan<byte> buffer)
|
||||
{
|
||||
Span<byte> hash = stackalloc byte[SHA256.HashSizeInBytes];
|
||||
SHA256.HashData(buffer, hash);
|
||||
return Convert.ToHexString(hash);
|
||||
}
|
||||
|
||||
private static string ComputeGraphHash(CallgraphParseResult result)
|
||||
{
|
||||
var builder = new StringBuilder();
|
||||
foreach (var node in result.Nodes.OrderBy(n => n.Id, StringComparer.Ordinal))
|
||||
{
|
||||
builder.Append(node.Id).Append('|').Append(node.Name).AppendLine();
|
||||
}
|
||||
|
||||
foreach (var edge in result.Edges.OrderBy(e => e.SourceId, StringComparer.Ordinal).ThenBy(e => e.TargetId, StringComparer.Ordinal))
|
||||
{
|
||||
builder.Append(edge.SourceId).Append("->").Append(edge.TargetId).AppendLine();
|
||||
}
|
||||
|
||||
return ComputeSha256(Encoding.UTF8.GetBytes(builder.ToString()));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Exception thrown when the ingestion request is invalid.
|
||||
|
||||
@@ -40,6 +40,14 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
|
||||
|
||||
ValidateRequest(request);
|
||||
|
||||
var subjectKey = request.Subject?.ToSubjectKey();
|
||||
if (string.IsNullOrWhiteSpace(subjectKey))
|
||||
{
|
||||
throw new ReachabilityScoringValidationException("Subject must include scanId, imageDigest, or component/version.");
|
||||
}
|
||||
|
||||
var existingFact = await factRepository.GetBySubjectAsync(subjectKey, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var callgraph = await callgraphRepository.GetByIdAsync(request.CallgraphId, cancellationToken).ConfigureAwait(false);
|
||||
if (callgraph is null)
|
||||
{
|
||||
@@ -57,10 +65,24 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
|
||||
throw new ReachabilityScoringValidationException("At least one target symbol is required.");
|
||||
}
|
||||
|
||||
var runtimeHits = request.RuntimeHits?.Where(hit => !string.IsNullOrWhiteSpace(hit))
|
||||
.Select(hit => hit.Trim())
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.ToList() ?? new List<string>();
|
||||
var runtimeHitSet = new HashSet<string>(StringComparer.Ordinal);
|
||||
if (existingFact?.RuntimeFacts is { Count: > 0 })
|
||||
{
|
||||
foreach (var fact in existingFact.RuntimeFacts.Where(f => !string.IsNullOrWhiteSpace(f.SymbolId)))
|
||||
{
|
||||
runtimeHitSet.Add(fact.SymbolId);
|
||||
}
|
||||
}
|
||||
|
||||
if (request.RuntimeHits is { Count: > 0 })
|
||||
{
|
||||
foreach (var hit in request.RuntimeHits.Where(h => !string.IsNullOrWhiteSpace(h)))
|
||||
{
|
||||
runtimeHitSet.Add(hit.Trim());
|
||||
}
|
||||
}
|
||||
|
||||
var runtimeHits = runtimeHitSet.ToList();
|
||||
|
||||
var states = new List<ReachabilityStateDocument>(targets.Count);
|
||||
foreach (var target in targets)
|
||||
@@ -95,12 +117,13 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
|
||||
var document = new ReachabilityFactDocument
|
||||
{
|
||||
CallgraphId = request.CallgraphId,
|
||||
Subject = request.Subject,
|
||||
Subject = request.Subject!,
|
||||
EntryPoints = entryPoints,
|
||||
States = states,
|
||||
Metadata = request.Metadata,
|
||||
ComputedAt = timeProvider.GetUtcNow(),
|
||||
SubjectKey = request.Subject.ToSubjectKey()
|
||||
SubjectKey = subjectKey,
|
||||
RuntimeFacts = existingFact?.RuntimeFacts
|
||||
};
|
||||
|
||||
logger.LogInformation("Computed reachability fact for subject {SubjectKey} with {StateCount} targets.", document.SubjectKey, states.Count);
|
||||
|
||||
@@ -115,6 +115,11 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
SymbolId = key.SymbolId,
|
||||
CodeId = key.CodeId,
|
||||
LoaderBase = key.LoaderBase,
|
||||
ProcessId = evt.ProcessId,
|
||||
ProcessName = Normalize(evt.ProcessName),
|
||||
SocketAddress = Normalize(evt.SocketAddress),
|
||||
ContainerId = Normalize(evt.ContainerId),
|
||||
EvidenceUri = Normalize(evt.EvidenceUri),
|
||||
Metadata = evt.Metadata != null
|
||||
? new Dictionary<string, string?>(evt.Metadata, StringComparer.Ordinal)
|
||||
: null
|
||||
@@ -152,9 +157,9 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
|
||||
if (incoming != null)
|
||||
{
|
||||
foreach (var (key, value) in incoming)
|
||||
foreach (var (metaKey, metaValue) in incoming)
|
||||
{
|
||||
merged[key] = value;
|
||||
merged[metaKey] = metaValue;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -177,6 +182,11 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
SymbolId = fact.SymbolId,
|
||||
CodeId = fact.CodeId,
|
||||
LoaderBase = fact.LoaderBase,
|
||||
ProcessId = fact.ProcessId,
|
||||
ProcessName = fact.ProcessName,
|
||||
SocketAddress = fact.SocketAddress,
|
||||
ContainerId = fact.ContainerId,
|
||||
EvidenceUri = fact.EvidenceUri,
|
||||
HitCount = fact.HitCount,
|
||||
Metadata = fact.Metadata is null
|
||||
? null
|
||||
@@ -197,6 +207,11 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
SymbolId = fact.SymbolId,
|
||||
CodeId = fact.CodeId,
|
||||
LoaderBase = fact.LoaderBase,
|
||||
ProcessId = fact.ProcessId,
|
||||
ProcessName = fact.ProcessName,
|
||||
SocketAddress = fact.SocketAddress,
|
||||
ContainerId = fact.ContainerId,
|
||||
EvidenceUri = fact.EvidenceUri,
|
||||
HitCount = fact.HitCount,
|
||||
Metadata = fact.Metadata is null
|
||||
? null
|
||||
@@ -206,12 +221,17 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
}
|
||||
|
||||
existingFact.HitCount = Math.Clamp(existingFact.HitCount + fact.HitCount, 1, int.MaxValue);
|
||||
existingFact.ProcessId ??= fact.ProcessId;
|
||||
existingFact.ProcessName ??= fact.ProcessName;
|
||||
existingFact.SocketAddress ??= fact.SocketAddress;
|
||||
existingFact.ContainerId ??= fact.ContainerId;
|
||||
existingFact.EvidenceUri ??= fact.EvidenceUri;
|
||||
if (fact.Metadata != null && fact.Metadata.Count > 0)
|
||||
{
|
||||
existingFact.Metadata ??= new Dictionary<string, string?>(StringComparer.Ordinal);
|
||||
foreach (var (key, value) in fact.Metadata)
|
||||
foreach (var (metaKey, metaValue) in fact.Metadata)
|
||||
{
|
||||
existingFact.Metadata[key] = value;
|
||||
existingFact.Metadata[metaKey] = metaValue;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -224,6 +244,9 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static string? Normalize(string? value) =>
|
||||
string.IsNullOrWhiteSpace(value) ? null : value.Trim();
|
||||
|
||||
private readonly record struct RuntimeFactKey(string SymbolId, string? CodeId, string? LoaderBase);
|
||||
|
||||
private sealed class RuntimeFactKeyComparer : IEqualityComparer<RuntimeFactKey>
|
||||
|
||||
@@ -48,15 +48,29 @@ internal sealed class FileSystemCallgraphArtifactStore : ICallgraphArtifactStore
|
||||
await content.CopyToAsync(fileStream, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var manifestPath = Path.Combine(casDirectory, "manifest.json");
|
||||
if (request.ManifestContent != null)
|
||||
{
|
||||
await using var manifestStream = File.Create(manifestPath);
|
||||
request.ManifestContent.Position = 0;
|
||||
await request.ManifestContent.CopyToAsync(manifestStream, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else if (!File.Exists(manifestPath))
|
||||
{
|
||||
await File.WriteAllTextAsync(manifestPath, "{}", cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var fileInfo = new FileInfo(destinationPath);
|
||||
logger.LogInformation("Stored callgraph artifact at {Path} (length={Length}).", destinationPath, fileInfo.Length);
|
||||
|
||||
return new StoredCallgraphArtifact(
|
||||
Path.GetRelativePath(root, destinationPath),
|
||||
fileInfo.Length,
|
||||
request.Hash,
|
||||
hash,
|
||||
request.ContentType,
|
||||
$"cas://reachability/graphs/{hash}");
|
||||
$"cas://reachability/graphs/{hash}",
|
||||
Path.GetRelativePath(root, manifestPath),
|
||||
$"cas://reachability/graphs/{hash}/manifest");
|
||||
}
|
||||
|
||||
private static string SanitizeFileName(string value)
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
namespace StellaOps.Signals.Storage.Models;
|
||||
using System.IO;
|
||||
|
||||
namespace StellaOps.Signals.Storage.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Context required to persist a callgraph artifact.
|
||||
/// </summary>
|
||||
public sealed record CallgraphArtifactSaveRequest(
|
||||
string Language,
|
||||
string Component,
|
||||
string Version,
|
||||
string FileName,
|
||||
string ContentType,
|
||||
string Hash);
|
||||
public sealed record CallgraphArtifactSaveRequest(
|
||||
string Language,
|
||||
string Component,
|
||||
string Version,
|
||||
string FileName,
|
||||
string ContentType,
|
||||
string Hash,
|
||||
Stream? ManifestContent);
|
||||
|
||||
@@ -8,4 +8,6 @@ public sealed record StoredCallgraphArtifact(
|
||||
long Length,
|
||||
string Hash,
|
||||
string ContentType,
|
||||
string CasUri);
|
||||
string CasUri,
|
||||
string ManifestPath,
|
||||
string ManifestCasUri);
|
||||
|
||||
Reference in New Issue
Block a user