feat: Implement Runtime Facts ingestion service and NDJSON reader
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Added RuntimeFactsNdjsonReader for reading NDJSON formatted runtime facts.
- Introduced IRuntimeFactsIngestionService interface and its implementation.
- Enhanced Program.cs to register new services and endpoints for runtime facts.
- Updated CallgraphIngestionService to include CAS URI in stored artifacts.
- Created RuntimeFactsValidationException for validation errors during ingestion.
- Added tests for RuntimeFactsIngestionService and RuntimeFactsNdjsonReader.
- Implemented SignalsSealedModeMonitor for compliance checks in sealed mode.
- Updated project dependencies for testing utilities.
This commit is contained in:
master
2025-11-10 07:56:15 +02:00
parent 9df52d84aa
commit 69c59defdc
132 changed files with 19718 additions and 9334 deletions

View File

@@ -0,0 +1,92 @@
using System;
using System.IO;
using Microsoft.Extensions.Logging;
using StellaOps.Signals.Options;
namespace StellaOps.Signals.Hosting;
public sealed class SignalsSealedModeMonitor
{
private readonly SignalsSealedModeOptions options;
private readonly TimeProvider timeProvider;
private readonly ILogger<SignalsSealedModeMonitor> logger;
private DateTimeOffset? lastCheck;
private bool lastResult = true;
private string? lastError;
public SignalsSealedModeMonitor(
SignalsOptions signalsOptions,
TimeProvider timeProvider,
ILogger<SignalsSealedModeMonitor> logger)
{
options = signalsOptions?.AirGap.SealedMode ?? throw new ArgumentNullException(nameof(signalsOptions));
this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public bool EnforcementEnabled => options.EnforcementEnabled;
public bool IsCompliant(out string? reason, bool forceRefresh = false)
{
if (!EnforcementEnabled)
{
reason = null;
return true;
}
var now = timeProvider.GetUtcNow();
if (!forceRefresh && lastCheck.HasValue && now - lastCheck < options.CacheLifetime)
{
reason = lastError;
return lastResult;
}
var compliant = ValidateEvidence(now, out var validationError);
lastCheck = now;
lastResult = compliant;
lastError = validationError;
if (!compliant)
{
logger.LogWarning(
"Signals sealed-mode evidence validation failed: {Reason}",
validationError ?? "unknown reason");
}
reason = validationError;
return compliant;
}
private bool ValidateEvidence(DateTimeOffset now, out string? error)
{
if (!options.RequireEvidenceHealth)
{
error = null;
return true;
}
if (string.IsNullOrWhiteSpace(options.EvidencePath))
{
error = "evidence path not configured";
return false;
}
var path = options.EvidencePath!;
if (!File.Exists(path))
{
error = $"evidence file '{path}' not found";
return false;
}
var lastWriteUtc = File.GetLastWriteTimeUtc(path);
var age = now - lastWriteUtc;
if (age > options.MaxEvidenceAge)
{
error = $"evidence file '{path}' is stale ({age:c} old, max allowed {options.MaxEvidenceAge:c})";
return false;
}
error = null;
return true;
}
}

View File

@@ -7,15 +7,18 @@ namespace StellaOps.Signals.Models;
/// </summary>
public sealed class CallgraphArtifactMetadata
{
[BsonElement("path")]
public string Path { get; set; } = string.Empty;
[BsonElement("hash")]
public string Hash { get; set; } = string.Empty;
[BsonElement("contentType")]
public string ContentType { get; set; } = string.Empty;
[BsonElement("path")]
public string Path { get; set; } = string.Empty;
[BsonElement("hash")]
public string Hash { get; set; } = string.Empty;
[BsonElement("casUri")]
public string CasUri { get; set; } = string.Empty;
[BsonElement("contentType")]
public string ContentType { get; set; } = string.Empty;
[BsonElement("length")]
public long Length { get; set; }
}

View File

@@ -3,7 +3,8 @@ namespace StellaOps.Signals.Models;
/// <summary>
/// Response returned after callgraph ingestion.
/// </summary>
public sealed record CallgraphIngestResponse(
string CallgraphId,
string ArtifactPath,
string ArtifactHash);
public sealed record CallgraphIngestResponse(
string CallgraphId,
string ArtifactPath,
string ArtifactHash,
string CasUri);

View File

@@ -23,6 +23,10 @@ public sealed class ReachabilityFactDocument
[BsonElement("states")]
public List<ReachabilityStateDocument> States { get; set; } = new();
[BsonElement("runtimeFacts")]
[BsonIgnoreIfNull]
public List<RuntimeFactDocument>? RuntimeFacts { get; set; }
[BsonElement("metadata")]
[BsonIgnoreIfNull]
public Dictionary<string, string?>? Metadata { get; set; }
@@ -96,3 +100,24 @@ public sealed class ReachabilitySubject
return string.Join('|', Component ?? string.Empty, Version ?? string.Empty).Trim('|');
}
}
public sealed class RuntimeFactDocument
{
[BsonElement("symbolId")]
public string SymbolId { get; set; } = string.Empty;
[BsonElement("codeId")]
[BsonIgnoreIfNull]
public string? CodeId { get; set; }
[BsonElement("loaderBase")]
[BsonIgnoreIfNull]
public string? LoaderBase { get; set; }
[BsonElement("hitCount")]
public int HitCount { get; set; }
[BsonElement("metadata")]
[BsonIgnoreIfNull]
public Dictionary<string, string?>? Metadata { get; set; }
}

View File

@@ -0,0 +1,47 @@
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
namespace StellaOps.Signals.Models;
public sealed class RuntimeFactsIngestRequest
{
[Required]
public ReachabilitySubject Subject { get; set; } = new();
[Required]
public string CallgraphId { get; set; } = string.Empty;
public List<RuntimeFactEvent> Events { get; set; } = new();
public Dictionary<string, string?>? Metadata { get; set; }
}
public sealed class RuntimeFactEvent
{
[Required]
public string SymbolId { get; set; } = string.Empty;
public string? CodeId { get; set; }
public string? LoaderBase { get; set; }
public int HitCount { get; set; } = 1;
public Dictionary<string, string?>? Metadata { get; set; }
}
public sealed class RuntimeFactsIngestResponse
{
public string FactId { get; init; } = string.Empty;
public string SubjectKey { get; init; } = string.Empty;
public string CallgraphId { get; init; } = string.Empty;
public int RuntimeFactCount { get; init; }
public int TotalHitCount { get; init; }
public DateTimeOffset StoredAt { get; init; }
}

View File

@@ -0,0 +1,14 @@
namespace StellaOps.Signals.Models;
public sealed class RuntimeFactsStreamMetadata
{
public string CallgraphId { get; set; } = string.Empty;
public string? ScanId { get; set; }
public string? ImageDigest { get; set; }
public string? Component { get; set; }
public string? Version { get; set; }
}

View File

@@ -0,0 +1,49 @@
using System;
namespace StellaOps.Signals.Options;
public sealed class SignalsAirGapOptions
{
public SignalsSealedModeOptions SealedMode { get; } = new();
public void Validate()
{
SealedMode.Validate();
}
}
public sealed class SignalsSealedModeOptions
{
public bool EnforcementEnabled { get; set; }
public string? EvidencePath { get; set; }
public TimeSpan MaxEvidenceAge { get; set; } = TimeSpan.FromHours(6);
public TimeSpan CacheLifetime { get; set; } = TimeSpan.FromMinutes(1);
public bool RequireEvidenceHealth { get; set; } = true;
public void Validate()
{
if (!EnforcementEnabled)
{
return;
}
if (string.IsNullOrWhiteSpace(EvidencePath))
{
throw new InvalidOperationException("Signals air-gap sealed-mode evidence path must be configured when enforcement is enabled.");
}
if (MaxEvidenceAge <= TimeSpan.Zero)
{
throw new InvalidOperationException("Signals air-gap sealed-mode max evidence age must be greater than zero.");
}
if (CacheLifetime <= TimeSpan.Zero)
{
throw new InvalidOperationException("Signals air-gap sealed-mode cache lifetime must be greater than zero.");
}
}
}

View File

@@ -21,17 +21,23 @@ public sealed class SignalsOptions
public SignalsMongoOptions Mongo { get; } = new();
/// <summary>
/// Artifact storage configuration.
/// </summary>
public SignalsArtifactStorageOptions Storage { get; } = new();
/// Artifact storage configuration.
/// </summary>
public SignalsArtifactStorageOptions Storage { get; } = new();
/// <summary>
/// Air-gap configuration.
/// </summary>
public SignalsAirGapOptions AirGap { get; } = new();
/// <summary>
/// Validates configured options.
/// </summary>
public void Validate()
{
Authority.Validate();
Mongo.Validate();
Storage.Validate();
}
}
Authority.Validate();
Mongo.Validate();
Storage.Validate();
AirGap.Validate();
}
}

View File

@@ -0,0 +1,52 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Signals.Models;
namespace StellaOps.Signals.Parsing;
internal static class RuntimeFactsNdjsonReader
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
public static async Task<List<RuntimeFactEvent>> ReadAsync(
Stream input,
bool gzipEncoded,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(input);
Stream effectiveStream = input;
if (gzipEncoded)
{
effectiveStream = new GZipStream(input, CompressionMode.Decompress, leaveOpen: true);
}
using var reader = new StreamReader(effectiveStream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, leaveOpen: !gzipEncoded);
var events = new List<RuntimeFactEvent>();
string? line;
while ((line = await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false)) != null)
{
if (string.IsNullOrWhiteSpace(line))
{
continue;
}
var evt = JsonSerializer.Deserialize<RuntimeFactEvent>(line, SerializerOptions);
if (evt is null || string.IsNullOrWhiteSpace(evt.SymbolId))
{
continue;
}
events.Add(evt);
}
return events;
}
}

View File

@@ -1,6 +1,7 @@
using System.IO;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
using MongoDB.Driver;
using NetEscapades.Configuration.Yaml;
@@ -12,10 +13,10 @@ using StellaOps.Signals.Hosting;
using StellaOps.Signals.Models;
using StellaOps.Signals.Options;
using StellaOps.Signals.Parsing;
using StellaOps.Signals.Persistence;
using StellaOps.Signals.Routing;
using StellaOps.Signals.Services;
using StellaOps.Signals.Storage;
using StellaOps.Signals.Persistence;
using StellaOps.Signals.Routing;
using StellaOps.Signals.Services;
using StellaOps.Signals.Storage;
var builder = WebApplication.CreateBuilder(args);
@@ -73,9 +74,10 @@ builder.Services.AddOptions<SignalsOptions>()
.ValidateOnStart();
builder.Services.AddSingleton(sp => sp.GetRequiredService<IOptions<SignalsOptions>>().Value);
builder.Services.AddSingleton<SignalsStartupState>();
builder.Services.AddSingleton(TimeProvider.System);
builder.Services.AddProblemDetails();
builder.Services.AddSingleton<SignalsStartupState>();
builder.Services.AddSingleton(TimeProvider.System);
builder.Services.AddSingleton<SignalsSealedModeMonitor>();
builder.Services.AddProblemDetails();
builder.Services.AddHealthChecks();
builder.Services.AddRouting(options => options.LowercaseUrls = true);
@@ -122,6 +124,7 @@ builder.Services.AddSingleton<ICallgraphParserResolver, CallgraphParserResolver>
builder.Services.AddSingleton<ICallgraphIngestionService, CallgraphIngestionService>();
builder.Services.AddSingleton<IReachabilityFactRepository, MongoReachabilityFactRepository>();
builder.Services.AddSingleton<IReachabilityScoringService, ReachabilityScoringService>();
builder.Services.AddSingleton<IRuntimeFactsIngestionService, RuntimeFactsIngestionService>();
if (bootstrap.Authority.Enabled)
{
@@ -189,39 +192,64 @@ app.UseAuthentication();
app.UseAuthorization();
app.MapHealthChecks("/healthz").AllowAnonymous();
app.MapGet("/readyz", static (SignalsStartupState state) =>
state.IsReady ? Results.Ok(new { status = "ready" }) : Results.StatusCode(StatusCodes.Status503ServiceUnavailable))
.AllowAnonymous();
app.MapGet("/readyz", (SignalsStartupState state, SignalsSealedModeMonitor sealedModeMonitor) =>
{
if (!sealedModeMonitor.IsCompliant(out var reason))
{
return Results.Json(
new { status = "sealed-mode-blocked", reason },
statusCode: StatusCodes.Status503ServiceUnavailable);
}
return state.IsReady
? Results.Ok(new { status = "ready" })
: Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
}).AllowAnonymous();
var fallbackAllowed = !bootstrap.Authority.Enabled || bootstrap.Authority.AllowAnonymousFallback;
var fallbackAllowed = !bootstrap.Authority.Enabled || bootstrap.Authority.AllowAnonymousFallback;
var signalsGroup = app.MapGroup("/signals");
signalsGroup.MapGet("/ping", (HttpContext context, SignalsOptions options, SignalsSealedModeMonitor sealedModeMonitor) =>
Program.TryAuthorize(context, requiredScope: SignalsPolicies.Read, fallbackAllowed: options.Authority.AllowAnonymousFallback, out var authFailure) &&
Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure)
? Results.NoContent()
: authFailure ?? sealedFailure ?? Results.Unauthorized()).WithName("SignalsPing");
var signalsGroup = app.MapGroup("/signals");
signalsGroup.MapGet("/status", (HttpContext context, SignalsOptions options, SignalsSealedModeMonitor sealedModeMonitor) =>
{
if (!Program.TryAuthorize(context, SignalsPolicies.Read, options.Authority.AllowAnonymousFallback, out var failure))
{
return failure ?? Results.Unauthorized();
}
var sealedCompliant = sealedModeMonitor.IsCompliant(out var sealedReason);
return Results.Ok(new
{
service = "signals",
version = typeof(Program).Assembly.GetName().Version?.ToString() ?? "unknown",
sealedMode = new
{
enforced = sealedModeMonitor.EnforcementEnabled,
compliant = sealedCompliant,
reason = sealedCompliant ? null : sealedReason
}
});
}).WithName("SignalsStatus");
signalsGroup.MapGet("/ping", (HttpContext context, SignalsOptions options) =>
Program.TryAuthorize(context, requiredScope: SignalsPolicies.Read, fallbackAllowed: options.Authority.AllowAnonymousFallback, out var failure)
? Results.NoContent()
: failure ?? Results.Unauthorized()).WithName("SignalsPing");
signalsGroup.MapGet("/status", (HttpContext context, SignalsOptions options) =>
Program.TryAuthorize(context, SignalsPolicies.Read, options.Authority.AllowAnonymousFallback, out var failure)
? Results.Ok(new
{
service = "signals",
version = typeof(Program).Assembly.GetName().Version?.ToString() ?? "unknown"
})
: failure ?? Results.Unauthorized()).WithName("SignalsStatus");
signalsGroup.MapPost("/callgraphs", async Task<IResult> (
HttpContext context,
SignalsOptions options,
CallgraphIngestRequest request,
ICallgraphIngestionService ingestionService,
CancellationToken cancellationToken) =>
{
if (!Program.TryAuthorize(context, SignalsPolicies.Write, options.Authority.AllowAnonymousFallback, out var failure))
{
return failure ?? Results.Unauthorized();
}
signalsGroup.MapPost("/callgraphs", async Task<IResult> (
HttpContext context,
SignalsOptions options,
CallgraphIngestRequest request,
ICallgraphIngestionService ingestionService,
SignalsSealedModeMonitor sealedModeMonitor,
CancellationToken cancellationToken) =>
{
if (!Program.TryAuthorize(context, SignalsPolicies.Write, options.Authority.AllowAnonymousFallback, out var authFailure) ||
!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
{
return authFailure ?? sealedFailure ?? Results.Unauthorized();
}
try
{
@@ -244,23 +272,143 @@ signalsGroup.MapPost("/callgraphs", async Task<IResult> (
{
return Results.BadRequest(new { error = ex.Message });
}
}).WithName("SignalsCallgraphIngest");
signalsGroup.MapPost("/runtime-facts", (HttpContext context, SignalsOptions options) =>
Program.TryAuthorize(context, SignalsPolicies.Write, options.Authority.AllowAnonymousFallback, out var failure)
? Results.StatusCode(StatusCodes.Status501NotImplemented)
: failure ?? Results.Unauthorized()).WithName("SignalsRuntimeIngest");
}).WithName("SignalsCallgraphIngest");
signalsGroup.MapGet("/callgraphs/{callgraphId}", async Task<IResult> (
HttpContext context,
SignalsOptions options,
string callgraphId,
ICallgraphRepository callgraphRepository,
SignalsSealedModeMonitor sealedModeMonitor,
CancellationToken cancellationToken) =>
{
if (!Program.TryAuthorize(context, SignalsPolicies.Read, options.Authority.AllowAnonymousFallback, out var authFailure) ||
!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
{
return authFailure ?? sealedFailure ?? Results.Unauthorized();
}
if (string.IsNullOrWhiteSpace(callgraphId))
{
return Results.BadRequest(new { error = "callgraphId is required." });
}
var document = await callgraphRepository.GetByIdAsync(callgraphId.Trim(), cancellationToken).ConfigureAwait(false);
return document is null ? Results.NotFound() : Results.Ok(document);
}).WithName("SignalsCallgraphGet");
signalsGroup.MapPost("/runtime-facts", async Task<IResult> (
HttpContext context,
SignalsOptions options,
RuntimeFactsIngestRequest request,
IRuntimeFactsIngestionService ingestionService,
SignalsSealedModeMonitor sealedModeMonitor,
CancellationToken cancellationToken) =>
{
if (!Program.TryAuthorize(context, SignalsPolicies.Write, options.Authority.AllowAnonymousFallback, out var authFailure) ||
!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
{
return authFailure ?? sealedFailure ?? Results.Unauthorized();
}
try
{
var response = await ingestionService.IngestAsync(request, cancellationToken).ConfigureAwait(false);
return Results.Accepted($"/signals/runtime-facts/{response.SubjectKey}", response);
}
catch (RuntimeFactsValidationException ex)
{
return Results.BadRequest(new { error = ex.Message });
}
}).WithName("SignalsRuntimeIngest");
signalsGroup.MapPost("/runtime-facts/ndjson", async Task<IResult> (
HttpContext context,
SignalsOptions options,
RuntimeFactsStreamMetadata metadata,
IRuntimeFactsIngestionService ingestionService,
SignalsSealedModeMonitor sealedModeMonitor,
CancellationToken cancellationToken) =>
{
if (!Program.TryAuthorize(context, SignalsPolicies.Write, options.Authority.AllowAnonymousFallback, out var authFailure) ||
!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
{
return authFailure ?? sealedFailure ?? Results.Unauthorized();
}
if (metadata is null || string.IsNullOrWhiteSpace(metadata.CallgraphId))
{
return Results.BadRequest(new { error = "callgraphId is required." });
}
var subject = new ReachabilitySubject
{
ScanId = metadata.ScanId,
ImageDigest = metadata.ImageDigest,
Component = metadata.Component,
Version = metadata.Version
};
var isGzip = string.Equals(context.Request.Headers.ContentEncoding, "gzip", StringComparison.OrdinalIgnoreCase);
var events = await RuntimeFactsNdjsonReader.ReadAsync(context.Request.Body, isGzip, cancellationToken).ConfigureAwait(false);
if (events.Count == 0)
{
return Results.BadRequest(new { error = "runtime fact stream was empty." });
}
var request = new RuntimeFactsIngestRequest
{
Subject = subject,
CallgraphId = metadata.CallgraphId,
Events = events
};
try
{
var response = await ingestionService.IngestAsync(request, cancellationToken).ConfigureAwait(false);
return Results.Accepted($"/signals/runtime-facts/{response.SubjectKey}", response);
}
catch (RuntimeFactsValidationException ex)
{
return Results.BadRequest(new { error = ex.Message });
}
}).WithName("SignalsRuntimeIngestNdjson");
signalsGroup.MapGet("/facts/{subjectKey}", async Task<IResult> (
HttpContext context,
SignalsOptions options,
string subjectKey,
IReachabilityFactRepository factRepository,
SignalsSealedModeMonitor sealedModeMonitor,
CancellationToken cancellationToken) =>
{
if (!Program.TryAuthorize(context, SignalsPolicies.Read, options.Authority.AllowAnonymousFallback, out var authFailure) ||
!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
{
return authFailure ?? sealedFailure ?? Results.Unauthorized();
}
if (string.IsNullOrWhiteSpace(subjectKey))
{
return Results.BadRequest(new { error = "subjectKey is required." });
}
var fact = await factRepository.GetBySubjectAsync(subjectKey.Trim(), cancellationToken).ConfigureAwait(false);
return fact is null ? Results.NotFound() : Results.Ok(fact);
}).WithName("SignalsFactsGet");
signalsGroup.MapPost("/reachability/recompute", async Task<IResult> (
HttpContext context,
SignalsOptions options,
ReachabilityRecomputeRequest request,
IReachabilityScoringService scoringService,
SignalsSealedModeMonitor sealedModeMonitor,
CancellationToken cancellationToken) =>
{
if (!Program.TryAuthorize(context, SignalsPolicies.Admin, options.Authority.AllowAnonymousFallback, out var failure))
if (!Program.TryAuthorize(context, SignalsPolicies.Admin, options.Authority.AllowAnonymousFallback, out var authFailure) ||
!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
{
return failure ?? Results.Unauthorized();
return authFailure ?? sealedFailure ?? Results.Unauthorized();
}
try
@@ -285,6 +433,27 @@ signalsGroup.MapPost("/reachability/recompute", async Task<IResult> (
return Results.NotFound(new { error = ex.Message });
}
}).WithName("SignalsReachabilityRecompute");
signalsGroup.MapGet("/facts/{subjectKey}", async Task<IResult> (
HttpContext context,
SignalsOptions options,
string subjectKey,
IReachabilityFactRepository factRepository,
CancellationToken cancellationToken) =>
{
if (!Program.TryAuthorize(context, SignalsPolicies.Read, options.Authority.AllowAnonymousFallback, out var failure))
{
return failure ?? Results.Unauthorized();
}
if (string.IsNullOrWhiteSpace(subjectKey))
{
return Results.BadRequest(new { error = "subjectKey is required." });
}
var fact = await factRepository.GetBySubjectAsync(subjectKey.Trim(), cancellationToken).ConfigureAwait(false);
return fact is null ? Results.NotFound() : Results.Ok(fact);
}).WithName("SignalsFactsGet");
app.Run();
@@ -376,4 +545,24 @@ public partial class Program
// Ignore when indexes already exist with different options to keep startup idempotent.
}
}
internal static bool TryEnsureSealedMode(SignalsSealedModeMonitor monitor, out IResult? failure)
{
if (!monitor.EnforcementEnabled)
{
failure = null;
return true;
}
if (monitor.IsCompliant(out var reason))
{
failure = null;
return true;
}
failure = Results.Json(
new { error = "sealed-mode evidence invalid", reason },
statusCode: StatusCodes.Status503ServiceUnavailable);
return false;
}
}

View File

@@ -82,12 +82,13 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
? null
: new Dictionary<string, string?>(request.Metadata, StringComparer.OrdinalIgnoreCase),
Artifact = new CallgraphArtifactMetadata
{
Path = artifactMetadata.Path,
Hash = artifactMetadata.Hash,
ContentType = artifactMetadata.ContentType,
Length = artifactMetadata.Length
},
{
Path = artifactMetadata.Path,
Hash = artifactMetadata.Hash,
CasUri = artifactMetadata.CasUri,
ContentType = artifactMetadata.ContentType,
Length = artifactMetadata.Length
},
IngestedAt = timeProvider.GetUtcNow()
};
@@ -105,8 +106,8 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
document.Nodes.Count,
document.Edges.Count);
return new CallgraphIngestResponse(document.Id, document.Artifact.Path, document.Artifact.Hash);
}
return new CallgraphIngestResponse(document.Id, document.Artifact.Path, document.Artifact.Hash, document.Artifact.CasUri);
}
private static void ValidateRequest(CallgraphIngestRequest request)
{

View File

@@ -0,0 +1,10 @@
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Signals.Models;
namespace StellaOps.Signals.Services;
public interface IRuntimeFactsIngestionService
{
Task<RuntimeFactsIngestResponse> IngestAsync(RuntimeFactsIngestRequest request, CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,255 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Signals.Models;
using StellaOps.Signals.Persistence;
namespace StellaOps.Signals.Services;
public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
{
private readonly IReachabilityFactRepository factRepository;
private readonly TimeProvider timeProvider;
private readonly ILogger<RuntimeFactsIngestionService> logger;
public RuntimeFactsIngestionService(
IReachabilityFactRepository factRepository,
TimeProvider timeProvider,
ILogger<RuntimeFactsIngestionService> logger)
{
this.factRepository = factRepository ?? throw new ArgumentNullException(nameof(factRepository));
this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<RuntimeFactsIngestResponse> IngestAsync(RuntimeFactsIngestRequest request, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
ValidateRequest(request);
var subjectKey = request.Subject.ToSubjectKey();
var existing = await factRepository.GetBySubjectAsync(subjectKey, cancellationToken).ConfigureAwait(false);
var document = existing ?? new ReachabilityFactDocument
{
Subject = request.Subject,
SubjectKey = subjectKey,
};
document.CallgraphId = request.CallgraphId;
document.Subject = request.Subject;
document.SubjectKey = subjectKey;
document.ComputedAt = timeProvider.GetUtcNow();
var aggregated = AggregateRuntimeFacts(request.Events);
document.RuntimeFacts = MergeRuntimeFacts(document.RuntimeFacts, aggregated);
document.Metadata = MergeMetadata(document.Metadata, request.Metadata);
var persisted = await factRepository.UpsertAsync(document, cancellationToken).ConfigureAwait(false);
logger.LogInformation(
"Stored {RuntimeFactCount} runtime fact(s) for subject {SubjectKey} (callgraph={CallgraphId}).",
persisted.RuntimeFacts?.Count ?? 0,
subjectKey,
request.CallgraphId);
return new RuntimeFactsIngestResponse
{
FactId = persisted.Id,
SubjectKey = subjectKey,
CallgraphId = request.CallgraphId,
RuntimeFactCount = persisted.RuntimeFacts?.Count ?? 0,
TotalHitCount = persisted.RuntimeFacts?.Sum(f => f.HitCount) ?? 0,
StoredAt = persisted.ComputedAt,
};
}
private static void ValidateRequest(RuntimeFactsIngestRequest request)
{
if (request.Subject is null)
{
throw new RuntimeFactsValidationException("Subject is required.");
}
var subjectKey = request.Subject.ToSubjectKey();
if (string.IsNullOrWhiteSpace(subjectKey))
{
throw new RuntimeFactsValidationException("Subject must include either scanId, imageDigest, or component/version.");
}
if (string.IsNullOrWhiteSpace(request.CallgraphId))
{
throw new RuntimeFactsValidationException("CallgraphId is required.");
}
if (request.Events is null || request.Events.Count == 0)
{
throw new RuntimeFactsValidationException("At least one runtime event is required.");
}
if (request.Events.Any(e => string.IsNullOrWhiteSpace(e.SymbolId)))
{
throw new RuntimeFactsValidationException("Runtime events must include symbolId.");
}
}
private static List<RuntimeFactDocument> AggregateRuntimeFacts(IEnumerable<RuntimeFactEvent> events)
{
var map = new Dictionary<RuntimeFactKey, RuntimeFactDocument>(RuntimeFactKeyComparer.Instance);
foreach (var evt in events)
{
if (string.IsNullOrWhiteSpace(evt.SymbolId))
{
continue;
}
var key = new RuntimeFactKey(evt.SymbolId.Trim(), evt.CodeId?.Trim(), evt.LoaderBase?.Trim());
if (!map.TryGetValue(key, out var document))
{
document = new RuntimeFactDocument
{
SymbolId = key.SymbolId,
CodeId = key.CodeId,
LoaderBase = key.LoaderBase,
Metadata = evt.Metadata != null
? new Dictionary<string, string?>(evt.Metadata, StringComparer.Ordinal)
: null
};
map[key] = document;
}
else if (evt.Metadata != null && evt.Metadata.Count > 0)
{
document.Metadata ??= new Dictionary<string, string?>(StringComparer.Ordinal);
foreach (var kvp in evt.Metadata)
{
document.Metadata[kvp.Key] = kvp.Value;
}
}
document.HitCount = Math.Clamp(document.HitCount + Math.Max(evt.HitCount, 1), 1, int.MaxValue);
}
return map.Values.ToList();
}
private static Dictionary<string, string?>? MergeMetadata(
Dictionary<string, string?>? existing,
Dictionary<string, string?>? incoming)
{
if (existing is null && incoming is null)
{
return null;
}
var merged = existing is null
? new Dictionary<string, string?>(StringComparer.Ordinal)
: new Dictionary<string, string?>(existing, StringComparer.Ordinal);
if (incoming != null)
{
foreach (var (key, value) in incoming)
{
merged[key] = value;
}
}
return merged;
}
private static List<RuntimeFactDocument> MergeRuntimeFacts(
List<RuntimeFactDocument>? existing,
List<RuntimeFactDocument> incoming)
{
var map = new Dictionary<RuntimeFactKey, RuntimeFactDocument>(RuntimeFactKeyComparer.Instance);
if (existing is { Count: > 0 })
{
foreach (var fact in existing)
{
var key = new RuntimeFactKey(fact.SymbolId, fact.CodeId, fact.LoaderBase);
map[key] = new RuntimeFactDocument
{
SymbolId = fact.SymbolId,
CodeId = fact.CodeId,
LoaderBase = fact.LoaderBase,
HitCount = fact.HitCount,
Metadata = fact.Metadata is null
? null
: new Dictionary<string, string?>(fact.Metadata, StringComparer.Ordinal)
};
}
}
if (incoming.Count > 0)
{
foreach (var fact in incoming)
{
var key = new RuntimeFactKey(fact.SymbolId, fact.CodeId, fact.LoaderBase);
if (!map.TryGetValue(key, out var existingFact))
{
map[key] = new RuntimeFactDocument
{
SymbolId = fact.SymbolId,
CodeId = fact.CodeId,
LoaderBase = fact.LoaderBase,
HitCount = fact.HitCount,
Metadata = fact.Metadata is null
? null
: new Dictionary<string, string?>(fact.Metadata, StringComparer.Ordinal)
};
continue;
}
existingFact.HitCount = Math.Clamp(existingFact.HitCount + fact.HitCount, 1, int.MaxValue);
if (fact.Metadata != null && fact.Metadata.Count > 0)
{
existingFact.Metadata ??= new Dictionary<string, string?>(StringComparer.Ordinal);
foreach (var (key, value) in fact.Metadata)
{
existingFact.Metadata[key] = value;
}
}
}
}
return map.Values
.OrderBy(doc => doc.SymbolId, StringComparer.Ordinal)
.ThenBy(doc => doc.CodeId, StringComparer.Ordinal)
.ThenBy(doc => doc.LoaderBase, StringComparer.Ordinal)
.ToList();
}
private readonly record struct RuntimeFactKey(string SymbolId, string? CodeId, string? LoaderBase);
private sealed class RuntimeFactKeyComparer : IEqualityComparer<RuntimeFactKey>
{
public static RuntimeFactKeyComparer Instance { get; } = new();
public bool Equals(RuntimeFactKey x, RuntimeFactKey y) =>
string.Equals(x.SymbolId, y.SymbolId, StringComparison.Ordinal) &&
string.Equals(x.CodeId, y.CodeId, StringComparison.Ordinal) &&
string.Equals(x.LoaderBase, y.LoaderBase, StringComparison.Ordinal);
public int GetHashCode(RuntimeFactKey obj)
{
var hash = new HashCode();
hash.Add(obj.SymbolId, StringComparer.Ordinal);
if (obj.CodeId is not null)
{
hash.Add(obj.CodeId, StringComparer.Ordinal);
}
if (obj.LoaderBase is not null)
{
hash.Add(obj.LoaderBase, StringComparer.Ordinal);
}
return hash.ToHashCode();
}
}
}

View File

@@ -0,0 +1,11 @@
using System;
namespace StellaOps.Signals.Services;
public sealed class RuntimeFactsValidationException : Exception
{
public RuntimeFactsValidationException(string message)
: base(message)
{
}
}

View File

@@ -25,36 +25,40 @@ internal sealed class FileSystemCallgraphArtifactStore : ICallgraphArtifactStore
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<StoredCallgraphArtifact> SaveAsync(CallgraphArtifactSaveRequest request, Stream content, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
ArgumentNullException.ThrowIfNull(content);
var root = storageOptions.RootPath;
var directory = Path.Combine(root, Sanitize(request.Language), Sanitize(request.Component), Sanitize(request.Version));
Directory.CreateDirectory(directory);
var fileName = string.IsNullOrWhiteSpace(request.FileName)
? FormattableString.Invariant($"artifact-{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}.bin")
: request.FileName.Trim();
var destinationPath = Path.Combine(directory, fileName);
await using (var fileStream = File.Create(destinationPath))
{
await content.CopyToAsync(fileStream, cancellationToken).ConfigureAwait(false);
}
var fileInfo = new FileInfo(destinationPath);
logger.LogInformation("Stored callgraph artifact at {Path} (length={Length}).", destinationPath, fileInfo.Length);
return new StoredCallgraphArtifact(
Path.GetRelativePath(root, destinationPath),
fileInfo.Length,
request.Hash,
request.ContentType);
}
private static string Sanitize(string value)
=> string.Join('_', value.Split(Path.GetInvalidFileNameChars(), StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)).ToLowerInvariant();
}
public async Task<StoredCallgraphArtifact> SaveAsync(CallgraphArtifactSaveRequest request, Stream content, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
ArgumentNullException.ThrowIfNull(content);
var root = storageOptions.RootPath;
var hash = request.Hash?.Trim().ToLowerInvariant();
if (string.IsNullOrWhiteSpace(hash))
{
throw new InvalidOperationException("Callgraph artifact hash is required for CAS storage.");
}
var casDirectory = Path.Combine(root, "cas", "reachability", "graphs", hash.Substring(0, Math.Min(hash.Length, 2)), hash);
Directory.CreateDirectory(casDirectory);
var fileName = SanitizeFileName(string.IsNullOrWhiteSpace(request.FileName) ? "callgraph.json" : request.FileName);
var destinationPath = Path.Combine(casDirectory, fileName);
await using (var fileStream = File.Create(destinationPath))
{
await content.CopyToAsync(fileStream, cancellationToken).ConfigureAwait(false);
}
var fileInfo = new FileInfo(destinationPath);
logger.LogInformation("Stored callgraph artifact at {Path} (length={Length}).", destinationPath, fileInfo.Length);
return new StoredCallgraphArtifact(
Path.GetRelativePath(root, destinationPath),
fileInfo.Length,
request.Hash,
request.ContentType,
$"cas://reachability/graphs/{hash}");
}
private static string SanitizeFileName(string value)
=> string.Join('_', value.Split(Path.GetInvalidFileNameChars(), StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)).ToLowerInvariant();
}

View File

@@ -3,8 +3,9 @@ namespace StellaOps.Signals.Storage.Models;
/// <summary>
/// Result returned after storing an artifact.
/// </summary>
public sealed record StoredCallgraphArtifact(
string Path,
long Length,
string Hash,
string ContentType);
public sealed record StoredCallgraphArtifact(
string Path,
long Length,
string Hash,
string ContentType,
string CasUri);