up
This commit is contained in:
105
src/Signals/StellaOps.Signals/Options/SignalsEventsOptions.cs
Normal file
105
src/Signals/StellaOps.Signals/Options/SignalsEventsOptions.cs
Normal file
@@ -0,0 +1,105 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Signals.Options;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for reachability fact events (SIGNALS-24-005).
|
||||
/// </summary>
|
||||
public sealed class SignalsEventsOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Enables event emission. When false, events are dropped.
|
||||
/// </summary>
|
||||
public bool Enabled { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Transport driver: "inmemory" or "redis".
|
||||
/// </summary>
|
||||
public string Driver { get; set; } = "inmemory";
|
||||
|
||||
/// <summary>
|
||||
/// Primary topic/stream name for fact updates.
|
||||
/// </summary>
|
||||
public string Stream { get; set; } = "signals.fact.updated.v1";
|
||||
|
||||
/// <summary>
|
||||
/// Dead-letter topic/stream used when publishing fails.
|
||||
/// </summary>
|
||||
public string DeadLetterStream { get; set; } = "signals.fact.updated.dlq";
|
||||
|
||||
/// <summary>
|
||||
/// Connection string for Redis streams (when Driver=redis).
|
||||
/// </summary>
|
||||
public string? ConnectionString { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional publish timeout (seconds). Set to 0 to disable.
|
||||
/// </summary>
|
||||
public int PublishTimeoutSeconds { get; set; } = 5;
|
||||
|
||||
/// <summary>
|
||||
/// Approximate maximum stream length (capped by Redis trimming).
|
||||
/// </summary>
|
||||
public long MaxStreamLength { get; set; } = 10_000;
|
||||
|
||||
/// <summary>
|
||||
/// Producer identifier for observability payloads.
|
||||
/// </summary>
|
||||
public string Producer { get; set; } = "StellaOps.Signals";
|
||||
|
||||
/// <summary>
|
||||
/// Pipeline name attached to event metadata.
|
||||
/// </summary>
|
||||
public string Pipeline { get; set; } = "signals";
|
||||
|
||||
/// <summary>
|
||||
/// Optional release string to stamp events with build provenance.
|
||||
/// </summary>
|
||||
public string? Release { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Default tenant when none is supplied in metadata.
|
||||
/// </summary>
|
||||
public string DefaultTenant { get; set; } = "tenant-default";
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
var normalizedDriver = Driver?.Trim();
|
||||
if (string.IsNullOrWhiteSpace(normalizedDriver))
|
||||
{
|
||||
throw new InvalidOperationException("Signals events driver is required.");
|
||||
}
|
||||
|
||||
if (!string.Equals(normalizedDriver, "redis", StringComparison.OrdinalIgnoreCase)
|
||||
&& !string.Equals(normalizedDriver, "inmemory", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException("Signals events driver must be 'redis' or 'inmemory'.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(Stream))
|
||||
{
|
||||
throw new InvalidOperationException("Signals events stream/topic is required.");
|
||||
}
|
||||
|
||||
if (PublishTimeoutSeconds < 0)
|
||||
{
|
||||
throw new InvalidOperationException("Signals events publish timeout must be >= 0 seconds.");
|
||||
}
|
||||
|
||||
if (MaxStreamLength < 0)
|
||||
{
|
||||
throw new InvalidOperationException("Signals events max stream length must be >= 0.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(DefaultTenant))
|
||||
{
|
||||
throw new InvalidOperationException("Signals events default tenant is required.");
|
||||
}
|
||||
|
||||
if (string.Equals(normalizedDriver, "redis", StringComparison.OrdinalIgnoreCase)
|
||||
&& string.IsNullOrWhiteSpace(ConnectionString))
|
||||
{
|
||||
throw new InvalidOperationException("Signals events Redis driver requires ConnectionString.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,26 +1,26 @@
|
||||
namespace StellaOps.Signals.Options;
|
||||
|
||||
/// <summary>
|
||||
/// Root configuration for the Signals service.
|
||||
/// </summary>
|
||||
public sealed class SignalsOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Configuration section name.
|
||||
/// </summary>
|
||||
public const string SectionName = "Signals";
|
||||
|
||||
/// <summary>
|
||||
/// Authority integration settings.
|
||||
/// </summary>
|
||||
public SignalsAuthorityOptions Authority { get; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// MongoDB configuration.
|
||||
/// </summary>
|
||||
public SignalsMongoOptions Mongo { get; } = new();
|
||||
|
||||
/// <summary>
|
||||
namespace StellaOps.Signals.Options;
|
||||
|
||||
/// <summary>
|
||||
/// Root configuration for the Signals service.
|
||||
/// </summary>
|
||||
public sealed class SignalsOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Configuration section name.
|
||||
/// </summary>
|
||||
public const string SectionName = "Signals";
|
||||
|
||||
/// <summary>
|
||||
/// Authority integration settings.
|
||||
/// </summary>
|
||||
public SignalsAuthorityOptions Authority { get; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// MongoDB configuration.
|
||||
/// </summary>
|
||||
public SignalsMongoOptions Mongo { get; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Artifact storage configuration.
|
||||
/// </summary>
|
||||
public SignalsArtifactStorageOptions Storage { get; } = new();
|
||||
@@ -40,22 +40,28 @@ public sealed class SignalsOptions
|
||||
/// </summary>
|
||||
public SignalsCacheOptions Cache { get; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Event transport configuration.
|
||||
/// </summary>
|
||||
public SignalsEventsOptions Events { get; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// OpenAPI exposure (if enabled).
|
||||
/// </summary>
|
||||
public SignalsOpenApiOptions OpenApi { get; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Validates configured options.
|
||||
/// </summary>
|
||||
public void Validate()
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// Validates configured options.
|
||||
/// </summary>
|
||||
public void Validate()
|
||||
{
|
||||
Authority.Validate();
|
||||
Mongo.Validate();
|
||||
Storage.Validate();
|
||||
AirGap.Validate();
|
||||
Scoring.Validate();
|
||||
Cache.Validate();
|
||||
Events.Validate();
|
||||
OpenApi.Validate();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -125,6 +125,7 @@ builder.Services.AddSingleton<IMongoCollection<UnknownSymbolDocument>>(sp =>
|
||||
});
|
||||
|
||||
builder.Services.AddSingleton<ICallgraphRepository, MongoCallgraphRepository>();
|
||||
builder.Services.AddSingleton<ICallgraphNormalizationService, CallgraphNormalizationService>();
|
||||
|
||||
// Configure callgraph artifact storage based on driver
|
||||
if (bootstrap.Storage.IsRustFsDriver())
|
||||
@@ -165,7 +166,31 @@ builder.Services.AddSingleton<IReachabilityCache>(sp =>
|
||||
var options = sp.GetRequiredService<IOptions<SignalsOptions>>().Value;
|
||||
return new RedisReachabilityCache(options.Cache);
|
||||
});
|
||||
builder.Services.AddSingleton<IEventsPublisher, InMemoryEventsPublisher>();
|
||||
builder.Services.AddSingleton<IRedisConnectionFactory, RedisConnectionFactory>();
|
||||
builder.Services.AddSingleton<ReachabilityFactEventBuilder>();
|
||||
builder.Services.AddSingleton<IEventsPublisher>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<SignalsOptions>();
|
||||
var eventBuilder = sp.GetRequiredService<ReachabilityFactEventBuilder>();
|
||||
|
||||
if (!options.Events.Enabled)
|
||||
{
|
||||
return new NullEventsPublisher();
|
||||
}
|
||||
|
||||
if (string.Equals(options.Events.Driver, "redis", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return new RedisEventsPublisher(
|
||||
options,
|
||||
sp.GetRequiredService<IRedisConnectionFactory>(),
|
||||
eventBuilder,
|
||||
sp.GetRequiredService<ILogger<RedisEventsPublisher>>());
|
||||
}
|
||||
|
||||
return new InMemoryEventsPublisher(
|
||||
sp.GetRequiredService<ILogger<InMemoryEventsPublisher>>(),
|
||||
eventBuilder);
|
||||
});
|
||||
builder.Services.AddSingleton<IReachabilityFactRepository>(sp =>
|
||||
{
|
||||
var inner = sp.GetRequiredService<MongoReachabilityFactRepository>();
|
||||
|
||||
@@ -1,72 +1,76 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Options;
|
||||
using StellaOps.Signals.Parsing;
|
||||
using StellaOps.Signals.Persistence;
|
||||
using StellaOps.Signals.Storage;
|
||||
using StellaOps.Signals.Storage.Models;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
{
|
||||
private static readonly HashSet<string> AllowedContentTypes = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"application/json",
|
||||
"application/vnd.stellaops.callgraph+json"
|
||||
};
|
||||
|
||||
private readonly ICallgraphParserResolver parserResolver;
|
||||
private readonly ICallgraphArtifactStore artifactStore;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Options;
|
||||
using StellaOps.Signals.Parsing;
|
||||
using StellaOps.Signals.Persistence;
|
||||
using StellaOps.Signals.Storage;
|
||||
using StellaOps.Signals.Storage.Models;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
{
|
||||
private static readonly HashSet<string> AllowedContentTypes = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"application/json",
|
||||
"application/vnd.stellaops.callgraph+json"
|
||||
};
|
||||
|
||||
private readonly ICallgraphParserResolver parserResolver;
|
||||
private readonly ICallgraphArtifactStore artifactStore;
|
||||
private readonly ICallgraphRepository repository;
|
||||
private readonly ICallgraphNormalizationService normalizer;
|
||||
private readonly ILogger<CallgraphIngestionService> logger;
|
||||
private readonly SignalsOptions options;
|
||||
private readonly TimeProvider timeProvider;
|
||||
private static readonly JsonSerializerOptions ManifestSerializerOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
public CallgraphIngestionService(
|
||||
ICallgraphParserResolver parserResolver,
|
||||
ICallgraphArtifactStore artifactStore,
|
||||
ICallgraphRepository repository,
|
||||
IOptions<SignalsOptions> options,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<CallgraphIngestionService> logger)
|
||||
{
|
||||
this.parserResolver = parserResolver ?? throw new ArgumentNullException(nameof(parserResolver));
|
||||
this.artifactStore = artifactStore ?? throw new ArgumentNullException(nameof(artifactStore));
|
||||
this.repository = repository ?? throw new ArgumentNullException(nameof(repository));
|
||||
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
this.options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
}
|
||||
|
||||
public async Task<CallgraphIngestResponse> IngestAsync(CallgraphIngestRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
ValidateRequest(request);
|
||||
|
||||
var parser = parserResolver.Resolve(request.Language);
|
||||
|
||||
|
||||
public CallgraphIngestionService(
|
||||
ICallgraphParserResolver parserResolver,
|
||||
ICallgraphArtifactStore artifactStore,
|
||||
ICallgraphRepository repository,
|
||||
ICallgraphNormalizationService normalizer,
|
||||
IOptions<SignalsOptions> options,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<CallgraphIngestionService> logger)
|
||||
{
|
||||
this.parserResolver = parserResolver ?? throw new ArgumentNullException(nameof(parserResolver));
|
||||
this.artifactStore = artifactStore ?? throw new ArgumentNullException(nameof(artifactStore));
|
||||
this.repository = repository ?? throw new ArgumentNullException(nameof(repository));
|
||||
this.normalizer = normalizer ?? throw new ArgumentNullException(nameof(normalizer));
|
||||
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
this.options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
}
|
||||
|
||||
public async Task<CallgraphIngestResponse> IngestAsync(CallgraphIngestRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
ValidateRequest(request);
|
||||
|
||||
var parser = parserResolver.Resolve(request.Language);
|
||||
|
||||
var artifactBytes = Convert.FromBase64String(request.ArtifactContentBase64);
|
||||
await using var parseStream = new MemoryStream(artifactBytes, writable: false);
|
||||
var parseResult = await parser.ParseAsync(parseStream, cancellationToken).ConfigureAwait(false);
|
||||
var parsed = await parser.ParseAsync(parseStream, cancellationToken).ConfigureAwait(false);
|
||||
var normalized = normalizer.Normalize(parser.Language, parsed);
|
||||
var schemaVersion = !string.IsNullOrWhiteSpace(request.SchemaVersion)
|
||||
? request.SchemaVersion!
|
||||
: parseResult.SchemaVersion;
|
||||
var analyzerMeta = request.Analyzer ?? parseResult.Analyzer;
|
||||
: normalized.SchemaVersion;
|
||||
var analyzerMeta = request.Analyzer ?? normalized.Analyzer;
|
||||
|
||||
parseStream.Position = 0;
|
||||
var artifactHash = ComputeSha256(artifactBytes);
|
||||
var graphHash = ComputeGraphHash(parseResult);
|
||||
var graphHash = ComputeGraphHash(normalized);
|
||||
|
||||
var manifest = new CallgraphManifest
|
||||
{
|
||||
@@ -76,9 +80,9 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
ArtifactHash = artifactHash,
|
||||
GraphHash = graphHash,
|
||||
SchemaVersion = schemaVersion,
|
||||
NodeCount = parseResult.Nodes.Count,
|
||||
EdgeCount = parseResult.Edges.Count,
|
||||
RootCount = parseResult.Roots.Count,
|
||||
NodeCount = normalized.Nodes.Count,
|
||||
EdgeCount = normalized.Edges.Count,
|
||||
RootCount = normalized.Roots.Count,
|
||||
CreatedAt = timeProvider.GetUtcNow()
|
||||
};
|
||||
|
||||
@@ -98,15 +102,15 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
manifestStream),
|
||||
parseStream,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var document = new CallgraphDocument
|
||||
{
|
||||
|
||||
var document = new CallgraphDocument
|
||||
{
|
||||
Language = parser.Language,
|
||||
Component = request.Component,
|
||||
Version = request.Version,
|
||||
Nodes = new List<CallgraphNode>(parseResult.Nodes),
|
||||
Edges = new List<CallgraphEdge>(parseResult.Edges),
|
||||
Roots = new List<CallgraphRoot>(parseResult.Roots),
|
||||
Nodes = new List<CallgraphNode>(normalized.Nodes),
|
||||
Edges = new List<CallgraphEdge>(normalized.Edges),
|
||||
Roots = new List<CallgraphRoot>(normalized.Roots),
|
||||
Metadata = request.Metadata is null
|
||||
? null
|
||||
: new Dictionary<string, string?>(request.Metadata, StringComparer.OrdinalIgnoreCase),
|
||||
@@ -125,7 +129,7 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
};
|
||||
|
||||
document.Metadata ??= new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
|
||||
document.Metadata["formatVersion"] = parseResult.FormatVersion;
|
||||
document.Metadata["formatVersion"] = normalized.FormatVersion;
|
||||
document.Metadata["schemaVersion"] = schemaVersion;
|
||||
if (analyzerMeta is not null)
|
||||
{
|
||||
@@ -138,16 +142,16 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
document.SchemaVersion = schemaVersion;
|
||||
|
||||
document = await repository.UpsertAsync(document, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
logger.LogInformation(
|
||||
"Ingested callgraph {Language}:{Component}:{Version} (id={Id}) with {NodeCount} nodes and {EdgeCount} edges.",
|
||||
document.Language,
|
||||
document.Component,
|
||||
document.Version,
|
||||
document.Id,
|
||||
document.Nodes.Count,
|
||||
document.Edges.Count);
|
||||
|
||||
|
||||
logger.LogInformation(
|
||||
"Ingested callgraph {Language}:{Component}:{Version} (id={Id}) with {NodeCount} nodes and {EdgeCount} edges.",
|
||||
document.Language,
|
||||
document.Component,
|
||||
document.Version,
|
||||
document.Id,
|
||||
document.Nodes.Count,
|
||||
document.Edges.Count);
|
||||
|
||||
return new CallgraphIngestResponse(
|
||||
document.Id,
|
||||
document.Artifact.Path,
|
||||
@@ -160,42 +164,42 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
document.Edges.Count,
|
||||
document.Roots?.Count ?? 0);
|
||||
}
|
||||
|
||||
private static void ValidateRequest(CallgraphIngestRequest request)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.Language))
|
||||
{
|
||||
throw new CallgraphIngestionValidationException("Language is required.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.Component))
|
||||
{
|
||||
throw new CallgraphIngestionValidationException("Component is required.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.Version))
|
||||
{
|
||||
throw new CallgraphIngestionValidationException("Version is required.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.ArtifactContentBase64))
|
||||
{
|
||||
throw new CallgraphIngestionValidationException("Artifact content is required.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.ArtifactFileName))
|
||||
{
|
||||
throw new CallgraphIngestionValidationException("Artifact file name is required.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.ArtifactContentType) || !AllowedContentTypes.Contains(request.ArtifactContentType))
|
||||
{
|
||||
throw new CallgraphIngestionValidationException($"Unsupported artifact content type '{request.ArtifactContentType}'.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static void ValidateRequest(CallgraphIngestRequest request)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.Language))
|
||||
{
|
||||
throw new CallgraphIngestionValidationException("Language is required.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.Component))
|
||||
{
|
||||
throw new CallgraphIngestionValidationException("Component is required.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.Version))
|
||||
{
|
||||
throw new CallgraphIngestionValidationException("Version is required.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.ArtifactContentBase64))
|
||||
{
|
||||
throw new CallgraphIngestionValidationException("Artifact content is required.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.ArtifactFileName))
|
||||
{
|
||||
throw new CallgraphIngestionValidationException("Artifact file name is required.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.ArtifactContentType) || !AllowedContentTypes.Contains(request.ArtifactContentType))
|
||||
{
|
||||
throw new CallgraphIngestionValidationException($"Unsupported artifact content type '{request.ArtifactContentType}'.");
|
||||
}
|
||||
}
|
||||
|
||||
private static string ComputeSha256(ReadOnlySpan<byte> buffer)
|
||||
{
|
||||
Span<byte> hash = stackalloc byte[SHA256.HashSizeInBytes];
|
||||
@@ -274,13 +278,13 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
return ordered.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Exception thrown when the ingestion request is invalid.
|
||||
/// </summary>
|
||||
public sealed class CallgraphIngestionValidationException : Exception
|
||||
{
|
||||
public CallgraphIngestionValidationException(string message) : base(message)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Exception thrown when the ingestion request is invalid.
|
||||
/// </summary>
|
||||
public sealed class CallgraphIngestionValidationException : Exception
|
||||
{
|
||||
public CallgraphIngestionValidationException(string message) : base(message)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Parsing;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
internal interface ICallgraphNormalizationService
|
||||
{
|
||||
CallgraphParseResult Normalize(string language, CallgraphParseResult result);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Normalizes language-specific callgraphs into deterministic graph documents.
|
||||
/// </summary>
|
||||
internal sealed class CallgraphNormalizationService : ICallgraphNormalizationService
|
||||
{
|
||||
public CallgraphParseResult Normalize(string language, CallgraphParseResult result)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(language);
|
||||
ArgumentNullException.ThrowIfNull(result);
|
||||
|
||||
var normalizedLanguage = language.Trim();
|
||||
var nodesById = new Dictionary<string, CallgraphNode>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var node in result.Nodes ?? Array.Empty<CallgraphNode>())
|
||||
{
|
||||
var normalizedNode = NormalizeNode(node, normalizedLanguage);
|
||||
if (!nodesById.ContainsKey(normalizedNode.Id))
|
||||
{
|
||||
nodesById[normalizedNode.Id] = normalizedNode;
|
||||
}
|
||||
}
|
||||
|
||||
var edges = NormalizeEdges(result.Edges, nodesById);
|
||||
var roots = NormalizeRoots(result.Roots);
|
||||
|
||||
return new CallgraphParseResult(
|
||||
Nodes: nodesById.Values.OrderBy(n => n.Id, StringComparer.Ordinal).ToList(),
|
||||
Edges: edges,
|
||||
Roots: roots,
|
||||
FormatVersion: string.IsNullOrWhiteSpace(result.FormatVersion) ? "1.0" : result.FormatVersion.Trim(),
|
||||
SchemaVersion: string.IsNullOrWhiteSpace(result.SchemaVersion) ? "1.0" : result.SchemaVersion.Trim(),
|
||||
Analyzer: result.Analyzer);
|
||||
}
|
||||
|
||||
private static CallgraphNode NormalizeNode(CallgraphNode node, string language)
|
||||
{
|
||||
var id = node.Id?.Trim();
|
||||
if (string.IsNullOrWhiteSpace(id))
|
||||
{
|
||||
throw new CallgraphParserValidationException("Callgraph node is missing an id.");
|
||||
}
|
||||
|
||||
var name = string.IsNullOrWhiteSpace(node.Name) ? id : node.Name.Trim();
|
||||
var kind = string.IsNullOrWhiteSpace(node.Kind) ? "function" : node.Kind.Trim();
|
||||
var normalizedLanguage = string.IsNullOrWhiteSpace(node.Language) ? language : node.Language.Trim();
|
||||
|
||||
var ns = string.IsNullOrWhiteSpace(node.Namespace)
|
||||
? DeriveNamespace(id, node.File, normalizedLanguage)
|
||||
: node.Namespace!.Trim();
|
||||
|
||||
return node with
|
||||
{
|
||||
Id = id,
|
||||
Name = name,
|
||||
Kind = kind,
|
||||
Namespace = ns,
|
||||
File = node.File?.Trim(),
|
||||
Purl = NormalizePurl(node.Purl),
|
||||
SymbolDigest = NormalizeDigest(node.SymbolDigest),
|
||||
BuildId = node.BuildId?.Trim(),
|
||||
Language = normalizedLanguage,
|
||||
Evidence = NormalizeList(node.Evidence),
|
||||
Analyzer = NormalizeDict(node.Analyzer),
|
||||
CodeId = node.CodeId?.Trim()
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyList<CallgraphEdge> NormalizeEdges(
|
||||
IReadOnlyList<CallgraphEdge>? edges,
|
||||
IReadOnlyDictionary<string, CallgraphNode> nodes)
|
||||
{
|
||||
var list = new List<CallgraphEdge>();
|
||||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var edge in edges ?? Array.Empty<CallgraphEdge>())
|
||||
{
|
||||
var source = edge.SourceId?.Trim();
|
||||
var target = edge.TargetId?.Trim();
|
||||
if (string.IsNullOrWhiteSpace(source) || string.IsNullOrWhiteSpace(target))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!nodes.ContainsKey(source) || !nodes.ContainsKey(target))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var type = string.IsNullOrWhiteSpace(edge.Type) ? "call" : edge.Type.Trim();
|
||||
var key = $"{source}|{target}|{type}";
|
||||
if (!seen.Add(key))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
list.Add(edge with
|
||||
{
|
||||
SourceId = source,
|
||||
TargetId = target,
|
||||
Type = type,
|
||||
Purl = NormalizePurl(edge.Purl),
|
||||
SymbolDigest = NormalizeDigest(edge.SymbolDigest),
|
||||
Confidence = ClampConfidence(edge.Confidence),
|
||||
Candidates = NormalizeList(edge.Candidates),
|
||||
Evidence = NormalizeList(edge.Evidence)
|
||||
});
|
||||
}
|
||||
|
||||
return list.OrderBy(e => e.SourceId, StringComparer.Ordinal)
|
||||
.ThenBy(e => e.TargetId, StringComparer.Ordinal)
|
||||
.ThenBy(e => e.Type, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<CallgraphRoot> NormalizeRoots(IReadOnlyList<CallgraphRoot>? roots)
|
||||
{
|
||||
var list = new List<CallgraphRoot>();
|
||||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var root in roots ?? Array.Empty<CallgraphRoot>())
|
||||
{
|
||||
var id = root.Id?.Trim();
|
||||
if (string.IsNullOrWhiteSpace(id))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var normalized = new CallgraphRoot(
|
||||
id,
|
||||
string.IsNullOrWhiteSpace(root.Phase) ? "runtime" : root.Phase.Trim(),
|
||||
root.Source?.Trim());
|
||||
|
||||
if (seen.Add($"{normalized.Id}|{normalized.Phase}|{normalized.Source}"))
|
||||
{
|
||||
list.Add(normalized);
|
||||
}
|
||||
}
|
||||
|
||||
return list.OrderBy(r => r.Id, StringComparer.Ordinal)
|
||||
.ThenBy(r => r.Phase, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static string? DeriveNamespace(string id, string? file, string language)
|
||||
{
|
||||
if (string.Equals(language, "java", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
var candidate = id.Replace('/', '.');
|
||||
var lastDot = candidate.LastIndexOf('.');
|
||||
if (lastDot > 0)
|
||||
{
|
||||
return candidate[..lastDot];
|
||||
}
|
||||
}
|
||||
|
||||
if (string.Equals(language, "go", StringComparison.OrdinalIgnoreCase) ||
|
||||
string.Equals(language, "nodejs", StringComparison.OrdinalIgnoreCase) ||
|
||||
string.Equals(language, "python", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(file))
|
||||
{
|
||||
var normalizedPath = file.Replace('\\', '/');
|
||||
var idx = normalizedPath.LastIndexOf('/');
|
||||
if (idx > 0)
|
||||
{
|
||||
return normalizedPath[..idx];
|
||||
}
|
||||
}
|
||||
|
||||
var sepIdx = id.LastIndexOfAny(new[] { '.', '/', ':' });
|
||||
if (sepIdx > 0)
|
||||
{
|
||||
return id[..sepIdx];
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string? NormalizePurl(string? value)
|
||||
{
|
||||
return string.IsNullOrWhiteSpace(value) ? null : value.Trim().ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static string? NormalizeDigest(string? value)
|
||||
{
|
||||
return string.IsNullOrWhiteSpace(value) ? null : value.Trim().ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static double? ClampConfidence(double? confidence)
|
||||
{
|
||||
if (!confidence.HasValue)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return Math.Clamp(confidence.Value, 0.0, 1.0);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string>? NormalizeList(IReadOnlyList<string>? values)
|
||||
{
|
||||
if (values is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return values
|
||||
.Where(v => !string.IsNullOrWhiteSpace(v))
|
||||
.Select(v => v.Trim())
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.OrderBy(v => v, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, string?>? NormalizeDict(IReadOnlyDictionary<string, string?>? values)
|
||||
{
|
||||
if (values is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var dict = new Dictionary<string, string?>(StringComparer.Ordinal);
|
||||
foreach (var kv in values)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(kv.Key))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
dict[kv.Key.Trim()] = kv.Value?.Trim();
|
||||
}
|
||||
|
||||
return dict.Count == 0 ? null : dict;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StackExchange.Redis;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
internal interface IRedisConnectionFactory
|
||||
{
|
||||
Task<IConnectionMultiplexer> ConnectAsync(ConfigurationOptions options, CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -1,11 +1,9 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Options;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
@@ -15,65 +13,27 @@ namespace StellaOps.Signals.Services;
|
||||
internal sealed class InMemoryEventsPublisher : IEventsPublisher
|
||||
{
|
||||
private readonly ILogger<InMemoryEventsPublisher> logger;
|
||||
private readonly string topic;
|
||||
private readonly ReachabilityFactEventBuilder eventBuilder;
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = false,
|
||||
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull
|
||||
};
|
||||
|
||||
public InMemoryEventsPublisher(ILogger<InMemoryEventsPublisher> logger, SignalsOptions options)
|
||||
public InMemoryEventsPublisher(ILogger<InMemoryEventsPublisher> logger, ReachabilityFactEventBuilder eventBuilder)
|
||||
{
|
||||
this.logger = logger;
|
||||
topic = string.IsNullOrWhiteSpace(options?.AirGap?.EventTopic)
|
||||
? "signals.fact.updated"
|
||||
: options!.AirGap.EventTopic!;
|
||||
this.eventBuilder = eventBuilder ?? throw new ArgumentNullException(nameof(eventBuilder));
|
||||
}
|
||||
|
||||
public Task PublishFactUpdatedAsync(ReachabilityFactDocument fact, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(fact);
|
||||
|
||||
var (reachable, unreachable) = CountStates(fact);
|
||||
var runtimeFactsCount = fact.RuntimeFacts?.Count ?? 0;
|
||||
var avgConfidence = fact.States.Count > 0 ? fact.States.Average(s => s.Confidence) : 0;
|
||||
var score = fact.Score;
|
||||
var unknownsCount = fact.UnknownsCount;
|
||||
var unknownsPressure = fact.UnknownsPressure;
|
||||
var topBucket = fact.States.Count > 0
|
||||
? fact.States
|
||||
.GroupBy(s => s.Bucket, StringComparer.OrdinalIgnoreCase)
|
||||
.OrderByDescending(g => g.Count())
|
||||
.ThenByDescending(g => g.Average(s => s.Weight))
|
||||
.First()
|
||||
: null;
|
||||
var payload = new ReachabilityFactUpdatedEvent(
|
||||
Version: "signals.fact.updated@v1",
|
||||
SubjectKey: fact.SubjectKey,
|
||||
CallgraphId: string.IsNullOrWhiteSpace(fact.CallgraphId) ? null : fact.CallgraphId,
|
||||
OccurredAtUtc: DateTimeOffset.UtcNow,
|
||||
ReachableCount: reachable,
|
||||
UnreachableCount: unreachable,
|
||||
RuntimeFactsCount: runtimeFactsCount,
|
||||
Bucket: topBucket?.Key ?? "unknown",
|
||||
Weight: topBucket?.Average(s => s.Weight) ?? 0,
|
||||
StateCount: fact.States.Count,
|
||||
FactScore: score,
|
||||
UnknownsCount: unknownsCount,
|
||||
UnknownsPressure: unknownsPressure,
|
||||
AverageConfidence: avgConfidence,
|
||||
ComputedAtUtc: fact.ComputedAt,
|
||||
Targets: fact.States.Select(s => s.Target).ToArray());
|
||||
var envelope = eventBuilder.Build(fact);
|
||||
var json = JsonSerializer.Serialize(envelope, SerializerOptions);
|
||||
|
||||
var json = JsonSerializer.Serialize(payload, new JsonSerializerOptions(JsonSerializerDefaults.Web));
|
||||
logger.LogInformation("{Topic} {Payload}", topic, json);
|
||||
logger.LogInformation(json);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private static (int reachable, int unreachable) CountStates(ReachabilityFactDocument fact)
|
||||
{
|
||||
if (fact.States is null || fact.States.Count == 0)
|
||||
{
|
||||
return (0, 0);
|
||||
}
|
||||
|
||||
var reachable = fact.States.Count(state => state.Reachable);
|
||||
var unreachable = fact.States.Count - reachable;
|
||||
return (reachable, unreachable);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Signals.Models;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
internal sealed class NullEventsPublisher : IEventsPublisher
|
||||
{
|
||||
public Task PublishFactUpdatedAsync(ReachabilityFactDocument fact, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using StellaOps.Signals.Models;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
internal static class ReachabilityFactDigestCalculator
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = false
|
||||
};
|
||||
|
||||
public static string Compute(ReachabilityFactDocument fact)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(fact);
|
||||
|
||||
var canonical = new CanonicalReachabilityFact(
|
||||
CallgraphId: fact.CallgraphId ?? string.Empty,
|
||||
SubjectKey: fact.SubjectKey ?? string.Empty,
|
||||
Subject: new CanonicalSubject(
|
||||
fact.Subject?.ImageDigest ?? string.Empty,
|
||||
fact.Subject?.Component ?? string.Empty,
|
||||
fact.Subject?.Version ?? string.Empty,
|
||||
fact.Subject?.ScanId ?? string.Empty),
|
||||
EntryPoints: NormalizeList(fact.EntryPoints),
|
||||
States: NormalizeStates(fact.States),
|
||||
RuntimeFacts: NormalizeRuntimeFacts(fact.RuntimeFacts),
|
||||
Metadata: NormalizeMetadata(fact.Metadata),
|
||||
Score: fact.Score,
|
||||
UnknownsCount: fact.UnknownsCount,
|
||||
UnknownsPressure: fact.UnknownsPressure,
|
||||
ComputedAt: fact.ComputedAt);
|
||||
|
||||
var json = JsonSerializer.Serialize(canonical, SerializerOptions);
|
||||
Span<byte> hash = stackalloc byte[SHA256.HashSizeInBytes];
|
||||
SHA256.HashData(Encoding.UTF8.GetBytes(json), hash);
|
||||
return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static List<string> NormalizeList(IEnumerable<string>? values) =>
|
||||
values?
|
||||
.Where(v => !string.IsNullOrWhiteSpace(v))
|
||||
.Select(v => v.Trim())
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.OrderBy(v => v, StringComparer.Ordinal)
|
||||
.ToList() ?? new List<string>();
|
||||
|
||||
private static List<CanonicalState> NormalizeStates(IEnumerable<ReachabilityStateDocument>? states)
|
||||
{
|
||||
if (states is null)
|
||||
{
|
||||
return new List<CanonicalState>();
|
||||
}
|
||||
|
||||
return states
|
||||
.OrderBy(s => s.Target, StringComparer.Ordinal)
|
||||
.Select(state => new CanonicalState(
|
||||
Target: state.Target ?? string.Empty,
|
||||
Reachable: state.Reachable,
|
||||
Confidence: state.Confidence,
|
||||
Bucket: state.Bucket ?? "unknown",
|
||||
Weight: state.Weight,
|
||||
Score: state.Score,
|
||||
Path: NormalizeList(state.Path),
|
||||
RuntimeHits: NormalizeList(state.Evidence?.RuntimeHits),
|
||||
BlockedEdges: NormalizeList(state.Evidence?.BlockedEdges)))
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static List<CanonicalRuntimeFact> NormalizeRuntimeFacts(IEnumerable<RuntimeFactDocument>? facts)
|
||||
{
|
||||
if (facts is null)
|
||||
{
|
||||
return new List<CanonicalRuntimeFact>();
|
||||
}
|
||||
|
||||
return facts
|
||||
.Select(f => new CanonicalRuntimeFact(
|
||||
SymbolId: f.SymbolId ?? string.Empty,
|
||||
CodeId: f.CodeId,
|
||||
SymbolDigest: f.SymbolDigest,
|
||||
Purl: f.Purl,
|
||||
BuildId: f.BuildId,
|
||||
LoaderBase: f.LoaderBase,
|
||||
ProcessId: f.ProcessId,
|
||||
ProcessName: f.ProcessName,
|
||||
SocketAddress: f.SocketAddress,
|
||||
ContainerId: f.ContainerId,
|
||||
EvidenceUri: f.EvidenceUri,
|
||||
HitCount: f.HitCount,
|
||||
ObservedAt: f.ObservedAt,
|
||||
Metadata: NormalizeMetadata(f.Metadata)))
|
||||
.OrderBy(f => f.SymbolId, StringComparer.Ordinal)
|
||||
.ThenBy(f => f.CodeId, StringComparer.Ordinal)
|
||||
.ThenBy(f => f.LoaderBase, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static SortedDictionary<string, string?> NormalizeMetadata(IDictionary<string, string?>? metadata)
|
||||
{
|
||||
var normalized = new SortedDictionary<string, string?>(StringComparer.Ordinal);
|
||||
if (metadata is null)
|
||||
{
|
||||
return normalized;
|
||||
}
|
||||
|
||||
foreach (var kvp in metadata)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(kvp.Key))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
normalized[kvp.Key.Trim()] = kvp.Value?.Trim();
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private sealed record CanonicalReachabilityFact(
|
||||
string CallgraphId,
|
||||
string SubjectKey,
|
||||
CanonicalSubject Subject,
|
||||
List<string> EntryPoints,
|
||||
List<CanonicalState> States,
|
||||
List<CanonicalRuntimeFact> RuntimeFacts,
|
||||
SortedDictionary<string, string?> Metadata,
|
||||
double Score,
|
||||
int UnknownsCount,
|
||||
double UnknownsPressure,
|
||||
DateTimeOffset ComputedAt);
|
||||
|
||||
private sealed record CanonicalSubject(
|
||||
string ImageDigest,
|
||||
string Component,
|
||||
string Version,
|
||||
string ScanId);
|
||||
|
||||
private sealed record CanonicalState(
|
||||
string Target,
|
||||
bool Reachable,
|
||||
double Confidence,
|
||||
string Bucket,
|
||||
double Weight,
|
||||
double Score,
|
||||
List<string> Path,
|
||||
List<string> RuntimeHits,
|
||||
List<string> BlockedEdges);
|
||||
|
||||
private sealed record CanonicalRuntimeFact(
|
||||
string SymbolId,
|
||||
string? CodeId,
|
||||
string? SymbolDigest,
|
||||
string? Purl,
|
||||
string? BuildId,
|
||||
string? LoaderBase,
|
||||
int? ProcessId,
|
||||
string? ProcessName,
|
||||
string? SocketAddress,
|
||||
string? ContainerId,
|
||||
string? EvidenceUri,
|
||||
int HitCount,
|
||||
DateTimeOffset? ObservedAt,
|
||||
SortedDictionary<string, string?> Metadata);
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Text.Json.Serialization;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Options;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
internal sealed class ReachabilityFactEventBuilder
|
||||
{
|
||||
private readonly SignalsOptions options;
|
||||
private readonly TimeProvider timeProvider;
|
||||
|
||||
public ReachabilityFactEventBuilder(SignalsOptions options, TimeProvider timeProvider)
|
||||
{
|
||||
this.options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
}
|
||||
|
||||
public ReachabilityFactUpdatedEnvelope Build(ReachabilityFactDocument fact)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(fact);
|
||||
|
||||
var summary = BuildSummary(fact);
|
||||
var digest = ResolveDigest(fact);
|
||||
var factVersion = ResolveFactVersion(fact);
|
||||
|
||||
return new ReachabilityFactUpdatedEnvelope(
|
||||
Topic: ResolveTopic(),
|
||||
EventId: Guid.NewGuid().ToString("n"),
|
||||
Version: "signals.fact.updated@v1",
|
||||
EmittedAtUtc: timeProvider.GetUtcNow(),
|
||||
Tenant: ResolveTenant(fact),
|
||||
SubjectKey: fact.SubjectKey,
|
||||
CallgraphId: string.IsNullOrWhiteSpace(fact.CallgraphId) ? null : fact.CallgraphId,
|
||||
FactKind: "reachability",
|
||||
FactVersion: factVersion,
|
||||
Digest: digest,
|
||||
ContentType: "application/json",
|
||||
Producer: new EventProducerMetadata(
|
||||
Service: options.Events.Producer,
|
||||
Pipeline: options.Events.Pipeline,
|
||||
Release: options.Events.Release ?? typeof(Program).Assembly.GetName().Version?.ToString() ?? "unknown"),
|
||||
Trace: ResolveTrace(fact),
|
||||
Summary: summary);
|
||||
}
|
||||
|
||||
private ReachabilityFactUpdatedEvent BuildSummary(ReachabilityFactDocument fact)
|
||||
{
|
||||
var (reachable, unreachable) = CountStates(fact);
|
||||
var runtimeFactsCount = fact.RuntimeFacts?.Count ?? 0;
|
||||
var avgConfidence = fact.States.Count > 0 ? fact.States.Average(s => s.Confidence) : 0;
|
||||
var topBucket = fact.States.Count > 0
|
||||
? fact.States
|
||||
.GroupBy(s => s.Bucket, StringComparer.OrdinalIgnoreCase)
|
||||
.OrderByDescending(g => g.Count())
|
||||
.ThenByDescending(g => g.Average(s => s.Weight))
|
||||
.First()
|
||||
: null;
|
||||
|
||||
return new ReachabilityFactUpdatedEvent(
|
||||
Version: "signals.fact.updated@v1",
|
||||
SubjectKey: fact.SubjectKey,
|
||||
CallgraphId: string.IsNullOrWhiteSpace(fact.CallgraphId) ? null : fact.CallgraphId,
|
||||
OccurredAtUtc: timeProvider.GetUtcNow(),
|
||||
ReachableCount: reachable,
|
||||
UnreachableCount: unreachable,
|
||||
RuntimeFactsCount: runtimeFactsCount,
|
||||
Bucket: topBucket?.Key ?? "unknown",
|
||||
Weight: topBucket?.Average(s => s.Weight) ?? 0,
|
||||
StateCount: fact.States.Count,
|
||||
FactScore: fact.Score,
|
||||
UnknownsCount: fact.UnknownsCount,
|
||||
UnknownsPressure: fact.UnknownsPressure,
|
||||
AverageConfidence: avgConfidence,
|
||||
ComputedAtUtc: fact.ComputedAt,
|
||||
Targets: fact.States.Select(s => s.Target).ToArray());
|
||||
}
|
||||
|
||||
private static (int reachable, int unreachable) CountStates(ReachabilityFactDocument fact)
|
||||
{
|
||||
if (fact.States is null || fact.States.Count == 0)
|
||||
{
|
||||
return (0, 0);
|
||||
}
|
||||
|
||||
var reachable = fact.States.Count(state => state.Reachable);
|
||||
var unreachable = fact.States.Count - reachable;
|
||||
return (reachable, unreachable);
|
||||
}
|
||||
|
||||
private string ResolveTopic()
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(options.Events.Stream))
|
||||
{
|
||||
return options.Events.Stream;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(options.AirGap.EventTopic))
|
||||
{
|
||||
return options.AirGap.EventTopic!;
|
||||
}
|
||||
|
||||
return "signals.fact.updated.v1";
|
||||
}
|
||||
|
||||
private string ResolveTenant(ReachabilityFactDocument fact)
|
||||
{
|
||||
if (fact.Metadata is not null)
|
||||
{
|
||||
if (fact.Metadata.TryGetValue("tenant", out var tenant) && !string.IsNullOrWhiteSpace(tenant))
|
||||
{
|
||||
return tenant!;
|
||||
}
|
||||
|
||||
if (fact.Metadata.TryGetValue("subject.tenant", out var subjectTenant) && !string.IsNullOrWhiteSpace(subjectTenant))
|
||||
{
|
||||
return subjectTenant!;
|
||||
}
|
||||
}
|
||||
|
||||
return options.Events.DefaultTenant;
|
||||
}
|
||||
|
||||
private static EventTraceMetadata ResolveTrace(ReachabilityFactDocument fact)
|
||||
{
|
||||
var metadata = fact.Metadata;
|
||||
string? traceId = null;
|
||||
string? spanId = null;
|
||||
|
||||
if (metadata is not null)
|
||||
{
|
||||
metadata.TryGetValue("trace_id", out traceId);
|
||||
metadata.TryGetValue("span_id", out spanId);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(traceId) && metadata.TryGetValue("trace.id", out var dottedTrace))
|
||||
{
|
||||
traceId = dottedTrace;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(spanId) && metadata.TryGetValue("trace.parent_span", out var dottedSpan))
|
||||
{
|
||||
spanId = dottedSpan;
|
||||
}
|
||||
}
|
||||
|
||||
return new EventTraceMetadata(traceId, spanId);
|
||||
}
|
||||
|
||||
private static int ResolveFactVersion(ReachabilityFactDocument fact)
|
||||
{
|
||||
if (fact.Metadata is not null &&
|
||||
fact.Metadata.TryGetValue("fact.version", out var versionValue) &&
|
||||
int.TryParse(versionValue, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed))
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
private static string ResolveDigest(ReachabilityFactDocument fact)
|
||||
{
|
||||
if (fact.Metadata is not null &&
|
||||
fact.Metadata.TryGetValue("fact.digest", out var digest) &&
|
||||
!string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
return digest!;
|
||||
}
|
||||
|
||||
return ReachabilityFactDigestCalculator.Compute(fact);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record ReachabilityFactUpdatedEnvelope(
|
||||
[property: JsonPropertyName("topic")] string Topic,
|
||||
[property: JsonPropertyName("event_id")] string EventId,
|
||||
[property: JsonPropertyName("version")] string Version,
|
||||
[property: JsonPropertyName("emitted_at")] DateTimeOffset EmittedAtUtc,
|
||||
[property: JsonPropertyName("tenant")] string Tenant,
|
||||
[property: JsonPropertyName("subject_key")] string SubjectKey,
|
||||
[property: JsonPropertyName("callgraph_id")] string? CallgraphId,
|
||||
[property: JsonPropertyName("fact_kind")] string FactKind,
|
||||
[property: JsonPropertyName("fact_version")] int FactVersion,
|
||||
[property: JsonPropertyName("digest")] string Digest,
|
||||
[property: JsonPropertyName("content_type")] string ContentType,
|
||||
[property: JsonPropertyName("producer")] EventProducerMetadata Producer,
|
||||
[property: JsonPropertyName("trace")] EventTraceMetadata Trace,
|
||||
[property: JsonPropertyName("summary")] ReachabilityFactUpdatedEvent Summary);
|
||||
|
||||
public sealed record EventProducerMetadata(
|
||||
[property: JsonPropertyName("service")] string Service,
|
||||
[property: JsonPropertyName("pipeline")] string Pipeline,
|
||||
[property: JsonPropertyName("release")] string? Release);
|
||||
|
||||
public sealed record EventTraceMetadata(
|
||||
[property: JsonPropertyName("trace_id")] string? TraceId,
|
||||
[property: JsonPropertyName("span_id")] string? SpanId);
|
||||
@@ -1,6 +1,7 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Globalization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -90,7 +91,7 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
|
||||
}
|
||||
}
|
||||
|
||||
var runtimeHits = runtimeHitSet.ToList();
|
||||
var runtimeHits = runtimeHitSet.OrderBy(h => h, StringComparer.Ordinal).ToList();
|
||||
|
||||
var states = new List<ReachabilityStateDocument>(targets.Count);
|
||||
foreach (var target in targets)
|
||||
@@ -108,6 +109,8 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
|
||||
|
||||
var score = confidence * weight;
|
||||
|
||||
runtimeEvidence = runtimeEvidence.OrderBy(hit => hit, StringComparer.Ordinal).ToList();
|
||||
|
||||
states.Add(new ReachabilityStateDocument
|
||||
{
|
||||
Target = target,
|
||||
@@ -120,11 +123,17 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
|
||||
Evidence = new ReachabilityEvidenceDocument
|
||||
{
|
||||
RuntimeHits = runtimeEvidence,
|
||||
BlockedEdges = request.BlockedEdges?.Select(edge => $"{edge.From} -> {edge.To}").ToList()
|
||||
BlockedEdges = request.BlockedEdges?
|
||||
.Select(edge => $"{edge.From} -> {edge.To}")
|
||||
.OrderBy(edge => edge, StringComparer.Ordinal)
|
||||
.ToList()
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
states = states.OrderBy(s => s.Target, StringComparer.Ordinal).ToList();
|
||||
entryPoints = entryPoints.OrderBy(ep => ep, StringComparer.Ordinal).ToList();
|
||||
|
||||
var baseScore = states.Count > 0 ? states.Average(s => s.Score) : 0;
|
||||
var unknownsCount = await unknownsRepository.CountBySubjectAsync(subjectKey, cancellationToken).ConfigureAwait(false);
|
||||
var pressure = states.Count + unknownsCount == 0
|
||||
@@ -148,6 +157,22 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
|
||||
RuntimeFacts = existingFact?.RuntimeFacts
|
||||
};
|
||||
|
||||
document.Metadata ??= new Dictionary<string, string?>(StringComparer.Ordinal);
|
||||
var priorVersion = 0;
|
||||
if (existingFact?.Metadata != null
|
||||
&& existingFact.Metadata.TryGetValue("fact.version", out var versionValue)
|
||||
&& int.TryParse(versionValue, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedVersion))
|
||||
{
|
||||
priorVersion = parsedVersion;
|
||||
}
|
||||
|
||||
var nextVersion = priorVersion + 1;
|
||||
document.Metadata["fact.version"] = nextVersion.ToString(CultureInfo.InvariantCulture);
|
||||
document.Metadata.Remove("fact.digest");
|
||||
document.Metadata.Remove("fact.digest.alg");
|
||||
document.Metadata["fact.digest"] = ReachabilityFactDigestCalculator.Compute(document);
|
||||
document.Metadata["fact.digest.alg"] = "sha256";
|
||||
|
||||
logger.LogInformation("Computed reachability fact for subject {SubjectKey} with {StateCount} targets.", document.SubjectKey, states.Count);
|
||||
var persisted = await factRepository.UpsertAsync(document, cancellationToken).ConfigureAwait(false);
|
||||
await cache.SetAsync(persisted, cancellationToken).ConfigureAwait(false);
|
||||
@@ -266,7 +291,7 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var neighbor in neighbors)
|
||||
foreach (var neighbor in neighbors.OrderBy(n => n, StringComparer.Ordinal))
|
||||
{
|
||||
if (visited.Add(neighbor))
|
||||
{
|
||||
|
||||
@@ -60,8 +60,14 @@ public sealed class ReachabilityUnionIngestionService : IReachabilityUnionIngest
|
||||
}
|
||||
|
||||
var metaEntry = entries["meta.json"];
|
||||
using var metaStream = metaEntry.Open();
|
||||
using var metaDoc = await JsonDocument.ParseAsync(metaStream, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
await using var metaBuffer = new MemoryStream();
|
||||
await using (var metaStream = metaEntry.Open())
|
||||
{
|
||||
await metaStream.CopyToAsync(metaBuffer, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
metaBuffer.Position = 0;
|
||||
using var metaDoc = await JsonDocument.ParseAsync(metaBuffer, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
var metaRoot = metaDoc.RootElement;
|
||||
|
||||
var filesElement = metaRoot.TryGetProperty("files", out var f) && f.ValueKind == JsonValueKind.Array
|
||||
@@ -77,6 +83,13 @@ public sealed class ReachabilityUnionIngestionService : IReachabilityUnionIngest
|
||||
})
|
||||
.ToList();
|
||||
|
||||
metaBuffer.Position = 0;
|
||||
var metaPath = Path.Combine(casRoot, "meta.json");
|
||||
await using (var metaDest = File.Create(metaPath))
|
||||
{
|
||||
await metaBuffer.CopyToAsync(metaDest, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var filesForResponse = new List<ReachabilityUnionFile>();
|
||||
|
||||
foreach (var file in recorded)
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StackExchange.Redis;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
internal sealed class RedisConnectionFactory : IRedisConnectionFactory
|
||||
{
|
||||
public Task<IConnectionMultiplexer> ConnectAsync(ConfigurationOptions options, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
return ConnectionMultiplexer.ConnectAsync(options)
|
||||
.ContinueWith(
|
||||
t => (IConnectionMultiplexer)t.Result,
|
||||
cancellationToken,
|
||||
TaskContinuationOptions.OnlyOnRanToCompletion | TaskContinuationOptions.ExecuteSynchronously,
|
||||
TaskScheduler.Current);
|
||||
}
|
||||
}
|
||||
185
src/Signals/StellaOps.Signals/Services/RedisEventsPublisher.cs
Normal file
185
src/Signals/StellaOps.Signals/Services/RedisEventsPublisher.cs
Normal file
@@ -0,0 +1,185 @@
|
||||
using System;
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StackExchange.Redis;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Options;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
internal sealed class RedisEventsPublisher : IEventsPublisher, IAsyncDisposable
|
||||
{
|
||||
private readonly SignalsEventsOptions options;
|
||||
private readonly ILogger<RedisEventsPublisher> logger;
|
||||
private readonly IRedisConnectionFactory connectionFactory;
|
||||
private readonly ReachabilityFactEventBuilder eventBuilder;
|
||||
private readonly TimeSpan publishTimeout;
|
||||
private readonly int? maxStreamLength;
|
||||
private readonly SemaphoreSlim connectionGate = new(1, 1);
|
||||
private IConnectionMultiplexer? connection;
|
||||
private bool disposed;
|
||||
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = false,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
|
||||
};
|
||||
|
||||
public RedisEventsPublisher(
|
||||
SignalsOptions options,
|
||||
IRedisConnectionFactory connectionFactory,
|
||||
ReachabilityFactEventBuilder eventBuilder,
|
||||
ILogger<RedisEventsPublisher> logger)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
this.options = options.Events ?? throw new InvalidOperationException("Signals events configuration is required.");
|
||||
this.connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory));
|
||||
this.eventBuilder = eventBuilder ?? throw new ArgumentNullException(nameof(eventBuilder));
|
||||
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
publishTimeout = this.options.PublishTimeoutSeconds > 0
|
||||
? TimeSpan.FromSeconds(this.options.PublishTimeoutSeconds)
|
||||
: TimeSpan.Zero;
|
||||
maxStreamLength = this.options.MaxStreamLength > 0
|
||||
? (int)Math.Min(this.options.MaxStreamLength, int.MaxValue)
|
||||
: null;
|
||||
}
|
||||
|
||||
public async Task PublishFactUpdatedAsync(ReachabilityFactDocument fact, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(fact);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (!options.Enabled)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var envelope = eventBuilder.Build(fact);
|
||||
var json = JsonSerializer.Serialize(envelope, SerializerOptions);
|
||||
|
||||
try
|
||||
{
|
||||
var database = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var entries = new[]
|
||||
{
|
||||
new NameValueEntry("event", json),
|
||||
new NameValueEntry("event_id", envelope.EventId),
|
||||
new NameValueEntry("subject_key", envelope.SubjectKey),
|
||||
new NameValueEntry("digest", envelope.Digest),
|
||||
new NameValueEntry("fact_version", envelope.FactVersion.ToString(CultureInfo.InvariantCulture))
|
||||
};
|
||||
|
||||
var publishTask = maxStreamLength.HasValue
|
||||
? database.StreamAddAsync(options.Stream, entries, maxLength: maxStreamLength, useApproximateMaxLength: true)
|
||||
: database.StreamAddAsync(options.Stream, entries);
|
||||
|
||||
if (publishTimeout > TimeSpan.Zero)
|
||||
{
|
||||
await publishTask.WaitAsync(publishTimeout, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await publishTask.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogError(ex, "Failed to publish reachability event to Redis stream {Stream}.", options.Stream);
|
||||
await TryPublishDeadLetterAsync(json, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IDatabase> GetDatabaseAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (connection is { IsConnected: true })
|
||||
{
|
||||
return connection.GetDatabase();
|
||||
}
|
||||
|
||||
await connectionGate.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (connection is null || !connection.IsConnected)
|
||||
{
|
||||
var configuration = ConfigurationOptions.Parse(options.ConnectionString!);
|
||||
configuration.AbortOnConnectFail = false;
|
||||
connection = await connectionFactory.ConnectAsync(configuration, cancellationToken).ConfigureAwait(false);
|
||||
logger.LogInformation("Connected Signals events publisher to Redis stream {Stream}.", options.Stream);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
connectionGate.Release();
|
||||
}
|
||||
|
||||
return connection!.GetDatabase();
|
||||
}
|
||||
|
||||
private async Task TryPublishDeadLetterAsync(string json, CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(options.DeadLetterStream) || connection is null || !connection.IsConnected)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var db = connection.GetDatabase();
|
||||
var entries = new[]
|
||||
{
|
||||
new NameValueEntry("event", json),
|
||||
new NameValueEntry("error", "publish-failed")
|
||||
};
|
||||
|
||||
var dlqTask = maxStreamLength.HasValue
|
||||
? db.StreamAddAsync(options.DeadLetterStream, entries, maxLength: maxStreamLength, useApproximateMaxLength: true)
|
||||
: db.StreamAddAsync(options.DeadLetterStream, entries);
|
||||
|
||||
if (publishTimeout > TimeSpan.Zero)
|
||||
{
|
||||
await dlqTask.WaitAsync(publishTimeout, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await dlqTask.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogWarning(ex, "Failed to publish reachability event to DLQ stream {Stream}.", options.DeadLetterStream);
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
disposed = true;
|
||||
|
||||
if (connection is not null)
|
||||
{
|
||||
try
|
||||
{
|
||||
await connection.CloseAsync();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogDebug(ex, "Error closing Redis events publisher connection.");
|
||||
}
|
||||
|
||||
connection.Dispose();
|
||||
}
|
||||
|
||||
connectionGate.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Options;
|
||||
using StellaOps.Signals.Parsing;
|
||||
using StellaOps.Signals.Persistence;
|
||||
using StellaOps.Signals.Services;
|
||||
using StellaOps.Signals.Storage;
|
||||
using StellaOps.Signals.Storage.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Signals.Tests;
|
||||
|
||||
public class CallgraphIngestionServiceTests
|
||||
{
|
||||
private readonly InMemoryCallgraphRepository _repository = new();
|
||||
private readonly InMemoryArtifactStore _artifactStore = new();
|
||||
private readonly CallgraphNormalizationService _normalizer = new();
|
||||
private readonly TimeProvider _timeProvider = TimeProvider.System;
|
||||
|
||||
[Fact]
|
||||
public async Task IngestAsync_normalizes_graph_and_persists_manifest_hash()
|
||||
{
|
||||
var parser = new StubParser("java");
|
||||
var resolver = new StubParserResolver(parser);
|
||||
var options = Microsoft.Extensions.Options.Options.Create(new SignalsOptions());
|
||||
var service = new CallgraphIngestionService(
|
||||
resolver,
|
||||
_artifactStore,
|
||||
_repository,
|
||||
_normalizer,
|
||||
options,
|
||||
_timeProvider,
|
||||
NullLogger<CallgraphIngestionService>.Instance);
|
||||
|
||||
var artifactJson = @"{""nodes"":[{""id"":""com/example/Foo.bar:(I)V"",""kind"":""fn""}],
|
||||
""edges"":[{""source"":""com/example/Foo.bar:(I)V"",""target"":""com/example/Foo.bar:(I)V""}]}";
|
||||
|
||||
var request = new CallgraphIngestRequest(
|
||||
Language: "java",
|
||||
Component: "demo",
|
||||
Version: "1.0.0",
|
||||
ArtifactFileName: "graph.json",
|
||||
ArtifactContentType: "application/json",
|
||||
ArtifactContentBase64: Convert.ToBase64String(Encoding.UTF8.GetBytes(artifactJson)),
|
||||
SchemaVersion: null,
|
||||
Metadata: new Dictionary<string, string?> { ["source"] = "test" },
|
||||
Analyzer: new Dictionary<string, string?> { ["name"] = "stub" });
|
||||
|
||||
var response = await service.IngestAsync(request, CancellationToken.None);
|
||||
|
||||
response.CallgraphId.Should().NotBeNullOrWhiteSpace();
|
||||
response.GraphHash.Should().NotBeNullOrWhiteSpace();
|
||||
response.NodeCount.Should().Be(1);
|
||||
response.EdgeCount.Should().Be(1);
|
||||
response.ManifestCasUri.Should().Be("cas://signals/manifests/graph.json");
|
||||
|
||||
var stored = _repository.LastUpserted!;
|
||||
stored.Artifact.Hash.Should().Be(response.ArtifactHash);
|
||||
stored.Nodes[0].Namespace.Should().Be("com.example.Foo");
|
||||
stored.Nodes[0].Language.Should().Be("java");
|
||||
stored.Metadata!["schemaVersion"].Should().Be("1.0");
|
||||
stored.Metadata!["analyzer.name"].Should().Be("stub");
|
||||
stored.Artifact.GraphHash.Should().Be(response.GraphHash);
|
||||
}
|
||||
|
||||
private sealed class StubParser : ICallgraphParser
|
||||
{
|
||||
public StubParser(string language) => Language = language;
|
||||
public string Language { get; }
|
||||
public Task<CallgraphParseResult> ParseAsync(Stream artifactStream, CancellationToken cancellationToken)
|
||||
{
|
||||
artifactStream.Position = 0;
|
||||
using var doc = JsonDocument.Parse(artifactStream);
|
||||
var nodes = new List<CallgraphNode>();
|
||||
foreach (var node in doc.RootElement.GetProperty("nodes").EnumerateArray())
|
||||
{
|
||||
nodes.Add(new CallgraphNode(node.GetProperty("id").GetString()!, "", "function", null, null, null));
|
||||
}
|
||||
|
||||
var edges = new List<CallgraphEdge>();
|
||||
foreach (var edge in doc.RootElement.GetProperty("edges").EnumerateArray())
|
||||
{
|
||||
edges.Add(new CallgraphEdge(
|
||||
edge.GetProperty("source").GetString()!,
|
||||
edge.GetProperty("target").GetString()!,
|
||||
"call"));
|
||||
}
|
||||
|
||||
return Task.FromResult(new CallgraphParseResult(nodes, edges, Array.Empty<CallgraphRoot>(), "1.0", "1.0", null));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StubParserResolver : ICallgraphParserResolver
|
||||
{
|
||||
private readonly ICallgraphParser _parser;
|
||||
public StubParserResolver(ICallgraphParser parser) => _parser = parser;
|
||||
public ICallgraphParser Resolve(string language) => _parser;
|
||||
}
|
||||
|
||||
private sealed class InMemoryArtifactStore : ICallgraphArtifactStore
|
||||
{
|
||||
private readonly Dictionary<string, byte[]> artifacts = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, byte[]> manifests = new(StringComparer.Ordinal);
|
||||
|
||||
public Task<StoredCallgraphArtifact> SaveAsync(CallgraphArtifactSaveRequest request, Stream content, CancellationToken cancellationToken)
|
||||
{
|
||||
using var ms = new MemoryStream();
|
||||
content.CopyTo(ms);
|
||||
artifacts[request.Hash] = ms.ToArray();
|
||||
|
||||
if (request.ManifestContent is not null)
|
||||
{
|
||||
using var manifestMs = new MemoryStream();
|
||||
request.ManifestContent.CopyTo(manifestMs);
|
||||
manifests[request.Hash] = manifestMs.ToArray();
|
||||
}
|
||||
|
||||
var path = $"cas://signals/artifacts/{request.FileName}";
|
||||
var manifestPath = "cas://signals/manifests/graph.json";
|
||||
|
||||
return Task.FromResult(new StoredCallgraphArtifact(
|
||||
Path: path,
|
||||
Length: ms.Length,
|
||||
Hash: request.Hash,
|
||||
ContentType: request.ContentType,
|
||||
CasUri: path,
|
||||
ManifestPath: manifestPath,
|
||||
ManifestCasUri: manifestPath));
|
||||
}
|
||||
|
||||
public Task<Stream?> GetAsync(string hash, string? fileName = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (artifacts.TryGetValue(hash, out var bytes))
|
||||
{
|
||||
return Task.FromResult<Stream?>(new MemoryStream(bytes, writable: false));
|
||||
}
|
||||
|
||||
return Task.FromResult<Stream?>(null);
|
||||
}
|
||||
|
||||
public Task<Stream?> GetManifestAsync(string hash, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (manifests.TryGetValue(hash, out var bytes))
|
||||
{
|
||||
return Task.FromResult<Stream?>(new MemoryStream(bytes, writable: false));
|
||||
}
|
||||
|
||||
return Task.FromResult<Stream?>(null);
|
||||
}
|
||||
|
||||
public Task<bool> ExistsAsync(string hash, CancellationToken cancellationToken = default)
|
||||
{
|
||||
return Task.FromResult(artifacts.ContainsKey(hash));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class InMemoryCallgraphRepository : ICallgraphRepository
|
||||
{
|
||||
public CallgraphDocument? LastUpserted { get; private set; }
|
||||
|
||||
public Task<CallgraphDocument?> GetByIdAsync(string id, CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(LastUpserted?.Id == id ? LastUpserted : null);
|
||||
}
|
||||
|
||||
public Task<CallgraphDocument> UpsertAsync(CallgraphDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
LastUpserted = document;
|
||||
return Task.FromResult(document);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using FluentAssertions;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Parsing;
|
||||
using StellaOps.Signals.Services;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Signals.Tests;
|
||||
|
||||
public class CallgraphNormalizationServiceTests
|
||||
{
|
||||
private readonly CallgraphNormalizationService _service = new();
|
||||
|
||||
[Fact]
|
||||
public void Normalize_adds_language_and_namespace_for_java()
|
||||
{
|
||||
var result = new CallgraphParseResult(
|
||||
Nodes: new[]
|
||||
{
|
||||
new CallgraphNode("com/example/Foo.bar:(I)V", "", "", null, null, null)
|
||||
},
|
||||
Edges: Array.Empty<CallgraphEdge>(),
|
||||
Roots: Array.Empty<CallgraphRoot>(),
|
||||
FormatVersion: "1.0",
|
||||
SchemaVersion: "1.0",
|
||||
Analyzer: null);
|
||||
|
||||
var normalized = _service.Normalize("java", result);
|
||||
|
||||
normalized.Nodes.Should().ContainSingle();
|
||||
var node = normalized.Nodes[0];
|
||||
node.Language.Should().Be("java");
|
||||
node.Namespace.Should().Be("com.example.Foo"); // dotted namespace derived from id
|
||||
node.Kind.Should().Be("function");
|
||||
node.Name.Should().Be("com/example/Foo.bar:(I)V");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Normalize_deduplicates_edges_and_clamps_confidence()
|
||||
{
|
||||
var result = new CallgraphParseResult(
|
||||
Nodes: new[]
|
||||
{
|
||||
new CallgraphNode("a", "a", "fn", null, null, null),
|
||||
new CallgraphNode("b", "b", "fn", null, null, null)
|
||||
},
|
||||
Edges: new[]
|
||||
{
|
||||
new CallgraphEdge(" a ", "b", "call", Confidence: 2.5, Evidence: new []{"x","x"}),
|
||||
new CallgraphEdge("a", "b", "call", Confidence: -1)
|
||||
},
|
||||
Roots: Array.Empty<CallgraphRoot>(),
|
||||
FormatVersion: "1.0",
|
||||
SchemaVersion: "1.0",
|
||||
Analyzer: null);
|
||||
|
||||
var normalized = _service.Normalize("python", result);
|
||||
|
||||
normalized.Edges.Should().ContainSingle();
|
||||
var edge = normalized.Edges[0];
|
||||
edge.SourceId.Should().Be("a");
|
||||
edge.TargetId.Should().Be("b");
|
||||
edge.Type.Should().Be("call");
|
||||
edge.Confidence.Should().Be(1.0);
|
||||
edge.Evidence.Should().BeEquivalentTo(new[] { "x" });
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,13 @@ public class InMemoryEventsPublisherTests
|
||||
public async Task PublishFactUpdatedAsync_EmitsStructuredEvent()
|
||||
{
|
||||
var logger = new TestLogger<InMemoryEventsPublisher>();
|
||||
var publisher = new InMemoryEventsPublisher(logger, new SignalsOptions());
|
||||
var options = new SignalsOptions();
|
||||
options.Events.Driver = "inmemory";
|
||||
options.Events.Stream = "signals.fact.updated.v1";
|
||||
options.Events.DefaultTenant = "tenant-default";
|
||||
|
||||
var builder = new ReachabilityFactEventBuilder(options, TimeProvider.System);
|
||||
var publisher = new InMemoryEventsPublisher(logger, builder);
|
||||
|
||||
var fact = new ReachabilityFactDocument
|
||||
{
|
||||
@@ -33,21 +39,23 @@ public class InMemoryEventsPublisherTests
|
||||
}
|
||||
};
|
||||
|
||||
var envelope = builder.Build(fact);
|
||||
await publisher.PublishFactUpdatedAsync(fact, CancellationToken.None);
|
||||
|
||||
Assert.Contains("signals.fact.updated", logger.LastMessage);
|
||||
Assert.Contains("\"subjectKey\":\"tenant:image@sha256:abc\"", logger.LastMessage);
|
||||
Assert.Contains("\"callgraphId\":\"cg-123\"", logger.LastMessage);
|
||||
Assert.Contains("\"reachableCount\":1", logger.LastMessage);
|
||||
Assert.Contains("\"unreachableCount\":1", logger.LastMessage);
|
||||
Assert.Contains("\"runtimeFactsCount\":1", logger.LastMessage);
|
||||
Assert.Contains("\"bucket\":\"runtime\"", logger.LastMessage);
|
||||
Assert.Contains("\"weight\":0.45", logger.LastMessage);
|
||||
Assert.Contains("\"factScore\":", logger.LastMessage);
|
||||
Assert.Contains("\"unknownsCount\":0", logger.LastMessage);
|
||||
Assert.Contains("\"unknownsPressure\":0", logger.LastMessage);
|
||||
Assert.Contains("\"stateCount\":2", logger.LastMessage);
|
||||
Assert.Contains("\"targets\":[\"pkg:pypi/django\",\"pkg:pypi/requests\"]", logger.LastMessage);
|
||||
Assert.Equal("signals.fact.updated.v1", envelope.Topic);
|
||||
Assert.Equal("signals.fact.updated@v1", envelope.Version);
|
||||
Assert.False(string.IsNullOrWhiteSpace(envelope.EventId));
|
||||
Assert.Equal("tenant-default", envelope.Tenant);
|
||||
Assert.Equal("tenant:image@sha256:abc", envelope.SubjectKey);
|
||||
Assert.Equal("cg-123", envelope.CallgraphId);
|
||||
Assert.Equal(1, envelope.Summary.ReachableCount);
|
||||
Assert.Equal(1, envelope.Summary.UnreachableCount);
|
||||
Assert.Equal(1, envelope.Summary.RuntimeFactsCount);
|
||||
Assert.Equal("runtime", envelope.Summary.Bucket);
|
||||
Assert.Equal(2, envelope.Summary.StateCount);
|
||||
Assert.Contains("pkg:pypi/django", envelope.Summary.Targets);
|
||||
Assert.Contains("pkg:pypi/requests", envelope.Summary.Targets);
|
||||
Assert.Contains("signals.fact.updated.v1", logger.LastMessage);
|
||||
}
|
||||
|
||||
private sealed class TestLogger<T> : ILogger<T>
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Services;
|
||||
using Xunit;
|
||||
|
||||
public class ReachabilityFactDigestCalculatorTests
|
||||
{
|
||||
[Fact]
|
||||
public void Compute_ReturnsDeterministicDigest_ForEquivalentFacts()
|
||||
{
|
||||
var factA = new ReachabilityFactDocument
|
||||
{
|
||||
CallgraphId = "cg-1",
|
||||
Subject = new ReachabilitySubject { Component = "demo", Version = "1.0.0" },
|
||||
SubjectKey = "demo|1.0.0",
|
||||
EntryPoints = new List<string> { "svc.main", "app.main" },
|
||||
States = new List<ReachabilityStateDocument>
|
||||
{
|
||||
new() { Target = "a", Reachable = true, Confidence = 0.9, Bucket = "runtime", Weight = 0.45, Path = new List<string> { "app.main", "a" }, Evidence = new ReachabilityEvidenceDocument { RuntimeHits = new List<string> { "a" } } },
|
||||
new() { Target = "b", Reachable = false, Confidence = 0.3, Bucket = "unreachable", Weight = 0.1, Path = new List<string> { "app.main", "b" } }
|
||||
},
|
||||
RuntimeFacts = new List<RuntimeFactDocument>
|
||||
{
|
||||
new() { SymbolId = "a", HitCount = 2 }
|
||||
},
|
||||
Metadata = new Dictionary<string, string?>(StringComparer.Ordinal) { { "tenant", "tenant-default" } },
|
||||
ComputedAt = DateTimeOffset.Parse("2025-12-09T00:00:00Z")
|
||||
};
|
||||
|
||||
var factB = new ReachabilityFactDocument
|
||||
{
|
||||
CallgraphId = "cg-1",
|
||||
Subject = new ReachabilitySubject { Component = "demo", Version = "1.0.0" },
|
||||
SubjectKey = "demo|1.0.0",
|
||||
EntryPoints = new List<string> { "app.main", "svc.main" }, // reversed
|
||||
States = new List<ReachabilityStateDocument>
|
||||
{
|
||||
new() { Target = "b", Reachable = false, Confidence = 0.3, Bucket = "unreachable", Weight = 0.1, Path = new List<string> { "app.main", "b" } },
|
||||
new() { Target = "a", Reachable = true, Confidence = 0.9, Bucket = "runtime", Weight = 0.45, Path = new List<string> { "app.main", "a" }, Evidence = new ReachabilityEvidenceDocument { RuntimeHits = new List<string> { "a" } } }
|
||||
},
|
||||
RuntimeFacts = new List<RuntimeFactDocument>
|
||||
{
|
||||
new() { SymbolId = "a", HitCount = 2 }
|
||||
},
|
||||
Metadata = new Dictionary<string, string?>(StringComparer.Ordinal) { { "tenant", "tenant-default" } },
|
||||
ComputedAt = DateTimeOffset.Parse("2025-12-09T00:00:00Z")
|
||||
};
|
||||
|
||||
var digestA = ReachabilityFactDigestCalculator.Compute(factA);
|
||||
var digestB = ReachabilityFactDigestCalculator.Compute(factB);
|
||||
|
||||
Assert.StartsWith("sha256:", digestA, StringComparison.Ordinal);
|
||||
Assert.Equal(digestA, digestB);
|
||||
}
|
||||
}
|
||||
@@ -82,6 +82,8 @@ public class ReachabilityScoringServiceTests
|
||||
Assert.Contains("target", state.Evidence.RuntimeHits);
|
||||
|
||||
Assert.Equal(0.405, fact.Score, 3);
|
||||
Assert.Equal("1", fact.Metadata?["fact.version"]);
|
||||
Assert.False(string.IsNullOrWhiteSpace(fact.Metadata?["fact.digest"]));
|
||||
}
|
||||
|
||||
private sealed class InMemoryCallgraphRepository : ICallgraphRepository
|
||||
|
||||
@@ -43,38 +43,41 @@ public class ReachabilityUnionIngestionServiceTests
|
||||
private static MemoryStream BuildSampleUnionZip()
|
||||
{
|
||||
var ms = new MemoryStream();
|
||||
using var archive = new ZipArchive(ms, ZipArchiveMode.Create, leaveOpen: true);
|
||||
|
||||
var nodes = archive.CreateEntry("nodes.ndjson");
|
||||
using (var writer = new StreamWriter(nodes.Open()))
|
||||
using (var archive = new ZipArchive(ms, ZipArchiveMode.Create, leaveOpen: true))
|
||||
{
|
||||
writer.WriteLine("{\"symbol_id\":\"sym:dotnet:abc\",\"lang\":\"dotnet\",\"kind\":\"function\",\"display\":\"abc\"}");
|
||||
}
|
||||
|
||||
var edges = archive.CreateEntry("edges.ndjson");
|
||||
using (var writer = new StreamWriter(edges.Open()))
|
||||
{
|
||||
writer.WriteLine("{\"from\":\"sym:dotnet:abc\",\"to\":\"sym:dotnet:def\",\"edge_type\":\"call\",\"source\":{\"origin\":\"static\",\"provenance\":\"il\"}}");
|
||||
}
|
||||
|
||||
// facts_runtime optional left out
|
||||
|
||||
var meta = archive.CreateEntry("meta.json");
|
||||
using (var writer = new StreamWriter(meta.Open()))
|
||||
{
|
||||
var files = new[]
|
||||
var nodes = archive.CreateEntry("nodes.ndjson");
|
||||
using (var writer = new StreamWriter(nodes.Open()))
|
||||
{
|
||||
new { path = "nodes.ndjson", sha256 = ComputeSha("{\"symbol_id\":\"sym:dotnet:abc\",\"lang\":\"dotnet\",\"kind\":\"function\",\"display\":\"abc\"}\n"), records = 1 },
|
||||
new { path = "edges.ndjson", sha256 = ComputeSha("{\"from\":\"sym:dotnet:abc\",\"to\":\"sym:dotnet:def\",\"edge_type\":\"call\",\"source\":{\"origin\":\"static\",\"provenance\":\"il\"}}\n"), records = 1 }
|
||||
};
|
||||
var metaObj = new
|
||||
writer.NewLine = "\n";
|
||||
writer.WriteLine("{\"symbol_id\":\"sym:dotnet:abc\",\"lang\":\"dotnet\",\"kind\":\"function\",\"display\":\"abc\"}");
|
||||
}
|
||||
|
||||
var edges = archive.CreateEntry("edges.ndjson");
|
||||
using (var writer = new StreamWriter(edges.Open()))
|
||||
{
|
||||
schema = "reachability-union@0.1",
|
||||
generated_at = "2025-11-23T00:00:00Z",
|
||||
produced_by = new { tool = "test", version = "0.0.1" },
|
||||
files
|
||||
};
|
||||
writer.Write(JsonSerializer.Serialize(metaObj));
|
||||
writer.NewLine = "\n";
|
||||
writer.WriteLine("{\"from\":\"sym:dotnet:abc\",\"to\":\"sym:dotnet:def\",\"edge_type\":\"call\",\"source\":{\"origin\":\"static\",\"provenance\":\"il\"}}");
|
||||
}
|
||||
|
||||
// facts_runtime optional left out
|
||||
|
||||
var meta = archive.CreateEntry("meta.json");
|
||||
using (var writer = new StreamWriter(meta.Open()))
|
||||
{
|
||||
var files = new[]
|
||||
{
|
||||
new { path = "nodes.ndjson", sha256 = ComputeSha("{\"symbol_id\":\"sym:dotnet:abc\",\"lang\":\"dotnet\",\"kind\":\"function\",\"display\":\"abc\"}\n"), records = 1 },
|
||||
new { path = "edges.ndjson", sha256 = ComputeSha("{\"from\":\"sym:dotnet:abc\",\"to\":\"sym:dotnet:def\",\"edge_type\":\"call\",\"source\":{\"origin\":\"static\",\"provenance\":\"il\"}}\n"), records = 1 }
|
||||
};
|
||||
var metaObj = new
|
||||
{
|
||||
schema = "reachability-union@0.1",
|
||||
generated_at = "2025-11-23T00:00:00Z",
|
||||
produced_by = new { tool = "test", version = "0.0.1" },
|
||||
files
|
||||
};
|
||||
writer.Write(JsonSerializer.Serialize(metaObj));
|
||||
}
|
||||
}
|
||||
|
||||
ms.Position = 0;
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.0" />
|
||||
<PackageReference Include="FluentAssertions" Version="6.12.0" />
|
||||
<PackageReference Include="xunit" Version="2.9.2" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
|
||||
<PackageReference Include="coverlet.collector" Version="6.0.4" />
|
||||
|
||||
Reference in New Issue
Block a user