up
This commit is contained in:
105
src/Signals/StellaOps.Signals/Options/SignalsEventsOptions.cs
Normal file
105
src/Signals/StellaOps.Signals/Options/SignalsEventsOptions.cs
Normal file
@@ -0,0 +1,105 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Signals.Options;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for reachability fact events (SIGNALS-24-005).
|
||||
/// </summary>
|
||||
public sealed class SignalsEventsOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Enables event emission. When false, events are dropped.
|
||||
/// </summary>
|
||||
public bool Enabled { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Transport driver: "inmemory" or "redis".
|
||||
/// </summary>
|
||||
public string Driver { get; set; } = "inmemory";
|
||||
|
||||
/// <summary>
|
||||
/// Primary topic/stream name for fact updates.
|
||||
/// </summary>
|
||||
public string Stream { get; set; } = "signals.fact.updated.v1";
|
||||
|
||||
/// <summary>
|
||||
/// Dead-letter topic/stream used when publishing fails.
|
||||
/// </summary>
|
||||
public string DeadLetterStream { get; set; } = "signals.fact.updated.dlq";
|
||||
|
||||
/// <summary>
|
||||
/// Connection string for Redis streams (when Driver=redis).
|
||||
/// </summary>
|
||||
public string? ConnectionString { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional publish timeout (seconds). Set to 0 to disable.
|
||||
/// </summary>
|
||||
public int PublishTimeoutSeconds { get; set; } = 5;
|
||||
|
||||
/// <summary>
|
||||
/// Approximate maximum stream length (capped by Redis trimming).
|
||||
/// </summary>
|
||||
public long MaxStreamLength { get; set; } = 10_000;
|
||||
|
||||
/// <summary>
|
||||
/// Producer identifier for observability payloads.
|
||||
/// </summary>
|
||||
public string Producer { get; set; } = "StellaOps.Signals";
|
||||
|
||||
/// <summary>
|
||||
/// Pipeline name attached to event metadata.
|
||||
/// </summary>
|
||||
public string Pipeline { get; set; } = "signals";
|
||||
|
||||
/// <summary>
|
||||
/// Optional release string to stamp events with build provenance.
|
||||
/// </summary>
|
||||
public string? Release { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Default tenant when none is supplied in metadata.
|
||||
/// </summary>
|
||||
public string DefaultTenant { get; set; } = "tenant-default";
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
var normalizedDriver = Driver?.Trim();
|
||||
if (string.IsNullOrWhiteSpace(normalizedDriver))
|
||||
{
|
||||
throw new InvalidOperationException("Signals events driver is required.");
|
||||
}
|
||||
|
||||
if (!string.Equals(normalizedDriver, "redis", StringComparison.OrdinalIgnoreCase)
|
||||
&& !string.Equals(normalizedDriver, "inmemory", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException("Signals events driver must be 'redis' or 'inmemory'.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(Stream))
|
||||
{
|
||||
throw new InvalidOperationException("Signals events stream/topic is required.");
|
||||
}
|
||||
|
||||
if (PublishTimeoutSeconds < 0)
|
||||
{
|
||||
throw new InvalidOperationException("Signals events publish timeout must be >= 0 seconds.");
|
||||
}
|
||||
|
||||
if (MaxStreamLength < 0)
|
||||
{
|
||||
throw new InvalidOperationException("Signals events max stream length must be >= 0.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(DefaultTenant))
|
||||
{
|
||||
throw new InvalidOperationException("Signals events default tenant is required.");
|
||||
}
|
||||
|
||||
if (string.Equals(normalizedDriver, "redis", StringComparison.OrdinalIgnoreCase)
|
||||
&& string.IsNullOrWhiteSpace(ConnectionString))
|
||||
{
|
||||
throw new InvalidOperationException("Signals events Redis driver requires ConnectionString.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,26 +1,26 @@
|
||||
namespace StellaOps.Signals.Options;
|
||||
|
||||
/// <summary>
|
||||
/// Root configuration for the Signals service.
|
||||
/// </summary>
|
||||
public sealed class SignalsOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Configuration section name.
|
||||
/// </summary>
|
||||
public const string SectionName = "Signals";
|
||||
|
||||
/// <summary>
|
||||
/// Authority integration settings.
|
||||
/// </summary>
|
||||
public SignalsAuthorityOptions Authority { get; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// MongoDB configuration.
|
||||
/// </summary>
|
||||
public SignalsMongoOptions Mongo { get; } = new();
|
||||
|
||||
/// <summary>
|
||||
namespace StellaOps.Signals.Options;
|
||||
|
||||
/// <summary>
|
||||
/// Root configuration for the Signals service.
|
||||
/// </summary>
|
||||
public sealed class SignalsOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Configuration section name.
|
||||
/// </summary>
|
||||
public const string SectionName = "Signals";
|
||||
|
||||
/// <summary>
|
||||
/// Authority integration settings.
|
||||
/// </summary>
|
||||
public SignalsAuthorityOptions Authority { get; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// MongoDB configuration.
|
||||
/// </summary>
|
||||
public SignalsMongoOptions Mongo { get; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Artifact storage configuration.
|
||||
/// </summary>
|
||||
public SignalsArtifactStorageOptions Storage { get; } = new();
|
||||
@@ -40,22 +40,28 @@ public sealed class SignalsOptions
|
||||
/// </summary>
|
||||
public SignalsCacheOptions Cache { get; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Event transport configuration.
|
||||
/// </summary>
|
||||
public SignalsEventsOptions Events { get; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// OpenAPI exposure (if enabled).
|
||||
/// </summary>
|
||||
public SignalsOpenApiOptions OpenApi { get; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Validates configured options.
|
||||
/// </summary>
|
||||
public void Validate()
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// Validates configured options.
|
||||
/// </summary>
|
||||
public void Validate()
|
||||
{
|
||||
Authority.Validate();
|
||||
Mongo.Validate();
|
||||
Storage.Validate();
|
||||
AirGap.Validate();
|
||||
Scoring.Validate();
|
||||
Cache.Validate();
|
||||
Events.Validate();
|
||||
OpenApi.Validate();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -125,6 +125,7 @@ builder.Services.AddSingleton<IMongoCollection<UnknownSymbolDocument>>(sp =>
|
||||
});
|
||||
|
||||
builder.Services.AddSingleton<ICallgraphRepository, MongoCallgraphRepository>();
|
||||
builder.Services.AddSingleton<ICallgraphNormalizationService, CallgraphNormalizationService>();
|
||||
|
||||
// Configure callgraph artifact storage based on driver
|
||||
if (bootstrap.Storage.IsRustFsDriver())
|
||||
@@ -165,7 +166,31 @@ builder.Services.AddSingleton<IReachabilityCache>(sp =>
|
||||
var options = sp.GetRequiredService<IOptions<SignalsOptions>>().Value;
|
||||
return new RedisReachabilityCache(options.Cache);
|
||||
});
|
||||
builder.Services.AddSingleton<IEventsPublisher, InMemoryEventsPublisher>();
|
||||
builder.Services.AddSingleton<IRedisConnectionFactory, RedisConnectionFactory>();
|
||||
builder.Services.AddSingleton<ReachabilityFactEventBuilder>();
|
||||
builder.Services.AddSingleton<IEventsPublisher>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<SignalsOptions>();
|
||||
var eventBuilder = sp.GetRequiredService<ReachabilityFactEventBuilder>();
|
||||
|
||||
if (!options.Events.Enabled)
|
||||
{
|
||||
return new NullEventsPublisher();
|
||||
}
|
||||
|
||||
if (string.Equals(options.Events.Driver, "redis", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return new RedisEventsPublisher(
|
||||
options,
|
||||
sp.GetRequiredService<IRedisConnectionFactory>(),
|
||||
eventBuilder,
|
||||
sp.GetRequiredService<ILogger<RedisEventsPublisher>>());
|
||||
}
|
||||
|
||||
return new InMemoryEventsPublisher(
|
||||
sp.GetRequiredService<ILogger<InMemoryEventsPublisher>>(),
|
||||
eventBuilder);
|
||||
});
|
||||
builder.Services.AddSingleton<IReachabilityFactRepository>(sp =>
|
||||
{
|
||||
var inner = sp.GetRequiredService<MongoReachabilityFactRepository>();
|
||||
|
||||
@@ -1,72 +1,76 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Options;
|
||||
using StellaOps.Signals.Parsing;
|
||||
using StellaOps.Signals.Persistence;
|
||||
using StellaOps.Signals.Storage;
|
||||
using StellaOps.Signals.Storage.Models;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
{
|
||||
private static readonly HashSet<string> AllowedContentTypes = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"application/json",
|
||||
"application/vnd.stellaops.callgraph+json"
|
||||
};
|
||||
|
||||
private readonly ICallgraphParserResolver parserResolver;
|
||||
private readonly ICallgraphArtifactStore artifactStore;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Options;
|
||||
using StellaOps.Signals.Parsing;
|
||||
using StellaOps.Signals.Persistence;
|
||||
using StellaOps.Signals.Storage;
|
||||
using StellaOps.Signals.Storage.Models;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
{
|
||||
private static readonly HashSet<string> AllowedContentTypes = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"application/json",
|
||||
"application/vnd.stellaops.callgraph+json"
|
||||
};
|
||||
|
||||
private readonly ICallgraphParserResolver parserResolver;
|
||||
private readonly ICallgraphArtifactStore artifactStore;
|
||||
private readonly ICallgraphRepository repository;
|
||||
private readonly ICallgraphNormalizationService normalizer;
|
||||
private readonly ILogger<CallgraphIngestionService> logger;
|
||||
private readonly SignalsOptions options;
|
||||
private readonly TimeProvider timeProvider;
|
||||
private static readonly JsonSerializerOptions ManifestSerializerOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
public CallgraphIngestionService(
|
||||
ICallgraphParserResolver parserResolver,
|
||||
ICallgraphArtifactStore artifactStore,
|
||||
ICallgraphRepository repository,
|
||||
IOptions<SignalsOptions> options,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<CallgraphIngestionService> logger)
|
||||
{
|
||||
this.parserResolver = parserResolver ?? throw new ArgumentNullException(nameof(parserResolver));
|
||||
this.artifactStore = artifactStore ?? throw new ArgumentNullException(nameof(artifactStore));
|
||||
this.repository = repository ?? throw new ArgumentNullException(nameof(repository));
|
||||
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
this.options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
}
|
||||
|
||||
public async Task<CallgraphIngestResponse> IngestAsync(CallgraphIngestRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
ValidateRequest(request);
|
||||
|
||||
var parser = parserResolver.Resolve(request.Language);
|
||||
|
||||
|
||||
public CallgraphIngestionService(
|
||||
ICallgraphParserResolver parserResolver,
|
||||
ICallgraphArtifactStore artifactStore,
|
||||
ICallgraphRepository repository,
|
||||
ICallgraphNormalizationService normalizer,
|
||||
IOptions<SignalsOptions> options,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<CallgraphIngestionService> logger)
|
||||
{
|
||||
this.parserResolver = parserResolver ?? throw new ArgumentNullException(nameof(parserResolver));
|
||||
this.artifactStore = artifactStore ?? throw new ArgumentNullException(nameof(artifactStore));
|
||||
this.repository = repository ?? throw new ArgumentNullException(nameof(repository));
|
||||
this.normalizer = normalizer ?? throw new ArgumentNullException(nameof(normalizer));
|
||||
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
this.options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
}
|
||||
|
||||
public async Task<CallgraphIngestResponse> IngestAsync(CallgraphIngestRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
ValidateRequest(request);
|
||||
|
||||
var parser = parserResolver.Resolve(request.Language);
|
||||
|
||||
var artifactBytes = Convert.FromBase64String(request.ArtifactContentBase64);
|
||||
await using var parseStream = new MemoryStream(artifactBytes, writable: false);
|
||||
var parseResult = await parser.ParseAsync(parseStream, cancellationToken).ConfigureAwait(false);
|
||||
var parsed = await parser.ParseAsync(parseStream, cancellationToken).ConfigureAwait(false);
|
||||
var normalized = normalizer.Normalize(parser.Language, parsed);
|
||||
var schemaVersion = !string.IsNullOrWhiteSpace(request.SchemaVersion)
|
||||
? request.SchemaVersion!
|
||||
: parseResult.SchemaVersion;
|
||||
var analyzerMeta = request.Analyzer ?? parseResult.Analyzer;
|
||||
: normalized.SchemaVersion;
|
||||
var analyzerMeta = request.Analyzer ?? normalized.Analyzer;
|
||||
|
||||
parseStream.Position = 0;
|
||||
var artifactHash = ComputeSha256(artifactBytes);
|
||||
var graphHash = ComputeGraphHash(parseResult);
|
||||
var graphHash = ComputeGraphHash(normalized);
|
||||
|
||||
var manifest = new CallgraphManifest
|
||||
{
|
||||
@@ -76,9 +80,9 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
ArtifactHash = artifactHash,
|
||||
GraphHash = graphHash,
|
||||
SchemaVersion = schemaVersion,
|
||||
NodeCount = parseResult.Nodes.Count,
|
||||
EdgeCount = parseResult.Edges.Count,
|
||||
RootCount = parseResult.Roots.Count,
|
||||
NodeCount = normalized.Nodes.Count,
|
||||
EdgeCount = normalized.Edges.Count,
|
||||
RootCount = normalized.Roots.Count,
|
||||
CreatedAt = timeProvider.GetUtcNow()
|
||||
};
|
||||
|
||||
@@ -98,15 +102,15 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
manifestStream),
|
||||
parseStream,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var document = new CallgraphDocument
|
||||
{
|
||||
|
||||
var document = new CallgraphDocument
|
||||
{
|
||||
Language = parser.Language,
|
||||
Component = request.Component,
|
||||
Version = request.Version,
|
||||
Nodes = new List<CallgraphNode>(parseResult.Nodes),
|
||||
Edges = new List<CallgraphEdge>(parseResult.Edges),
|
||||
Roots = new List<CallgraphRoot>(parseResult.Roots),
|
||||
Nodes = new List<CallgraphNode>(normalized.Nodes),
|
||||
Edges = new List<CallgraphEdge>(normalized.Edges),
|
||||
Roots = new List<CallgraphRoot>(normalized.Roots),
|
||||
Metadata = request.Metadata is null
|
||||
? null
|
||||
: new Dictionary<string, string?>(request.Metadata, StringComparer.OrdinalIgnoreCase),
|
||||
@@ -125,7 +129,7 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
};
|
||||
|
||||
document.Metadata ??= new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
|
||||
document.Metadata["formatVersion"] = parseResult.FormatVersion;
|
||||
document.Metadata["formatVersion"] = normalized.FormatVersion;
|
||||
document.Metadata["schemaVersion"] = schemaVersion;
|
||||
if (analyzerMeta is not null)
|
||||
{
|
||||
@@ -138,16 +142,16 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
document.SchemaVersion = schemaVersion;
|
||||
|
||||
document = await repository.UpsertAsync(document, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
logger.LogInformation(
|
||||
"Ingested callgraph {Language}:{Component}:{Version} (id={Id}) with {NodeCount} nodes and {EdgeCount} edges.",
|
||||
document.Language,
|
||||
document.Component,
|
||||
document.Version,
|
||||
document.Id,
|
||||
document.Nodes.Count,
|
||||
document.Edges.Count);
|
||||
|
||||
|
||||
logger.LogInformation(
|
||||
"Ingested callgraph {Language}:{Component}:{Version} (id={Id}) with {NodeCount} nodes and {EdgeCount} edges.",
|
||||
document.Language,
|
||||
document.Component,
|
||||
document.Version,
|
||||
document.Id,
|
||||
document.Nodes.Count,
|
||||
document.Edges.Count);
|
||||
|
||||
return new CallgraphIngestResponse(
|
||||
document.Id,
|
||||
document.Artifact.Path,
|
||||
@@ -160,42 +164,42 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
document.Edges.Count,
|
||||
document.Roots?.Count ?? 0);
|
||||
}
|
||||
|
||||
private static void ValidateRequest(CallgraphIngestRequest request)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.Language))
|
||||
{
|
||||
throw new CallgraphIngestionValidationException("Language is required.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.Component))
|
||||
{
|
||||
throw new CallgraphIngestionValidationException("Component is required.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.Version))
|
||||
{
|
||||
throw new CallgraphIngestionValidationException("Version is required.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.ArtifactContentBase64))
|
||||
{
|
||||
throw new CallgraphIngestionValidationException("Artifact content is required.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.ArtifactFileName))
|
||||
{
|
||||
throw new CallgraphIngestionValidationException("Artifact file name is required.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.ArtifactContentType) || !AllowedContentTypes.Contains(request.ArtifactContentType))
|
||||
{
|
||||
throw new CallgraphIngestionValidationException($"Unsupported artifact content type '{request.ArtifactContentType}'.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static void ValidateRequest(CallgraphIngestRequest request)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.Language))
|
||||
{
|
||||
throw new CallgraphIngestionValidationException("Language is required.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.Component))
|
||||
{
|
||||
throw new CallgraphIngestionValidationException("Component is required.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.Version))
|
||||
{
|
||||
throw new CallgraphIngestionValidationException("Version is required.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.ArtifactContentBase64))
|
||||
{
|
||||
throw new CallgraphIngestionValidationException("Artifact content is required.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.ArtifactFileName))
|
||||
{
|
||||
throw new CallgraphIngestionValidationException("Artifact file name is required.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.ArtifactContentType) || !AllowedContentTypes.Contains(request.ArtifactContentType))
|
||||
{
|
||||
throw new CallgraphIngestionValidationException($"Unsupported artifact content type '{request.ArtifactContentType}'.");
|
||||
}
|
||||
}
|
||||
|
||||
private static string ComputeSha256(ReadOnlySpan<byte> buffer)
|
||||
{
|
||||
Span<byte> hash = stackalloc byte[SHA256.HashSizeInBytes];
|
||||
@@ -274,13 +278,13 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
return ordered.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Exception thrown when the ingestion request is invalid.
|
||||
/// </summary>
|
||||
public sealed class CallgraphIngestionValidationException : Exception
|
||||
{
|
||||
public CallgraphIngestionValidationException(string message) : base(message)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Exception thrown when the ingestion request is invalid.
|
||||
/// </summary>
|
||||
public sealed class CallgraphIngestionValidationException : Exception
|
||||
{
|
||||
public CallgraphIngestionValidationException(string message) : base(message)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Parsing;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
internal interface ICallgraphNormalizationService
|
||||
{
|
||||
CallgraphParseResult Normalize(string language, CallgraphParseResult result);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Normalizes language-specific callgraphs into deterministic graph documents.
|
||||
/// </summary>
|
||||
internal sealed class CallgraphNormalizationService : ICallgraphNormalizationService
|
||||
{
|
||||
public CallgraphParseResult Normalize(string language, CallgraphParseResult result)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(language);
|
||||
ArgumentNullException.ThrowIfNull(result);
|
||||
|
||||
var normalizedLanguage = language.Trim();
|
||||
var nodesById = new Dictionary<string, CallgraphNode>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var node in result.Nodes ?? Array.Empty<CallgraphNode>())
|
||||
{
|
||||
var normalizedNode = NormalizeNode(node, normalizedLanguage);
|
||||
if (!nodesById.ContainsKey(normalizedNode.Id))
|
||||
{
|
||||
nodesById[normalizedNode.Id] = normalizedNode;
|
||||
}
|
||||
}
|
||||
|
||||
var edges = NormalizeEdges(result.Edges, nodesById);
|
||||
var roots = NormalizeRoots(result.Roots);
|
||||
|
||||
return new CallgraphParseResult(
|
||||
Nodes: nodesById.Values.OrderBy(n => n.Id, StringComparer.Ordinal).ToList(),
|
||||
Edges: edges,
|
||||
Roots: roots,
|
||||
FormatVersion: string.IsNullOrWhiteSpace(result.FormatVersion) ? "1.0" : result.FormatVersion.Trim(),
|
||||
SchemaVersion: string.IsNullOrWhiteSpace(result.SchemaVersion) ? "1.0" : result.SchemaVersion.Trim(),
|
||||
Analyzer: result.Analyzer);
|
||||
}
|
||||
|
||||
private static CallgraphNode NormalizeNode(CallgraphNode node, string language)
|
||||
{
|
||||
var id = node.Id?.Trim();
|
||||
if (string.IsNullOrWhiteSpace(id))
|
||||
{
|
||||
throw new CallgraphParserValidationException("Callgraph node is missing an id.");
|
||||
}
|
||||
|
||||
var name = string.IsNullOrWhiteSpace(node.Name) ? id : node.Name.Trim();
|
||||
var kind = string.IsNullOrWhiteSpace(node.Kind) ? "function" : node.Kind.Trim();
|
||||
var normalizedLanguage = string.IsNullOrWhiteSpace(node.Language) ? language : node.Language.Trim();
|
||||
|
||||
var ns = string.IsNullOrWhiteSpace(node.Namespace)
|
||||
? DeriveNamespace(id, node.File, normalizedLanguage)
|
||||
: node.Namespace!.Trim();
|
||||
|
||||
return node with
|
||||
{
|
||||
Id = id,
|
||||
Name = name,
|
||||
Kind = kind,
|
||||
Namespace = ns,
|
||||
File = node.File?.Trim(),
|
||||
Purl = NormalizePurl(node.Purl),
|
||||
SymbolDigest = NormalizeDigest(node.SymbolDigest),
|
||||
BuildId = node.BuildId?.Trim(),
|
||||
Language = normalizedLanguage,
|
||||
Evidence = NormalizeList(node.Evidence),
|
||||
Analyzer = NormalizeDict(node.Analyzer),
|
||||
CodeId = node.CodeId?.Trim()
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyList<CallgraphEdge> NormalizeEdges(
|
||||
IReadOnlyList<CallgraphEdge>? edges,
|
||||
IReadOnlyDictionary<string, CallgraphNode> nodes)
|
||||
{
|
||||
var list = new List<CallgraphEdge>();
|
||||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var edge in edges ?? Array.Empty<CallgraphEdge>())
|
||||
{
|
||||
var source = edge.SourceId?.Trim();
|
||||
var target = edge.TargetId?.Trim();
|
||||
if (string.IsNullOrWhiteSpace(source) || string.IsNullOrWhiteSpace(target))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!nodes.ContainsKey(source) || !nodes.ContainsKey(target))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var type = string.IsNullOrWhiteSpace(edge.Type) ? "call" : edge.Type.Trim();
|
||||
var key = $"{source}|{target}|{type}";
|
||||
if (!seen.Add(key))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
list.Add(edge with
|
||||
{
|
||||
SourceId = source,
|
||||
TargetId = target,
|
||||
Type = type,
|
||||
Purl = NormalizePurl(edge.Purl),
|
||||
SymbolDigest = NormalizeDigest(edge.SymbolDigest),
|
||||
Confidence = ClampConfidence(edge.Confidence),
|
||||
Candidates = NormalizeList(edge.Candidates),
|
||||
Evidence = NormalizeList(edge.Evidence)
|
||||
});
|
||||
}
|
||||
|
||||
return list.OrderBy(e => e.SourceId, StringComparer.Ordinal)
|
||||
.ThenBy(e => e.TargetId, StringComparer.Ordinal)
|
||||
.ThenBy(e => e.Type, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<CallgraphRoot> NormalizeRoots(IReadOnlyList<CallgraphRoot>? roots)
|
||||
{
|
||||
var list = new List<CallgraphRoot>();
|
||||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var root in roots ?? Array.Empty<CallgraphRoot>())
|
||||
{
|
||||
var id = root.Id?.Trim();
|
||||
if (string.IsNullOrWhiteSpace(id))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var normalized = new CallgraphRoot(
|
||||
id,
|
||||
string.IsNullOrWhiteSpace(root.Phase) ? "runtime" : root.Phase.Trim(),
|
||||
root.Source?.Trim());
|
||||
|
||||
if (seen.Add($"{normalized.Id}|{normalized.Phase}|{normalized.Source}"))
|
||||
{
|
||||
list.Add(normalized);
|
||||
}
|
||||
}
|
||||
|
||||
return list.OrderBy(r => r.Id, StringComparer.Ordinal)
|
||||
.ThenBy(r => r.Phase, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static string? DeriveNamespace(string id, string? file, string language)
|
||||
{
|
||||
if (string.Equals(language, "java", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
var candidate = id.Replace('/', '.');
|
||||
var lastDot = candidate.LastIndexOf('.');
|
||||
if (lastDot > 0)
|
||||
{
|
||||
return candidate[..lastDot];
|
||||
}
|
||||
}
|
||||
|
||||
if (string.Equals(language, "go", StringComparison.OrdinalIgnoreCase) ||
|
||||
string.Equals(language, "nodejs", StringComparison.OrdinalIgnoreCase) ||
|
||||
string.Equals(language, "python", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(file))
|
||||
{
|
||||
var normalizedPath = file.Replace('\\', '/');
|
||||
var idx = normalizedPath.LastIndexOf('/');
|
||||
if (idx > 0)
|
||||
{
|
||||
return normalizedPath[..idx];
|
||||
}
|
||||
}
|
||||
|
||||
var sepIdx = id.LastIndexOfAny(new[] { '.', '/', ':' });
|
||||
if (sepIdx > 0)
|
||||
{
|
||||
return id[..sepIdx];
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string? NormalizePurl(string? value)
|
||||
{
|
||||
return string.IsNullOrWhiteSpace(value) ? null : value.Trim().ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static string? NormalizeDigest(string? value)
|
||||
{
|
||||
return string.IsNullOrWhiteSpace(value) ? null : value.Trim().ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static double? ClampConfidence(double? confidence)
|
||||
{
|
||||
if (!confidence.HasValue)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return Math.Clamp(confidence.Value, 0.0, 1.0);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string>? NormalizeList(IReadOnlyList<string>? values)
|
||||
{
|
||||
if (values is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return values
|
||||
.Where(v => !string.IsNullOrWhiteSpace(v))
|
||||
.Select(v => v.Trim())
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.OrderBy(v => v, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, string?>? NormalizeDict(IReadOnlyDictionary<string, string?>? values)
|
||||
{
|
||||
if (values is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var dict = new Dictionary<string, string?>(StringComparer.Ordinal);
|
||||
foreach (var kv in values)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(kv.Key))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
dict[kv.Key.Trim()] = kv.Value?.Trim();
|
||||
}
|
||||
|
||||
return dict.Count == 0 ? null : dict;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StackExchange.Redis;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
internal interface IRedisConnectionFactory
|
||||
{
|
||||
Task<IConnectionMultiplexer> ConnectAsync(ConfigurationOptions options, CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -1,11 +1,9 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Options;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
@@ -15,65 +13,27 @@ namespace StellaOps.Signals.Services;
|
||||
internal sealed class InMemoryEventsPublisher : IEventsPublisher
|
||||
{
|
||||
private readonly ILogger<InMemoryEventsPublisher> logger;
|
||||
private readonly string topic;
|
||||
private readonly ReachabilityFactEventBuilder eventBuilder;
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = false,
|
||||
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull
|
||||
};
|
||||
|
||||
public InMemoryEventsPublisher(ILogger<InMemoryEventsPublisher> logger, SignalsOptions options)
|
||||
public InMemoryEventsPublisher(ILogger<InMemoryEventsPublisher> logger, ReachabilityFactEventBuilder eventBuilder)
|
||||
{
|
||||
this.logger = logger;
|
||||
topic = string.IsNullOrWhiteSpace(options?.AirGap?.EventTopic)
|
||||
? "signals.fact.updated"
|
||||
: options!.AirGap.EventTopic!;
|
||||
this.eventBuilder = eventBuilder ?? throw new ArgumentNullException(nameof(eventBuilder));
|
||||
}
|
||||
|
||||
public Task PublishFactUpdatedAsync(ReachabilityFactDocument fact, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(fact);
|
||||
|
||||
var (reachable, unreachable) = CountStates(fact);
|
||||
var runtimeFactsCount = fact.RuntimeFacts?.Count ?? 0;
|
||||
var avgConfidence = fact.States.Count > 0 ? fact.States.Average(s => s.Confidence) : 0;
|
||||
var score = fact.Score;
|
||||
var unknownsCount = fact.UnknownsCount;
|
||||
var unknownsPressure = fact.UnknownsPressure;
|
||||
var topBucket = fact.States.Count > 0
|
||||
? fact.States
|
||||
.GroupBy(s => s.Bucket, StringComparer.OrdinalIgnoreCase)
|
||||
.OrderByDescending(g => g.Count())
|
||||
.ThenByDescending(g => g.Average(s => s.Weight))
|
||||
.First()
|
||||
: null;
|
||||
var payload = new ReachabilityFactUpdatedEvent(
|
||||
Version: "signals.fact.updated@v1",
|
||||
SubjectKey: fact.SubjectKey,
|
||||
CallgraphId: string.IsNullOrWhiteSpace(fact.CallgraphId) ? null : fact.CallgraphId,
|
||||
OccurredAtUtc: DateTimeOffset.UtcNow,
|
||||
ReachableCount: reachable,
|
||||
UnreachableCount: unreachable,
|
||||
RuntimeFactsCount: runtimeFactsCount,
|
||||
Bucket: topBucket?.Key ?? "unknown",
|
||||
Weight: topBucket?.Average(s => s.Weight) ?? 0,
|
||||
StateCount: fact.States.Count,
|
||||
FactScore: score,
|
||||
UnknownsCount: unknownsCount,
|
||||
UnknownsPressure: unknownsPressure,
|
||||
AverageConfidence: avgConfidence,
|
||||
ComputedAtUtc: fact.ComputedAt,
|
||||
Targets: fact.States.Select(s => s.Target).ToArray());
|
||||
var envelope = eventBuilder.Build(fact);
|
||||
var json = JsonSerializer.Serialize(envelope, SerializerOptions);
|
||||
|
||||
var json = JsonSerializer.Serialize(payload, new JsonSerializerOptions(JsonSerializerDefaults.Web));
|
||||
logger.LogInformation("{Topic} {Payload}", topic, json);
|
||||
logger.LogInformation(json);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private static (int reachable, int unreachable) CountStates(ReachabilityFactDocument fact)
|
||||
{
|
||||
if (fact.States is null || fact.States.Count == 0)
|
||||
{
|
||||
return (0, 0);
|
||||
}
|
||||
|
||||
var reachable = fact.States.Count(state => state.Reachable);
|
||||
var unreachable = fact.States.Count - reachable;
|
||||
return (reachable, unreachable);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Signals.Models;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
internal sealed class NullEventsPublisher : IEventsPublisher
|
||||
{
|
||||
public Task PublishFactUpdatedAsync(ReachabilityFactDocument fact, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using StellaOps.Signals.Models;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
internal static class ReachabilityFactDigestCalculator
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = false
|
||||
};
|
||||
|
||||
public static string Compute(ReachabilityFactDocument fact)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(fact);
|
||||
|
||||
var canonical = new CanonicalReachabilityFact(
|
||||
CallgraphId: fact.CallgraphId ?? string.Empty,
|
||||
SubjectKey: fact.SubjectKey ?? string.Empty,
|
||||
Subject: new CanonicalSubject(
|
||||
fact.Subject?.ImageDigest ?? string.Empty,
|
||||
fact.Subject?.Component ?? string.Empty,
|
||||
fact.Subject?.Version ?? string.Empty,
|
||||
fact.Subject?.ScanId ?? string.Empty),
|
||||
EntryPoints: NormalizeList(fact.EntryPoints),
|
||||
States: NormalizeStates(fact.States),
|
||||
RuntimeFacts: NormalizeRuntimeFacts(fact.RuntimeFacts),
|
||||
Metadata: NormalizeMetadata(fact.Metadata),
|
||||
Score: fact.Score,
|
||||
UnknownsCount: fact.UnknownsCount,
|
||||
UnknownsPressure: fact.UnknownsPressure,
|
||||
ComputedAt: fact.ComputedAt);
|
||||
|
||||
var json = JsonSerializer.Serialize(canonical, SerializerOptions);
|
||||
Span<byte> hash = stackalloc byte[SHA256.HashSizeInBytes];
|
||||
SHA256.HashData(Encoding.UTF8.GetBytes(json), hash);
|
||||
return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static List<string> NormalizeList(IEnumerable<string>? values) =>
|
||||
values?
|
||||
.Where(v => !string.IsNullOrWhiteSpace(v))
|
||||
.Select(v => v.Trim())
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.OrderBy(v => v, StringComparer.Ordinal)
|
||||
.ToList() ?? new List<string>();
|
||||
|
||||
private static List<CanonicalState> NormalizeStates(IEnumerable<ReachabilityStateDocument>? states)
|
||||
{
|
||||
if (states is null)
|
||||
{
|
||||
return new List<CanonicalState>();
|
||||
}
|
||||
|
||||
return states
|
||||
.OrderBy(s => s.Target, StringComparer.Ordinal)
|
||||
.Select(state => new CanonicalState(
|
||||
Target: state.Target ?? string.Empty,
|
||||
Reachable: state.Reachable,
|
||||
Confidence: state.Confidence,
|
||||
Bucket: state.Bucket ?? "unknown",
|
||||
Weight: state.Weight,
|
||||
Score: state.Score,
|
||||
Path: NormalizeList(state.Path),
|
||||
RuntimeHits: NormalizeList(state.Evidence?.RuntimeHits),
|
||||
BlockedEdges: NormalizeList(state.Evidence?.BlockedEdges)))
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static List<CanonicalRuntimeFact> NormalizeRuntimeFacts(IEnumerable<RuntimeFactDocument>? facts)
|
||||
{
|
||||
if (facts is null)
|
||||
{
|
||||
return new List<CanonicalRuntimeFact>();
|
||||
}
|
||||
|
||||
return facts
|
||||
.Select(f => new CanonicalRuntimeFact(
|
||||
SymbolId: f.SymbolId ?? string.Empty,
|
||||
CodeId: f.CodeId,
|
||||
SymbolDigest: f.SymbolDigest,
|
||||
Purl: f.Purl,
|
||||
BuildId: f.BuildId,
|
||||
LoaderBase: f.LoaderBase,
|
||||
ProcessId: f.ProcessId,
|
||||
ProcessName: f.ProcessName,
|
||||
SocketAddress: f.SocketAddress,
|
||||
ContainerId: f.ContainerId,
|
||||
EvidenceUri: f.EvidenceUri,
|
||||
HitCount: f.HitCount,
|
||||
ObservedAt: f.ObservedAt,
|
||||
Metadata: NormalizeMetadata(f.Metadata)))
|
||||
.OrderBy(f => f.SymbolId, StringComparer.Ordinal)
|
||||
.ThenBy(f => f.CodeId, StringComparer.Ordinal)
|
||||
.ThenBy(f => f.LoaderBase, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static SortedDictionary<string, string?> NormalizeMetadata(IDictionary<string, string?>? metadata)
|
||||
{
|
||||
var normalized = new SortedDictionary<string, string?>(StringComparer.Ordinal);
|
||||
if (metadata is null)
|
||||
{
|
||||
return normalized;
|
||||
}
|
||||
|
||||
foreach (var kvp in metadata)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(kvp.Key))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
normalized[kvp.Key.Trim()] = kvp.Value?.Trim();
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private sealed record CanonicalReachabilityFact(
|
||||
string CallgraphId,
|
||||
string SubjectKey,
|
||||
CanonicalSubject Subject,
|
||||
List<string> EntryPoints,
|
||||
List<CanonicalState> States,
|
||||
List<CanonicalRuntimeFact> RuntimeFacts,
|
||||
SortedDictionary<string, string?> Metadata,
|
||||
double Score,
|
||||
int UnknownsCount,
|
||||
double UnknownsPressure,
|
||||
DateTimeOffset ComputedAt);
|
||||
|
||||
private sealed record CanonicalSubject(
|
||||
string ImageDigest,
|
||||
string Component,
|
||||
string Version,
|
||||
string ScanId);
|
||||
|
||||
private sealed record CanonicalState(
|
||||
string Target,
|
||||
bool Reachable,
|
||||
double Confidence,
|
||||
string Bucket,
|
||||
double Weight,
|
||||
double Score,
|
||||
List<string> Path,
|
||||
List<string> RuntimeHits,
|
||||
List<string> BlockedEdges);
|
||||
|
||||
private sealed record CanonicalRuntimeFact(
|
||||
string SymbolId,
|
||||
string? CodeId,
|
||||
string? SymbolDigest,
|
||||
string? Purl,
|
||||
string? BuildId,
|
||||
string? LoaderBase,
|
||||
int? ProcessId,
|
||||
string? ProcessName,
|
||||
string? SocketAddress,
|
||||
string? ContainerId,
|
||||
string? EvidenceUri,
|
||||
int HitCount,
|
||||
DateTimeOffset? ObservedAt,
|
||||
SortedDictionary<string, string?> Metadata);
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Text.Json.Serialization;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Options;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
internal sealed class ReachabilityFactEventBuilder
|
||||
{
|
||||
private readonly SignalsOptions options;
|
||||
private readonly TimeProvider timeProvider;
|
||||
|
||||
public ReachabilityFactEventBuilder(SignalsOptions options, TimeProvider timeProvider)
|
||||
{
|
||||
this.options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
}
|
||||
|
||||
public ReachabilityFactUpdatedEnvelope Build(ReachabilityFactDocument fact)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(fact);
|
||||
|
||||
var summary = BuildSummary(fact);
|
||||
var digest = ResolveDigest(fact);
|
||||
var factVersion = ResolveFactVersion(fact);
|
||||
|
||||
return new ReachabilityFactUpdatedEnvelope(
|
||||
Topic: ResolveTopic(),
|
||||
EventId: Guid.NewGuid().ToString("n"),
|
||||
Version: "signals.fact.updated@v1",
|
||||
EmittedAtUtc: timeProvider.GetUtcNow(),
|
||||
Tenant: ResolveTenant(fact),
|
||||
SubjectKey: fact.SubjectKey,
|
||||
CallgraphId: string.IsNullOrWhiteSpace(fact.CallgraphId) ? null : fact.CallgraphId,
|
||||
FactKind: "reachability",
|
||||
FactVersion: factVersion,
|
||||
Digest: digest,
|
||||
ContentType: "application/json",
|
||||
Producer: new EventProducerMetadata(
|
||||
Service: options.Events.Producer,
|
||||
Pipeline: options.Events.Pipeline,
|
||||
Release: options.Events.Release ?? typeof(Program).Assembly.GetName().Version?.ToString() ?? "unknown"),
|
||||
Trace: ResolveTrace(fact),
|
||||
Summary: summary);
|
||||
}
|
||||
|
||||
private ReachabilityFactUpdatedEvent BuildSummary(ReachabilityFactDocument fact)
|
||||
{
|
||||
var (reachable, unreachable) = CountStates(fact);
|
||||
var runtimeFactsCount = fact.RuntimeFacts?.Count ?? 0;
|
||||
var avgConfidence = fact.States.Count > 0 ? fact.States.Average(s => s.Confidence) : 0;
|
||||
var topBucket = fact.States.Count > 0
|
||||
? fact.States
|
||||
.GroupBy(s => s.Bucket, StringComparer.OrdinalIgnoreCase)
|
||||
.OrderByDescending(g => g.Count())
|
||||
.ThenByDescending(g => g.Average(s => s.Weight))
|
||||
.First()
|
||||
: null;
|
||||
|
||||
return new ReachabilityFactUpdatedEvent(
|
||||
Version: "signals.fact.updated@v1",
|
||||
SubjectKey: fact.SubjectKey,
|
||||
CallgraphId: string.IsNullOrWhiteSpace(fact.CallgraphId) ? null : fact.CallgraphId,
|
||||
OccurredAtUtc: timeProvider.GetUtcNow(),
|
||||
ReachableCount: reachable,
|
||||
UnreachableCount: unreachable,
|
||||
RuntimeFactsCount: runtimeFactsCount,
|
||||
Bucket: topBucket?.Key ?? "unknown",
|
||||
Weight: topBucket?.Average(s => s.Weight) ?? 0,
|
||||
StateCount: fact.States.Count,
|
||||
FactScore: fact.Score,
|
||||
UnknownsCount: fact.UnknownsCount,
|
||||
UnknownsPressure: fact.UnknownsPressure,
|
||||
AverageConfidence: avgConfidence,
|
||||
ComputedAtUtc: fact.ComputedAt,
|
||||
Targets: fact.States.Select(s => s.Target).ToArray());
|
||||
}
|
||||
|
||||
private static (int reachable, int unreachable) CountStates(ReachabilityFactDocument fact)
|
||||
{
|
||||
if (fact.States is null || fact.States.Count == 0)
|
||||
{
|
||||
return (0, 0);
|
||||
}
|
||||
|
||||
var reachable = fact.States.Count(state => state.Reachable);
|
||||
var unreachable = fact.States.Count - reachable;
|
||||
return (reachable, unreachable);
|
||||
}
|
||||
|
||||
private string ResolveTopic()
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(options.Events.Stream))
|
||||
{
|
||||
return options.Events.Stream;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(options.AirGap.EventTopic))
|
||||
{
|
||||
return options.AirGap.EventTopic!;
|
||||
}
|
||||
|
||||
return "signals.fact.updated.v1";
|
||||
}
|
||||
|
||||
private string ResolveTenant(ReachabilityFactDocument fact)
|
||||
{
|
||||
if (fact.Metadata is not null)
|
||||
{
|
||||
if (fact.Metadata.TryGetValue("tenant", out var tenant) && !string.IsNullOrWhiteSpace(tenant))
|
||||
{
|
||||
return tenant!;
|
||||
}
|
||||
|
||||
if (fact.Metadata.TryGetValue("subject.tenant", out var subjectTenant) && !string.IsNullOrWhiteSpace(subjectTenant))
|
||||
{
|
||||
return subjectTenant!;
|
||||
}
|
||||
}
|
||||
|
||||
return options.Events.DefaultTenant;
|
||||
}
|
||||
|
||||
private static EventTraceMetadata ResolveTrace(ReachabilityFactDocument fact)
|
||||
{
|
||||
var metadata = fact.Metadata;
|
||||
string? traceId = null;
|
||||
string? spanId = null;
|
||||
|
||||
if (metadata is not null)
|
||||
{
|
||||
metadata.TryGetValue("trace_id", out traceId);
|
||||
metadata.TryGetValue("span_id", out spanId);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(traceId) && metadata.TryGetValue("trace.id", out var dottedTrace))
|
||||
{
|
||||
traceId = dottedTrace;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(spanId) && metadata.TryGetValue("trace.parent_span", out var dottedSpan))
|
||||
{
|
||||
spanId = dottedSpan;
|
||||
}
|
||||
}
|
||||
|
||||
return new EventTraceMetadata(traceId, spanId);
|
||||
}
|
||||
|
||||
private static int ResolveFactVersion(ReachabilityFactDocument fact)
|
||||
{
|
||||
if (fact.Metadata is not null &&
|
||||
fact.Metadata.TryGetValue("fact.version", out var versionValue) &&
|
||||
int.TryParse(versionValue, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed))
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
private static string ResolveDigest(ReachabilityFactDocument fact)
|
||||
{
|
||||
if (fact.Metadata is not null &&
|
||||
fact.Metadata.TryGetValue("fact.digest", out var digest) &&
|
||||
!string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
return digest!;
|
||||
}
|
||||
|
||||
return ReachabilityFactDigestCalculator.Compute(fact);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record ReachabilityFactUpdatedEnvelope(
|
||||
[property: JsonPropertyName("topic")] string Topic,
|
||||
[property: JsonPropertyName("event_id")] string EventId,
|
||||
[property: JsonPropertyName("version")] string Version,
|
||||
[property: JsonPropertyName("emitted_at")] DateTimeOffset EmittedAtUtc,
|
||||
[property: JsonPropertyName("tenant")] string Tenant,
|
||||
[property: JsonPropertyName("subject_key")] string SubjectKey,
|
||||
[property: JsonPropertyName("callgraph_id")] string? CallgraphId,
|
||||
[property: JsonPropertyName("fact_kind")] string FactKind,
|
||||
[property: JsonPropertyName("fact_version")] int FactVersion,
|
||||
[property: JsonPropertyName("digest")] string Digest,
|
||||
[property: JsonPropertyName("content_type")] string ContentType,
|
||||
[property: JsonPropertyName("producer")] EventProducerMetadata Producer,
|
||||
[property: JsonPropertyName("trace")] EventTraceMetadata Trace,
|
||||
[property: JsonPropertyName("summary")] ReachabilityFactUpdatedEvent Summary);
|
||||
|
||||
public sealed record EventProducerMetadata(
|
||||
[property: JsonPropertyName("service")] string Service,
|
||||
[property: JsonPropertyName("pipeline")] string Pipeline,
|
||||
[property: JsonPropertyName("release")] string? Release);
|
||||
|
||||
public sealed record EventTraceMetadata(
|
||||
[property: JsonPropertyName("trace_id")] string? TraceId,
|
||||
[property: JsonPropertyName("span_id")] string? SpanId);
|
||||
@@ -1,6 +1,7 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Globalization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -90,7 +91,7 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
|
||||
}
|
||||
}
|
||||
|
||||
var runtimeHits = runtimeHitSet.ToList();
|
||||
var runtimeHits = runtimeHitSet.OrderBy(h => h, StringComparer.Ordinal).ToList();
|
||||
|
||||
var states = new List<ReachabilityStateDocument>(targets.Count);
|
||||
foreach (var target in targets)
|
||||
@@ -108,6 +109,8 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
|
||||
|
||||
var score = confidence * weight;
|
||||
|
||||
runtimeEvidence = runtimeEvidence.OrderBy(hit => hit, StringComparer.Ordinal).ToList();
|
||||
|
||||
states.Add(new ReachabilityStateDocument
|
||||
{
|
||||
Target = target,
|
||||
@@ -120,11 +123,17 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
|
||||
Evidence = new ReachabilityEvidenceDocument
|
||||
{
|
||||
RuntimeHits = runtimeEvidence,
|
||||
BlockedEdges = request.BlockedEdges?.Select(edge => $"{edge.From} -> {edge.To}").ToList()
|
||||
BlockedEdges = request.BlockedEdges?
|
||||
.Select(edge => $"{edge.From} -> {edge.To}")
|
||||
.OrderBy(edge => edge, StringComparer.Ordinal)
|
||||
.ToList()
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
states = states.OrderBy(s => s.Target, StringComparer.Ordinal).ToList();
|
||||
entryPoints = entryPoints.OrderBy(ep => ep, StringComparer.Ordinal).ToList();
|
||||
|
||||
var baseScore = states.Count > 0 ? states.Average(s => s.Score) : 0;
|
||||
var unknownsCount = await unknownsRepository.CountBySubjectAsync(subjectKey, cancellationToken).ConfigureAwait(false);
|
||||
var pressure = states.Count + unknownsCount == 0
|
||||
@@ -148,6 +157,22 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
|
||||
RuntimeFacts = existingFact?.RuntimeFacts
|
||||
};
|
||||
|
||||
document.Metadata ??= new Dictionary<string, string?>(StringComparer.Ordinal);
|
||||
var priorVersion = 0;
|
||||
if (existingFact?.Metadata != null
|
||||
&& existingFact.Metadata.TryGetValue("fact.version", out var versionValue)
|
||||
&& int.TryParse(versionValue, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedVersion))
|
||||
{
|
||||
priorVersion = parsedVersion;
|
||||
}
|
||||
|
||||
var nextVersion = priorVersion + 1;
|
||||
document.Metadata["fact.version"] = nextVersion.ToString(CultureInfo.InvariantCulture);
|
||||
document.Metadata.Remove("fact.digest");
|
||||
document.Metadata.Remove("fact.digest.alg");
|
||||
document.Metadata["fact.digest"] = ReachabilityFactDigestCalculator.Compute(document);
|
||||
document.Metadata["fact.digest.alg"] = "sha256";
|
||||
|
||||
logger.LogInformation("Computed reachability fact for subject {SubjectKey} with {StateCount} targets.", document.SubjectKey, states.Count);
|
||||
var persisted = await factRepository.UpsertAsync(document, cancellationToken).ConfigureAwait(false);
|
||||
await cache.SetAsync(persisted, cancellationToken).ConfigureAwait(false);
|
||||
@@ -266,7 +291,7 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var neighbor in neighbors)
|
||||
foreach (var neighbor in neighbors.OrderBy(n => n, StringComparer.Ordinal))
|
||||
{
|
||||
if (visited.Add(neighbor))
|
||||
{
|
||||
|
||||
@@ -60,8 +60,14 @@ public sealed class ReachabilityUnionIngestionService : IReachabilityUnionIngest
|
||||
}
|
||||
|
||||
var metaEntry = entries["meta.json"];
|
||||
using var metaStream = metaEntry.Open();
|
||||
using var metaDoc = await JsonDocument.ParseAsync(metaStream, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
await using var metaBuffer = new MemoryStream();
|
||||
await using (var metaStream = metaEntry.Open())
|
||||
{
|
||||
await metaStream.CopyToAsync(metaBuffer, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
metaBuffer.Position = 0;
|
||||
using var metaDoc = await JsonDocument.ParseAsync(metaBuffer, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
var metaRoot = metaDoc.RootElement;
|
||||
|
||||
var filesElement = metaRoot.TryGetProperty("files", out var f) && f.ValueKind == JsonValueKind.Array
|
||||
@@ -77,6 +83,13 @@ public sealed class ReachabilityUnionIngestionService : IReachabilityUnionIngest
|
||||
})
|
||||
.ToList();
|
||||
|
||||
metaBuffer.Position = 0;
|
||||
var metaPath = Path.Combine(casRoot, "meta.json");
|
||||
await using (var metaDest = File.Create(metaPath))
|
||||
{
|
||||
await metaBuffer.CopyToAsync(metaDest, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var filesForResponse = new List<ReachabilityUnionFile>();
|
||||
|
||||
foreach (var file in recorded)
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StackExchange.Redis;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
internal sealed class RedisConnectionFactory : IRedisConnectionFactory
|
||||
{
|
||||
public Task<IConnectionMultiplexer> ConnectAsync(ConfigurationOptions options, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
return ConnectionMultiplexer.ConnectAsync(options)
|
||||
.ContinueWith(
|
||||
t => (IConnectionMultiplexer)t.Result,
|
||||
cancellationToken,
|
||||
TaskContinuationOptions.OnlyOnRanToCompletion | TaskContinuationOptions.ExecuteSynchronously,
|
||||
TaskScheduler.Current);
|
||||
}
|
||||
}
|
||||
185
src/Signals/StellaOps.Signals/Services/RedisEventsPublisher.cs
Normal file
185
src/Signals/StellaOps.Signals/Services/RedisEventsPublisher.cs
Normal file
@@ -0,0 +1,185 @@
|
||||
using System;
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StackExchange.Redis;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Options;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
internal sealed class RedisEventsPublisher : IEventsPublisher, IAsyncDisposable
|
||||
{
|
||||
private readonly SignalsEventsOptions options;
|
||||
private readonly ILogger<RedisEventsPublisher> logger;
|
||||
private readonly IRedisConnectionFactory connectionFactory;
|
||||
private readonly ReachabilityFactEventBuilder eventBuilder;
|
||||
private readonly TimeSpan publishTimeout;
|
||||
private readonly int? maxStreamLength;
|
||||
private readonly SemaphoreSlim connectionGate = new(1, 1);
|
||||
private IConnectionMultiplexer? connection;
|
||||
private bool disposed;
|
||||
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = false,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
|
||||
};
|
||||
|
||||
public RedisEventsPublisher(
|
||||
SignalsOptions options,
|
||||
IRedisConnectionFactory connectionFactory,
|
||||
ReachabilityFactEventBuilder eventBuilder,
|
||||
ILogger<RedisEventsPublisher> logger)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
this.options = options.Events ?? throw new InvalidOperationException("Signals events configuration is required.");
|
||||
this.connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory));
|
||||
this.eventBuilder = eventBuilder ?? throw new ArgumentNullException(nameof(eventBuilder));
|
||||
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
publishTimeout = this.options.PublishTimeoutSeconds > 0
|
||||
? TimeSpan.FromSeconds(this.options.PublishTimeoutSeconds)
|
||||
: TimeSpan.Zero;
|
||||
maxStreamLength = this.options.MaxStreamLength > 0
|
||||
? (int)Math.Min(this.options.MaxStreamLength, int.MaxValue)
|
||||
: null;
|
||||
}
|
||||
|
||||
public async Task PublishFactUpdatedAsync(ReachabilityFactDocument fact, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(fact);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (!options.Enabled)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var envelope = eventBuilder.Build(fact);
|
||||
var json = JsonSerializer.Serialize(envelope, SerializerOptions);
|
||||
|
||||
try
|
||||
{
|
||||
var database = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var entries = new[]
|
||||
{
|
||||
new NameValueEntry("event", json),
|
||||
new NameValueEntry("event_id", envelope.EventId),
|
||||
new NameValueEntry("subject_key", envelope.SubjectKey),
|
||||
new NameValueEntry("digest", envelope.Digest),
|
||||
new NameValueEntry("fact_version", envelope.FactVersion.ToString(CultureInfo.InvariantCulture))
|
||||
};
|
||||
|
||||
var publishTask = maxStreamLength.HasValue
|
||||
? database.StreamAddAsync(options.Stream, entries, maxLength: maxStreamLength, useApproximateMaxLength: true)
|
||||
: database.StreamAddAsync(options.Stream, entries);
|
||||
|
||||
if (publishTimeout > TimeSpan.Zero)
|
||||
{
|
||||
await publishTask.WaitAsync(publishTimeout, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await publishTask.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogError(ex, "Failed to publish reachability event to Redis stream {Stream}.", options.Stream);
|
||||
await TryPublishDeadLetterAsync(json, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IDatabase> GetDatabaseAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (connection is { IsConnected: true })
|
||||
{
|
||||
return connection.GetDatabase();
|
||||
}
|
||||
|
||||
await connectionGate.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (connection is null || !connection.IsConnected)
|
||||
{
|
||||
var configuration = ConfigurationOptions.Parse(options.ConnectionString!);
|
||||
configuration.AbortOnConnectFail = false;
|
||||
connection = await connectionFactory.ConnectAsync(configuration, cancellationToken).ConfigureAwait(false);
|
||||
logger.LogInformation("Connected Signals events publisher to Redis stream {Stream}.", options.Stream);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
connectionGate.Release();
|
||||
}
|
||||
|
||||
return connection!.GetDatabase();
|
||||
}
|
||||
|
||||
private async Task TryPublishDeadLetterAsync(string json, CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(options.DeadLetterStream) || connection is null || !connection.IsConnected)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var db = connection.GetDatabase();
|
||||
var entries = new[]
|
||||
{
|
||||
new NameValueEntry("event", json),
|
||||
new NameValueEntry("error", "publish-failed")
|
||||
};
|
||||
|
||||
var dlqTask = maxStreamLength.HasValue
|
||||
? db.StreamAddAsync(options.DeadLetterStream, entries, maxLength: maxStreamLength, useApproximateMaxLength: true)
|
||||
: db.StreamAddAsync(options.DeadLetterStream, entries);
|
||||
|
||||
if (publishTimeout > TimeSpan.Zero)
|
||||
{
|
||||
await dlqTask.WaitAsync(publishTimeout, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await dlqTask.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogWarning(ex, "Failed to publish reachability event to DLQ stream {Stream}.", options.DeadLetterStream);
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
disposed = true;
|
||||
|
||||
if (connection is not null)
|
||||
{
|
||||
try
|
||||
{
|
||||
await connection.CloseAsync();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogDebug(ex, "Error closing Redis events publisher connection.");
|
||||
}
|
||||
|
||||
connection.Dispose();
|
||||
}
|
||||
|
||||
connectionGate.Dispose();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user