This commit is contained in:
StellaOps Bot
2025-12-09 00:20:52 +02:00
parent 3d01bf9edc
commit bc0762e97d
261 changed files with 14033 additions and 4427 deletions

View File

@@ -0,0 +1,105 @@
using System;
namespace StellaOps.Signals.Options;
/// <summary>
/// Configuration for reachability fact events (SIGNALS-24-005).
/// </summary>
public sealed class SignalsEventsOptions
{
/// <summary>
/// Enables event emission. When false, events are dropped.
/// </summary>
public bool Enabled { get; set; } = true;
/// <summary>
/// Transport driver: "inmemory" or "redis".
/// </summary>
public string Driver { get; set; } = "inmemory";
/// <summary>
/// Primary topic/stream name for fact updates.
/// </summary>
public string Stream { get; set; } = "signals.fact.updated.v1";
/// <summary>
/// Dead-letter topic/stream used when publishing fails.
/// </summary>
public string DeadLetterStream { get; set; } = "signals.fact.updated.dlq";
/// <summary>
/// Connection string for Redis streams (when Driver=redis).
/// </summary>
public string? ConnectionString { get; set; }
/// <summary>
/// Optional publish timeout (seconds). Set to 0 to disable.
/// </summary>
public int PublishTimeoutSeconds { get; set; } = 5;
/// <summary>
/// Approximate maximum stream length (capped by Redis trimming).
/// </summary>
public long MaxStreamLength { get; set; } = 10_000;
/// <summary>
/// Producer identifier for observability payloads.
/// </summary>
public string Producer { get; set; } = "StellaOps.Signals";
/// <summary>
/// Pipeline name attached to event metadata.
/// </summary>
public string Pipeline { get; set; } = "signals";
/// <summary>
/// Optional release string to stamp events with build provenance.
/// </summary>
public string? Release { get; set; }
/// <summary>
/// Default tenant when none is supplied in metadata.
/// </summary>
public string DefaultTenant { get; set; } = "tenant-default";
public void Validate()
{
var normalizedDriver = Driver?.Trim();
if (string.IsNullOrWhiteSpace(normalizedDriver))
{
throw new InvalidOperationException("Signals events driver is required.");
}
if (!string.Equals(normalizedDriver, "redis", StringComparison.OrdinalIgnoreCase)
&& !string.Equals(normalizedDriver, "inmemory", StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException("Signals events driver must be 'redis' or 'inmemory'.");
}
if (string.IsNullOrWhiteSpace(Stream))
{
throw new InvalidOperationException("Signals events stream/topic is required.");
}
if (PublishTimeoutSeconds < 0)
{
throw new InvalidOperationException("Signals events publish timeout must be >= 0 seconds.");
}
if (MaxStreamLength < 0)
{
throw new InvalidOperationException("Signals events max stream length must be >= 0.");
}
if (string.IsNullOrWhiteSpace(DefaultTenant))
{
throw new InvalidOperationException("Signals events default tenant is required.");
}
if (string.Equals(normalizedDriver, "redis", StringComparison.OrdinalIgnoreCase)
&& string.IsNullOrWhiteSpace(ConnectionString))
{
throw new InvalidOperationException("Signals events Redis driver requires ConnectionString.");
}
}
}

View File

@@ -1,26 +1,26 @@
namespace StellaOps.Signals.Options;
/// <summary>
/// Root configuration for the Signals service.
/// </summary>
public sealed class SignalsOptions
{
/// <summary>
/// Configuration section name.
/// </summary>
public const string SectionName = "Signals";
/// <summary>
/// Authority integration settings.
/// </summary>
public SignalsAuthorityOptions Authority { get; } = new();
/// <summary>
/// MongoDB configuration.
/// </summary>
public SignalsMongoOptions Mongo { get; } = new();
/// <summary>
namespace StellaOps.Signals.Options;
/// <summary>
/// Root configuration for the Signals service.
/// </summary>
public sealed class SignalsOptions
{
/// <summary>
/// Configuration section name.
/// </summary>
public const string SectionName = "Signals";
/// <summary>
/// Authority integration settings.
/// </summary>
public SignalsAuthorityOptions Authority { get; } = new();
/// <summary>
/// MongoDB configuration.
/// </summary>
public SignalsMongoOptions Mongo { get; } = new();
/// <summary>
/// Artifact storage configuration.
/// </summary>
public SignalsArtifactStorageOptions Storage { get; } = new();
@@ -40,22 +40,28 @@ public sealed class SignalsOptions
/// </summary>
public SignalsCacheOptions Cache { get; } = new();
/// <summary>
/// Event transport configuration.
/// </summary>
public SignalsEventsOptions Events { get; } = new();
/// <summary>
/// OpenAPI exposure (if enabled).
/// </summary>
public SignalsOpenApiOptions OpenApi { get; } = new();
/// <summary>
/// Validates configured options.
/// </summary>
public void Validate()
{
/// <summary>
/// Validates configured options.
/// </summary>
public void Validate()
{
Authority.Validate();
Mongo.Validate();
Storage.Validate();
AirGap.Validate();
Scoring.Validate();
Cache.Validate();
Events.Validate();
OpenApi.Validate();
}
}

View File

@@ -125,6 +125,7 @@ builder.Services.AddSingleton<IMongoCollection<UnknownSymbolDocument>>(sp =>
});
builder.Services.AddSingleton<ICallgraphRepository, MongoCallgraphRepository>();
builder.Services.AddSingleton<ICallgraphNormalizationService, CallgraphNormalizationService>();
// Configure callgraph artifact storage based on driver
if (bootstrap.Storage.IsRustFsDriver())
@@ -165,7 +166,31 @@ builder.Services.AddSingleton<IReachabilityCache>(sp =>
var options = sp.GetRequiredService<IOptions<SignalsOptions>>().Value;
return new RedisReachabilityCache(options.Cache);
});
builder.Services.AddSingleton<IEventsPublisher, InMemoryEventsPublisher>();
builder.Services.AddSingleton<IRedisConnectionFactory, RedisConnectionFactory>();
builder.Services.AddSingleton<ReachabilityFactEventBuilder>();
builder.Services.AddSingleton<IEventsPublisher>(sp =>
{
var options = sp.GetRequiredService<SignalsOptions>();
var eventBuilder = sp.GetRequiredService<ReachabilityFactEventBuilder>();
if (!options.Events.Enabled)
{
return new NullEventsPublisher();
}
if (string.Equals(options.Events.Driver, "redis", StringComparison.OrdinalIgnoreCase))
{
return new RedisEventsPublisher(
options,
sp.GetRequiredService<IRedisConnectionFactory>(),
eventBuilder,
sp.GetRequiredService<ILogger<RedisEventsPublisher>>());
}
return new InMemoryEventsPublisher(
sp.GetRequiredService<ILogger<InMemoryEventsPublisher>>(),
eventBuilder);
});
builder.Services.AddSingleton<IReachabilityFactRepository>(sp =>
{
var inner = sp.GetRequiredService<MongoReachabilityFactRepository>();

View File

@@ -1,72 +1,76 @@
using System;
using System.Collections.Generic;
using System.IO;
using System;
using System.Collections.Generic;
using System.IO;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Signals.Models;
using StellaOps.Signals.Options;
using StellaOps.Signals.Parsing;
using StellaOps.Signals.Persistence;
using StellaOps.Signals.Storage;
using StellaOps.Signals.Storage.Models;
namespace StellaOps.Signals.Services;
internal sealed class CallgraphIngestionService : ICallgraphIngestionService
{
private static readonly HashSet<string> AllowedContentTypes = new(StringComparer.OrdinalIgnoreCase)
{
"application/json",
"application/vnd.stellaops.callgraph+json"
};
private readonly ICallgraphParserResolver parserResolver;
private readonly ICallgraphArtifactStore artifactStore;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Signals.Models;
using StellaOps.Signals.Options;
using StellaOps.Signals.Parsing;
using StellaOps.Signals.Persistence;
using StellaOps.Signals.Storage;
using StellaOps.Signals.Storage.Models;
namespace StellaOps.Signals.Services;
internal sealed class CallgraphIngestionService : ICallgraphIngestionService
{
private static readonly HashSet<string> AllowedContentTypes = new(StringComparer.OrdinalIgnoreCase)
{
"application/json",
"application/vnd.stellaops.callgraph+json"
};
private readonly ICallgraphParserResolver parserResolver;
private readonly ICallgraphArtifactStore artifactStore;
private readonly ICallgraphRepository repository;
private readonly ICallgraphNormalizationService normalizer;
private readonly ILogger<CallgraphIngestionService> logger;
private readonly SignalsOptions options;
private readonly TimeProvider timeProvider;
private static readonly JsonSerializerOptions ManifestSerializerOptions = new(JsonSerializerDefaults.Web);
public CallgraphIngestionService(
ICallgraphParserResolver parserResolver,
ICallgraphArtifactStore artifactStore,
ICallgraphRepository repository,
IOptions<SignalsOptions> options,
TimeProvider timeProvider,
ILogger<CallgraphIngestionService> logger)
{
this.parserResolver = parserResolver ?? throw new ArgumentNullException(nameof(parserResolver));
this.artifactStore = artifactStore ?? throw new ArgumentNullException(nameof(artifactStore));
this.repository = repository ?? throw new ArgumentNullException(nameof(repository));
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
this.options = options?.Value ?? throw new ArgumentNullException(nameof(options));
}
public async Task<CallgraphIngestResponse> IngestAsync(CallgraphIngestRequest request, CancellationToken cancellationToken)
{
ValidateRequest(request);
var parser = parserResolver.Resolve(request.Language);
public CallgraphIngestionService(
ICallgraphParserResolver parserResolver,
ICallgraphArtifactStore artifactStore,
ICallgraphRepository repository,
ICallgraphNormalizationService normalizer,
IOptions<SignalsOptions> options,
TimeProvider timeProvider,
ILogger<CallgraphIngestionService> logger)
{
this.parserResolver = parserResolver ?? throw new ArgumentNullException(nameof(parserResolver));
this.artifactStore = artifactStore ?? throw new ArgumentNullException(nameof(artifactStore));
this.repository = repository ?? throw new ArgumentNullException(nameof(repository));
this.normalizer = normalizer ?? throw new ArgumentNullException(nameof(normalizer));
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
this.options = options?.Value ?? throw new ArgumentNullException(nameof(options));
}
public async Task<CallgraphIngestResponse> IngestAsync(CallgraphIngestRequest request, CancellationToken cancellationToken)
{
ValidateRequest(request);
var parser = parserResolver.Resolve(request.Language);
var artifactBytes = Convert.FromBase64String(request.ArtifactContentBase64);
await using var parseStream = new MemoryStream(artifactBytes, writable: false);
var parseResult = await parser.ParseAsync(parseStream, cancellationToken).ConfigureAwait(false);
var parsed = await parser.ParseAsync(parseStream, cancellationToken).ConfigureAwait(false);
var normalized = normalizer.Normalize(parser.Language, parsed);
var schemaVersion = !string.IsNullOrWhiteSpace(request.SchemaVersion)
? request.SchemaVersion!
: parseResult.SchemaVersion;
var analyzerMeta = request.Analyzer ?? parseResult.Analyzer;
: normalized.SchemaVersion;
var analyzerMeta = request.Analyzer ?? normalized.Analyzer;
parseStream.Position = 0;
var artifactHash = ComputeSha256(artifactBytes);
var graphHash = ComputeGraphHash(parseResult);
var graphHash = ComputeGraphHash(normalized);
var manifest = new CallgraphManifest
{
@@ -76,9 +80,9 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
ArtifactHash = artifactHash,
GraphHash = graphHash,
SchemaVersion = schemaVersion,
NodeCount = parseResult.Nodes.Count,
EdgeCount = parseResult.Edges.Count,
RootCount = parseResult.Roots.Count,
NodeCount = normalized.Nodes.Count,
EdgeCount = normalized.Edges.Count,
RootCount = normalized.Roots.Count,
CreatedAt = timeProvider.GetUtcNow()
};
@@ -98,15 +102,15 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
manifestStream),
parseStream,
cancellationToken).ConfigureAwait(false);
var document = new CallgraphDocument
{
var document = new CallgraphDocument
{
Language = parser.Language,
Component = request.Component,
Version = request.Version,
Nodes = new List<CallgraphNode>(parseResult.Nodes),
Edges = new List<CallgraphEdge>(parseResult.Edges),
Roots = new List<CallgraphRoot>(parseResult.Roots),
Nodes = new List<CallgraphNode>(normalized.Nodes),
Edges = new List<CallgraphEdge>(normalized.Edges),
Roots = new List<CallgraphRoot>(normalized.Roots),
Metadata = request.Metadata is null
? null
: new Dictionary<string, string?>(request.Metadata, StringComparer.OrdinalIgnoreCase),
@@ -125,7 +129,7 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
};
document.Metadata ??= new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
document.Metadata["formatVersion"] = parseResult.FormatVersion;
document.Metadata["formatVersion"] = normalized.FormatVersion;
document.Metadata["schemaVersion"] = schemaVersion;
if (analyzerMeta is not null)
{
@@ -138,16 +142,16 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
document.SchemaVersion = schemaVersion;
document = await repository.UpsertAsync(document, cancellationToken).ConfigureAwait(false);
logger.LogInformation(
"Ingested callgraph {Language}:{Component}:{Version} (id={Id}) with {NodeCount} nodes and {EdgeCount} edges.",
document.Language,
document.Component,
document.Version,
document.Id,
document.Nodes.Count,
document.Edges.Count);
logger.LogInformation(
"Ingested callgraph {Language}:{Component}:{Version} (id={Id}) with {NodeCount} nodes and {EdgeCount} edges.",
document.Language,
document.Component,
document.Version,
document.Id,
document.Nodes.Count,
document.Edges.Count);
return new CallgraphIngestResponse(
document.Id,
document.Artifact.Path,
@@ -160,42 +164,42 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
document.Edges.Count,
document.Roots?.Count ?? 0);
}
private static void ValidateRequest(CallgraphIngestRequest request)
{
ArgumentNullException.ThrowIfNull(request);
if (string.IsNullOrWhiteSpace(request.Language))
{
throw new CallgraphIngestionValidationException("Language is required.");
}
if (string.IsNullOrWhiteSpace(request.Component))
{
throw new CallgraphIngestionValidationException("Component is required.");
}
if (string.IsNullOrWhiteSpace(request.Version))
{
throw new CallgraphIngestionValidationException("Version is required.");
}
if (string.IsNullOrWhiteSpace(request.ArtifactContentBase64))
{
throw new CallgraphIngestionValidationException("Artifact content is required.");
}
if (string.IsNullOrWhiteSpace(request.ArtifactFileName))
{
throw new CallgraphIngestionValidationException("Artifact file name is required.");
}
if (string.IsNullOrWhiteSpace(request.ArtifactContentType) || !AllowedContentTypes.Contains(request.ArtifactContentType))
{
throw new CallgraphIngestionValidationException($"Unsupported artifact content type '{request.ArtifactContentType}'.");
}
}
private static void ValidateRequest(CallgraphIngestRequest request)
{
ArgumentNullException.ThrowIfNull(request);
if (string.IsNullOrWhiteSpace(request.Language))
{
throw new CallgraphIngestionValidationException("Language is required.");
}
if (string.IsNullOrWhiteSpace(request.Component))
{
throw new CallgraphIngestionValidationException("Component is required.");
}
if (string.IsNullOrWhiteSpace(request.Version))
{
throw new CallgraphIngestionValidationException("Version is required.");
}
if (string.IsNullOrWhiteSpace(request.ArtifactContentBase64))
{
throw new CallgraphIngestionValidationException("Artifact content is required.");
}
if (string.IsNullOrWhiteSpace(request.ArtifactFileName))
{
throw new CallgraphIngestionValidationException("Artifact file name is required.");
}
if (string.IsNullOrWhiteSpace(request.ArtifactContentType) || !AllowedContentTypes.Contains(request.ArtifactContentType))
{
throw new CallgraphIngestionValidationException($"Unsupported artifact content type '{request.ArtifactContentType}'.");
}
}
private static string ComputeSha256(ReadOnlySpan<byte> buffer)
{
Span<byte> hash = stackalloc byte[SHA256.HashSizeInBytes];
@@ -274,13 +278,13 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
return ordered.ToString();
}
}
/// <summary>
/// Exception thrown when the ingestion request is invalid.
/// </summary>
public sealed class CallgraphIngestionValidationException : Exception
{
public CallgraphIngestionValidationException(string message) : base(message)
{
}
}
/// <summary>
/// Exception thrown when the ingestion request is invalid.
/// </summary>
public sealed class CallgraphIngestionValidationException : Exception
{
public CallgraphIngestionValidationException(string message) : base(message)
{
}
}

View File

@@ -0,0 +1,248 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Signals.Models;
using StellaOps.Signals.Parsing;
namespace StellaOps.Signals.Services;
internal interface ICallgraphNormalizationService
{
CallgraphParseResult Normalize(string language, CallgraphParseResult result);
}
/// <summary>
/// Normalizes language-specific callgraphs into deterministic graph documents.
/// </summary>
internal sealed class CallgraphNormalizationService : ICallgraphNormalizationService
{
public CallgraphParseResult Normalize(string language, CallgraphParseResult result)
{
ArgumentException.ThrowIfNullOrWhiteSpace(language);
ArgumentNullException.ThrowIfNull(result);
var normalizedLanguage = language.Trim();
var nodesById = new Dictionary<string, CallgraphNode>(StringComparer.Ordinal);
foreach (var node in result.Nodes ?? Array.Empty<CallgraphNode>())
{
var normalizedNode = NormalizeNode(node, normalizedLanguage);
if (!nodesById.ContainsKey(normalizedNode.Id))
{
nodesById[normalizedNode.Id] = normalizedNode;
}
}
var edges = NormalizeEdges(result.Edges, nodesById);
var roots = NormalizeRoots(result.Roots);
return new CallgraphParseResult(
Nodes: nodesById.Values.OrderBy(n => n.Id, StringComparer.Ordinal).ToList(),
Edges: edges,
Roots: roots,
FormatVersion: string.IsNullOrWhiteSpace(result.FormatVersion) ? "1.0" : result.FormatVersion.Trim(),
SchemaVersion: string.IsNullOrWhiteSpace(result.SchemaVersion) ? "1.0" : result.SchemaVersion.Trim(),
Analyzer: result.Analyzer);
}
private static CallgraphNode NormalizeNode(CallgraphNode node, string language)
{
var id = node.Id?.Trim();
if (string.IsNullOrWhiteSpace(id))
{
throw new CallgraphParserValidationException("Callgraph node is missing an id.");
}
var name = string.IsNullOrWhiteSpace(node.Name) ? id : node.Name.Trim();
var kind = string.IsNullOrWhiteSpace(node.Kind) ? "function" : node.Kind.Trim();
var normalizedLanguage = string.IsNullOrWhiteSpace(node.Language) ? language : node.Language.Trim();
var ns = string.IsNullOrWhiteSpace(node.Namespace)
? DeriveNamespace(id, node.File, normalizedLanguage)
: node.Namespace!.Trim();
return node with
{
Id = id,
Name = name,
Kind = kind,
Namespace = ns,
File = node.File?.Trim(),
Purl = NormalizePurl(node.Purl),
SymbolDigest = NormalizeDigest(node.SymbolDigest),
BuildId = node.BuildId?.Trim(),
Language = normalizedLanguage,
Evidence = NormalizeList(node.Evidence),
Analyzer = NormalizeDict(node.Analyzer),
CodeId = node.CodeId?.Trim()
};
}
private static IReadOnlyList<CallgraphEdge> NormalizeEdges(
IReadOnlyList<CallgraphEdge>? edges,
IReadOnlyDictionary<string, CallgraphNode> nodes)
{
var list = new List<CallgraphEdge>();
var seen = new HashSet<string>(StringComparer.Ordinal);
foreach (var edge in edges ?? Array.Empty<CallgraphEdge>())
{
var source = edge.SourceId?.Trim();
var target = edge.TargetId?.Trim();
if (string.IsNullOrWhiteSpace(source) || string.IsNullOrWhiteSpace(target))
{
continue;
}
if (!nodes.ContainsKey(source) || !nodes.ContainsKey(target))
{
continue;
}
var type = string.IsNullOrWhiteSpace(edge.Type) ? "call" : edge.Type.Trim();
var key = $"{source}|{target}|{type}";
if (!seen.Add(key))
{
continue;
}
list.Add(edge with
{
SourceId = source,
TargetId = target,
Type = type,
Purl = NormalizePurl(edge.Purl),
SymbolDigest = NormalizeDigest(edge.SymbolDigest),
Confidence = ClampConfidence(edge.Confidence),
Candidates = NormalizeList(edge.Candidates),
Evidence = NormalizeList(edge.Evidence)
});
}
return list.OrderBy(e => e.SourceId, StringComparer.Ordinal)
.ThenBy(e => e.TargetId, StringComparer.Ordinal)
.ThenBy(e => e.Type, StringComparer.Ordinal)
.ToList();
}
private static IReadOnlyList<CallgraphRoot> NormalizeRoots(IReadOnlyList<CallgraphRoot>? roots)
{
var list = new List<CallgraphRoot>();
var seen = new HashSet<string>(StringComparer.Ordinal);
foreach (var root in roots ?? Array.Empty<CallgraphRoot>())
{
var id = root.Id?.Trim();
if (string.IsNullOrWhiteSpace(id))
{
continue;
}
var normalized = new CallgraphRoot(
id,
string.IsNullOrWhiteSpace(root.Phase) ? "runtime" : root.Phase.Trim(),
root.Source?.Trim());
if (seen.Add($"{normalized.Id}|{normalized.Phase}|{normalized.Source}"))
{
list.Add(normalized);
}
}
return list.OrderBy(r => r.Id, StringComparer.Ordinal)
.ThenBy(r => r.Phase, StringComparer.Ordinal)
.ToList();
}
private static string? DeriveNamespace(string id, string? file, string language)
{
if (string.Equals(language, "java", StringComparison.OrdinalIgnoreCase))
{
var candidate = id.Replace('/', '.');
var lastDot = candidate.LastIndexOf('.');
if (lastDot > 0)
{
return candidate[..lastDot];
}
}
if (string.Equals(language, "go", StringComparison.OrdinalIgnoreCase) ||
string.Equals(language, "nodejs", StringComparison.OrdinalIgnoreCase) ||
string.Equals(language, "python", StringComparison.OrdinalIgnoreCase))
{
if (!string.IsNullOrWhiteSpace(file))
{
var normalizedPath = file.Replace('\\', '/');
var idx = normalizedPath.LastIndexOf('/');
if (idx > 0)
{
return normalizedPath[..idx];
}
}
var sepIdx = id.LastIndexOfAny(new[] { '.', '/', ':' });
if (sepIdx > 0)
{
return id[..sepIdx];
}
}
return null;
}
private static string? NormalizePurl(string? value)
{
return string.IsNullOrWhiteSpace(value) ? null : value.Trim().ToLowerInvariant();
}
private static string? NormalizeDigest(string? value)
{
return string.IsNullOrWhiteSpace(value) ? null : value.Trim().ToLowerInvariant();
}
private static double? ClampConfidence(double? confidence)
{
if (!confidence.HasValue)
{
return null;
}
return Math.Clamp(confidence.Value, 0.0, 1.0);
}
private static IReadOnlyList<string>? NormalizeList(IReadOnlyList<string>? values)
{
if (values is null)
{
return null;
}
return values
.Where(v => !string.IsNullOrWhiteSpace(v))
.Select(v => v.Trim())
.Distinct(StringComparer.Ordinal)
.OrderBy(v => v, StringComparer.Ordinal)
.ToList();
}
private static IReadOnlyDictionary<string, string?>? NormalizeDict(IReadOnlyDictionary<string, string?>? values)
{
if (values is null)
{
return null;
}
var dict = new Dictionary<string, string?>(StringComparer.Ordinal);
foreach (var kv in values)
{
if (string.IsNullOrWhiteSpace(kv.Key))
{
continue;
}
dict[kv.Key.Trim()] = kv.Value?.Trim();
}
return dict.Count == 0 ? null : dict;
}
}

View File

@@ -0,0 +1,10 @@
using System.Threading;
using System.Threading.Tasks;
using StackExchange.Redis;
namespace StellaOps.Signals.Services;
internal interface IRedisConnectionFactory
{
Task<IConnectionMultiplexer> ConnectAsync(ConfigurationOptions options, CancellationToken cancellationToken);
}

View File

@@ -1,11 +1,9 @@
using System;
using System.Linq;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Signals.Models;
using StellaOps.Signals.Options;
namespace StellaOps.Signals.Services;
@@ -15,65 +13,27 @@ namespace StellaOps.Signals.Services;
internal sealed class InMemoryEventsPublisher : IEventsPublisher
{
private readonly ILogger<InMemoryEventsPublisher> logger;
private readonly string topic;
private readonly ReachabilityFactEventBuilder eventBuilder;
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
WriteIndented = false,
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull
};
public InMemoryEventsPublisher(ILogger<InMemoryEventsPublisher> logger, SignalsOptions options)
public InMemoryEventsPublisher(ILogger<InMemoryEventsPublisher> logger, ReachabilityFactEventBuilder eventBuilder)
{
this.logger = logger;
topic = string.IsNullOrWhiteSpace(options?.AirGap?.EventTopic)
? "signals.fact.updated"
: options!.AirGap.EventTopic!;
this.eventBuilder = eventBuilder ?? throw new ArgumentNullException(nameof(eventBuilder));
}
public Task PublishFactUpdatedAsync(ReachabilityFactDocument fact, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(fact);
var (reachable, unreachable) = CountStates(fact);
var runtimeFactsCount = fact.RuntimeFacts?.Count ?? 0;
var avgConfidence = fact.States.Count > 0 ? fact.States.Average(s => s.Confidence) : 0;
var score = fact.Score;
var unknownsCount = fact.UnknownsCount;
var unknownsPressure = fact.UnknownsPressure;
var topBucket = fact.States.Count > 0
? fact.States
.GroupBy(s => s.Bucket, StringComparer.OrdinalIgnoreCase)
.OrderByDescending(g => g.Count())
.ThenByDescending(g => g.Average(s => s.Weight))
.First()
: null;
var payload = new ReachabilityFactUpdatedEvent(
Version: "signals.fact.updated@v1",
SubjectKey: fact.SubjectKey,
CallgraphId: string.IsNullOrWhiteSpace(fact.CallgraphId) ? null : fact.CallgraphId,
OccurredAtUtc: DateTimeOffset.UtcNow,
ReachableCount: reachable,
UnreachableCount: unreachable,
RuntimeFactsCount: runtimeFactsCount,
Bucket: topBucket?.Key ?? "unknown",
Weight: topBucket?.Average(s => s.Weight) ?? 0,
StateCount: fact.States.Count,
FactScore: score,
UnknownsCount: unknownsCount,
UnknownsPressure: unknownsPressure,
AverageConfidence: avgConfidence,
ComputedAtUtc: fact.ComputedAt,
Targets: fact.States.Select(s => s.Target).ToArray());
var envelope = eventBuilder.Build(fact);
var json = JsonSerializer.Serialize(envelope, SerializerOptions);
var json = JsonSerializer.Serialize(payload, new JsonSerializerOptions(JsonSerializerDefaults.Web));
logger.LogInformation("{Topic} {Payload}", topic, json);
logger.LogInformation(json);
return Task.CompletedTask;
}
private static (int reachable, int unreachable) CountStates(ReachabilityFactDocument fact)
{
if (fact.States is null || fact.States.Count == 0)
{
return (0, 0);
}
var reachable = fact.States.Count(state => state.Reachable);
var unreachable = fact.States.Count - reachable;
return (reachable, unreachable);
}
}

View File

@@ -0,0 +1,10 @@
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Signals.Models;
namespace StellaOps.Signals.Services;
internal sealed class NullEventsPublisher : IEventsPublisher
{
public Task PublishFactUpdatedAsync(ReachabilityFactDocument fact, CancellationToken cancellationToken) => Task.CompletedTask;
}

View File

@@ -0,0 +1,170 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using StellaOps.Signals.Models;
namespace StellaOps.Signals.Services;
internal static class ReachabilityFactDigestCalculator
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
WriteIndented = false
};
public static string Compute(ReachabilityFactDocument fact)
{
ArgumentNullException.ThrowIfNull(fact);
var canonical = new CanonicalReachabilityFact(
CallgraphId: fact.CallgraphId ?? string.Empty,
SubjectKey: fact.SubjectKey ?? string.Empty,
Subject: new CanonicalSubject(
fact.Subject?.ImageDigest ?? string.Empty,
fact.Subject?.Component ?? string.Empty,
fact.Subject?.Version ?? string.Empty,
fact.Subject?.ScanId ?? string.Empty),
EntryPoints: NormalizeList(fact.EntryPoints),
States: NormalizeStates(fact.States),
RuntimeFacts: NormalizeRuntimeFacts(fact.RuntimeFacts),
Metadata: NormalizeMetadata(fact.Metadata),
Score: fact.Score,
UnknownsCount: fact.UnknownsCount,
UnknownsPressure: fact.UnknownsPressure,
ComputedAt: fact.ComputedAt);
var json = JsonSerializer.Serialize(canonical, SerializerOptions);
Span<byte> hash = stackalloc byte[SHA256.HashSizeInBytes];
SHA256.HashData(Encoding.UTF8.GetBytes(json), hash);
return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant();
}
private static List<string> NormalizeList(IEnumerable<string>? values) =>
values?
.Where(v => !string.IsNullOrWhiteSpace(v))
.Select(v => v.Trim())
.Distinct(StringComparer.Ordinal)
.OrderBy(v => v, StringComparer.Ordinal)
.ToList() ?? new List<string>();
private static List<CanonicalState> NormalizeStates(IEnumerable<ReachabilityStateDocument>? states)
{
if (states is null)
{
return new List<CanonicalState>();
}
return states
.OrderBy(s => s.Target, StringComparer.Ordinal)
.Select(state => new CanonicalState(
Target: state.Target ?? string.Empty,
Reachable: state.Reachable,
Confidence: state.Confidence,
Bucket: state.Bucket ?? "unknown",
Weight: state.Weight,
Score: state.Score,
Path: NormalizeList(state.Path),
RuntimeHits: NormalizeList(state.Evidence?.RuntimeHits),
BlockedEdges: NormalizeList(state.Evidence?.BlockedEdges)))
.ToList();
}
private static List<CanonicalRuntimeFact> NormalizeRuntimeFacts(IEnumerable<RuntimeFactDocument>? facts)
{
if (facts is null)
{
return new List<CanonicalRuntimeFact>();
}
return facts
.Select(f => new CanonicalRuntimeFact(
SymbolId: f.SymbolId ?? string.Empty,
CodeId: f.CodeId,
SymbolDigest: f.SymbolDigest,
Purl: f.Purl,
BuildId: f.BuildId,
LoaderBase: f.LoaderBase,
ProcessId: f.ProcessId,
ProcessName: f.ProcessName,
SocketAddress: f.SocketAddress,
ContainerId: f.ContainerId,
EvidenceUri: f.EvidenceUri,
HitCount: f.HitCount,
ObservedAt: f.ObservedAt,
Metadata: NormalizeMetadata(f.Metadata)))
.OrderBy(f => f.SymbolId, StringComparer.Ordinal)
.ThenBy(f => f.CodeId, StringComparer.Ordinal)
.ThenBy(f => f.LoaderBase, StringComparer.Ordinal)
.ToList();
}
private static SortedDictionary<string, string?> NormalizeMetadata(IDictionary<string, string?>? metadata)
{
var normalized = new SortedDictionary<string, string?>(StringComparer.Ordinal);
if (metadata is null)
{
return normalized;
}
foreach (var kvp in metadata)
{
if (string.IsNullOrWhiteSpace(kvp.Key))
{
continue;
}
normalized[kvp.Key.Trim()] = kvp.Value?.Trim();
}
return normalized;
}
private sealed record CanonicalReachabilityFact(
string CallgraphId,
string SubjectKey,
CanonicalSubject Subject,
List<string> EntryPoints,
List<CanonicalState> States,
List<CanonicalRuntimeFact> RuntimeFacts,
SortedDictionary<string, string?> Metadata,
double Score,
int UnknownsCount,
double UnknownsPressure,
DateTimeOffset ComputedAt);
private sealed record CanonicalSubject(
string ImageDigest,
string Component,
string Version,
string ScanId);
private sealed record CanonicalState(
string Target,
bool Reachable,
double Confidence,
string Bucket,
double Weight,
double Score,
List<string> Path,
List<string> RuntimeHits,
List<string> BlockedEdges);
private sealed record CanonicalRuntimeFact(
string SymbolId,
string? CodeId,
string? SymbolDigest,
string? Purl,
string? BuildId,
string? LoaderBase,
int? ProcessId,
string? ProcessName,
string? SocketAddress,
string? ContainerId,
string? EvidenceUri,
int HitCount,
DateTimeOffset? ObservedAt,
SortedDictionary<string, string?> Metadata);
}

View File

@@ -0,0 +1,200 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text.Json.Serialization;
using StellaOps.Signals.Models;
using StellaOps.Signals.Options;
namespace StellaOps.Signals.Services;
internal sealed class ReachabilityFactEventBuilder
{
private readonly SignalsOptions options;
private readonly TimeProvider timeProvider;
public ReachabilityFactEventBuilder(SignalsOptions options, TimeProvider timeProvider)
{
this.options = options ?? throw new ArgumentNullException(nameof(options));
this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
}
public ReachabilityFactUpdatedEnvelope Build(ReachabilityFactDocument fact)
{
ArgumentNullException.ThrowIfNull(fact);
var summary = BuildSummary(fact);
var digest = ResolveDigest(fact);
var factVersion = ResolveFactVersion(fact);
return new ReachabilityFactUpdatedEnvelope(
Topic: ResolveTopic(),
EventId: Guid.NewGuid().ToString("n"),
Version: "signals.fact.updated@v1",
EmittedAtUtc: timeProvider.GetUtcNow(),
Tenant: ResolveTenant(fact),
SubjectKey: fact.SubjectKey,
CallgraphId: string.IsNullOrWhiteSpace(fact.CallgraphId) ? null : fact.CallgraphId,
FactKind: "reachability",
FactVersion: factVersion,
Digest: digest,
ContentType: "application/json",
Producer: new EventProducerMetadata(
Service: options.Events.Producer,
Pipeline: options.Events.Pipeline,
Release: options.Events.Release ?? typeof(Program).Assembly.GetName().Version?.ToString() ?? "unknown"),
Trace: ResolveTrace(fact),
Summary: summary);
}
private ReachabilityFactUpdatedEvent BuildSummary(ReachabilityFactDocument fact)
{
var (reachable, unreachable) = CountStates(fact);
var runtimeFactsCount = fact.RuntimeFacts?.Count ?? 0;
var avgConfidence = fact.States.Count > 0 ? fact.States.Average(s => s.Confidence) : 0;
var topBucket = fact.States.Count > 0
? fact.States
.GroupBy(s => s.Bucket, StringComparer.OrdinalIgnoreCase)
.OrderByDescending(g => g.Count())
.ThenByDescending(g => g.Average(s => s.Weight))
.First()
: null;
return new ReachabilityFactUpdatedEvent(
Version: "signals.fact.updated@v1",
SubjectKey: fact.SubjectKey,
CallgraphId: string.IsNullOrWhiteSpace(fact.CallgraphId) ? null : fact.CallgraphId,
OccurredAtUtc: timeProvider.GetUtcNow(),
ReachableCount: reachable,
UnreachableCount: unreachable,
RuntimeFactsCount: runtimeFactsCount,
Bucket: topBucket?.Key ?? "unknown",
Weight: topBucket?.Average(s => s.Weight) ?? 0,
StateCount: fact.States.Count,
FactScore: fact.Score,
UnknownsCount: fact.UnknownsCount,
UnknownsPressure: fact.UnknownsPressure,
AverageConfidence: avgConfidence,
ComputedAtUtc: fact.ComputedAt,
Targets: fact.States.Select(s => s.Target).ToArray());
}
private static (int reachable, int unreachable) CountStates(ReachabilityFactDocument fact)
{
if (fact.States is null || fact.States.Count == 0)
{
return (0, 0);
}
var reachable = fact.States.Count(state => state.Reachable);
var unreachable = fact.States.Count - reachable;
return (reachable, unreachable);
}
private string ResolveTopic()
{
if (!string.IsNullOrWhiteSpace(options.Events.Stream))
{
return options.Events.Stream;
}
if (!string.IsNullOrWhiteSpace(options.AirGap.EventTopic))
{
return options.AirGap.EventTopic!;
}
return "signals.fact.updated.v1";
}
private string ResolveTenant(ReachabilityFactDocument fact)
{
if (fact.Metadata is not null)
{
if (fact.Metadata.TryGetValue("tenant", out var tenant) && !string.IsNullOrWhiteSpace(tenant))
{
return tenant!;
}
if (fact.Metadata.TryGetValue("subject.tenant", out var subjectTenant) && !string.IsNullOrWhiteSpace(subjectTenant))
{
return subjectTenant!;
}
}
return options.Events.DefaultTenant;
}
private static EventTraceMetadata ResolveTrace(ReachabilityFactDocument fact)
{
var metadata = fact.Metadata;
string? traceId = null;
string? spanId = null;
if (metadata is not null)
{
metadata.TryGetValue("trace_id", out traceId);
metadata.TryGetValue("span_id", out spanId);
if (string.IsNullOrWhiteSpace(traceId) && metadata.TryGetValue("trace.id", out var dottedTrace))
{
traceId = dottedTrace;
}
if (string.IsNullOrWhiteSpace(spanId) && metadata.TryGetValue("trace.parent_span", out var dottedSpan))
{
spanId = dottedSpan;
}
}
return new EventTraceMetadata(traceId, spanId);
}
private static int ResolveFactVersion(ReachabilityFactDocument fact)
{
if (fact.Metadata is not null &&
fact.Metadata.TryGetValue("fact.version", out var versionValue) &&
int.TryParse(versionValue, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed))
{
return parsed;
}
return 1;
}
private static string ResolveDigest(ReachabilityFactDocument fact)
{
if (fact.Metadata is not null &&
fact.Metadata.TryGetValue("fact.digest", out var digest) &&
!string.IsNullOrWhiteSpace(digest))
{
return digest!;
}
return ReachabilityFactDigestCalculator.Compute(fact);
}
}
public sealed record ReachabilityFactUpdatedEnvelope(
[property: JsonPropertyName("topic")] string Topic,
[property: JsonPropertyName("event_id")] string EventId,
[property: JsonPropertyName("version")] string Version,
[property: JsonPropertyName("emitted_at")] DateTimeOffset EmittedAtUtc,
[property: JsonPropertyName("tenant")] string Tenant,
[property: JsonPropertyName("subject_key")] string SubjectKey,
[property: JsonPropertyName("callgraph_id")] string? CallgraphId,
[property: JsonPropertyName("fact_kind")] string FactKind,
[property: JsonPropertyName("fact_version")] int FactVersion,
[property: JsonPropertyName("digest")] string Digest,
[property: JsonPropertyName("content_type")] string ContentType,
[property: JsonPropertyName("producer")] EventProducerMetadata Producer,
[property: JsonPropertyName("trace")] EventTraceMetadata Trace,
[property: JsonPropertyName("summary")] ReachabilityFactUpdatedEvent Summary);
public sealed record EventProducerMetadata(
[property: JsonPropertyName("service")] string Service,
[property: JsonPropertyName("pipeline")] string Pipeline,
[property: JsonPropertyName("release")] string? Release);
public sealed record EventTraceMetadata(
[property: JsonPropertyName("trace_id")] string? TraceId,
[property: JsonPropertyName("span_id")] string? SpanId);

View File

@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Globalization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
@@ -90,7 +91,7 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
}
}
var runtimeHits = runtimeHitSet.ToList();
var runtimeHits = runtimeHitSet.OrderBy(h => h, StringComparer.Ordinal).ToList();
var states = new List<ReachabilityStateDocument>(targets.Count);
foreach (var target in targets)
@@ -108,6 +109,8 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
var score = confidence * weight;
runtimeEvidence = runtimeEvidence.OrderBy(hit => hit, StringComparer.Ordinal).ToList();
states.Add(new ReachabilityStateDocument
{
Target = target,
@@ -120,11 +123,17 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
Evidence = new ReachabilityEvidenceDocument
{
RuntimeHits = runtimeEvidence,
BlockedEdges = request.BlockedEdges?.Select(edge => $"{edge.From} -> {edge.To}").ToList()
BlockedEdges = request.BlockedEdges?
.Select(edge => $"{edge.From} -> {edge.To}")
.OrderBy(edge => edge, StringComparer.Ordinal)
.ToList()
}
});
}
states = states.OrderBy(s => s.Target, StringComparer.Ordinal).ToList();
entryPoints = entryPoints.OrderBy(ep => ep, StringComparer.Ordinal).ToList();
var baseScore = states.Count > 0 ? states.Average(s => s.Score) : 0;
var unknownsCount = await unknownsRepository.CountBySubjectAsync(subjectKey, cancellationToken).ConfigureAwait(false);
var pressure = states.Count + unknownsCount == 0
@@ -148,6 +157,22 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
RuntimeFacts = existingFact?.RuntimeFacts
};
document.Metadata ??= new Dictionary<string, string?>(StringComparer.Ordinal);
var priorVersion = 0;
if (existingFact?.Metadata != null
&& existingFact.Metadata.TryGetValue("fact.version", out var versionValue)
&& int.TryParse(versionValue, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedVersion))
{
priorVersion = parsedVersion;
}
var nextVersion = priorVersion + 1;
document.Metadata["fact.version"] = nextVersion.ToString(CultureInfo.InvariantCulture);
document.Metadata.Remove("fact.digest");
document.Metadata.Remove("fact.digest.alg");
document.Metadata["fact.digest"] = ReachabilityFactDigestCalculator.Compute(document);
document.Metadata["fact.digest.alg"] = "sha256";
logger.LogInformation("Computed reachability fact for subject {SubjectKey} with {StateCount} targets.", document.SubjectKey, states.Count);
var persisted = await factRepository.UpsertAsync(document, cancellationToken).ConfigureAwait(false);
await cache.SetAsync(persisted, cancellationToken).ConfigureAwait(false);
@@ -266,7 +291,7 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
continue;
}
foreach (var neighbor in neighbors)
foreach (var neighbor in neighbors.OrderBy(n => n, StringComparer.Ordinal))
{
if (visited.Add(neighbor))
{

View File

@@ -60,8 +60,14 @@ public sealed class ReachabilityUnionIngestionService : IReachabilityUnionIngest
}
var metaEntry = entries["meta.json"];
using var metaStream = metaEntry.Open();
using var metaDoc = await JsonDocument.ParseAsync(metaStream, cancellationToken: cancellationToken).ConfigureAwait(false);
await using var metaBuffer = new MemoryStream();
await using (var metaStream = metaEntry.Open())
{
await metaStream.CopyToAsync(metaBuffer, cancellationToken).ConfigureAwait(false);
}
metaBuffer.Position = 0;
using var metaDoc = await JsonDocument.ParseAsync(metaBuffer, cancellationToken: cancellationToken).ConfigureAwait(false);
var metaRoot = metaDoc.RootElement;
var filesElement = metaRoot.TryGetProperty("files", out var f) && f.ValueKind == JsonValueKind.Array
@@ -77,6 +83,13 @@ public sealed class ReachabilityUnionIngestionService : IReachabilityUnionIngest
})
.ToList();
metaBuffer.Position = 0;
var metaPath = Path.Combine(casRoot, "meta.json");
await using (var metaDest = File.Create(metaPath))
{
await metaBuffer.CopyToAsync(metaDest, cancellationToken).ConfigureAwait(false);
}
var filesForResponse = new List<ReachabilityUnionFile>();
foreach (var file in recorded)

View File

@@ -0,0 +1,19 @@
using System.Threading;
using System.Threading.Tasks;
using StackExchange.Redis;
namespace StellaOps.Signals.Services;
internal sealed class RedisConnectionFactory : IRedisConnectionFactory
{
public Task<IConnectionMultiplexer> ConnectAsync(ConfigurationOptions options, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
return ConnectionMultiplexer.ConnectAsync(options)
.ContinueWith(
t => (IConnectionMultiplexer)t.Result,
cancellationToken,
TaskContinuationOptions.OnlyOnRanToCompletion | TaskContinuationOptions.ExecuteSynchronously,
TaskScheduler.Current);
}
}

View File

@@ -0,0 +1,185 @@
using System;
using System.Globalization;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StackExchange.Redis;
using StellaOps.Signals.Models;
using StellaOps.Signals.Options;
namespace StellaOps.Signals.Services;
internal sealed class RedisEventsPublisher : IEventsPublisher, IAsyncDisposable
{
private readonly SignalsEventsOptions options;
private readonly ILogger<RedisEventsPublisher> logger;
private readonly IRedisConnectionFactory connectionFactory;
private readonly ReachabilityFactEventBuilder eventBuilder;
private readonly TimeSpan publishTimeout;
private readonly int? maxStreamLength;
private readonly SemaphoreSlim connectionGate = new(1, 1);
private IConnectionMultiplexer? connection;
private bool disposed;
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
WriteIndented = false,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
public RedisEventsPublisher(
SignalsOptions options,
IRedisConnectionFactory connectionFactory,
ReachabilityFactEventBuilder eventBuilder,
ILogger<RedisEventsPublisher> logger)
{
ArgumentNullException.ThrowIfNull(options);
this.options = options.Events ?? throw new InvalidOperationException("Signals events configuration is required.");
this.connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory));
this.eventBuilder = eventBuilder ?? throw new ArgumentNullException(nameof(eventBuilder));
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
publishTimeout = this.options.PublishTimeoutSeconds > 0
? TimeSpan.FromSeconds(this.options.PublishTimeoutSeconds)
: TimeSpan.Zero;
maxStreamLength = this.options.MaxStreamLength > 0
? (int)Math.Min(this.options.MaxStreamLength, int.MaxValue)
: null;
}
public async Task PublishFactUpdatedAsync(ReachabilityFactDocument fact, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(fact);
cancellationToken.ThrowIfCancellationRequested();
if (!options.Enabled)
{
return;
}
var envelope = eventBuilder.Build(fact);
var json = JsonSerializer.Serialize(envelope, SerializerOptions);
try
{
var database = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
var entries = new[]
{
new NameValueEntry("event", json),
new NameValueEntry("event_id", envelope.EventId),
new NameValueEntry("subject_key", envelope.SubjectKey),
new NameValueEntry("digest", envelope.Digest),
new NameValueEntry("fact_version", envelope.FactVersion.ToString(CultureInfo.InvariantCulture))
};
var publishTask = maxStreamLength.HasValue
? database.StreamAddAsync(options.Stream, entries, maxLength: maxStreamLength, useApproximateMaxLength: true)
: database.StreamAddAsync(options.Stream, entries);
if (publishTimeout > TimeSpan.Zero)
{
await publishTask.WaitAsync(publishTimeout, cancellationToken).ConfigureAwait(false);
}
else
{
await publishTask.ConfigureAwait(false);
}
}
catch (Exception ex)
{
logger.LogError(ex, "Failed to publish reachability event to Redis stream {Stream}.", options.Stream);
await TryPublishDeadLetterAsync(json, cancellationToken).ConfigureAwait(false);
}
}
private async Task<IDatabase> GetDatabaseAsync(CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
if (connection is { IsConnected: true })
{
return connection.GetDatabase();
}
await connectionGate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (connection is null || !connection.IsConnected)
{
var configuration = ConfigurationOptions.Parse(options.ConnectionString!);
configuration.AbortOnConnectFail = false;
connection = await connectionFactory.ConnectAsync(configuration, cancellationToken).ConfigureAwait(false);
logger.LogInformation("Connected Signals events publisher to Redis stream {Stream}.", options.Stream);
}
}
finally
{
connectionGate.Release();
}
return connection!.GetDatabase();
}
private async Task TryPublishDeadLetterAsync(string json, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(options.DeadLetterStream) || connection is null || !connection.IsConnected)
{
return;
}
try
{
var db = connection.GetDatabase();
var entries = new[]
{
new NameValueEntry("event", json),
new NameValueEntry("error", "publish-failed")
};
var dlqTask = maxStreamLength.HasValue
? db.StreamAddAsync(options.DeadLetterStream, entries, maxLength: maxStreamLength, useApproximateMaxLength: true)
: db.StreamAddAsync(options.DeadLetterStream, entries);
if (publishTimeout > TimeSpan.Zero)
{
await dlqTask.WaitAsync(publishTimeout, cancellationToken).ConfigureAwait(false);
}
else
{
await dlqTask.ConfigureAwait(false);
}
}
catch (Exception ex)
{
logger.LogWarning(ex, "Failed to publish reachability event to DLQ stream {Stream}.", options.DeadLetterStream);
}
}
public async ValueTask DisposeAsync()
{
if (disposed)
{
return;
}
disposed = true;
if (connection is not null)
{
try
{
await connection.CloseAsync();
}
catch (Exception ex)
{
logger.LogDebug(ex, "Error closing Redis events publisher connection.");
}
connection.Dispose();
}
connectionGate.Dispose();
}
}