up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled

This commit is contained in:
StellaOps Bot
2025-12-13 00:20:26 +02:00
parent e1f1bef4c1
commit 564df71bfb
2376 changed files with 334389 additions and 328032 deletions

View File

@@ -1,300 +1,300 @@
using System.Collections.Immutable;
using System.Text.Json.Nodes;
using StellaOps.Notify.Engine;
using StellaOps.Notify.Models;
namespace StellaOps.Notifier.Worker.Processing;
public sealed class DefaultNotifyRuleEvaluator : INotifyRuleEvaluator
{
private static readonly IDictionary<string, int> SeverityRank = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase)
{
["none"] = 0,
["info"] = 1,
["low"] = 2,
["medium"] = 3,
["moderate"] = 3,
["high"] = 4,
["critical"] = 5,
["blocker"] = 6,
};
public NotifyRuleEvaluationOutcome Evaluate(NotifyRule rule, NotifyEvent @event, DateTimeOffset? evaluationTimestamp = null)
{
ArgumentNullException.ThrowIfNull(rule);
ArgumentNullException.ThrowIfNull(@event);
if (!rule.Enabled)
{
return NotifyRuleEvaluationOutcome.NotMatched(rule, "rule_disabled");
}
var match = rule.Match;
if (!match.EventKinds.IsDefaultOrEmpty && !match.EventKinds.Contains(@event.Kind))
{
return NotifyRuleEvaluationOutcome.NotMatched(rule, "event_kind_mismatch");
}
if (!match.Namespaces.IsDefaultOrEmpty)
{
var ns = @event.Scope?.Namespace ?? string.Empty;
if (!match.Namespaces.Contains(ns))
{
return NotifyRuleEvaluationOutcome.NotMatched(rule, "namespace_mismatch");
}
}
if (!match.Repositories.IsDefaultOrEmpty)
{
var repo = @event.Scope?.Repo ?? string.Empty;
if (!match.Repositories.Contains(repo))
{
return NotifyRuleEvaluationOutcome.NotMatched(rule, "repository_mismatch");
}
}
if (!match.Digests.IsDefaultOrEmpty)
{
var digest = @event.Scope?.Digest ?? string.Empty;
if (!match.Digests.Contains(digest))
{
return NotifyRuleEvaluationOutcome.NotMatched(rule, "digest_mismatch");
}
}
if (!match.ComponentPurls.IsDefaultOrEmpty)
{
var components = ExtractComponentPurls(@event.Payload);
if (!components.Overlaps(match.ComponentPurls))
{
return NotifyRuleEvaluationOutcome.NotMatched(rule, "component_mismatch");
}
}
if (match.KevOnly == true && !ExtractLabels(@event).Contains("kev"))
{
return NotifyRuleEvaluationOutcome.NotMatched(rule, "kev_required");
}
if (!match.Labels.IsDefaultOrEmpty)
{
var labels = ExtractLabels(@event);
if (!labels.IsSupersetOf(match.Labels))
{
return NotifyRuleEvaluationOutcome.NotMatched(rule, "label_mismatch");
}
}
if (!string.IsNullOrWhiteSpace(match.MinSeverity))
{
var eventSeverity = ResolveSeverity(@event);
if (!MeetsSeverity(match.MinSeverity!, eventSeverity))
{
return NotifyRuleEvaluationOutcome.NotMatched(rule, "severity_below_threshold");
}
}
if (!match.Verdicts.IsDefaultOrEmpty)
{
var verdict = ResolveVerdict(@event);
if (verdict is null || !match.Verdicts.Contains(verdict))
{
return NotifyRuleEvaluationOutcome.NotMatched(rule, "verdict_mismatch");
}
}
var actions = rule.Actions
.Where(static action => action is not null && action.Enabled)
.Distinct()
.OrderBy(static action => action.ActionId, StringComparer.Ordinal)
.ToImmutableArray();
if (actions.IsDefaultOrEmpty)
{
return NotifyRuleEvaluationOutcome.NotMatched(rule, "no_enabled_actions");
}
var matchedAt = evaluationTimestamp ?? DateTimeOffset.UtcNow;
return NotifyRuleEvaluationOutcome.Matched(rule, actions, matchedAt);
}
public ImmutableArray<NotifyRuleEvaluationOutcome> Evaluate(
IEnumerable<NotifyRule> rules,
NotifyEvent @event,
DateTimeOffset? evaluationTimestamp = null)
{
if (rules is null)
{
return ImmutableArray<NotifyRuleEvaluationOutcome>.Empty;
}
return rules
.Select(rule => Evaluate(rule, @event, evaluationTimestamp))
.Where(static outcome => outcome.IsMatch)
.ToImmutableArray();
}
private static bool MeetsSeverity(string required, string actual)
{
if (!SeverityRank.TryGetValue(required, out var requiredRank))
{
return true;
}
if (!SeverityRank.TryGetValue(actual, out var actualRank))
{
actualRank = 0;
}
return actualRank >= requiredRank;
}
private static string ResolveSeverity(NotifyEvent @event)
{
if (@event.Attributes.TryGetValue("severity", out var attributeSeverity) && !string.IsNullOrWhiteSpace(attributeSeverity))
{
return attributeSeverity.ToLowerInvariant();
}
if (@event.Payload is JsonObject obj)
{
if (TryGetString(obj, "severity", out var severity))
{
return severity.ToLowerInvariant();
}
if (obj.TryGetPropertyValue("summary", out var summaryNode) && summaryNode is JsonObject summaryObj)
{
if (TryGetString(summaryObj, "highestSeverity", out var summarySeverity))
{
return summarySeverity.ToLowerInvariant();
}
}
}
return "unknown";
}
private static string? ResolveVerdict(NotifyEvent @event)
{
if (@event.Attributes.TryGetValue("verdict", out var attributeVerdict) && !string.IsNullOrWhiteSpace(attributeVerdict))
{
return attributeVerdict.ToLowerInvariant();
}
if (@event.Payload is JsonObject obj)
{
if (TryGetString(obj, "verdict", out var verdict))
{
return verdict.ToLowerInvariant();
}
if (obj.TryGetPropertyValue("summary", out var summaryNode) && summaryNode is JsonObject summaryObj)
{
if (TryGetString(summaryObj, "verdict", out var summaryVerdict))
{
return summaryVerdict.ToLowerInvariant();
}
}
}
return null;
}
private static bool TryGetString(JsonObject obj, string propertyName, out string value)
{
if (obj.TryGetPropertyValue(propertyName, out var node) && node is JsonValue jsonValue && jsonValue.TryGetValue(out string? str) && !string.IsNullOrWhiteSpace(str))
{
value = str.Trim();
return true;
}
value = string.Empty;
return false;
}
private static ImmutableHashSet<string> ExtractComponentPurls(JsonNode? payload)
{
var builder = ImmutableHashSet.CreateBuilder<string>(StringComparer.OrdinalIgnoreCase);
if (payload is JsonObject obj && obj.TryGetPropertyValue("componentPurls", out var arrayNode) && arrayNode is JsonArray array)
{
foreach (var item in array)
{
if (item is JsonValue value && value.TryGetValue(out string? str) && !string.IsNullOrWhiteSpace(str))
{
builder.Add(str.Trim());
}
}
}
return builder.ToImmutable();
}
private static ImmutableHashSet<string> ExtractLabels(NotifyEvent @event)
{
var builder = ImmutableHashSet.CreateBuilder<string>(StringComparer.OrdinalIgnoreCase);
foreach (var (key, value) in @event.Attributes)
{
if (!string.IsNullOrWhiteSpace(key))
{
builder.Add(key.Trim());
}
if (!string.IsNullOrWhiteSpace(value))
{
builder.Add(value.Trim());
}
}
if (@event.Scope?.Labels is { Count: > 0 } scopeLabels)
{
foreach (var (key, value) in scopeLabels)
{
if (!string.IsNullOrWhiteSpace(key))
{
builder.Add(key.Trim());
}
if (!string.IsNullOrWhiteSpace(value))
{
builder.Add(value.Trim());
}
}
}
if (@event.Payload is JsonObject obj && obj.TryGetPropertyValue("labels", out var labelsNode))
{
switch (labelsNode)
{
case JsonArray array:
foreach (var item in array)
{
if (item is JsonValue value && value.TryGetValue(out string? str) && !string.IsNullOrWhiteSpace(str))
{
builder.Add(str.Trim());
}
}
break;
case JsonObject labelObj:
foreach (var (key, value) in labelObj)
{
if (!string.IsNullOrWhiteSpace(key))
{
builder.Add(key.Trim());
}
if (value is JsonValue v && v.TryGetValue(out string? str) && !string.IsNullOrWhiteSpace(str))
{
builder.Add(str.Trim());
}
}
break;
}
}
return builder.ToImmutable();
}
}
using System.Collections.Immutable;
using System.Text.Json.Nodes;
using StellaOps.Notify.Engine;
using StellaOps.Notify.Models;
namespace StellaOps.Notifier.Worker.Processing;
public sealed class DefaultNotifyRuleEvaluator : INotifyRuleEvaluator
{
private static readonly IDictionary<string, int> SeverityRank = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase)
{
["none"] = 0,
["info"] = 1,
["low"] = 2,
["medium"] = 3,
["moderate"] = 3,
["high"] = 4,
["critical"] = 5,
["blocker"] = 6,
};
public NotifyRuleEvaluationOutcome Evaluate(NotifyRule rule, NotifyEvent @event, DateTimeOffset? evaluationTimestamp = null)
{
ArgumentNullException.ThrowIfNull(rule);
ArgumentNullException.ThrowIfNull(@event);
if (!rule.Enabled)
{
return NotifyRuleEvaluationOutcome.NotMatched(rule, "rule_disabled");
}
var match = rule.Match;
if (!match.EventKinds.IsDefaultOrEmpty && !match.EventKinds.Contains(@event.Kind))
{
return NotifyRuleEvaluationOutcome.NotMatched(rule, "event_kind_mismatch");
}
if (!match.Namespaces.IsDefaultOrEmpty)
{
var ns = @event.Scope?.Namespace ?? string.Empty;
if (!match.Namespaces.Contains(ns))
{
return NotifyRuleEvaluationOutcome.NotMatched(rule, "namespace_mismatch");
}
}
if (!match.Repositories.IsDefaultOrEmpty)
{
var repo = @event.Scope?.Repo ?? string.Empty;
if (!match.Repositories.Contains(repo))
{
return NotifyRuleEvaluationOutcome.NotMatched(rule, "repository_mismatch");
}
}
if (!match.Digests.IsDefaultOrEmpty)
{
var digest = @event.Scope?.Digest ?? string.Empty;
if (!match.Digests.Contains(digest))
{
return NotifyRuleEvaluationOutcome.NotMatched(rule, "digest_mismatch");
}
}
if (!match.ComponentPurls.IsDefaultOrEmpty)
{
var components = ExtractComponentPurls(@event.Payload);
if (!components.Overlaps(match.ComponentPurls))
{
return NotifyRuleEvaluationOutcome.NotMatched(rule, "component_mismatch");
}
}
if (match.KevOnly == true && !ExtractLabels(@event).Contains("kev"))
{
return NotifyRuleEvaluationOutcome.NotMatched(rule, "kev_required");
}
if (!match.Labels.IsDefaultOrEmpty)
{
var labels = ExtractLabels(@event);
if (!labels.IsSupersetOf(match.Labels))
{
return NotifyRuleEvaluationOutcome.NotMatched(rule, "label_mismatch");
}
}
if (!string.IsNullOrWhiteSpace(match.MinSeverity))
{
var eventSeverity = ResolveSeverity(@event);
if (!MeetsSeverity(match.MinSeverity!, eventSeverity))
{
return NotifyRuleEvaluationOutcome.NotMatched(rule, "severity_below_threshold");
}
}
if (!match.Verdicts.IsDefaultOrEmpty)
{
var verdict = ResolveVerdict(@event);
if (verdict is null || !match.Verdicts.Contains(verdict))
{
return NotifyRuleEvaluationOutcome.NotMatched(rule, "verdict_mismatch");
}
}
var actions = rule.Actions
.Where(static action => action is not null && action.Enabled)
.Distinct()
.OrderBy(static action => action.ActionId, StringComparer.Ordinal)
.ToImmutableArray();
if (actions.IsDefaultOrEmpty)
{
return NotifyRuleEvaluationOutcome.NotMatched(rule, "no_enabled_actions");
}
var matchedAt = evaluationTimestamp ?? DateTimeOffset.UtcNow;
return NotifyRuleEvaluationOutcome.Matched(rule, actions, matchedAt);
}
public ImmutableArray<NotifyRuleEvaluationOutcome> Evaluate(
IEnumerable<NotifyRule> rules,
NotifyEvent @event,
DateTimeOffset? evaluationTimestamp = null)
{
if (rules is null)
{
return ImmutableArray<NotifyRuleEvaluationOutcome>.Empty;
}
return rules
.Select(rule => Evaluate(rule, @event, evaluationTimestamp))
.Where(static outcome => outcome.IsMatch)
.ToImmutableArray();
}
private static bool MeetsSeverity(string required, string actual)
{
if (!SeverityRank.TryGetValue(required, out var requiredRank))
{
return true;
}
if (!SeverityRank.TryGetValue(actual, out var actualRank))
{
actualRank = 0;
}
return actualRank >= requiredRank;
}
private static string ResolveSeverity(NotifyEvent @event)
{
if (@event.Attributes.TryGetValue("severity", out var attributeSeverity) && !string.IsNullOrWhiteSpace(attributeSeverity))
{
return attributeSeverity.ToLowerInvariant();
}
if (@event.Payload is JsonObject obj)
{
if (TryGetString(obj, "severity", out var severity))
{
return severity.ToLowerInvariant();
}
if (obj.TryGetPropertyValue("summary", out var summaryNode) && summaryNode is JsonObject summaryObj)
{
if (TryGetString(summaryObj, "highestSeverity", out var summarySeverity))
{
return summarySeverity.ToLowerInvariant();
}
}
}
return "unknown";
}
private static string? ResolveVerdict(NotifyEvent @event)
{
if (@event.Attributes.TryGetValue("verdict", out var attributeVerdict) && !string.IsNullOrWhiteSpace(attributeVerdict))
{
return attributeVerdict.ToLowerInvariant();
}
if (@event.Payload is JsonObject obj)
{
if (TryGetString(obj, "verdict", out var verdict))
{
return verdict.ToLowerInvariant();
}
if (obj.TryGetPropertyValue("summary", out var summaryNode) && summaryNode is JsonObject summaryObj)
{
if (TryGetString(summaryObj, "verdict", out var summaryVerdict))
{
return summaryVerdict.ToLowerInvariant();
}
}
}
return null;
}
private static bool TryGetString(JsonObject obj, string propertyName, out string value)
{
if (obj.TryGetPropertyValue(propertyName, out var node) && node is JsonValue jsonValue && jsonValue.TryGetValue(out string? str) && !string.IsNullOrWhiteSpace(str))
{
value = str.Trim();
return true;
}
value = string.Empty;
return false;
}
private static ImmutableHashSet<string> ExtractComponentPurls(JsonNode? payload)
{
var builder = ImmutableHashSet.CreateBuilder<string>(StringComparer.OrdinalIgnoreCase);
if (payload is JsonObject obj && obj.TryGetPropertyValue("componentPurls", out var arrayNode) && arrayNode is JsonArray array)
{
foreach (var item in array)
{
if (item is JsonValue value && value.TryGetValue(out string? str) && !string.IsNullOrWhiteSpace(str))
{
builder.Add(str.Trim());
}
}
}
return builder.ToImmutable();
}
private static ImmutableHashSet<string> ExtractLabels(NotifyEvent @event)
{
var builder = ImmutableHashSet.CreateBuilder<string>(StringComparer.OrdinalIgnoreCase);
foreach (var (key, value) in @event.Attributes)
{
if (!string.IsNullOrWhiteSpace(key))
{
builder.Add(key.Trim());
}
if (!string.IsNullOrWhiteSpace(value))
{
builder.Add(value.Trim());
}
}
if (@event.Scope?.Labels is { Count: > 0 } scopeLabels)
{
foreach (var (key, value) in scopeLabels)
{
if (!string.IsNullOrWhiteSpace(key))
{
builder.Add(key.Trim());
}
if (!string.IsNullOrWhiteSpace(value))
{
builder.Add(value.Trim());
}
}
}
if (@event.Payload is JsonObject obj && obj.TryGetPropertyValue("labels", out var labelsNode))
{
switch (labelsNode)
{
case JsonArray array:
foreach (var item in array)
{
if (item is JsonValue value && value.TryGetValue(out string? str) && !string.IsNullOrWhiteSpace(str))
{
builder.Add(str.Trim());
}
}
break;
case JsonObject labelObj:
foreach (var (key, value) in labelObj)
{
if (!string.IsNullOrWhiteSpace(key))
{
builder.Add(key.Trim());
}
if (value is JsonValue v && v.TryGetValue(out string? str) && !string.IsNullOrWhiteSpace(str))
{
builder.Add(str.Trim());
}
}
break;
}
}
return builder.ToImmutable();
}
}

View File

@@ -1,45 +1,45 @@
using System.Collections.Generic;
using StellaOps.Notify.Models;
namespace StellaOps.Notifier.Worker.Processing;
/// <summary>
/// Tracks per-event delivery intents for SLO evaluation and webhook emission.
/// </summary>
internal sealed class EgressSloContext
{
private readonly List<EgressSloSignal> _signals = new();
public IReadOnlyList<EgressSloSignal> Signals => _signals;
public static EgressSloContext FromNotifyEvent(NotifyEvent notifyEvent)
{
ArgumentNullException.ThrowIfNull(notifyEvent);
return new EgressSloContext
{
EventId = notifyEvent.EventId,
TenantId = notifyEvent.Tenant,
EventKind = notifyEvent.Kind,
OccurredAt = notifyEvent.Ts
};
}
public Guid EventId { get; private set; }
public string TenantId { get; private set; } = string.Empty;
public string EventKind { get; private set; } = string.Empty;
public DateTimeOffset OccurredAt { get; private set; }
public void AddDelivery(string channelType, string template, string kind)
{
_signals.Add(new EgressSloSignal(channelType, template, kind, OccurredAt));
}
}
internal sealed record EgressSloSignal(
string Channel,
string Template,
string Kind,
DateTimeOffset OccurredAt);
using System.Collections.Generic;
using StellaOps.Notify.Models;
namespace StellaOps.Notifier.Worker.Processing;
/// <summary>
/// Tracks per-event delivery intents for SLO evaluation and webhook emission.
/// </summary>
internal sealed class EgressSloContext
{
private readonly List<EgressSloSignal> _signals = new();
public IReadOnlyList<EgressSloSignal> Signals => _signals;
public static EgressSloContext FromNotifyEvent(NotifyEvent notifyEvent)
{
ArgumentNullException.ThrowIfNull(notifyEvent);
return new EgressSloContext
{
EventId = notifyEvent.EventId,
TenantId = notifyEvent.Tenant,
EventKind = notifyEvent.Kind,
OccurredAt = notifyEvent.Ts
};
}
public Guid EventId { get; private set; }
public string TenantId { get; private set; } = string.Empty;
public string EventKind { get; private set; } = string.Empty;
public DateTimeOffset OccurredAt { get; private set; }
public void AddDelivery(string channelType, string template, string kind)
{
_signals.Add(new EgressSloSignal(channelType, template, kind, OccurredAt));
}
}
internal sealed record EgressSloSignal(
string Channel,
string Template,
string Kind,
DateTimeOffset OccurredAt);

View File

@@ -1,93 +1,93 @@
using System.Net.Http;
using System.Net.Http.Json;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Notifier.Worker.Options;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Notifier.Worker.Processing;
internal sealed class HttpEgressSloSink : IEgressSloSink
{
private readonly IHttpClientFactory _clientFactory;
private readonly EgressSloOptions _options;
private readonly ILogger<HttpEgressSloSink> _logger;
public HttpEgressSloSink(
IHttpClientFactory clientFactory,
IOptions<EgressSloOptions> options,
ILogger<HttpEgressSloSink> logger)
{
_clientFactory = clientFactory ?? throw new ArgumentNullException(nameof(clientFactory));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task PublishAsync(EgressSloContext context, CancellationToken cancellationToken)
{
if (context is null)
{
throw new ArgumentNullException(nameof(context));
}
if (!_options.Enabled)
{
return;
}
using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
if (_options.TimeoutSeconds > 0)
{
linkedCts.CancelAfter(TimeSpan.FromSeconds(_options.TimeoutSeconds));
}
try
{
var client = _clientFactory.CreateClient("notifier-slo-webhook");
var payload = Map(context);
var response = await client.PostAsJsonAsync(_options.Webhook, payload, linkedCts.Token).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
_logger.LogWarning(
"SLO webhook returned non-success status {StatusCode} for event {EventId} (tenant {TenantId}).",
(int)response.StatusCode,
context.EventId,
context.TenantId);
}
}
catch (OperationCanceledException) when (linkedCts.IsCancellationRequested)
{
_logger.LogWarning(
"SLO webhook timed out after {TimeoutSeconds}s for event {EventId} (tenant {TenantId}).",
_options.TimeoutSeconds,
context.EventId,
context.TenantId);
}
catch (Exception ex)
{
_logger.LogWarning(
ex,
"Failed to publish SLO webhook for event {EventId} (tenant {TenantId}).",
context.EventId,
context.TenantId);
}
}
private static object Map(EgressSloContext context)
=> new
{
context.EventId,
context.TenantId,
context.EventKind,
context.OccurredAt,
deliveries = context.Signals.Select(signal => new
{
signal.Channel,
signal.Template,
signal.Kind,
signal.OccurredAt
})
};
}
using System.Net.Http;
using System.Net.Http.Json;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Notifier.Worker.Options;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Notifier.Worker.Processing;
internal sealed class HttpEgressSloSink : IEgressSloSink
{
private readonly IHttpClientFactory _clientFactory;
private readonly EgressSloOptions _options;
private readonly ILogger<HttpEgressSloSink> _logger;
public HttpEgressSloSink(
IHttpClientFactory clientFactory,
IOptions<EgressSloOptions> options,
ILogger<HttpEgressSloSink> logger)
{
_clientFactory = clientFactory ?? throw new ArgumentNullException(nameof(clientFactory));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task PublishAsync(EgressSloContext context, CancellationToken cancellationToken)
{
if (context is null)
{
throw new ArgumentNullException(nameof(context));
}
if (!_options.Enabled)
{
return;
}
using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
if (_options.TimeoutSeconds > 0)
{
linkedCts.CancelAfter(TimeSpan.FromSeconds(_options.TimeoutSeconds));
}
try
{
var client = _clientFactory.CreateClient("notifier-slo-webhook");
var payload = Map(context);
var response = await client.PostAsJsonAsync(_options.Webhook, payload, linkedCts.Token).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
_logger.LogWarning(
"SLO webhook returned non-success status {StatusCode} for event {EventId} (tenant {TenantId}).",
(int)response.StatusCode,
context.EventId,
context.TenantId);
}
}
catch (OperationCanceledException) when (linkedCts.IsCancellationRequested)
{
_logger.LogWarning(
"SLO webhook timed out after {TimeoutSeconds}s for event {EventId} (tenant {TenantId}).",
_options.TimeoutSeconds,
context.EventId,
context.TenantId);
}
catch (Exception ex)
{
_logger.LogWarning(
ex,
"Failed to publish SLO webhook for event {EventId} (tenant {TenantId}).",
context.EventId,
context.TenantId);
}
}
private static object Map(EgressSloContext context)
=> new
{
context.EventId,
context.TenantId,
context.EventKind,
context.OccurredAt,
deliveries = context.Signals.Select(signal => new
{
signal.Channel,
signal.Template,
signal.Kind,
signal.OccurredAt
})
};
}

View File

@@ -1,17 +1,17 @@
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Notifier.Worker.Processing;
internal interface IEgressSloSink
{
Task PublishAsync(EgressSloContext context, CancellationToken cancellationToken);
}
internal sealed class NullEgressSloSink : IEgressSloSink
{
public Task PublishAsync(EgressSloContext context, CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
}
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Notifier.Worker.Processing;
internal interface IEgressSloSink
{
Task PublishAsync(EgressSloContext context, CancellationToken cancellationToken);
}
internal sealed class NullEgressSloSink : IEgressSloSink
{
public Task PublishAsync(EgressSloContext context, CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
}

View File

@@ -1,30 +1,30 @@
using System.Security.Cryptography;
using System.Text;
using StellaOps.Notify.Models;
namespace StellaOps.Notifier.Worker.Processing;
internal static class IdempotencyKeyBuilder
{
public static string Build(string tenantId, string ruleId, string actionId, NotifyEvent notifyEvent)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
ArgumentException.ThrowIfNullOrWhiteSpace(ruleId);
ArgumentException.ThrowIfNullOrWhiteSpace(actionId);
ArgumentNullException.ThrowIfNull(notifyEvent);
var scopeDigest = notifyEvent.Scope?.Digest ?? string.Empty;
var source = string.Join(
'|',
tenantId,
ruleId,
actionId,
notifyEvent.Kind,
scopeDigest,
notifyEvent.EventId.ToString("N"));
var bytes = Encoding.UTF8.GetBytes(source);
var hash = SHA256.HashData(bytes);
return Convert.ToHexString(hash).ToLowerInvariant();
}
}
using System.Security.Cryptography;
using System.Text;
using StellaOps.Notify.Models;
namespace StellaOps.Notifier.Worker.Processing;
internal static class IdempotencyKeyBuilder
{
public static string Build(string tenantId, string ruleId, string actionId, NotifyEvent notifyEvent)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
ArgumentException.ThrowIfNullOrWhiteSpace(ruleId);
ArgumentException.ThrowIfNullOrWhiteSpace(actionId);
ArgumentNullException.ThrowIfNull(notifyEvent);
var scopeDigest = notifyEvent.Scope?.Digest ?? string.Empty;
var source = string.Join(
'|',
tenantId,
ruleId,
actionId,
notifyEvent.Kind,
scopeDigest,
notifyEvent.EventId.ToString("N"));
var bytes = Encoding.UTF8.GetBytes(source);
var hash = SHA256.HashData(bytes);
return Convert.ToHexString(hash).ToLowerInvariant();
}
}

View File

@@ -1,296 +1,296 @@
using System.Collections.Immutable;
using System.Text.Json.Nodes;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Notify.Models;
using StellaOps.Notifier.Worker.Storage;
using StellaOps.Notifier.Worker.Channels;
using StellaOps.Notifier.Worker.Dispatch;
using StellaOps.Notifier.Worker.Options;
namespace StellaOps.Notifier.Worker.Processing;
/// <summary>
/// Background worker that picks up pending deliveries, renders templates, and dispatches through channels.
/// </summary>
public sealed class NotifierDispatchWorker : BackgroundService
{
private readonly INotifyDeliveryRepository _deliveryRepository;
private readonly INotifyTemplateRepository _templateRepository;
private readonly INotifyChannelRepository _channelRepository;
private readonly INotifyTemplateRenderer _templateRenderer;
private readonly IReadOnlyDictionary<NotifyChannelType, INotifyChannelAdapter> _channelAdapters;
private readonly NotifierWorkerOptions _options;
private readonly TimeProvider _timeProvider;
private readonly ILogger<NotifierDispatchWorker> _logger;
private readonly string _workerId;
public NotifierDispatchWorker(
INotifyDeliveryRepository deliveryRepository,
INotifyTemplateRepository templateRepository,
INotifyChannelRepository channelRepository,
INotifyTemplateRenderer templateRenderer,
IEnumerable<INotifyChannelAdapter> channelAdapters,
IOptions<NotifierWorkerOptions> options,
TimeProvider timeProvider,
ILogger<NotifierDispatchWorker> logger)
{
_deliveryRepository = deliveryRepository ?? throw new ArgumentNullException(nameof(deliveryRepository));
_templateRepository = templateRepository ?? throw new ArgumentNullException(nameof(templateRepository));
_channelRepository = channelRepository ?? throw new ArgumentNullException(nameof(channelRepository));
_templateRenderer = templateRenderer ?? throw new ArgumentNullException(nameof(templateRenderer));
_channelAdapters = BuildAdapterMap(channelAdapters);
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_workerId = $"notifier-dispatch-{Environment.MachineName}-{Guid.NewGuid():N}";
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Notifier dispatch worker {WorkerId} started.", _workerId);
var pollInterval = _options.DispatchInterval > TimeSpan.Zero
? _options.DispatchInterval
: TimeSpan.FromSeconds(5);
while (!stoppingToken.IsCancellationRequested)
{
try
{
var processed = await ProcessPendingDeliveriesAsync(stoppingToken).ConfigureAwait(false);
if (processed == 0)
{
await Task.Delay(pollInterval, stoppingToken).ConfigureAwait(false);
}
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
catch (Exception ex)
{
_logger.LogError(ex, "Unhandled exception in dispatch worker loop.");
await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken).ConfigureAwait(false);
}
}
_logger.LogInformation("Notifier dispatch worker {WorkerId} stopping.", _workerId);
}
private async Task<int> ProcessPendingDeliveriesAsync(CancellationToken cancellationToken)
{
// Query for pending deliveries across all tenants (simplified - production would partition)
var result = await _deliveryRepository.QueryAsync(
tenantId: "tenant-sample", // In production, would iterate tenants
since: null,
status: "pending",
limit: _options.DispatchBatchSize > 0 ? _options.DispatchBatchSize : 10,
cancellationToken: cancellationToken).ConfigureAwait(false);
if (result.Items.Count == 0)
{
return 0;
}
var processed = 0;
foreach (var delivery in result.Items)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
await ProcessDeliveryAsync(delivery, cancellationToken).ConfigureAwait(false);
processed++;
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to process delivery {DeliveryId}.", delivery.DeliveryId);
}
}
return processed;
}
private async Task ProcessDeliveryAsync(NotifyDelivery delivery, CancellationToken cancellationToken)
{
var tenantId = delivery.TenantId;
// Look up channel from metadata
if (!delivery.Metadata.TryGetValue("channel", out var channelId) || string.IsNullOrWhiteSpace(channelId))
{
await MarkDeliveryFailedAsync(delivery, "Channel reference missing in delivery metadata", cancellationToken)
.ConfigureAwait(false);
return;
}
var channel = await _channelRepository.GetAsync(tenantId, channelId, cancellationToken).ConfigureAwait(false);
if (channel is null)
{
await MarkDeliveryFailedAsync(delivery, $"Channel {channelId} not found", cancellationToken)
.ConfigureAwait(false);
return;
}
// Look up template from metadata
delivery.Metadata.TryGetValue("template", out var templateKey);
delivery.Metadata.TryGetValue("locale", out var locale);
locale ??= "en-us";
NotifyTemplate? template = null;
if (!string.IsNullOrWhiteSpace(templateKey))
{
// GetAsync uses templateId, so we look up by the template reference from metadata
template = await _templateRepository.GetAsync(tenantId, templateKey, cancellationToken)
.ConfigureAwait(false);
}
// Build rendered content
NotifyDeliveryRendered rendered;
if (template is not null)
{
var notifyEvent = BuildEventFromDelivery(delivery);
var renderedContent = await _templateRenderer
.RenderAsync(template, notifyEvent, cancellationToken)
.ConfigureAwait(false);
var subject = renderedContent.Subject ?? $"Notification: {delivery.Kind}";
rendered = NotifyDeliveryRendered.Create(
channelType: channel.Type,
format: renderedContent.Format,
target: channel.Config?.Target ?? string.Empty,
title: subject,
body: renderedContent.Body,
locale: locale,
bodyHash: renderedContent.BodyHash);
}
else
{
// Fallback rendering without template
rendered = NotifyDeliveryRendered.Create(
channelType: channel.Type,
format: NotifyDeliveryFormat.Json,
target: channel.Config?.Target ?? string.Empty,
title: $"Notification: {delivery.Kind}",
body: $"Event {delivery.EventId} triggered rule {delivery.RuleId}",
locale: locale);
}
// Dispatch through channel adapter
if (!_channelAdapters.TryGetValue(channel.Type, out var adapter))
{
await MarkDeliveryFailedAsync(delivery, $"No adapter for channel type {channel.Type}", cancellationToken)
.ConfigureAwait(false);
return;
}
var dispatchResult = await adapter.SendAsync(channel, rendered, cancellationToken).ConfigureAwait(false);
// Update delivery with result
var attempt = new NotifyDeliveryAttempt(
timestamp: _timeProvider.GetUtcNow(),
status: dispatchResult.Success ? NotifyDeliveryAttemptStatus.Succeeded : NotifyDeliveryAttemptStatus.Failed,
statusCode: dispatchResult.HttpStatusCode,
reason: dispatchResult.Message);
var shouldRetry = !dispatchResult.Success && (dispatchResult.Status == ChannelDispatchStatus.Throttled
|| dispatchResult.Status == ChannelDispatchStatus.Timeout
|| dispatchResult.Status == ChannelDispatchStatus.NetworkError);
var newStatus = dispatchResult.Success
? NotifyDeliveryStatus.Delivered
: (shouldRetry ? NotifyDeliveryStatus.Pending : NotifyDeliveryStatus.Failed);
var updatedDelivery = NotifyDelivery.Create(
deliveryId: delivery.DeliveryId,
tenantId: delivery.TenantId,
ruleId: delivery.RuleId,
actionId: delivery.ActionId,
eventId: delivery.EventId,
kind: delivery.Kind,
status: newStatus,
statusReason: dispatchResult.Message,
rendered: rendered,
attempts: delivery.Attempts.Add(attempt),
metadata: delivery.Metadata,
createdAt: delivery.CreatedAt,
sentAt: dispatchResult.Success ? _timeProvider.GetUtcNow() : delivery.SentAt,
completedAt: newStatus == NotifyDeliveryStatus.Delivered || newStatus == NotifyDeliveryStatus.Failed
? _timeProvider.GetUtcNow()
: null);
await _deliveryRepository.UpdateAsync(updatedDelivery, cancellationToken).ConfigureAwait(false);
_logger.LogInformation(
"Delivery {DeliveryId} dispatched via {ChannelType}: {Status}",
delivery.DeliveryId,
channel.Type,
newStatus);
}
private async Task MarkDeliveryFailedAsync(
NotifyDelivery delivery,
string reason,
CancellationToken cancellationToken)
{
var failedDelivery = NotifyDelivery.Create(
deliveryId: delivery.DeliveryId,
tenantId: delivery.TenantId,
ruleId: delivery.RuleId,
actionId: delivery.ActionId,
eventId: delivery.EventId,
kind: delivery.Kind,
status: NotifyDeliveryStatus.Failed,
statusReason: reason,
attempts: delivery.Attempts,
metadata: delivery.Metadata,
createdAt: delivery.CreatedAt,
completedAt: _timeProvider.GetUtcNow());
await _deliveryRepository.UpdateAsync(failedDelivery, cancellationToken).ConfigureAwait(false);
_logger.LogWarning("Delivery {DeliveryId} marked failed: {Reason}", delivery.DeliveryId, reason);
}
private static NotifyEvent BuildEventFromDelivery(NotifyDelivery delivery)
{
var payload = new JsonObject
{
["eventId"] = delivery.EventId.ToString(),
["kind"] = delivery.Kind,
["ruleId"] = delivery.RuleId,
["actionId"] = delivery.ActionId
};
foreach (var (key, value) in delivery.Metadata)
{
payload[key] = value;
}
delivery.Metadata.TryGetValue("version", out var version);
delivery.Metadata.TryGetValue("actor", out var actor);
return NotifyEvent.Create(
eventId: delivery.EventId,
kind: delivery.Kind,
tenant: delivery.TenantId,
ts: delivery.CreatedAt,
payload: payload,
version: version,
actor: actor,
attributes: delivery.Metadata);
}
private static IReadOnlyDictionary<NotifyChannelType, INotifyChannelAdapter> BuildAdapterMap(
IEnumerable<INotifyChannelAdapter> adapters)
{
var builder = ImmutableDictionary.CreateBuilder<NotifyChannelType, INotifyChannelAdapter>();
foreach (var adapter in adapters)
{
builder[adapter.ChannelType] = adapter;
}
return builder.ToImmutable();
}
}
using System.Collections.Immutable;
using System.Text.Json.Nodes;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Notify.Models;
using StellaOps.Notifier.Worker.Storage;
using StellaOps.Notifier.Worker.Channels;
using StellaOps.Notifier.Worker.Dispatch;
using StellaOps.Notifier.Worker.Options;
namespace StellaOps.Notifier.Worker.Processing;
/// <summary>
/// Background worker that picks up pending deliveries, renders templates, and dispatches through channels.
/// </summary>
public sealed class NotifierDispatchWorker : BackgroundService
{
private readonly INotifyDeliveryRepository _deliveryRepository;
private readonly INotifyTemplateRepository _templateRepository;
private readonly INotifyChannelRepository _channelRepository;
private readonly INotifyTemplateRenderer _templateRenderer;
private readonly IReadOnlyDictionary<NotifyChannelType, INotifyChannelAdapter> _channelAdapters;
private readonly NotifierWorkerOptions _options;
private readonly TimeProvider _timeProvider;
private readonly ILogger<NotifierDispatchWorker> _logger;
private readonly string _workerId;
public NotifierDispatchWorker(
INotifyDeliveryRepository deliveryRepository,
INotifyTemplateRepository templateRepository,
INotifyChannelRepository channelRepository,
INotifyTemplateRenderer templateRenderer,
IEnumerable<INotifyChannelAdapter> channelAdapters,
IOptions<NotifierWorkerOptions> options,
TimeProvider timeProvider,
ILogger<NotifierDispatchWorker> logger)
{
_deliveryRepository = deliveryRepository ?? throw new ArgumentNullException(nameof(deliveryRepository));
_templateRepository = templateRepository ?? throw new ArgumentNullException(nameof(templateRepository));
_channelRepository = channelRepository ?? throw new ArgumentNullException(nameof(channelRepository));
_templateRenderer = templateRenderer ?? throw new ArgumentNullException(nameof(templateRenderer));
_channelAdapters = BuildAdapterMap(channelAdapters);
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_workerId = $"notifier-dispatch-{Environment.MachineName}-{Guid.NewGuid():N}";
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Notifier dispatch worker {WorkerId} started.", _workerId);
var pollInterval = _options.DispatchInterval > TimeSpan.Zero
? _options.DispatchInterval
: TimeSpan.FromSeconds(5);
while (!stoppingToken.IsCancellationRequested)
{
try
{
var processed = await ProcessPendingDeliveriesAsync(stoppingToken).ConfigureAwait(false);
if (processed == 0)
{
await Task.Delay(pollInterval, stoppingToken).ConfigureAwait(false);
}
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
catch (Exception ex)
{
_logger.LogError(ex, "Unhandled exception in dispatch worker loop.");
await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken).ConfigureAwait(false);
}
}
_logger.LogInformation("Notifier dispatch worker {WorkerId} stopping.", _workerId);
}
private async Task<int> ProcessPendingDeliveriesAsync(CancellationToken cancellationToken)
{
// Query for pending deliveries across all tenants (simplified - production would partition)
var result = await _deliveryRepository.QueryAsync(
tenantId: "tenant-sample", // In production, would iterate tenants
since: null,
status: "pending",
limit: _options.DispatchBatchSize > 0 ? _options.DispatchBatchSize : 10,
cancellationToken: cancellationToken).ConfigureAwait(false);
if (result.Items.Count == 0)
{
return 0;
}
var processed = 0;
foreach (var delivery in result.Items)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
await ProcessDeliveryAsync(delivery, cancellationToken).ConfigureAwait(false);
processed++;
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to process delivery {DeliveryId}.", delivery.DeliveryId);
}
}
return processed;
}
private async Task ProcessDeliveryAsync(NotifyDelivery delivery, CancellationToken cancellationToken)
{
var tenantId = delivery.TenantId;
// Look up channel from metadata
if (!delivery.Metadata.TryGetValue("channel", out var channelId) || string.IsNullOrWhiteSpace(channelId))
{
await MarkDeliveryFailedAsync(delivery, "Channel reference missing in delivery metadata", cancellationToken)
.ConfigureAwait(false);
return;
}
var channel = await _channelRepository.GetAsync(tenantId, channelId, cancellationToken).ConfigureAwait(false);
if (channel is null)
{
await MarkDeliveryFailedAsync(delivery, $"Channel {channelId} not found", cancellationToken)
.ConfigureAwait(false);
return;
}
// Look up template from metadata
delivery.Metadata.TryGetValue("template", out var templateKey);
delivery.Metadata.TryGetValue("locale", out var locale);
locale ??= "en-us";
NotifyTemplate? template = null;
if (!string.IsNullOrWhiteSpace(templateKey))
{
// GetAsync uses templateId, so we look up by the template reference from metadata
template = await _templateRepository.GetAsync(tenantId, templateKey, cancellationToken)
.ConfigureAwait(false);
}
// Build rendered content
NotifyDeliveryRendered rendered;
if (template is not null)
{
var notifyEvent = BuildEventFromDelivery(delivery);
var renderedContent = await _templateRenderer
.RenderAsync(template, notifyEvent, cancellationToken)
.ConfigureAwait(false);
var subject = renderedContent.Subject ?? $"Notification: {delivery.Kind}";
rendered = NotifyDeliveryRendered.Create(
channelType: channel.Type,
format: renderedContent.Format,
target: channel.Config?.Target ?? string.Empty,
title: subject,
body: renderedContent.Body,
locale: locale,
bodyHash: renderedContent.BodyHash);
}
else
{
// Fallback rendering without template
rendered = NotifyDeliveryRendered.Create(
channelType: channel.Type,
format: NotifyDeliveryFormat.Json,
target: channel.Config?.Target ?? string.Empty,
title: $"Notification: {delivery.Kind}",
body: $"Event {delivery.EventId} triggered rule {delivery.RuleId}",
locale: locale);
}
// Dispatch through channel adapter
if (!_channelAdapters.TryGetValue(channel.Type, out var adapter))
{
await MarkDeliveryFailedAsync(delivery, $"No adapter for channel type {channel.Type}", cancellationToken)
.ConfigureAwait(false);
return;
}
var dispatchResult = await adapter.SendAsync(channel, rendered, cancellationToken).ConfigureAwait(false);
// Update delivery with result
var attempt = new NotifyDeliveryAttempt(
timestamp: _timeProvider.GetUtcNow(),
status: dispatchResult.Success ? NotifyDeliveryAttemptStatus.Succeeded : NotifyDeliveryAttemptStatus.Failed,
statusCode: dispatchResult.HttpStatusCode,
reason: dispatchResult.Message);
var shouldRetry = !dispatchResult.Success && (dispatchResult.Status == ChannelDispatchStatus.Throttled
|| dispatchResult.Status == ChannelDispatchStatus.Timeout
|| dispatchResult.Status == ChannelDispatchStatus.NetworkError);
var newStatus = dispatchResult.Success
? NotifyDeliveryStatus.Delivered
: (shouldRetry ? NotifyDeliveryStatus.Pending : NotifyDeliveryStatus.Failed);
var updatedDelivery = NotifyDelivery.Create(
deliveryId: delivery.DeliveryId,
tenantId: delivery.TenantId,
ruleId: delivery.RuleId,
actionId: delivery.ActionId,
eventId: delivery.EventId,
kind: delivery.Kind,
status: newStatus,
statusReason: dispatchResult.Message,
rendered: rendered,
attempts: delivery.Attempts.Add(attempt),
metadata: delivery.Metadata,
createdAt: delivery.CreatedAt,
sentAt: dispatchResult.Success ? _timeProvider.GetUtcNow() : delivery.SentAt,
completedAt: newStatus == NotifyDeliveryStatus.Delivered || newStatus == NotifyDeliveryStatus.Failed
? _timeProvider.GetUtcNow()
: null);
await _deliveryRepository.UpdateAsync(updatedDelivery, cancellationToken).ConfigureAwait(false);
_logger.LogInformation(
"Delivery {DeliveryId} dispatched via {ChannelType}: {Status}",
delivery.DeliveryId,
channel.Type,
newStatus);
}
private async Task MarkDeliveryFailedAsync(
NotifyDelivery delivery,
string reason,
CancellationToken cancellationToken)
{
var failedDelivery = NotifyDelivery.Create(
deliveryId: delivery.DeliveryId,
tenantId: delivery.TenantId,
ruleId: delivery.RuleId,
actionId: delivery.ActionId,
eventId: delivery.EventId,
kind: delivery.Kind,
status: NotifyDeliveryStatus.Failed,
statusReason: reason,
attempts: delivery.Attempts,
metadata: delivery.Metadata,
createdAt: delivery.CreatedAt,
completedAt: _timeProvider.GetUtcNow());
await _deliveryRepository.UpdateAsync(failedDelivery, cancellationToken).ConfigureAwait(false);
_logger.LogWarning("Delivery {DeliveryId} marked failed: {Reason}", delivery.DeliveryId, reason);
}
private static NotifyEvent BuildEventFromDelivery(NotifyDelivery delivery)
{
var payload = new JsonObject
{
["eventId"] = delivery.EventId.ToString(),
["kind"] = delivery.Kind,
["ruleId"] = delivery.RuleId,
["actionId"] = delivery.ActionId
};
foreach (var (key, value) in delivery.Metadata)
{
payload[key] = value;
}
delivery.Metadata.TryGetValue("version", out var version);
delivery.Metadata.TryGetValue("actor", out var actor);
return NotifyEvent.Create(
eventId: delivery.EventId,
kind: delivery.Kind,
tenant: delivery.TenantId,
ts: delivery.CreatedAt,
payload: payload,
version: version,
actor: actor,
attributes: delivery.Metadata);
}
private static IReadOnlyDictionary<NotifyChannelType, INotifyChannelAdapter> BuildAdapterMap(
IEnumerable<INotifyChannelAdapter> adapters)
{
var builder = ImmutableDictionary.CreateBuilder<NotifyChannelType, INotifyChannelAdapter>();
foreach (var adapter in adapters)
{
builder[adapter.ChannelType] = adapter;
}
return builder.ToImmutable();
}
}

View File

@@ -6,21 +6,21 @@ using StellaOps.Notify.Engine;
using StellaOps.Notify.Models;
using StellaOps.Notifier.Worker.Storage;
using StellaOps.Notifier.Worker.Options;
namespace StellaOps.Notifier.Worker.Processing;
internal sealed class NotifierEventProcessor
{
private readonly INotifyRuleRepository _ruleRepository;
namespace StellaOps.Notifier.Worker.Processing;
internal sealed class NotifierEventProcessor
{
private readonly INotifyRuleRepository _ruleRepository;
private readonly INotifyDeliveryRepository _deliveryRepository;
private readonly INotifyLockRepository _lockRepository;
private readonly INotifyChannelRepository _channelRepository;
private readonly INotifyRuleEvaluator _ruleEvaluator;
private readonly IEgressPolicy _egressPolicy;
private readonly NotifierWorkerOptions _options;
private readonly TimeProvider _timeProvider;
private readonly ILogger<NotifierEventProcessor> _logger;
private readonly NotifierWorkerOptions _options;
private readonly TimeProvider _timeProvider;
private readonly ILogger<NotifierEventProcessor> _logger;
public NotifierEventProcessor(
INotifyRuleRepository ruleRepository,
INotifyDeliveryRepository deliveryRepository,
@@ -31,60 +31,60 @@ internal sealed class NotifierEventProcessor
IOptions<NotifierWorkerOptions> options,
TimeProvider timeProvider,
ILogger<NotifierEventProcessor> logger)
{
_ruleRepository = ruleRepository ?? throw new ArgumentNullException(nameof(ruleRepository));
{
_ruleRepository = ruleRepository ?? throw new ArgumentNullException(nameof(ruleRepository));
_deliveryRepository = deliveryRepository ?? throw new ArgumentNullException(nameof(deliveryRepository));
_lockRepository = lockRepository ?? throw new ArgumentNullException(nameof(lockRepository));
_channelRepository = channelRepository ?? throw new ArgumentNullException(nameof(channelRepository));
_ruleEvaluator = ruleEvaluator ?? throw new ArgumentNullException(nameof(ruleEvaluator));
_egressPolicy = egressPolicy ?? throw new ArgumentNullException(nameof(egressPolicy));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<int> ProcessAsync(NotifyEvent notifyEvent, string workerId, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(notifyEvent);
ArgumentException.ThrowIfNullOrWhiteSpace(workerId);
var tenantId = notifyEvent.Tenant;
var evaluationTime = _timeProvider.GetUtcNow();
IReadOnlyList<NotifyRule> rules;
try
{
rules = await _ruleRepository.ListAsync(tenantId, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to load rules for tenant {TenantId}.", tenantId);
throw;
}
if (rules.Count == 0)
{
_logger.LogDebug("No rules found for tenant {TenantId}.", tenantId);
return 0;
}
var enabledRules = rules.Where(static rule => rule.Enabled).ToArray();
if (enabledRules.Length == 0)
{
_logger.LogDebug("All rules are disabled for tenant {TenantId}.", tenantId);
return 0;
}
var outcomes = _ruleEvaluator.Evaluate(enabledRules, notifyEvent, evaluationTime);
if (outcomes.IsDefaultOrEmpty)
{
_logger.LogDebug(
"Event {EventId} produced no matches for tenant {TenantId}.",
notifyEvent.EventId,
tenantId);
return 0;
}
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<int> ProcessAsync(NotifyEvent notifyEvent, string workerId, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(notifyEvent);
ArgumentException.ThrowIfNullOrWhiteSpace(workerId);
var tenantId = notifyEvent.Tenant;
var evaluationTime = _timeProvider.GetUtcNow();
IReadOnlyList<NotifyRule> rules;
try
{
rules = await _ruleRepository.ListAsync(tenantId, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to load rules for tenant {TenantId}.", tenantId);
throw;
}
if (rules.Count == 0)
{
_logger.LogDebug("No rules found for tenant {TenantId}.", tenantId);
return 0;
}
var enabledRules = rules.Where(static rule => rule.Enabled).ToArray();
if (enabledRules.Length == 0)
{
_logger.LogDebug("All rules are disabled for tenant {TenantId}.", tenantId);
return 0;
}
var outcomes = _ruleEvaluator.Evaluate(enabledRules, notifyEvent, evaluationTime);
if (outcomes.IsDefaultOrEmpty)
{
_logger.LogDebug(
"Event {EventId} produced no matches for tenant {TenantId}.",
notifyEvent.EventId,
tenantId);
return 0;
}
var channelCache = new Dictionary<string, NotifyChannel?>(StringComparer.Ordinal);
var created = 0;
@@ -153,78 +153,78 @@ internal sealed class NotifierEventProcessor
var ttl = ResolveIdempotencyTtl(action);
var idempotencyKey = IdempotencyKeyBuilder.Build(tenantId, outcome.Rule.RuleId, action.ActionId, notifyEvent);
bool reserved;
try
{
reserved = await _lockRepository.TryAcquireAsync(tenantId, idempotencyKey, workerId, ttl, cancellationToken)
.ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogError(
ex,
"Failed to reserve idempotency token for tenant {TenantId}, rule {RuleId}, action {ActionId}.",
tenantId,
outcome.Rule.RuleId,
action.ActionId);
throw;
}
if (!reserved)
{
_logger.LogInformation(
"Skipped event {EventId} for tenant {TenantId}, rule {RuleId}, action {ActionId} due to idempotency.",
notifyEvent.EventId,
tenantId,
outcome.Rule.RuleId,
action.ActionId);
continue;
}
var delivery = NotifyDelivery.Create(
deliveryId: Guid.NewGuid().ToString("N"),
tenantId: tenantId,
ruleId: outcome.Rule.RuleId,
actionId: action.ActionId,
eventId: notifyEvent.EventId,
kind: notifyEvent.Kind,
status: NotifyDeliveryStatus.Pending,
metadata: BuildDeliveryMetadata(action));
try
{
await _deliveryRepository.AppendAsync(delivery, cancellationToken).ConfigureAwait(false);
created++;
}
catch (Exception ex)
{
_logger.LogError(
ex,
"Failed to persist delivery record for tenant {TenantId}, rule {RuleId}, action {ActionId}.",
tenantId,
outcome.Rule.RuleId,
action.ActionId);
throw;
}
}
}
return created;
}
bool reserved;
try
{
reserved = await _lockRepository.TryAcquireAsync(tenantId, idempotencyKey, workerId, ttl, cancellationToken)
.ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogError(
ex,
"Failed to reserve idempotency token for tenant {TenantId}, rule {RuleId}, action {ActionId}.",
tenantId,
outcome.Rule.RuleId,
action.ActionId);
throw;
}
if (!reserved)
{
_logger.LogInformation(
"Skipped event {EventId} for tenant {TenantId}, rule {RuleId}, action {ActionId} due to idempotency.",
notifyEvent.EventId,
tenantId,
outcome.Rule.RuleId,
action.ActionId);
continue;
}
var delivery = NotifyDelivery.Create(
deliveryId: Guid.NewGuid().ToString("N"),
tenantId: tenantId,
ruleId: outcome.Rule.RuleId,
actionId: action.ActionId,
eventId: notifyEvent.EventId,
kind: notifyEvent.Kind,
status: NotifyDeliveryStatus.Pending,
metadata: BuildDeliveryMetadata(action));
try
{
await _deliveryRepository.AppendAsync(delivery, cancellationToken).ConfigureAwait(false);
created++;
}
catch (Exception ex)
{
_logger.LogError(
ex,
"Failed to persist delivery record for tenant {TenantId}, rule {RuleId}, action {ActionId}.",
tenantId,
outcome.Rule.RuleId,
action.ActionId);
throw;
}
}
}
return created;
}
private TimeSpan ResolveIdempotencyTtl(NotifyRuleAction action)
{
if (action.Throttle is { Ticks: > 0 } throttle)
{
return throttle;
}
if (_options.DefaultIdempotencyTtl > TimeSpan.Zero)
{
return _options.DefaultIdempotencyTtl;
}
if (_options.DefaultIdempotencyTtl > TimeSpan.Zero)
{
return _options.DefaultIdempotencyTtl;
}
return TimeSpan.FromMinutes(5);
}
@@ -299,36 +299,36 @@ internal sealed class NotifierEventProcessor
}
private static IEnumerable<KeyValuePair<string, string>> BuildDeliveryMetadata(NotifyRuleAction action)
{
var metadata = new List<KeyValuePair<string, string>>
{
new("channel", action.Channel)
};
if (!string.IsNullOrWhiteSpace(action.Template))
{
metadata.Add(new("template", action.Template));
}
if (!string.IsNullOrWhiteSpace(action.Digest))
{
metadata.Add(new("digest", action.Digest));
}
if (!string.IsNullOrWhiteSpace(action.Locale))
{
metadata.Add(new("locale", action.Locale));
}
foreach (var (key, value) in action.Metadata)
{
if (!string.IsNullOrWhiteSpace(key) && !string.IsNullOrWhiteSpace(value))
{
metadata.Add(new(key, value));
}
}
return metadata;
}
}
{
var metadata = new List<KeyValuePair<string, string>>
{
new("channel", action.Channel)
};
if (!string.IsNullOrWhiteSpace(action.Template))
{
metadata.Add(new("template", action.Template));
}
if (!string.IsNullOrWhiteSpace(action.Digest))
{
metadata.Add(new("digest", action.Digest));
}
if (!string.IsNullOrWhiteSpace(action.Locale))
{
metadata.Add(new("locale", action.Locale));
}
foreach (var (key, value) in action.Metadata)
{
if (!string.IsNullOrWhiteSpace(key) && !string.IsNullOrWhiteSpace(value))
{
metadata.Add(new(key, value));
}
}
return metadata;
}
}

View File

@@ -1,120 +1,120 @@
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Notify.Queue;
using StellaOps.Notifier.Worker.Options;
namespace StellaOps.Notifier.Worker.Processing;
internal sealed class NotifierEventWorker : BackgroundService
{
private readonly INotifyEventQueue _queue;
private readonly NotifierEventProcessor _processor;
private readonly NotifierWorkerOptions _options;
private readonly ILogger<NotifierEventWorker> _logger;
private readonly string _workerId;
private readonly TimeProvider _timeProvider;
public NotifierEventWorker(
INotifyEventQueue queue,
NotifierEventProcessor processor,
IOptions<NotifierWorkerOptions> options,
TimeProvider timeProvider,
ILogger<NotifierEventWorker> logger)
{
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
_processor = processor ?? throw new ArgumentNullException(nameof(processor));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_workerId = $"notifier-worker-{Environment.MachineName}-{Guid.NewGuid():N}";
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Notifier event worker {WorkerId} started.", _workerId);
while (!stoppingToken.IsCancellationRequested)
{
try
{
var leases = await _queue.LeaseAsync(BuildLeaseRequest(), stoppingToken).ConfigureAwait(false);
if (leases.Count == 0)
{
await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken).ConfigureAwait(false);
continue;
}
foreach (var lease in leases)
{
stoppingToken.ThrowIfCancellationRequested();
try
{
var processed = await _processor
.ProcessAsync(lease.Message.Event, _workerId, stoppingToken)
.ConfigureAwait(false);
await lease.AcknowledgeAsync(stoppingToken).ConfigureAwait(false);
_logger.LogInformation(
"Processed event {EventId} for tenant {TenantId}; created {DeliveryCount} deliveries.",
lease.Message.Event.EventId,
lease.Message.TenantId,
processed);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
await SafeReleaseAsync(lease, NotifyQueueReleaseDisposition.Retry).ConfigureAwait(false);
throw;
}
catch (Exception ex)
{
_logger.LogError(
ex,
"Failed processing event {EventId} (tenant {TenantId}); scheduling retry.",
lease.Message.Event.EventId,
lease.Message.TenantId);
await SafeReleaseAsync(lease, NotifyQueueReleaseDisposition.Retry).ConfigureAwait(false);
}
}
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
catch (Exception ex)
{
_logger.LogError(ex, "Unhandled exception within notifier event worker loop.");
await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken).ConfigureAwait(false);
}
}
_logger.LogInformation("Notifier event worker {WorkerId} stopping.", _workerId);
}
private NotifyQueueLeaseRequest BuildLeaseRequest()
{
var batchSize = Math.Max(1, _options.LeaseBatchSize);
var leaseDuration = _options.LeaseDuration > TimeSpan.Zero
? _options.LeaseDuration
: TimeSpan.FromSeconds(60);
return new NotifyQueueLeaseRequest(_workerId, batchSize, leaseDuration);
}
private static async Task SafeReleaseAsync(
INotifyQueueLease<NotifyQueueEventMessage> lease,
NotifyQueueReleaseDisposition disposition)
{
try
{
await lease.ReleaseAsync(disposition, CancellationToken.None).ConfigureAwait(false);
}
catch
{
// Suppress release errors during shutdown/cleanup.
}
}
}
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Notify.Queue;
using StellaOps.Notifier.Worker.Options;
namespace StellaOps.Notifier.Worker.Processing;
internal sealed class NotifierEventWorker : BackgroundService
{
private readonly INotifyEventQueue _queue;
private readonly NotifierEventProcessor _processor;
private readonly NotifierWorkerOptions _options;
private readonly ILogger<NotifierEventWorker> _logger;
private readonly string _workerId;
private readonly TimeProvider _timeProvider;
public NotifierEventWorker(
INotifyEventQueue queue,
NotifierEventProcessor processor,
IOptions<NotifierWorkerOptions> options,
TimeProvider timeProvider,
ILogger<NotifierEventWorker> logger)
{
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
_processor = processor ?? throw new ArgumentNullException(nameof(processor));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_workerId = $"notifier-worker-{Environment.MachineName}-{Guid.NewGuid():N}";
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Notifier event worker {WorkerId} started.", _workerId);
while (!stoppingToken.IsCancellationRequested)
{
try
{
var leases = await _queue.LeaseAsync(BuildLeaseRequest(), stoppingToken).ConfigureAwait(false);
if (leases.Count == 0)
{
await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken).ConfigureAwait(false);
continue;
}
foreach (var lease in leases)
{
stoppingToken.ThrowIfCancellationRequested();
try
{
var processed = await _processor
.ProcessAsync(lease.Message.Event, _workerId, stoppingToken)
.ConfigureAwait(false);
await lease.AcknowledgeAsync(stoppingToken).ConfigureAwait(false);
_logger.LogInformation(
"Processed event {EventId} for tenant {TenantId}; created {DeliveryCount} deliveries.",
lease.Message.Event.EventId,
lease.Message.TenantId,
processed);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
await SafeReleaseAsync(lease, NotifyQueueReleaseDisposition.Retry).ConfigureAwait(false);
throw;
}
catch (Exception ex)
{
_logger.LogError(
ex,
"Failed processing event {EventId} (tenant {TenantId}); scheduling retry.",
lease.Message.Event.EventId,
lease.Message.TenantId);
await SafeReleaseAsync(lease, NotifyQueueReleaseDisposition.Retry).ConfigureAwait(false);
}
}
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
catch (Exception ex)
{
_logger.LogError(ex, "Unhandled exception within notifier event worker loop.");
await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken).ConfigureAwait(false);
}
}
_logger.LogInformation("Notifier event worker {WorkerId} stopping.", _workerId);
}
private NotifyQueueLeaseRequest BuildLeaseRequest()
{
var batchSize = Math.Max(1, _options.LeaseBatchSize);
var leaseDuration = _options.LeaseDuration > TimeSpan.Zero
? _options.LeaseDuration
: TimeSpan.FromSeconds(60);
return new NotifyQueueLeaseRequest(_workerId, batchSize, leaseDuration);
}
private static async Task SafeReleaseAsync(
INotifyQueueLease<NotifyQueueEventMessage> lease,
NotifyQueueReleaseDisposition disposition)
{
try
{
await lease.ReleaseAsync(disposition, CancellationToken.None).ConfigureAwait(false);
}
catch
{
// Suppress release errors during shutdown/cleanup.
}
}
}