up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
This commit is contained in:
@@ -1,300 +1,300 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Nodes;
|
||||
using StellaOps.Notify.Engine;
|
||||
using StellaOps.Notify.Models;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Processing;
|
||||
|
||||
public sealed class DefaultNotifyRuleEvaluator : INotifyRuleEvaluator
|
||||
{
|
||||
private static readonly IDictionary<string, int> SeverityRank = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["none"] = 0,
|
||||
["info"] = 1,
|
||||
["low"] = 2,
|
||||
["medium"] = 3,
|
||||
["moderate"] = 3,
|
||||
["high"] = 4,
|
||||
["critical"] = 5,
|
||||
["blocker"] = 6,
|
||||
};
|
||||
|
||||
public NotifyRuleEvaluationOutcome Evaluate(NotifyRule rule, NotifyEvent @event, DateTimeOffset? evaluationTimestamp = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(rule);
|
||||
ArgumentNullException.ThrowIfNull(@event);
|
||||
|
||||
if (!rule.Enabled)
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "rule_disabled");
|
||||
}
|
||||
|
||||
var match = rule.Match;
|
||||
|
||||
if (!match.EventKinds.IsDefaultOrEmpty && !match.EventKinds.Contains(@event.Kind))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "event_kind_mismatch");
|
||||
}
|
||||
|
||||
if (!match.Namespaces.IsDefaultOrEmpty)
|
||||
{
|
||||
var ns = @event.Scope?.Namespace ?? string.Empty;
|
||||
if (!match.Namespaces.Contains(ns))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "namespace_mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
if (!match.Repositories.IsDefaultOrEmpty)
|
||||
{
|
||||
var repo = @event.Scope?.Repo ?? string.Empty;
|
||||
if (!match.Repositories.Contains(repo))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "repository_mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
if (!match.Digests.IsDefaultOrEmpty)
|
||||
{
|
||||
var digest = @event.Scope?.Digest ?? string.Empty;
|
||||
if (!match.Digests.Contains(digest))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "digest_mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
if (!match.ComponentPurls.IsDefaultOrEmpty)
|
||||
{
|
||||
var components = ExtractComponentPurls(@event.Payload);
|
||||
if (!components.Overlaps(match.ComponentPurls))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "component_mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
if (match.KevOnly == true && !ExtractLabels(@event).Contains("kev"))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "kev_required");
|
||||
}
|
||||
|
||||
if (!match.Labels.IsDefaultOrEmpty)
|
||||
{
|
||||
var labels = ExtractLabels(@event);
|
||||
if (!labels.IsSupersetOf(match.Labels))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "label_mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(match.MinSeverity))
|
||||
{
|
||||
var eventSeverity = ResolveSeverity(@event);
|
||||
if (!MeetsSeverity(match.MinSeverity!, eventSeverity))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "severity_below_threshold");
|
||||
}
|
||||
}
|
||||
|
||||
if (!match.Verdicts.IsDefaultOrEmpty)
|
||||
{
|
||||
var verdict = ResolveVerdict(@event);
|
||||
if (verdict is null || !match.Verdicts.Contains(verdict))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "verdict_mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
var actions = rule.Actions
|
||||
.Where(static action => action is not null && action.Enabled)
|
||||
.Distinct()
|
||||
.OrderBy(static action => action.ActionId, StringComparer.Ordinal)
|
||||
.ToImmutableArray();
|
||||
|
||||
if (actions.IsDefaultOrEmpty)
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "no_enabled_actions");
|
||||
}
|
||||
|
||||
var matchedAt = evaluationTimestamp ?? DateTimeOffset.UtcNow;
|
||||
return NotifyRuleEvaluationOutcome.Matched(rule, actions, matchedAt);
|
||||
}
|
||||
|
||||
public ImmutableArray<NotifyRuleEvaluationOutcome> Evaluate(
|
||||
IEnumerable<NotifyRule> rules,
|
||||
NotifyEvent @event,
|
||||
DateTimeOffset? evaluationTimestamp = null)
|
||||
{
|
||||
if (rules is null)
|
||||
{
|
||||
return ImmutableArray<NotifyRuleEvaluationOutcome>.Empty;
|
||||
}
|
||||
|
||||
return rules
|
||||
.Select(rule => Evaluate(rule, @event, evaluationTimestamp))
|
||||
.Where(static outcome => outcome.IsMatch)
|
||||
.ToImmutableArray();
|
||||
}
|
||||
|
||||
private static bool MeetsSeverity(string required, string actual)
|
||||
{
|
||||
if (!SeverityRank.TryGetValue(required, out var requiredRank))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!SeverityRank.TryGetValue(actual, out var actualRank))
|
||||
{
|
||||
actualRank = 0;
|
||||
}
|
||||
|
||||
return actualRank >= requiredRank;
|
||||
}
|
||||
|
||||
private static string ResolveSeverity(NotifyEvent @event)
|
||||
{
|
||||
if (@event.Attributes.TryGetValue("severity", out var attributeSeverity) && !string.IsNullOrWhiteSpace(attributeSeverity))
|
||||
{
|
||||
return attributeSeverity.ToLowerInvariant();
|
||||
}
|
||||
|
||||
if (@event.Payload is JsonObject obj)
|
||||
{
|
||||
if (TryGetString(obj, "severity", out var severity))
|
||||
{
|
||||
return severity.ToLowerInvariant();
|
||||
}
|
||||
|
||||
if (obj.TryGetPropertyValue("summary", out var summaryNode) && summaryNode is JsonObject summaryObj)
|
||||
{
|
||||
if (TryGetString(summaryObj, "highestSeverity", out var summarySeverity))
|
||||
{
|
||||
return summarySeverity.ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
private static string? ResolveVerdict(NotifyEvent @event)
|
||||
{
|
||||
if (@event.Attributes.TryGetValue("verdict", out var attributeVerdict) && !string.IsNullOrWhiteSpace(attributeVerdict))
|
||||
{
|
||||
return attributeVerdict.ToLowerInvariant();
|
||||
}
|
||||
|
||||
if (@event.Payload is JsonObject obj)
|
||||
{
|
||||
if (TryGetString(obj, "verdict", out var verdict))
|
||||
{
|
||||
return verdict.ToLowerInvariant();
|
||||
}
|
||||
|
||||
if (obj.TryGetPropertyValue("summary", out var summaryNode) && summaryNode is JsonObject summaryObj)
|
||||
{
|
||||
if (TryGetString(summaryObj, "verdict", out var summaryVerdict))
|
||||
{
|
||||
return summaryVerdict.ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool TryGetString(JsonObject obj, string propertyName, out string value)
|
||||
{
|
||||
if (obj.TryGetPropertyValue(propertyName, out var node) && node is JsonValue jsonValue && jsonValue.TryGetValue(out string? str) && !string.IsNullOrWhiteSpace(str))
|
||||
{
|
||||
value = str.Trim();
|
||||
return true;
|
||||
}
|
||||
|
||||
value = string.Empty;
|
||||
return false;
|
||||
}
|
||||
|
||||
private static ImmutableHashSet<string> ExtractComponentPurls(JsonNode? payload)
|
||||
{
|
||||
var builder = ImmutableHashSet.CreateBuilder<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
if (payload is JsonObject obj && obj.TryGetPropertyValue("componentPurls", out var arrayNode) && arrayNode is JsonArray array)
|
||||
{
|
||||
foreach (var item in array)
|
||||
{
|
||||
if (item is JsonValue value && value.TryGetValue(out string? str) && !string.IsNullOrWhiteSpace(str))
|
||||
{
|
||||
builder.Add(str.Trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
|
||||
private static ImmutableHashSet<string> ExtractLabels(NotifyEvent @event)
|
||||
{
|
||||
var builder = ImmutableHashSet.CreateBuilder<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var (key, value) in @event.Attributes)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(key))
|
||||
{
|
||||
builder.Add(key.Trim());
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
builder.Add(value.Trim());
|
||||
}
|
||||
}
|
||||
|
||||
if (@event.Scope?.Labels is { Count: > 0 } scopeLabels)
|
||||
{
|
||||
foreach (var (key, value) in scopeLabels)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(key))
|
||||
{
|
||||
builder.Add(key.Trim());
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
builder.Add(value.Trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (@event.Payload is JsonObject obj && obj.TryGetPropertyValue("labels", out var labelsNode))
|
||||
{
|
||||
switch (labelsNode)
|
||||
{
|
||||
case JsonArray array:
|
||||
foreach (var item in array)
|
||||
{
|
||||
if (item is JsonValue value && value.TryGetValue(out string? str) && !string.IsNullOrWhiteSpace(str))
|
||||
{
|
||||
builder.Add(str.Trim());
|
||||
}
|
||||
}
|
||||
break;
|
||||
case JsonObject labelObj:
|
||||
foreach (var (key, value) in labelObj)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(key))
|
||||
{
|
||||
builder.Add(key.Trim());
|
||||
}
|
||||
|
||||
if (value is JsonValue v && v.TryGetValue(out string? str) && !string.IsNullOrWhiteSpace(str))
|
||||
{
|
||||
builder.Add(str.Trim());
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
}
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Nodes;
|
||||
using StellaOps.Notify.Engine;
|
||||
using StellaOps.Notify.Models;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Processing;
|
||||
|
||||
public sealed class DefaultNotifyRuleEvaluator : INotifyRuleEvaluator
|
||||
{
|
||||
private static readonly IDictionary<string, int> SeverityRank = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["none"] = 0,
|
||||
["info"] = 1,
|
||||
["low"] = 2,
|
||||
["medium"] = 3,
|
||||
["moderate"] = 3,
|
||||
["high"] = 4,
|
||||
["critical"] = 5,
|
||||
["blocker"] = 6,
|
||||
};
|
||||
|
||||
public NotifyRuleEvaluationOutcome Evaluate(NotifyRule rule, NotifyEvent @event, DateTimeOffset? evaluationTimestamp = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(rule);
|
||||
ArgumentNullException.ThrowIfNull(@event);
|
||||
|
||||
if (!rule.Enabled)
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "rule_disabled");
|
||||
}
|
||||
|
||||
var match = rule.Match;
|
||||
|
||||
if (!match.EventKinds.IsDefaultOrEmpty && !match.EventKinds.Contains(@event.Kind))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "event_kind_mismatch");
|
||||
}
|
||||
|
||||
if (!match.Namespaces.IsDefaultOrEmpty)
|
||||
{
|
||||
var ns = @event.Scope?.Namespace ?? string.Empty;
|
||||
if (!match.Namespaces.Contains(ns))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "namespace_mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
if (!match.Repositories.IsDefaultOrEmpty)
|
||||
{
|
||||
var repo = @event.Scope?.Repo ?? string.Empty;
|
||||
if (!match.Repositories.Contains(repo))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "repository_mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
if (!match.Digests.IsDefaultOrEmpty)
|
||||
{
|
||||
var digest = @event.Scope?.Digest ?? string.Empty;
|
||||
if (!match.Digests.Contains(digest))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "digest_mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
if (!match.ComponentPurls.IsDefaultOrEmpty)
|
||||
{
|
||||
var components = ExtractComponentPurls(@event.Payload);
|
||||
if (!components.Overlaps(match.ComponentPurls))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "component_mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
if (match.KevOnly == true && !ExtractLabels(@event).Contains("kev"))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "kev_required");
|
||||
}
|
||||
|
||||
if (!match.Labels.IsDefaultOrEmpty)
|
||||
{
|
||||
var labels = ExtractLabels(@event);
|
||||
if (!labels.IsSupersetOf(match.Labels))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "label_mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(match.MinSeverity))
|
||||
{
|
||||
var eventSeverity = ResolveSeverity(@event);
|
||||
if (!MeetsSeverity(match.MinSeverity!, eventSeverity))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "severity_below_threshold");
|
||||
}
|
||||
}
|
||||
|
||||
if (!match.Verdicts.IsDefaultOrEmpty)
|
||||
{
|
||||
var verdict = ResolveVerdict(@event);
|
||||
if (verdict is null || !match.Verdicts.Contains(verdict))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "verdict_mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
var actions = rule.Actions
|
||||
.Where(static action => action is not null && action.Enabled)
|
||||
.Distinct()
|
||||
.OrderBy(static action => action.ActionId, StringComparer.Ordinal)
|
||||
.ToImmutableArray();
|
||||
|
||||
if (actions.IsDefaultOrEmpty)
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "no_enabled_actions");
|
||||
}
|
||||
|
||||
var matchedAt = evaluationTimestamp ?? DateTimeOffset.UtcNow;
|
||||
return NotifyRuleEvaluationOutcome.Matched(rule, actions, matchedAt);
|
||||
}
|
||||
|
||||
public ImmutableArray<NotifyRuleEvaluationOutcome> Evaluate(
|
||||
IEnumerable<NotifyRule> rules,
|
||||
NotifyEvent @event,
|
||||
DateTimeOffset? evaluationTimestamp = null)
|
||||
{
|
||||
if (rules is null)
|
||||
{
|
||||
return ImmutableArray<NotifyRuleEvaluationOutcome>.Empty;
|
||||
}
|
||||
|
||||
return rules
|
||||
.Select(rule => Evaluate(rule, @event, evaluationTimestamp))
|
||||
.Where(static outcome => outcome.IsMatch)
|
||||
.ToImmutableArray();
|
||||
}
|
||||
|
||||
private static bool MeetsSeverity(string required, string actual)
|
||||
{
|
||||
if (!SeverityRank.TryGetValue(required, out var requiredRank))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!SeverityRank.TryGetValue(actual, out var actualRank))
|
||||
{
|
||||
actualRank = 0;
|
||||
}
|
||||
|
||||
return actualRank >= requiredRank;
|
||||
}
|
||||
|
||||
private static string ResolveSeverity(NotifyEvent @event)
|
||||
{
|
||||
if (@event.Attributes.TryGetValue("severity", out var attributeSeverity) && !string.IsNullOrWhiteSpace(attributeSeverity))
|
||||
{
|
||||
return attributeSeverity.ToLowerInvariant();
|
||||
}
|
||||
|
||||
if (@event.Payload is JsonObject obj)
|
||||
{
|
||||
if (TryGetString(obj, "severity", out var severity))
|
||||
{
|
||||
return severity.ToLowerInvariant();
|
||||
}
|
||||
|
||||
if (obj.TryGetPropertyValue("summary", out var summaryNode) && summaryNode is JsonObject summaryObj)
|
||||
{
|
||||
if (TryGetString(summaryObj, "highestSeverity", out var summarySeverity))
|
||||
{
|
||||
return summarySeverity.ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
private static string? ResolveVerdict(NotifyEvent @event)
|
||||
{
|
||||
if (@event.Attributes.TryGetValue("verdict", out var attributeVerdict) && !string.IsNullOrWhiteSpace(attributeVerdict))
|
||||
{
|
||||
return attributeVerdict.ToLowerInvariant();
|
||||
}
|
||||
|
||||
if (@event.Payload is JsonObject obj)
|
||||
{
|
||||
if (TryGetString(obj, "verdict", out var verdict))
|
||||
{
|
||||
return verdict.ToLowerInvariant();
|
||||
}
|
||||
|
||||
if (obj.TryGetPropertyValue("summary", out var summaryNode) && summaryNode is JsonObject summaryObj)
|
||||
{
|
||||
if (TryGetString(summaryObj, "verdict", out var summaryVerdict))
|
||||
{
|
||||
return summaryVerdict.ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool TryGetString(JsonObject obj, string propertyName, out string value)
|
||||
{
|
||||
if (obj.TryGetPropertyValue(propertyName, out var node) && node is JsonValue jsonValue && jsonValue.TryGetValue(out string? str) && !string.IsNullOrWhiteSpace(str))
|
||||
{
|
||||
value = str.Trim();
|
||||
return true;
|
||||
}
|
||||
|
||||
value = string.Empty;
|
||||
return false;
|
||||
}
|
||||
|
||||
private static ImmutableHashSet<string> ExtractComponentPurls(JsonNode? payload)
|
||||
{
|
||||
var builder = ImmutableHashSet.CreateBuilder<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
if (payload is JsonObject obj && obj.TryGetPropertyValue("componentPurls", out var arrayNode) && arrayNode is JsonArray array)
|
||||
{
|
||||
foreach (var item in array)
|
||||
{
|
||||
if (item is JsonValue value && value.TryGetValue(out string? str) && !string.IsNullOrWhiteSpace(str))
|
||||
{
|
||||
builder.Add(str.Trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
|
||||
private static ImmutableHashSet<string> ExtractLabels(NotifyEvent @event)
|
||||
{
|
||||
var builder = ImmutableHashSet.CreateBuilder<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var (key, value) in @event.Attributes)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(key))
|
||||
{
|
||||
builder.Add(key.Trim());
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
builder.Add(value.Trim());
|
||||
}
|
||||
}
|
||||
|
||||
if (@event.Scope?.Labels is { Count: > 0 } scopeLabels)
|
||||
{
|
||||
foreach (var (key, value) in scopeLabels)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(key))
|
||||
{
|
||||
builder.Add(key.Trim());
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
builder.Add(value.Trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (@event.Payload is JsonObject obj && obj.TryGetPropertyValue("labels", out var labelsNode))
|
||||
{
|
||||
switch (labelsNode)
|
||||
{
|
||||
case JsonArray array:
|
||||
foreach (var item in array)
|
||||
{
|
||||
if (item is JsonValue value && value.TryGetValue(out string? str) && !string.IsNullOrWhiteSpace(str))
|
||||
{
|
||||
builder.Add(str.Trim());
|
||||
}
|
||||
}
|
||||
break;
|
||||
case JsonObject labelObj:
|
||||
foreach (var (key, value) in labelObj)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(key))
|
||||
{
|
||||
builder.Add(key.Trim());
|
||||
}
|
||||
|
||||
if (value is JsonValue v && v.TryGetValue(out string? str) && !string.IsNullOrWhiteSpace(str))
|
||||
{
|
||||
builder.Add(str.Trim());
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,45 +1,45 @@
|
||||
using System.Collections.Generic;
|
||||
using StellaOps.Notify.Models;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Processing;
|
||||
|
||||
/// <summary>
|
||||
/// Tracks per-event delivery intents for SLO evaluation and webhook emission.
|
||||
/// </summary>
|
||||
internal sealed class EgressSloContext
|
||||
{
|
||||
private readonly List<EgressSloSignal> _signals = new();
|
||||
|
||||
public IReadOnlyList<EgressSloSignal> Signals => _signals;
|
||||
|
||||
public static EgressSloContext FromNotifyEvent(NotifyEvent notifyEvent)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(notifyEvent);
|
||||
return new EgressSloContext
|
||||
{
|
||||
EventId = notifyEvent.EventId,
|
||||
TenantId = notifyEvent.Tenant,
|
||||
EventKind = notifyEvent.Kind,
|
||||
OccurredAt = notifyEvent.Ts
|
||||
};
|
||||
}
|
||||
|
||||
public Guid EventId { get; private set; }
|
||||
|
||||
public string TenantId { get; private set; } = string.Empty;
|
||||
|
||||
public string EventKind { get; private set; } = string.Empty;
|
||||
|
||||
public DateTimeOffset OccurredAt { get; private set; }
|
||||
|
||||
public void AddDelivery(string channelType, string template, string kind)
|
||||
{
|
||||
_signals.Add(new EgressSloSignal(channelType, template, kind, OccurredAt));
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record EgressSloSignal(
|
||||
string Channel,
|
||||
string Template,
|
||||
string Kind,
|
||||
DateTimeOffset OccurredAt);
|
||||
using System.Collections.Generic;
|
||||
using StellaOps.Notify.Models;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Processing;
|
||||
|
||||
/// <summary>
|
||||
/// Tracks per-event delivery intents for SLO evaluation and webhook emission.
|
||||
/// </summary>
|
||||
internal sealed class EgressSloContext
|
||||
{
|
||||
private readonly List<EgressSloSignal> _signals = new();
|
||||
|
||||
public IReadOnlyList<EgressSloSignal> Signals => _signals;
|
||||
|
||||
public static EgressSloContext FromNotifyEvent(NotifyEvent notifyEvent)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(notifyEvent);
|
||||
return new EgressSloContext
|
||||
{
|
||||
EventId = notifyEvent.EventId,
|
||||
TenantId = notifyEvent.Tenant,
|
||||
EventKind = notifyEvent.Kind,
|
||||
OccurredAt = notifyEvent.Ts
|
||||
};
|
||||
}
|
||||
|
||||
public Guid EventId { get; private set; }
|
||||
|
||||
public string TenantId { get; private set; } = string.Empty;
|
||||
|
||||
public string EventKind { get; private set; } = string.Empty;
|
||||
|
||||
public DateTimeOffset OccurredAt { get; private set; }
|
||||
|
||||
public void AddDelivery(string channelType, string template, string kind)
|
||||
{
|
||||
_signals.Add(new EgressSloSignal(channelType, template, kind, OccurredAt));
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record EgressSloSignal(
|
||||
string Channel,
|
||||
string Template,
|
||||
string Kind,
|
||||
DateTimeOffset OccurredAt);
|
||||
|
||||
@@ -1,93 +1,93 @@
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notifier.Worker.Options;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Processing;
|
||||
|
||||
internal sealed class HttpEgressSloSink : IEgressSloSink
|
||||
{
|
||||
private readonly IHttpClientFactory _clientFactory;
|
||||
private readonly EgressSloOptions _options;
|
||||
private readonly ILogger<HttpEgressSloSink> _logger;
|
||||
|
||||
public HttpEgressSloSink(
|
||||
IHttpClientFactory clientFactory,
|
||||
IOptions<EgressSloOptions> options,
|
||||
ILogger<HttpEgressSloSink> logger)
|
||||
{
|
||||
_clientFactory = clientFactory ?? throw new ArgumentNullException(nameof(clientFactory));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task PublishAsync(EgressSloContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
if (context is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(context));
|
||||
}
|
||||
|
||||
if (!_options.Enabled)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
if (_options.TimeoutSeconds > 0)
|
||||
{
|
||||
linkedCts.CancelAfter(TimeSpan.FromSeconds(_options.TimeoutSeconds));
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var client = _clientFactory.CreateClient("notifier-slo-webhook");
|
||||
var payload = Map(context);
|
||||
|
||||
var response = await client.PostAsJsonAsync(_options.Webhook, payload, linkedCts.Token).ConfigureAwait(false);
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"SLO webhook returned non-success status {StatusCode} for event {EventId} (tenant {TenantId}).",
|
||||
(int)response.StatusCode,
|
||||
context.EventId,
|
||||
context.TenantId);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (linkedCts.IsCancellationRequested)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"SLO webhook timed out after {TimeoutSeconds}s for event {EventId} (tenant {TenantId}).",
|
||||
_options.TimeoutSeconds,
|
||||
context.EventId,
|
||||
context.TenantId);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
ex,
|
||||
"Failed to publish SLO webhook for event {EventId} (tenant {TenantId}).",
|
||||
context.EventId,
|
||||
context.TenantId);
|
||||
}
|
||||
}
|
||||
|
||||
private static object Map(EgressSloContext context)
|
||||
=> new
|
||||
{
|
||||
context.EventId,
|
||||
context.TenantId,
|
||||
context.EventKind,
|
||||
context.OccurredAt,
|
||||
deliveries = context.Signals.Select(signal => new
|
||||
{
|
||||
signal.Channel,
|
||||
signal.Template,
|
||||
signal.Kind,
|
||||
signal.OccurredAt
|
||||
})
|
||||
};
|
||||
}
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notifier.Worker.Options;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Processing;
|
||||
|
||||
internal sealed class HttpEgressSloSink : IEgressSloSink
|
||||
{
|
||||
private readonly IHttpClientFactory _clientFactory;
|
||||
private readonly EgressSloOptions _options;
|
||||
private readonly ILogger<HttpEgressSloSink> _logger;
|
||||
|
||||
public HttpEgressSloSink(
|
||||
IHttpClientFactory clientFactory,
|
||||
IOptions<EgressSloOptions> options,
|
||||
ILogger<HttpEgressSloSink> logger)
|
||||
{
|
||||
_clientFactory = clientFactory ?? throw new ArgumentNullException(nameof(clientFactory));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task PublishAsync(EgressSloContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
if (context is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(context));
|
||||
}
|
||||
|
||||
if (!_options.Enabled)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
if (_options.TimeoutSeconds > 0)
|
||||
{
|
||||
linkedCts.CancelAfter(TimeSpan.FromSeconds(_options.TimeoutSeconds));
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var client = _clientFactory.CreateClient("notifier-slo-webhook");
|
||||
var payload = Map(context);
|
||||
|
||||
var response = await client.PostAsJsonAsync(_options.Webhook, payload, linkedCts.Token).ConfigureAwait(false);
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"SLO webhook returned non-success status {StatusCode} for event {EventId} (tenant {TenantId}).",
|
||||
(int)response.StatusCode,
|
||||
context.EventId,
|
||||
context.TenantId);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (linkedCts.IsCancellationRequested)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"SLO webhook timed out after {TimeoutSeconds}s for event {EventId} (tenant {TenantId}).",
|
||||
_options.TimeoutSeconds,
|
||||
context.EventId,
|
||||
context.TenantId);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
ex,
|
||||
"Failed to publish SLO webhook for event {EventId} (tenant {TenantId}).",
|
||||
context.EventId,
|
||||
context.TenantId);
|
||||
}
|
||||
}
|
||||
|
||||
private static object Map(EgressSloContext context)
|
||||
=> new
|
||||
{
|
||||
context.EventId,
|
||||
context.TenantId,
|
||||
context.EventKind,
|
||||
context.OccurredAt,
|
||||
deliveries = context.Signals.Select(signal => new
|
||||
{
|
||||
signal.Channel,
|
||||
signal.Template,
|
||||
signal.Kind,
|
||||
signal.OccurredAt
|
||||
})
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Processing;
|
||||
|
||||
internal interface IEgressSloSink
|
||||
{
|
||||
Task PublishAsync(EgressSloContext context, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class NullEgressSloSink : IEgressSloSink
|
||||
{
|
||||
public Task PublishAsync(EgressSloContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Processing;
|
||||
|
||||
internal interface IEgressSloSink
|
||||
{
|
||||
Task PublishAsync(EgressSloContext context, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class NullEgressSloSink : IEgressSloSink
|
||||
{
|
||||
public Task PublishAsync(EgressSloContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,30 +1,30 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using StellaOps.Notify.Models;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Processing;
|
||||
|
||||
internal static class IdempotencyKeyBuilder
|
||||
{
|
||||
public static string Build(string tenantId, string ruleId, string actionId, NotifyEvent notifyEvent)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(ruleId);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(actionId);
|
||||
ArgumentNullException.ThrowIfNull(notifyEvent);
|
||||
|
||||
var scopeDigest = notifyEvent.Scope?.Digest ?? string.Empty;
|
||||
var source = string.Join(
|
||||
'|',
|
||||
tenantId,
|
||||
ruleId,
|
||||
actionId,
|
||||
notifyEvent.Kind,
|
||||
scopeDigest,
|
||||
notifyEvent.EventId.ToString("N"));
|
||||
|
||||
var bytes = Encoding.UTF8.GetBytes(source);
|
||||
var hash = SHA256.HashData(bytes);
|
||||
return Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using StellaOps.Notify.Models;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Processing;
|
||||
|
||||
internal static class IdempotencyKeyBuilder
|
||||
{
|
||||
public static string Build(string tenantId, string ruleId, string actionId, NotifyEvent notifyEvent)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(ruleId);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(actionId);
|
||||
ArgumentNullException.ThrowIfNull(notifyEvent);
|
||||
|
||||
var scopeDigest = notifyEvent.Scope?.Digest ?? string.Empty;
|
||||
var source = string.Join(
|
||||
'|',
|
||||
tenantId,
|
||||
ruleId,
|
||||
actionId,
|
||||
notifyEvent.Kind,
|
||||
scopeDigest,
|
||||
notifyEvent.EventId.ToString("N"));
|
||||
|
||||
var bytes = Encoding.UTF8.GetBytes(source);
|
||||
var hash = SHA256.HashData(bytes);
|
||||
return Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,296 +1,296 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Nodes;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notifier.Worker.Storage;
|
||||
using StellaOps.Notifier.Worker.Channels;
|
||||
using StellaOps.Notifier.Worker.Dispatch;
|
||||
using StellaOps.Notifier.Worker.Options;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Processing;
|
||||
|
||||
/// <summary>
|
||||
/// Background worker that picks up pending deliveries, renders templates, and dispatches through channels.
|
||||
/// </summary>
|
||||
public sealed class NotifierDispatchWorker : BackgroundService
|
||||
{
|
||||
private readonly INotifyDeliveryRepository _deliveryRepository;
|
||||
private readonly INotifyTemplateRepository _templateRepository;
|
||||
private readonly INotifyChannelRepository _channelRepository;
|
||||
private readonly INotifyTemplateRenderer _templateRenderer;
|
||||
private readonly IReadOnlyDictionary<NotifyChannelType, INotifyChannelAdapter> _channelAdapters;
|
||||
private readonly NotifierWorkerOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<NotifierDispatchWorker> _logger;
|
||||
private readonly string _workerId;
|
||||
|
||||
public NotifierDispatchWorker(
|
||||
INotifyDeliveryRepository deliveryRepository,
|
||||
INotifyTemplateRepository templateRepository,
|
||||
INotifyChannelRepository channelRepository,
|
||||
INotifyTemplateRenderer templateRenderer,
|
||||
IEnumerable<INotifyChannelAdapter> channelAdapters,
|
||||
IOptions<NotifierWorkerOptions> options,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<NotifierDispatchWorker> logger)
|
||||
{
|
||||
_deliveryRepository = deliveryRepository ?? throw new ArgumentNullException(nameof(deliveryRepository));
|
||||
_templateRepository = templateRepository ?? throw new ArgumentNullException(nameof(templateRepository));
|
||||
_channelRepository = channelRepository ?? throw new ArgumentNullException(nameof(channelRepository));
|
||||
_templateRenderer = templateRenderer ?? throw new ArgumentNullException(nameof(templateRenderer));
|
||||
_channelAdapters = BuildAdapterMap(channelAdapters);
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_workerId = $"notifier-dispatch-{Environment.MachineName}-{Guid.NewGuid():N}";
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
_logger.LogInformation("Notifier dispatch worker {WorkerId} started.", _workerId);
|
||||
|
||||
var pollInterval = _options.DispatchInterval > TimeSpan.Zero
|
||||
? _options.DispatchInterval
|
||||
: TimeSpan.FromSeconds(5);
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
var processed = await ProcessPendingDeliveriesAsync(stoppingToken).ConfigureAwait(false);
|
||||
if (processed == 0)
|
||||
{
|
||||
await Task.Delay(pollInterval, stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Unhandled exception in dispatch worker loop.");
|
||||
await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogInformation("Notifier dispatch worker {WorkerId} stopping.", _workerId);
|
||||
}
|
||||
|
||||
private async Task<int> ProcessPendingDeliveriesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
// Query for pending deliveries across all tenants (simplified - production would partition)
|
||||
var result = await _deliveryRepository.QueryAsync(
|
||||
tenantId: "tenant-sample", // In production, would iterate tenants
|
||||
since: null,
|
||||
status: "pending",
|
||||
limit: _options.DispatchBatchSize > 0 ? _options.DispatchBatchSize : 10,
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (result.Items.Count == 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var processed = 0;
|
||||
foreach (var delivery in result.Items)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
try
|
||||
{
|
||||
await ProcessDeliveryAsync(delivery, cancellationToken).ConfigureAwait(false);
|
||||
processed++;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to process delivery {DeliveryId}.", delivery.DeliveryId);
|
||||
}
|
||||
}
|
||||
|
||||
return processed;
|
||||
}
|
||||
|
||||
private async Task ProcessDeliveryAsync(NotifyDelivery delivery, CancellationToken cancellationToken)
|
||||
{
|
||||
var tenantId = delivery.TenantId;
|
||||
|
||||
// Look up channel from metadata
|
||||
if (!delivery.Metadata.TryGetValue("channel", out var channelId) || string.IsNullOrWhiteSpace(channelId))
|
||||
{
|
||||
await MarkDeliveryFailedAsync(delivery, "Channel reference missing in delivery metadata", cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
var channel = await _channelRepository.GetAsync(tenantId, channelId, cancellationToken).ConfigureAwait(false);
|
||||
if (channel is null)
|
||||
{
|
||||
await MarkDeliveryFailedAsync(delivery, $"Channel {channelId} not found", cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
// Look up template from metadata
|
||||
delivery.Metadata.TryGetValue("template", out var templateKey);
|
||||
delivery.Metadata.TryGetValue("locale", out var locale);
|
||||
locale ??= "en-us";
|
||||
|
||||
NotifyTemplate? template = null;
|
||||
if (!string.IsNullOrWhiteSpace(templateKey))
|
||||
{
|
||||
// GetAsync uses templateId, so we look up by the template reference from metadata
|
||||
template = await _templateRepository.GetAsync(tenantId, templateKey, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// Build rendered content
|
||||
NotifyDeliveryRendered rendered;
|
||||
if (template is not null)
|
||||
{
|
||||
var notifyEvent = BuildEventFromDelivery(delivery);
|
||||
var renderedContent = await _templateRenderer
|
||||
.RenderAsync(template, notifyEvent, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var subject = renderedContent.Subject ?? $"Notification: {delivery.Kind}";
|
||||
|
||||
rendered = NotifyDeliveryRendered.Create(
|
||||
channelType: channel.Type,
|
||||
format: renderedContent.Format,
|
||||
target: channel.Config?.Target ?? string.Empty,
|
||||
title: subject,
|
||||
body: renderedContent.Body,
|
||||
locale: locale,
|
||||
bodyHash: renderedContent.BodyHash);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Fallback rendering without template
|
||||
rendered = NotifyDeliveryRendered.Create(
|
||||
channelType: channel.Type,
|
||||
format: NotifyDeliveryFormat.Json,
|
||||
target: channel.Config?.Target ?? string.Empty,
|
||||
title: $"Notification: {delivery.Kind}",
|
||||
body: $"Event {delivery.EventId} triggered rule {delivery.RuleId}",
|
||||
locale: locale);
|
||||
}
|
||||
|
||||
// Dispatch through channel adapter
|
||||
if (!_channelAdapters.TryGetValue(channel.Type, out var adapter))
|
||||
{
|
||||
await MarkDeliveryFailedAsync(delivery, $"No adapter for channel type {channel.Type}", cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
var dispatchResult = await adapter.SendAsync(channel, rendered, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Update delivery with result
|
||||
var attempt = new NotifyDeliveryAttempt(
|
||||
timestamp: _timeProvider.GetUtcNow(),
|
||||
status: dispatchResult.Success ? NotifyDeliveryAttemptStatus.Succeeded : NotifyDeliveryAttemptStatus.Failed,
|
||||
statusCode: dispatchResult.HttpStatusCode,
|
||||
reason: dispatchResult.Message);
|
||||
|
||||
var shouldRetry = !dispatchResult.Success && (dispatchResult.Status == ChannelDispatchStatus.Throttled
|
||||
|| dispatchResult.Status == ChannelDispatchStatus.Timeout
|
||||
|| dispatchResult.Status == ChannelDispatchStatus.NetworkError);
|
||||
|
||||
var newStatus = dispatchResult.Success
|
||||
? NotifyDeliveryStatus.Delivered
|
||||
: (shouldRetry ? NotifyDeliveryStatus.Pending : NotifyDeliveryStatus.Failed);
|
||||
|
||||
var updatedDelivery = NotifyDelivery.Create(
|
||||
deliveryId: delivery.DeliveryId,
|
||||
tenantId: delivery.TenantId,
|
||||
ruleId: delivery.RuleId,
|
||||
actionId: delivery.ActionId,
|
||||
eventId: delivery.EventId,
|
||||
kind: delivery.Kind,
|
||||
status: newStatus,
|
||||
statusReason: dispatchResult.Message,
|
||||
rendered: rendered,
|
||||
attempts: delivery.Attempts.Add(attempt),
|
||||
metadata: delivery.Metadata,
|
||||
createdAt: delivery.CreatedAt,
|
||||
sentAt: dispatchResult.Success ? _timeProvider.GetUtcNow() : delivery.SentAt,
|
||||
completedAt: newStatus == NotifyDeliveryStatus.Delivered || newStatus == NotifyDeliveryStatus.Failed
|
||||
? _timeProvider.GetUtcNow()
|
||||
: null);
|
||||
|
||||
await _deliveryRepository.UpdateAsync(updatedDelivery, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Delivery {DeliveryId} dispatched via {ChannelType}: {Status}",
|
||||
delivery.DeliveryId,
|
||||
channel.Type,
|
||||
newStatus);
|
||||
}
|
||||
|
||||
private async Task MarkDeliveryFailedAsync(
|
||||
NotifyDelivery delivery,
|
||||
string reason,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var failedDelivery = NotifyDelivery.Create(
|
||||
deliveryId: delivery.DeliveryId,
|
||||
tenantId: delivery.TenantId,
|
||||
ruleId: delivery.RuleId,
|
||||
actionId: delivery.ActionId,
|
||||
eventId: delivery.EventId,
|
||||
kind: delivery.Kind,
|
||||
status: NotifyDeliveryStatus.Failed,
|
||||
statusReason: reason,
|
||||
attempts: delivery.Attempts,
|
||||
metadata: delivery.Metadata,
|
||||
createdAt: delivery.CreatedAt,
|
||||
completedAt: _timeProvider.GetUtcNow());
|
||||
|
||||
await _deliveryRepository.UpdateAsync(failedDelivery, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogWarning("Delivery {DeliveryId} marked failed: {Reason}", delivery.DeliveryId, reason);
|
||||
}
|
||||
|
||||
private static NotifyEvent BuildEventFromDelivery(NotifyDelivery delivery)
|
||||
{
|
||||
var payload = new JsonObject
|
||||
{
|
||||
["eventId"] = delivery.EventId.ToString(),
|
||||
["kind"] = delivery.Kind,
|
||||
["ruleId"] = delivery.RuleId,
|
||||
["actionId"] = delivery.ActionId
|
||||
};
|
||||
|
||||
foreach (var (key, value) in delivery.Metadata)
|
||||
{
|
||||
payload[key] = value;
|
||||
}
|
||||
|
||||
delivery.Metadata.TryGetValue("version", out var version);
|
||||
delivery.Metadata.TryGetValue("actor", out var actor);
|
||||
|
||||
return NotifyEvent.Create(
|
||||
eventId: delivery.EventId,
|
||||
kind: delivery.Kind,
|
||||
tenant: delivery.TenantId,
|
||||
ts: delivery.CreatedAt,
|
||||
payload: payload,
|
||||
version: version,
|
||||
actor: actor,
|
||||
attributes: delivery.Metadata);
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<NotifyChannelType, INotifyChannelAdapter> BuildAdapterMap(
|
||||
IEnumerable<INotifyChannelAdapter> adapters)
|
||||
{
|
||||
var builder = ImmutableDictionary.CreateBuilder<NotifyChannelType, INotifyChannelAdapter>();
|
||||
foreach (var adapter in adapters)
|
||||
{
|
||||
builder[adapter.ChannelType] = adapter;
|
||||
}
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
}
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Nodes;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notifier.Worker.Storage;
|
||||
using StellaOps.Notifier.Worker.Channels;
|
||||
using StellaOps.Notifier.Worker.Dispatch;
|
||||
using StellaOps.Notifier.Worker.Options;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Processing;
|
||||
|
||||
/// <summary>
|
||||
/// Background worker that picks up pending deliveries, renders templates, and dispatches through channels.
|
||||
/// </summary>
|
||||
public sealed class NotifierDispatchWorker : BackgroundService
|
||||
{
|
||||
private readonly INotifyDeliveryRepository _deliveryRepository;
|
||||
private readonly INotifyTemplateRepository _templateRepository;
|
||||
private readonly INotifyChannelRepository _channelRepository;
|
||||
private readonly INotifyTemplateRenderer _templateRenderer;
|
||||
private readonly IReadOnlyDictionary<NotifyChannelType, INotifyChannelAdapter> _channelAdapters;
|
||||
private readonly NotifierWorkerOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<NotifierDispatchWorker> _logger;
|
||||
private readonly string _workerId;
|
||||
|
||||
public NotifierDispatchWorker(
|
||||
INotifyDeliveryRepository deliveryRepository,
|
||||
INotifyTemplateRepository templateRepository,
|
||||
INotifyChannelRepository channelRepository,
|
||||
INotifyTemplateRenderer templateRenderer,
|
||||
IEnumerable<INotifyChannelAdapter> channelAdapters,
|
||||
IOptions<NotifierWorkerOptions> options,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<NotifierDispatchWorker> logger)
|
||||
{
|
||||
_deliveryRepository = deliveryRepository ?? throw new ArgumentNullException(nameof(deliveryRepository));
|
||||
_templateRepository = templateRepository ?? throw new ArgumentNullException(nameof(templateRepository));
|
||||
_channelRepository = channelRepository ?? throw new ArgumentNullException(nameof(channelRepository));
|
||||
_templateRenderer = templateRenderer ?? throw new ArgumentNullException(nameof(templateRenderer));
|
||||
_channelAdapters = BuildAdapterMap(channelAdapters);
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_workerId = $"notifier-dispatch-{Environment.MachineName}-{Guid.NewGuid():N}";
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
_logger.LogInformation("Notifier dispatch worker {WorkerId} started.", _workerId);
|
||||
|
||||
var pollInterval = _options.DispatchInterval > TimeSpan.Zero
|
||||
? _options.DispatchInterval
|
||||
: TimeSpan.FromSeconds(5);
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
var processed = await ProcessPendingDeliveriesAsync(stoppingToken).ConfigureAwait(false);
|
||||
if (processed == 0)
|
||||
{
|
||||
await Task.Delay(pollInterval, stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Unhandled exception in dispatch worker loop.");
|
||||
await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogInformation("Notifier dispatch worker {WorkerId} stopping.", _workerId);
|
||||
}
|
||||
|
||||
private async Task<int> ProcessPendingDeliveriesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
// Query for pending deliveries across all tenants (simplified - production would partition)
|
||||
var result = await _deliveryRepository.QueryAsync(
|
||||
tenantId: "tenant-sample", // In production, would iterate tenants
|
||||
since: null,
|
||||
status: "pending",
|
||||
limit: _options.DispatchBatchSize > 0 ? _options.DispatchBatchSize : 10,
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (result.Items.Count == 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var processed = 0;
|
||||
foreach (var delivery in result.Items)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
try
|
||||
{
|
||||
await ProcessDeliveryAsync(delivery, cancellationToken).ConfigureAwait(false);
|
||||
processed++;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to process delivery {DeliveryId}.", delivery.DeliveryId);
|
||||
}
|
||||
}
|
||||
|
||||
return processed;
|
||||
}
|
||||
|
||||
private async Task ProcessDeliveryAsync(NotifyDelivery delivery, CancellationToken cancellationToken)
|
||||
{
|
||||
var tenantId = delivery.TenantId;
|
||||
|
||||
// Look up channel from metadata
|
||||
if (!delivery.Metadata.TryGetValue("channel", out var channelId) || string.IsNullOrWhiteSpace(channelId))
|
||||
{
|
||||
await MarkDeliveryFailedAsync(delivery, "Channel reference missing in delivery metadata", cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
var channel = await _channelRepository.GetAsync(tenantId, channelId, cancellationToken).ConfigureAwait(false);
|
||||
if (channel is null)
|
||||
{
|
||||
await MarkDeliveryFailedAsync(delivery, $"Channel {channelId} not found", cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
// Look up template from metadata
|
||||
delivery.Metadata.TryGetValue("template", out var templateKey);
|
||||
delivery.Metadata.TryGetValue("locale", out var locale);
|
||||
locale ??= "en-us";
|
||||
|
||||
NotifyTemplate? template = null;
|
||||
if (!string.IsNullOrWhiteSpace(templateKey))
|
||||
{
|
||||
// GetAsync uses templateId, so we look up by the template reference from metadata
|
||||
template = await _templateRepository.GetAsync(tenantId, templateKey, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// Build rendered content
|
||||
NotifyDeliveryRendered rendered;
|
||||
if (template is not null)
|
||||
{
|
||||
var notifyEvent = BuildEventFromDelivery(delivery);
|
||||
var renderedContent = await _templateRenderer
|
||||
.RenderAsync(template, notifyEvent, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var subject = renderedContent.Subject ?? $"Notification: {delivery.Kind}";
|
||||
|
||||
rendered = NotifyDeliveryRendered.Create(
|
||||
channelType: channel.Type,
|
||||
format: renderedContent.Format,
|
||||
target: channel.Config?.Target ?? string.Empty,
|
||||
title: subject,
|
||||
body: renderedContent.Body,
|
||||
locale: locale,
|
||||
bodyHash: renderedContent.BodyHash);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Fallback rendering without template
|
||||
rendered = NotifyDeliveryRendered.Create(
|
||||
channelType: channel.Type,
|
||||
format: NotifyDeliveryFormat.Json,
|
||||
target: channel.Config?.Target ?? string.Empty,
|
||||
title: $"Notification: {delivery.Kind}",
|
||||
body: $"Event {delivery.EventId} triggered rule {delivery.RuleId}",
|
||||
locale: locale);
|
||||
}
|
||||
|
||||
// Dispatch through channel adapter
|
||||
if (!_channelAdapters.TryGetValue(channel.Type, out var adapter))
|
||||
{
|
||||
await MarkDeliveryFailedAsync(delivery, $"No adapter for channel type {channel.Type}", cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
var dispatchResult = await adapter.SendAsync(channel, rendered, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Update delivery with result
|
||||
var attempt = new NotifyDeliveryAttempt(
|
||||
timestamp: _timeProvider.GetUtcNow(),
|
||||
status: dispatchResult.Success ? NotifyDeliveryAttemptStatus.Succeeded : NotifyDeliveryAttemptStatus.Failed,
|
||||
statusCode: dispatchResult.HttpStatusCode,
|
||||
reason: dispatchResult.Message);
|
||||
|
||||
var shouldRetry = !dispatchResult.Success && (dispatchResult.Status == ChannelDispatchStatus.Throttled
|
||||
|| dispatchResult.Status == ChannelDispatchStatus.Timeout
|
||||
|| dispatchResult.Status == ChannelDispatchStatus.NetworkError);
|
||||
|
||||
var newStatus = dispatchResult.Success
|
||||
? NotifyDeliveryStatus.Delivered
|
||||
: (shouldRetry ? NotifyDeliveryStatus.Pending : NotifyDeliveryStatus.Failed);
|
||||
|
||||
var updatedDelivery = NotifyDelivery.Create(
|
||||
deliveryId: delivery.DeliveryId,
|
||||
tenantId: delivery.TenantId,
|
||||
ruleId: delivery.RuleId,
|
||||
actionId: delivery.ActionId,
|
||||
eventId: delivery.EventId,
|
||||
kind: delivery.Kind,
|
||||
status: newStatus,
|
||||
statusReason: dispatchResult.Message,
|
||||
rendered: rendered,
|
||||
attempts: delivery.Attempts.Add(attempt),
|
||||
metadata: delivery.Metadata,
|
||||
createdAt: delivery.CreatedAt,
|
||||
sentAt: dispatchResult.Success ? _timeProvider.GetUtcNow() : delivery.SentAt,
|
||||
completedAt: newStatus == NotifyDeliveryStatus.Delivered || newStatus == NotifyDeliveryStatus.Failed
|
||||
? _timeProvider.GetUtcNow()
|
||||
: null);
|
||||
|
||||
await _deliveryRepository.UpdateAsync(updatedDelivery, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Delivery {DeliveryId} dispatched via {ChannelType}: {Status}",
|
||||
delivery.DeliveryId,
|
||||
channel.Type,
|
||||
newStatus);
|
||||
}
|
||||
|
||||
private async Task MarkDeliveryFailedAsync(
|
||||
NotifyDelivery delivery,
|
||||
string reason,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var failedDelivery = NotifyDelivery.Create(
|
||||
deliveryId: delivery.DeliveryId,
|
||||
tenantId: delivery.TenantId,
|
||||
ruleId: delivery.RuleId,
|
||||
actionId: delivery.ActionId,
|
||||
eventId: delivery.EventId,
|
||||
kind: delivery.Kind,
|
||||
status: NotifyDeliveryStatus.Failed,
|
||||
statusReason: reason,
|
||||
attempts: delivery.Attempts,
|
||||
metadata: delivery.Metadata,
|
||||
createdAt: delivery.CreatedAt,
|
||||
completedAt: _timeProvider.GetUtcNow());
|
||||
|
||||
await _deliveryRepository.UpdateAsync(failedDelivery, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogWarning("Delivery {DeliveryId} marked failed: {Reason}", delivery.DeliveryId, reason);
|
||||
}
|
||||
|
||||
private static NotifyEvent BuildEventFromDelivery(NotifyDelivery delivery)
|
||||
{
|
||||
var payload = new JsonObject
|
||||
{
|
||||
["eventId"] = delivery.EventId.ToString(),
|
||||
["kind"] = delivery.Kind,
|
||||
["ruleId"] = delivery.RuleId,
|
||||
["actionId"] = delivery.ActionId
|
||||
};
|
||||
|
||||
foreach (var (key, value) in delivery.Metadata)
|
||||
{
|
||||
payload[key] = value;
|
||||
}
|
||||
|
||||
delivery.Metadata.TryGetValue("version", out var version);
|
||||
delivery.Metadata.TryGetValue("actor", out var actor);
|
||||
|
||||
return NotifyEvent.Create(
|
||||
eventId: delivery.EventId,
|
||||
kind: delivery.Kind,
|
||||
tenant: delivery.TenantId,
|
||||
ts: delivery.CreatedAt,
|
||||
payload: payload,
|
||||
version: version,
|
||||
actor: actor,
|
||||
attributes: delivery.Metadata);
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<NotifyChannelType, INotifyChannelAdapter> BuildAdapterMap(
|
||||
IEnumerable<INotifyChannelAdapter> adapters)
|
||||
{
|
||||
var builder = ImmutableDictionary.CreateBuilder<NotifyChannelType, INotifyChannelAdapter>();
|
||||
foreach (var adapter in adapters)
|
||||
{
|
||||
builder[adapter.ChannelType] = adapter;
|
||||
}
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,21 +6,21 @@ using StellaOps.Notify.Engine;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notifier.Worker.Storage;
|
||||
using StellaOps.Notifier.Worker.Options;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Processing;
|
||||
|
||||
internal sealed class NotifierEventProcessor
|
||||
{
|
||||
private readonly INotifyRuleRepository _ruleRepository;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Processing;
|
||||
|
||||
internal sealed class NotifierEventProcessor
|
||||
{
|
||||
private readonly INotifyRuleRepository _ruleRepository;
|
||||
private readonly INotifyDeliveryRepository _deliveryRepository;
|
||||
private readonly INotifyLockRepository _lockRepository;
|
||||
private readonly INotifyChannelRepository _channelRepository;
|
||||
private readonly INotifyRuleEvaluator _ruleEvaluator;
|
||||
private readonly IEgressPolicy _egressPolicy;
|
||||
private readonly NotifierWorkerOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<NotifierEventProcessor> _logger;
|
||||
|
||||
private readonly NotifierWorkerOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<NotifierEventProcessor> _logger;
|
||||
|
||||
public NotifierEventProcessor(
|
||||
INotifyRuleRepository ruleRepository,
|
||||
INotifyDeliveryRepository deliveryRepository,
|
||||
@@ -31,60 +31,60 @@ internal sealed class NotifierEventProcessor
|
||||
IOptions<NotifierWorkerOptions> options,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<NotifierEventProcessor> logger)
|
||||
{
|
||||
_ruleRepository = ruleRepository ?? throw new ArgumentNullException(nameof(ruleRepository));
|
||||
{
|
||||
_ruleRepository = ruleRepository ?? throw new ArgumentNullException(nameof(ruleRepository));
|
||||
_deliveryRepository = deliveryRepository ?? throw new ArgumentNullException(nameof(deliveryRepository));
|
||||
_lockRepository = lockRepository ?? throw new ArgumentNullException(nameof(lockRepository));
|
||||
_channelRepository = channelRepository ?? throw new ArgumentNullException(nameof(channelRepository));
|
||||
_ruleEvaluator = ruleEvaluator ?? throw new ArgumentNullException(nameof(ruleEvaluator));
|
||||
_egressPolicy = egressPolicy ?? throw new ArgumentNullException(nameof(egressPolicy));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<int> ProcessAsync(NotifyEvent notifyEvent, string workerId, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(notifyEvent);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(workerId);
|
||||
|
||||
var tenantId = notifyEvent.Tenant;
|
||||
var evaluationTime = _timeProvider.GetUtcNow();
|
||||
|
||||
IReadOnlyList<NotifyRule> rules;
|
||||
try
|
||||
{
|
||||
rules = await _ruleRepository.ListAsync(tenantId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to load rules for tenant {TenantId}.", tenantId);
|
||||
throw;
|
||||
}
|
||||
|
||||
if (rules.Count == 0)
|
||||
{
|
||||
_logger.LogDebug("No rules found for tenant {TenantId}.", tenantId);
|
||||
return 0;
|
||||
}
|
||||
|
||||
var enabledRules = rules.Where(static rule => rule.Enabled).ToArray();
|
||||
if (enabledRules.Length == 0)
|
||||
{
|
||||
_logger.LogDebug("All rules are disabled for tenant {TenantId}.", tenantId);
|
||||
return 0;
|
||||
}
|
||||
|
||||
var outcomes = _ruleEvaluator.Evaluate(enabledRules, notifyEvent, evaluationTime);
|
||||
if (outcomes.IsDefaultOrEmpty)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Event {EventId} produced no matches for tenant {TenantId}.",
|
||||
notifyEvent.EventId,
|
||||
tenantId);
|
||||
return 0;
|
||||
}
|
||||
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<int> ProcessAsync(NotifyEvent notifyEvent, string workerId, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(notifyEvent);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(workerId);
|
||||
|
||||
var tenantId = notifyEvent.Tenant;
|
||||
var evaluationTime = _timeProvider.GetUtcNow();
|
||||
|
||||
IReadOnlyList<NotifyRule> rules;
|
||||
try
|
||||
{
|
||||
rules = await _ruleRepository.ListAsync(tenantId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to load rules for tenant {TenantId}.", tenantId);
|
||||
throw;
|
||||
}
|
||||
|
||||
if (rules.Count == 0)
|
||||
{
|
||||
_logger.LogDebug("No rules found for tenant {TenantId}.", tenantId);
|
||||
return 0;
|
||||
}
|
||||
|
||||
var enabledRules = rules.Where(static rule => rule.Enabled).ToArray();
|
||||
if (enabledRules.Length == 0)
|
||||
{
|
||||
_logger.LogDebug("All rules are disabled for tenant {TenantId}.", tenantId);
|
||||
return 0;
|
||||
}
|
||||
|
||||
var outcomes = _ruleEvaluator.Evaluate(enabledRules, notifyEvent, evaluationTime);
|
||||
if (outcomes.IsDefaultOrEmpty)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Event {EventId} produced no matches for tenant {TenantId}.",
|
||||
notifyEvent.EventId,
|
||||
tenantId);
|
||||
return 0;
|
||||
}
|
||||
|
||||
var channelCache = new Dictionary<string, NotifyChannel?>(StringComparer.Ordinal);
|
||||
|
||||
var created = 0;
|
||||
@@ -153,78 +153,78 @@ internal sealed class NotifierEventProcessor
|
||||
|
||||
var ttl = ResolveIdempotencyTtl(action);
|
||||
var idempotencyKey = IdempotencyKeyBuilder.Build(tenantId, outcome.Rule.RuleId, action.ActionId, notifyEvent);
|
||||
|
||||
bool reserved;
|
||||
try
|
||||
{
|
||||
reserved = await _lockRepository.TryAcquireAsync(tenantId, idempotencyKey, workerId, ttl, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"Failed to reserve idempotency token for tenant {TenantId}, rule {RuleId}, action {ActionId}.",
|
||||
tenantId,
|
||||
outcome.Rule.RuleId,
|
||||
action.ActionId);
|
||||
throw;
|
||||
}
|
||||
|
||||
if (!reserved)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Skipped event {EventId} for tenant {TenantId}, rule {RuleId}, action {ActionId} due to idempotency.",
|
||||
notifyEvent.EventId,
|
||||
tenantId,
|
||||
outcome.Rule.RuleId,
|
||||
action.ActionId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var delivery = NotifyDelivery.Create(
|
||||
deliveryId: Guid.NewGuid().ToString("N"),
|
||||
tenantId: tenantId,
|
||||
ruleId: outcome.Rule.RuleId,
|
||||
actionId: action.ActionId,
|
||||
eventId: notifyEvent.EventId,
|
||||
kind: notifyEvent.Kind,
|
||||
status: NotifyDeliveryStatus.Pending,
|
||||
metadata: BuildDeliveryMetadata(action));
|
||||
|
||||
try
|
||||
{
|
||||
await _deliveryRepository.AppendAsync(delivery, cancellationToken).ConfigureAwait(false);
|
||||
created++;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"Failed to persist delivery record for tenant {TenantId}, rule {RuleId}, action {ActionId}.",
|
||||
tenantId,
|
||||
outcome.Rule.RuleId,
|
||||
action.ActionId);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return created;
|
||||
}
|
||||
|
||||
|
||||
bool reserved;
|
||||
try
|
||||
{
|
||||
reserved = await _lockRepository.TryAcquireAsync(tenantId, idempotencyKey, workerId, ttl, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"Failed to reserve idempotency token for tenant {TenantId}, rule {RuleId}, action {ActionId}.",
|
||||
tenantId,
|
||||
outcome.Rule.RuleId,
|
||||
action.ActionId);
|
||||
throw;
|
||||
}
|
||||
|
||||
if (!reserved)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Skipped event {EventId} for tenant {TenantId}, rule {RuleId}, action {ActionId} due to idempotency.",
|
||||
notifyEvent.EventId,
|
||||
tenantId,
|
||||
outcome.Rule.RuleId,
|
||||
action.ActionId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var delivery = NotifyDelivery.Create(
|
||||
deliveryId: Guid.NewGuid().ToString("N"),
|
||||
tenantId: tenantId,
|
||||
ruleId: outcome.Rule.RuleId,
|
||||
actionId: action.ActionId,
|
||||
eventId: notifyEvent.EventId,
|
||||
kind: notifyEvent.Kind,
|
||||
status: NotifyDeliveryStatus.Pending,
|
||||
metadata: BuildDeliveryMetadata(action));
|
||||
|
||||
try
|
||||
{
|
||||
await _deliveryRepository.AppendAsync(delivery, cancellationToken).ConfigureAwait(false);
|
||||
created++;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"Failed to persist delivery record for tenant {TenantId}, rule {RuleId}, action {ActionId}.",
|
||||
tenantId,
|
||||
outcome.Rule.RuleId,
|
||||
action.ActionId);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return created;
|
||||
}
|
||||
|
||||
private TimeSpan ResolveIdempotencyTtl(NotifyRuleAction action)
|
||||
{
|
||||
if (action.Throttle is { Ticks: > 0 } throttle)
|
||||
{
|
||||
return throttle;
|
||||
}
|
||||
|
||||
if (_options.DefaultIdempotencyTtl > TimeSpan.Zero)
|
||||
{
|
||||
return _options.DefaultIdempotencyTtl;
|
||||
}
|
||||
|
||||
|
||||
if (_options.DefaultIdempotencyTtl > TimeSpan.Zero)
|
||||
{
|
||||
return _options.DefaultIdempotencyTtl;
|
||||
}
|
||||
|
||||
return TimeSpan.FromMinutes(5);
|
||||
}
|
||||
|
||||
@@ -299,36 +299,36 @@ internal sealed class NotifierEventProcessor
|
||||
}
|
||||
|
||||
private static IEnumerable<KeyValuePair<string, string>> BuildDeliveryMetadata(NotifyRuleAction action)
|
||||
{
|
||||
var metadata = new List<KeyValuePair<string, string>>
|
||||
{
|
||||
new("channel", action.Channel)
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(action.Template))
|
||||
{
|
||||
metadata.Add(new("template", action.Template));
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(action.Digest))
|
||||
{
|
||||
metadata.Add(new("digest", action.Digest));
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(action.Locale))
|
||||
{
|
||||
metadata.Add(new("locale", action.Locale));
|
||||
}
|
||||
|
||||
foreach (var (key, value) in action.Metadata)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(key) && !string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
metadata.Add(new(key, value));
|
||||
}
|
||||
}
|
||||
|
||||
return metadata;
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
var metadata = new List<KeyValuePair<string, string>>
|
||||
{
|
||||
new("channel", action.Channel)
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(action.Template))
|
||||
{
|
||||
metadata.Add(new("template", action.Template));
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(action.Digest))
|
||||
{
|
||||
metadata.Add(new("digest", action.Digest));
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(action.Locale))
|
||||
{
|
||||
metadata.Add(new("locale", action.Locale));
|
||||
}
|
||||
|
||||
foreach (var (key, value) in action.Metadata)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(key) && !string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
metadata.Add(new(key, value));
|
||||
}
|
||||
}
|
||||
|
||||
return metadata;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,120 +1,120 @@
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notify.Queue;
|
||||
using StellaOps.Notifier.Worker.Options;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Processing;
|
||||
|
||||
internal sealed class NotifierEventWorker : BackgroundService
|
||||
{
|
||||
private readonly INotifyEventQueue _queue;
|
||||
private readonly NotifierEventProcessor _processor;
|
||||
private readonly NotifierWorkerOptions _options;
|
||||
private readonly ILogger<NotifierEventWorker> _logger;
|
||||
private readonly string _workerId;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public NotifierEventWorker(
|
||||
INotifyEventQueue queue,
|
||||
NotifierEventProcessor processor,
|
||||
IOptions<NotifierWorkerOptions> options,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<NotifierEventWorker> logger)
|
||||
{
|
||||
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
|
||||
_processor = processor ?? throw new ArgumentNullException(nameof(processor));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_workerId = $"notifier-worker-{Environment.MachineName}-{Guid.NewGuid():N}";
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
_logger.LogInformation("Notifier event worker {WorkerId} started.", _workerId);
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
var leases = await _queue.LeaseAsync(BuildLeaseRequest(), stoppingToken).ConfigureAwait(false);
|
||||
if (leases.Count == 0)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var lease in leases)
|
||||
{
|
||||
stoppingToken.ThrowIfCancellationRequested();
|
||||
|
||||
try
|
||||
{
|
||||
var processed = await _processor
|
||||
.ProcessAsync(lease.Message.Event, _workerId, stoppingToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await lease.AcknowledgeAsync(stoppingToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Processed event {EventId} for tenant {TenantId}; created {DeliveryCount} deliveries.",
|
||||
lease.Message.Event.EventId,
|
||||
lease.Message.TenantId,
|
||||
processed);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
await SafeReleaseAsync(lease, NotifyQueueReleaseDisposition.Retry).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"Failed processing event {EventId} (tenant {TenantId}); scheduling retry.",
|
||||
lease.Message.Event.EventId,
|
||||
lease.Message.TenantId);
|
||||
|
||||
await SafeReleaseAsync(lease, NotifyQueueReleaseDisposition.Retry).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Unhandled exception within notifier event worker loop.");
|
||||
await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogInformation("Notifier event worker {WorkerId} stopping.", _workerId);
|
||||
}
|
||||
|
||||
private NotifyQueueLeaseRequest BuildLeaseRequest()
|
||||
{
|
||||
var batchSize = Math.Max(1, _options.LeaseBatchSize);
|
||||
var leaseDuration = _options.LeaseDuration > TimeSpan.Zero
|
||||
? _options.LeaseDuration
|
||||
: TimeSpan.FromSeconds(60);
|
||||
|
||||
return new NotifyQueueLeaseRequest(_workerId, batchSize, leaseDuration);
|
||||
}
|
||||
|
||||
private static async Task SafeReleaseAsync(
|
||||
INotifyQueueLease<NotifyQueueEventMessage> lease,
|
||||
NotifyQueueReleaseDisposition disposition)
|
||||
{
|
||||
try
|
||||
{
|
||||
await lease.ReleaseAsync(disposition, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Suppress release errors during shutdown/cleanup.
|
||||
}
|
||||
}
|
||||
}
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notify.Queue;
|
||||
using StellaOps.Notifier.Worker.Options;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Processing;
|
||||
|
||||
internal sealed class NotifierEventWorker : BackgroundService
|
||||
{
|
||||
private readonly INotifyEventQueue _queue;
|
||||
private readonly NotifierEventProcessor _processor;
|
||||
private readonly NotifierWorkerOptions _options;
|
||||
private readonly ILogger<NotifierEventWorker> _logger;
|
||||
private readonly string _workerId;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public NotifierEventWorker(
|
||||
INotifyEventQueue queue,
|
||||
NotifierEventProcessor processor,
|
||||
IOptions<NotifierWorkerOptions> options,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<NotifierEventWorker> logger)
|
||||
{
|
||||
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
|
||||
_processor = processor ?? throw new ArgumentNullException(nameof(processor));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_workerId = $"notifier-worker-{Environment.MachineName}-{Guid.NewGuid():N}";
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
_logger.LogInformation("Notifier event worker {WorkerId} started.", _workerId);
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
var leases = await _queue.LeaseAsync(BuildLeaseRequest(), stoppingToken).ConfigureAwait(false);
|
||||
if (leases.Count == 0)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var lease in leases)
|
||||
{
|
||||
stoppingToken.ThrowIfCancellationRequested();
|
||||
|
||||
try
|
||||
{
|
||||
var processed = await _processor
|
||||
.ProcessAsync(lease.Message.Event, _workerId, stoppingToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await lease.AcknowledgeAsync(stoppingToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Processed event {EventId} for tenant {TenantId}; created {DeliveryCount} deliveries.",
|
||||
lease.Message.Event.EventId,
|
||||
lease.Message.TenantId,
|
||||
processed);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
await SafeReleaseAsync(lease, NotifyQueueReleaseDisposition.Retry).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"Failed processing event {EventId} (tenant {TenantId}); scheduling retry.",
|
||||
lease.Message.Event.EventId,
|
||||
lease.Message.TenantId);
|
||||
|
||||
await SafeReleaseAsync(lease, NotifyQueueReleaseDisposition.Retry).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Unhandled exception within notifier event worker loop.");
|
||||
await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogInformation("Notifier event worker {WorkerId} stopping.", _workerId);
|
||||
}
|
||||
|
||||
private NotifyQueueLeaseRequest BuildLeaseRequest()
|
||||
{
|
||||
var batchSize = Math.Max(1, _options.LeaseBatchSize);
|
||||
var leaseDuration = _options.LeaseDuration > TimeSpan.Zero
|
||||
? _options.LeaseDuration
|
||||
: TimeSpan.FromSeconds(60);
|
||||
|
||||
return new NotifyQueueLeaseRequest(_workerId, batchSize, leaseDuration);
|
||||
}
|
||||
|
||||
private static async Task SafeReleaseAsync(
|
||||
INotifyQueueLease<NotifyQueueEventMessage> lease,
|
||||
NotifyQueueReleaseDisposition disposition)
|
||||
{
|
||||
try
|
||||
{
|
||||
await lease.ReleaseAsync(disposition, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Suppress release errors during shutdown/cleanup.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user