Restructure solution layout by module
This commit is contained in:
master
@@ -0,0 +1,300 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Nodes;
|
||||
using StellaOps.Notify.Engine;
|
||||
using StellaOps.Notify.Models;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Processing;
|
||||
|
||||
internal sealed class DefaultNotifyRuleEvaluator : INotifyRuleEvaluator
|
||||
{
|
||||
private static readonly IDictionary<string, int> SeverityRank = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["none"] = 0,
|
||||
["info"] = 1,
|
||||
["low"] = 2,
|
||||
["medium"] = 3,
|
||||
["moderate"] = 3,
|
||||
["high"] = 4,
|
||||
["critical"] = 5,
|
||||
["blocker"] = 6,
|
||||
};
|
||||
|
||||
public NotifyRuleEvaluationOutcome Evaluate(NotifyRule rule, NotifyEvent @event, DateTimeOffset? evaluationTimestamp = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(rule);
|
||||
ArgumentNullException.ThrowIfNull(@event);
|
||||
|
||||
if (!rule.Enabled)
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "rule_disabled");
|
||||
}
|
||||
|
||||
var match = rule.Match;
|
||||
|
||||
if (!match.EventKinds.IsDefaultOrEmpty && !match.EventKinds.Contains(@event.Kind))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "event_kind_mismatch");
|
||||
}
|
||||
|
||||
if (!match.Namespaces.IsDefaultOrEmpty)
|
||||
{
|
||||
var ns = @event.Scope?.Namespace ?? string.Empty;
|
||||
if (!match.Namespaces.Contains(ns))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "namespace_mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
if (!match.Repositories.IsDefaultOrEmpty)
|
||||
{
|
||||
var repo = @event.Scope?.Repo ?? string.Empty;
|
||||
if (!match.Repositories.Contains(repo))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "repository_mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
if (!match.Digests.IsDefaultOrEmpty)
|
||||
{
|
||||
var digest = @event.Scope?.Digest ?? string.Empty;
|
||||
if (!match.Digests.Contains(digest))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "digest_mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
if (!match.ComponentPurls.IsDefaultOrEmpty)
|
||||
{
|
||||
var components = ExtractComponentPurls(@event.Payload);
|
||||
if (!components.Overlaps(match.ComponentPurls))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "component_mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
if (match.KevOnly == true && !ExtractLabels(@event).Contains("kev"))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "kev_required");
|
||||
}
|
||||
|
||||
if (!match.Labels.IsDefaultOrEmpty)
|
||||
{
|
||||
var labels = ExtractLabels(@event);
|
||||
if (!labels.IsSupersetOf(match.Labels))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "label_mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(match.MinSeverity))
|
||||
{
|
||||
var eventSeverity = ResolveSeverity(@event);
|
||||
if (!MeetsSeverity(match.MinSeverity!, eventSeverity))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "severity_below_threshold");
|
||||
}
|
||||
}
|
||||
|
||||
if (!match.Verdicts.IsDefaultOrEmpty)
|
||||
{
|
||||
var verdict = ResolveVerdict(@event);
|
||||
if (verdict is null || !match.Verdicts.Contains(verdict))
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "verdict_mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
var actions = rule.Actions
|
||||
.Where(static action => action is not null && action.Enabled)
|
||||
.Distinct()
|
||||
.OrderBy(static action => action.ActionId, StringComparer.Ordinal)
|
||||
.ToImmutableArray();
|
||||
|
||||
if (actions.IsDefaultOrEmpty)
|
||||
{
|
||||
return NotifyRuleEvaluationOutcome.NotMatched(rule, "no_enabled_actions");
|
||||
}
|
||||
|
||||
var matchedAt = evaluationTimestamp ?? DateTimeOffset.UtcNow;
|
||||
return NotifyRuleEvaluationOutcome.Matched(rule, actions, matchedAt);
|
||||
}
|
||||
|
||||
public ImmutableArray<NotifyRuleEvaluationOutcome> Evaluate(
|
||||
IEnumerable<NotifyRule> rules,
|
||||
NotifyEvent @event,
|
||||
DateTimeOffset? evaluationTimestamp = null)
|
||||
{
|
||||
if (rules is null)
|
||||
{
|
||||
return ImmutableArray<NotifyRuleEvaluationOutcome>.Empty;
|
||||
}
|
||||
|
||||
return rules
|
||||
.Select(rule => Evaluate(rule, @event, evaluationTimestamp))
|
||||
.Where(static outcome => outcome.IsMatch)
|
||||
.ToImmutableArray();
|
||||
}
|
||||
|
||||
private static bool MeetsSeverity(string required, string actual)
|
||||
{
|
||||
if (!SeverityRank.TryGetValue(required, out var requiredRank))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!SeverityRank.TryGetValue(actual, out var actualRank))
|
||||
{
|
||||
actualRank = 0;
|
||||
}
|
||||
|
||||
return actualRank >= requiredRank;
|
||||
}
|
||||
|
||||
private static string ResolveSeverity(NotifyEvent @event)
|
||||
{
|
||||
if (@event.Attributes.TryGetValue("severity", out var attributeSeverity) && !string.IsNullOrWhiteSpace(attributeSeverity))
|
||||
{
|
||||
return attributeSeverity.ToLowerInvariant();
|
||||
}
|
||||
|
||||
if (@event.Payload is JsonObject obj)
|
||||
{
|
||||
if (TryGetString(obj, "severity", out var severity))
|
||||
{
|
||||
return severity.ToLowerInvariant();
|
||||
}
|
||||
|
||||
if (obj.TryGetPropertyValue("summary", out var summaryNode) && summaryNode is JsonObject summaryObj)
|
||||
{
|
||||
if (TryGetString(summaryObj, "highestSeverity", out var summarySeverity))
|
||||
{
|
||||
return summarySeverity.ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
private static string? ResolveVerdict(NotifyEvent @event)
|
||||
{
|
||||
if (@event.Attributes.TryGetValue("verdict", out var attributeVerdict) && !string.IsNullOrWhiteSpace(attributeVerdict))
|
||||
{
|
||||
return attributeVerdict.ToLowerInvariant();
|
||||
}
|
||||
|
||||
if (@event.Payload is JsonObject obj)
|
||||
{
|
||||
if (TryGetString(obj, "verdict", out var verdict))
|
||||
{
|
||||
return verdict.ToLowerInvariant();
|
||||
}
|
||||
|
||||
if (obj.TryGetPropertyValue("summary", out var summaryNode) && summaryNode is JsonObject summaryObj)
|
||||
{
|
||||
if (TryGetString(summaryObj, "verdict", out var summaryVerdict))
|
||||
{
|
||||
return summaryVerdict.ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool TryGetString(JsonObject obj, string propertyName, out string value)
|
||||
{
|
||||
if (obj.TryGetPropertyValue(propertyName, out var node) && node is JsonValue jsonValue && jsonValue.TryGetValue(out string? str) && !string.IsNullOrWhiteSpace(str))
|
||||
{
|
||||
value = str.Trim();
|
||||
return true;
|
||||
}
|
||||
|
||||
value = string.Empty;
|
||||
return false;
|
||||
}
|
||||
|
||||
private static ImmutableHashSet<string> ExtractComponentPurls(JsonNode? payload)
|
||||
{
|
||||
var builder = ImmutableHashSet.CreateBuilder<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
if (payload is JsonObject obj && obj.TryGetPropertyValue("componentPurls", out var arrayNode) && arrayNode is JsonArray array)
|
||||
{
|
||||
foreach (var item in array)
|
||||
{
|
||||
if (item is JsonValue value && value.TryGetValue(out string? str) && !string.IsNullOrWhiteSpace(str))
|
||||
{
|
||||
builder.Add(str.Trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
|
||||
private static ImmutableHashSet<string> ExtractLabels(NotifyEvent @event)
|
||||
{
|
||||
var builder = ImmutableHashSet.CreateBuilder<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var (key, value) in @event.Attributes)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(key))
|
||||
{
|
||||
builder.Add(key.Trim());
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
builder.Add(value.Trim());
|
||||
}
|
||||
}
|
||||
|
||||
if (@event.Scope?.Labels is { Count: > 0 } scopeLabels)
|
||||
{
|
||||
foreach (var (key, value) in scopeLabels)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(key))
|
||||
{
|
||||
builder.Add(key.Trim());
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
builder.Add(value.Trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (@event.Payload is JsonObject obj && obj.TryGetPropertyValue("labels", out var labelsNode))
|
||||
{
|
||||
switch (labelsNode)
|
||||
{
|
||||
case JsonArray array:
|
||||
foreach (var item in array)
|
||||
{
|
||||
if (item is JsonValue value && value.TryGetValue(out string? str) && !string.IsNullOrWhiteSpace(str))
|
||||
{
|
||||
builder.Add(str.Trim());
|
||||
}
|
||||
}
|
||||
break;
|
||||
case JsonObject labelObj:
|
||||
foreach (var (key, value) in labelObj)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(key))
|
||||
{
|
||||
builder.Add(key.Trim());
|
||||
}
|
||||
|
||||
if (value is JsonValue v && v.TryGetValue(out string? str) && !string.IsNullOrWhiteSpace(str))
|
||||
{
|
||||
builder.Add(str.Trim());
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using StellaOps.Notify.Models;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Processing;
|
||||
|
||||
internal static class IdempotencyKeyBuilder
|
||||
{
|
||||
public static string Build(string tenantId, string ruleId, string actionId, NotifyEvent notifyEvent)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(ruleId);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(actionId);
|
||||
ArgumentNullException.ThrowIfNull(notifyEvent);
|
||||
|
||||
var scopeDigest = notifyEvent.Scope?.Digest ?? string.Empty;
|
||||
var source = string.Join(
|
||||
'|',
|
||||
tenantId,
|
||||
ruleId,
|
||||
actionId,
|
||||
notifyEvent.Kind,
|
||||
scopeDigest,
|
||||
notifyEvent.EventId.ToString("N"));
|
||||
|
||||
var bytes = Encoding.UTF8.GetBytes(source);
|
||||
var hash = SHA256.HashData(bytes);
|
||||
return Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Processing;
|
||||
|
||||
internal sealed class MongoInitializationHostedService : IHostedService
|
||||
{
|
||||
private const string InitializerTypeName = "StellaOps.Notify.Storage.Mongo.Internal.NotifyMongoInitializer, StellaOps.Notify.Storage.Mongo";
|
||||
|
||||
private readonly IServiceProvider _serviceProvider;
|
||||
private readonly ILogger<MongoInitializationHostedService> _logger;
|
||||
|
||||
public MongoInitializationHostedService(IServiceProvider serviceProvider, ILogger<MongoInitializationHostedService> logger)
|
||||
{
|
||||
_serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var initializerType = Type.GetType(InitializerTypeName, throwOnError: false, ignoreCase: false);
|
||||
if (initializerType is null)
|
||||
{
|
||||
_logger.LogWarning("Notify Mongo initializer type {TypeName} was not found; skipping migration run.", InitializerTypeName);
|
||||
return;
|
||||
}
|
||||
|
||||
using var scope = _serviceProvider.CreateScope();
|
||||
var initializer = scope.ServiceProvider.GetService(initializerType);
|
||||
if (initializer is null)
|
||||
{
|
||||
_logger.LogWarning("Notify Mongo initializer could not be resolved from the service provider.");
|
||||
return;
|
||||
}
|
||||
|
||||
var method = initializerType.GetMethod("EnsureIndexesAsync");
|
||||
if (method is null)
|
||||
{
|
||||
_logger.LogWarning("Notify Mongo initializer does not expose EnsureIndexesAsync; skipping migration run.");
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var task = method.Invoke(initializer, new object?[] { cancellationToken }) as Task;
|
||||
if (task is not null)
|
||||
{
|
||||
await task.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to run Notify Mongo migrations.");
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notify.Engine;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notify.Storage.Mongo.Repositories;
|
||||
using StellaOps.Notifier.Worker.Options;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Processing;
|
||||
|
||||
internal sealed class NotifierEventProcessor
|
||||
{
|
||||
private readonly INotifyRuleRepository _ruleRepository;
|
||||
private readonly INotifyDeliveryRepository _deliveryRepository;
|
||||
private readonly INotifyLockRepository _lockRepository;
|
||||
private readonly INotifyRuleEvaluator _ruleEvaluator;
|
||||
private readonly NotifierWorkerOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<NotifierEventProcessor> _logger;
|
||||
|
||||
public NotifierEventProcessor(
|
||||
INotifyRuleRepository ruleRepository,
|
||||
INotifyDeliveryRepository deliveryRepository,
|
||||
INotifyLockRepository lockRepository,
|
||||
INotifyRuleEvaluator ruleEvaluator,
|
||||
IOptions<NotifierWorkerOptions> options,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<NotifierEventProcessor> logger)
|
||||
{
|
||||
_ruleRepository = ruleRepository ?? throw new ArgumentNullException(nameof(ruleRepository));
|
||||
_deliveryRepository = deliveryRepository ?? throw new ArgumentNullException(nameof(deliveryRepository));
|
||||
_lockRepository = lockRepository ?? throw new ArgumentNullException(nameof(lockRepository));
|
||||
_ruleEvaluator = ruleEvaluator ?? throw new ArgumentNullException(nameof(ruleEvaluator));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<int> ProcessAsync(NotifyEvent notifyEvent, string workerId, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(notifyEvent);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(workerId);
|
||||
|
||||
var tenantId = notifyEvent.Tenant;
|
||||
var evaluationTime = _timeProvider.GetUtcNow();
|
||||
|
||||
IReadOnlyList<NotifyRule> rules;
|
||||
try
|
||||
{
|
||||
rules = await _ruleRepository.ListAsync(tenantId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to load rules for tenant {TenantId}.", tenantId);
|
||||
throw;
|
||||
}
|
||||
|
||||
if (rules.Count == 0)
|
||||
{
|
||||
_logger.LogDebug("No rules found for tenant {TenantId}.", tenantId);
|
||||
return 0;
|
||||
}
|
||||
|
||||
var enabledRules = rules.Where(static rule => rule.Enabled).ToArray();
|
||||
if (enabledRules.Length == 0)
|
||||
{
|
||||
_logger.LogDebug("All rules are disabled for tenant {TenantId}.", tenantId);
|
||||
return 0;
|
||||
}
|
||||
|
||||
var outcomes = _ruleEvaluator.Evaluate(enabledRules, notifyEvent, evaluationTime);
|
||||
if (outcomes.IsDefaultOrEmpty)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Event {EventId} produced no matches for tenant {TenantId}.",
|
||||
notifyEvent.EventId,
|
||||
tenantId);
|
||||
return 0;
|
||||
}
|
||||
|
||||
var created = 0;
|
||||
foreach (var outcome in outcomes)
|
||||
{
|
||||
foreach (var action in outcome.Actions)
|
||||
{
|
||||
var ttl = ResolveIdempotencyTtl(action);
|
||||
var idempotencyKey = IdempotencyKeyBuilder.Build(tenantId, outcome.Rule.RuleId, action.ActionId, notifyEvent);
|
||||
|
||||
bool reserved;
|
||||
try
|
||||
{
|
||||
reserved = await _lockRepository.TryAcquireAsync(tenantId, idempotencyKey, workerId, ttl, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"Failed to reserve idempotency token for tenant {TenantId}, rule {RuleId}, action {ActionId}.",
|
||||
tenantId,
|
||||
outcome.Rule.RuleId,
|
||||
action.ActionId);
|
||||
throw;
|
||||
}
|
||||
|
||||
if (!reserved)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Skipped event {EventId} for tenant {TenantId}, rule {RuleId}, action {ActionId} due to idempotency.",
|
||||
notifyEvent.EventId,
|
||||
tenantId,
|
||||
outcome.Rule.RuleId,
|
||||
action.ActionId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var delivery = NotifyDelivery.Create(
|
||||
deliveryId: Guid.NewGuid().ToString("N"),
|
||||
tenantId: tenantId,
|
||||
ruleId: outcome.Rule.RuleId,
|
||||
actionId: action.ActionId,
|
||||
eventId: notifyEvent.EventId,
|
||||
kind: notifyEvent.Kind,
|
||||
status: NotifyDeliveryStatus.Pending,
|
||||
metadata: BuildDeliveryMetadata(action));
|
||||
|
||||
try
|
||||
{
|
||||
await _deliveryRepository.AppendAsync(delivery, cancellationToken).ConfigureAwait(false);
|
||||
created++;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"Failed to persist delivery record for tenant {TenantId}, rule {RuleId}, action {ActionId}.",
|
||||
tenantId,
|
||||
outcome.Rule.RuleId,
|
||||
action.ActionId);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return created;
|
||||
}
|
||||
|
||||
private TimeSpan ResolveIdempotencyTtl(NotifyRuleAction action)
|
||||
{
|
||||
if (action.Throttle is { Ticks: > 0 } throttle)
|
||||
{
|
||||
return throttle;
|
||||
}
|
||||
|
||||
if (_options.DefaultIdempotencyTtl > TimeSpan.Zero)
|
||||
{
|
||||
return _options.DefaultIdempotencyTtl;
|
||||
}
|
||||
|
||||
return TimeSpan.FromMinutes(5);
|
||||
}
|
||||
|
||||
private static IEnumerable<KeyValuePair<string, string>> BuildDeliveryMetadata(NotifyRuleAction action)
|
||||
{
|
||||
var metadata = new List<KeyValuePair<string, string>>
|
||||
{
|
||||
new("channel", action.Channel)
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(action.Template))
|
||||
{
|
||||
metadata.Add(new("template", action.Template));
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(action.Digest))
|
||||
{
|
||||
metadata.Add(new("digest", action.Digest));
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(action.Locale))
|
||||
{
|
||||
metadata.Add(new("locale", action.Locale));
|
||||
}
|
||||
|
||||
foreach (var (key, value) in action.Metadata)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(key) && !string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
metadata.Add(new(key, value));
|
||||
}
|
||||
}
|
||||
|
||||
return metadata;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notify.Queue;
|
||||
using StellaOps.Notifier.Worker.Options;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Processing;
|
||||
|
||||
internal sealed class NotifierEventWorker : BackgroundService
|
||||
{
|
||||
private readonly INotifyEventQueue _queue;
|
||||
private readonly NotifierEventProcessor _processor;
|
||||
private readonly NotifierWorkerOptions _options;
|
||||
private readonly ILogger<NotifierEventWorker> _logger;
|
||||
private readonly string _workerId;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public NotifierEventWorker(
|
||||
INotifyEventQueue queue,
|
||||
NotifierEventProcessor processor,
|
||||
IOptions<NotifierWorkerOptions> options,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<NotifierEventWorker> logger)
|
||||
{
|
||||
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
|
||||
_processor = processor ?? throw new ArgumentNullException(nameof(processor));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_workerId = $"notifier-worker-{Environment.MachineName}-{Guid.NewGuid():N}";
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
_logger.LogInformation("Notifier event worker {WorkerId} started.", _workerId);
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
var leases = await _queue.LeaseAsync(BuildLeaseRequest(), stoppingToken).ConfigureAwait(false);
|
||||
if (leases.Count == 0)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var lease in leases)
|
||||
{
|
||||
stoppingToken.ThrowIfCancellationRequested();
|
||||
|
||||
try
|
||||
{
|
||||
var processed = await _processor
|
||||
.ProcessAsync(lease.Message.Event, _workerId, stoppingToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await lease.AcknowledgeAsync(stoppingToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Processed event {EventId} for tenant {TenantId}; created {DeliveryCount} deliveries.",
|
||||
lease.Message.Event.EventId,
|
||||
lease.Message.TenantId,
|
||||
processed);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
await SafeReleaseAsync(lease, NotifyQueueReleaseDisposition.Retry).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"Failed processing event {EventId} (tenant {TenantId}); scheduling retry.",
|
||||
lease.Message.Event.EventId,
|
||||
lease.Message.TenantId);
|
||||
|
||||
await SafeReleaseAsync(lease, NotifyQueueReleaseDisposition.Retry).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Unhandled exception within notifier event worker loop.");
|
||||
await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogInformation("Notifier event worker {WorkerId} stopping.", _workerId);
|
||||
}
|
||||
|
||||
private NotifyQueueLeaseRequest BuildLeaseRequest()
|
||||
{
|
||||
var batchSize = Math.Max(1, _options.LeaseBatchSize);
|
||||
var leaseDuration = _options.LeaseDuration > TimeSpan.Zero
|
||||
? _options.LeaseDuration
|
||||
: TimeSpan.FromSeconds(60);
|
||||
|
||||
return new NotifyQueueLeaseRequest(_workerId, batchSize, leaseDuration);
|
||||
}
|
||||
|
||||
private static async Task SafeReleaseAsync(
|
||||
INotifyQueueLease<NotifyQueueEventMessage> lease,
|
||||
NotifyQueueReleaseDisposition disposition)
|
||||
{
|
||||
try
|
||||
{
|
||||
await lease.ReleaseAsync(disposition, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Suppress release errors during shutdown/cleanup.
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user