up the blokcing tasks
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Manifest Integrity / Validate Schema Integrity (push) Has been cancelled
Manifest Integrity / Validate Contract Documents (push) Has been cancelled
Manifest Integrity / Validate Pack Fixtures (push) Has been cancelled
Manifest Integrity / Audit SHA256SUMS Files (push) Has been cancelled
Manifest Integrity / Verify Merkle Roots (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Risk Bundle CI / risk-bundle-build (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Risk Bundle CI / risk-bundle-offline-kit (push) Has been cancelled
Risk Bundle CI / publish-checksums (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
devportal-offline / build-offline (push) Has been cancelled
Mirror Thin Bundle Sign & Verify / mirror-sign (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Manifest Integrity / Validate Schema Integrity (push) Has been cancelled
Manifest Integrity / Validate Contract Documents (push) Has been cancelled
Manifest Integrity / Validate Pack Fixtures (push) Has been cancelled
Manifest Integrity / Audit SHA256SUMS Files (push) Has been cancelled
Manifest Integrity / Verify Merkle Roots (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Risk Bundle CI / risk-bundle-build (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Risk Bundle CI / risk-bundle-offline-kit (push) Has been cancelled
Risk Bundle CI / publish-checksums (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
devportal-offline / build-offline (push) Has been cancelled
Mirror Thin Bundle Sign & Verify / mirror-sign (push) Has been cancelled
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics;
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
@@ -6,7 +6,7 @@ using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notify.Storage.Mongo.Repositories;
|
||||
using StellaOps.Notifier.Worker.Storage;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Channels;
|
||||
|
||||
@@ -404,3 +404,4 @@ public sealed class ChatWebhookChannelAdapter : IChannelAdapter
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -141,8 +141,8 @@ public sealed class CliChannelAdapter : INotifyChannelAdapter
|
||||
// Non-zero exit codes are typically not retryable
|
||||
return ChannelDispatchResult.Fail(
|
||||
$"Exit code {process.ExitCode}: {stderr}",
|
||||
process.ExitCode,
|
||||
shouldRetry: false);
|
||||
shouldRetry: false,
|
||||
httpStatusCode: process.ExitCode);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics;
|
||||
using System.Net;
|
||||
using System.Net.Mail;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notify.Storage.Mongo.Repositories;
|
||||
using StellaOps.Notifier.Worker.Storage;
|
||||
using StellaOps.Notifier.Worker.Options;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Channels;
|
||||
@@ -376,3 +376,4 @@ public sealed class EmailChannelAdapter : IChannelAdapter, IDisposable
|
||||
string? Password,
|
||||
bool EnableSsl);
|
||||
}
|
||||
|
||||
|
||||
@@ -69,6 +69,21 @@ public sealed record ChannelDispatchResult
|
||||
Metadata = metadata ?? new Dictionary<string, string>()
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Creates a simple success result (legacy helper).
|
||||
/// </summary>
|
||||
public static ChannelDispatchResult Ok(
|
||||
int? httpStatusCode = null,
|
||||
string? message = null,
|
||||
IReadOnlyDictionary<string, string>? metadata = null) => new()
|
||||
{
|
||||
Success = true,
|
||||
Status = ChannelDispatchStatus.Sent,
|
||||
HttpStatusCode = httpStatusCode,
|
||||
Message = message ?? "ok",
|
||||
Metadata = metadata ?? new Dictionary<string, string>()
|
||||
};
|
||||
|
||||
public static ChannelDispatchResult Failed(
|
||||
string message,
|
||||
ChannelDispatchStatus status = ChannelDispatchStatus.Failed,
|
||||
@@ -86,6 +101,28 @@ public sealed record ChannelDispatchResult
|
||||
Metadata = metadata ?? new Dictionary<string, string>()
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Creates a simplified failure result (legacy helper).
|
||||
/// </summary>
|
||||
public static ChannelDispatchResult Fail(
|
||||
string message,
|
||||
bool shouldRetry = false,
|
||||
int? httpStatusCode = null,
|
||||
Exception? exception = null,
|
||||
IReadOnlyDictionary<string, string>? metadata = null)
|
||||
{
|
||||
var status = shouldRetry ? ChannelDispatchStatus.Timeout : ChannelDispatchStatus.Failed;
|
||||
return new()
|
||||
{
|
||||
Success = false,
|
||||
Status = status,
|
||||
Message = message,
|
||||
HttpStatusCode = httpStatusCode,
|
||||
Exception = exception,
|
||||
Metadata = metadata ?? new Dictionary<string, string>()
|
||||
};
|
||||
}
|
||||
|
||||
public static ChannelDispatchResult Throttled(
|
||||
string message,
|
||||
TimeSpan? retryAfter = null,
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notify.Storage.Mongo.Repositories;
|
||||
using StellaOps.Notifier.Worker.Storage;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Channels;
|
||||
|
||||
@@ -481,3 +481,4 @@ public enum InAppNotificationPriority
|
||||
High,
|
||||
Urgent
|
||||
}
|
||||
|
||||
|
||||
@@ -1,101 +0,0 @@
|
||||
using StellaOps.Notify.Storage.Mongo.Repositories;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Channels;
|
||||
|
||||
/// <summary>
|
||||
/// Adapter that bridges IInAppInboxStore to INotifyInboxRepository.
|
||||
/// </summary>
|
||||
public sealed class MongoInboxStoreAdapter : IInAppInboxStore
|
||||
{
|
||||
private readonly INotifyInboxRepository _repository;
|
||||
|
||||
public MongoInboxStoreAdapter(INotifyInboxRepository repository)
|
||||
{
|
||||
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
|
||||
}
|
||||
|
||||
public async Task StoreAsync(InAppInboxMessage message, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(message);
|
||||
|
||||
var repoMessage = new NotifyInboxMessage
|
||||
{
|
||||
MessageId = message.MessageId,
|
||||
TenantId = message.TenantId,
|
||||
UserId = message.UserId,
|
||||
Title = message.Title,
|
||||
Body = message.Body,
|
||||
Summary = message.Summary,
|
||||
Category = message.Category,
|
||||
Priority = (int)message.Priority,
|
||||
Metadata = message.Metadata,
|
||||
CreatedAt = message.CreatedAt,
|
||||
ExpiresAt = message.ExpiresAt,
|
||||
ReadAt = message.ReadAt,
|
||||
SourceChannel = message.SourceChannel,
|
||||
DeliveryId = message.DeliveryId
|
||||
};
|
||||
|
||||
await _repository.StoreAsync(repoMessage, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<InAppInboxMessage>> GetForUserAsync(
|
||||
string tenantId,
|
||||
string userId,
|
||||
int limit = 50,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var repoMessages = await _repository.GetForUserAsync(tenantId, userId, limit, cancellationToken).ConfigureAwait(false);
|
||||
return repoMessages.Select(MapToInboxMessage).ToList();
|
||||
}
|
||||
|
||||
public async Task<InAppInboxMessage?> GetAsync(
|
||||
string tenantId,
|
||||
string messageId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var repoMessage = await _repository.GetAsync(tenantId, messageId, cancellationToken).ConfigureAwait(false);
|
||||
return repoMessage is null ? null : MapToInboxMessage(repoMessage);
|
||||
}
|
||||
|
||||
public Task MarkReadAsync(string tenantId, string messageId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
return _repository.MarkReadAsync(tenantId, messageId, cancellationToken);
|
||||
}
|
||||
|
||||
public Task MarkAllReadAsync(string tenantId, string userId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
return _repository.MarkAllReadAsync(tenantId, userId, cancellationToken);
|
||||
}
|
||||
|
||||
public Task DeleteAsync(string tenantId, string messageId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
return _repository.DeleteAsync(tenantId, messageId, cancellationToken);
|
||||
}
|
||||
|
||||
public Task<int> GetUnreadCountAsync(string tenantId, string userId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
return _repository.GetUnreadCountAsync(tenantId, userId, cancellationToken);
|
||||
}
|
||||
|
||||
private static InAppInboxMessage MapToInboxMessage(NotifyInboxMessage repo)
|
||||
{
|
||||
return new InAppInboxMessage
|
||||
{
|
||||
MessageId = repo.MessageId,
|
||||
TenantId = repo.TenantId,
|
||||
UserId = repo.UserId,
|
||||
Title = repo.Title,
|
||||
Body = repo.Body,
|
||||
Summary = repo.Summary,
|
||||
Category = repo.Category,
|
||||
Priority = (InAppInboxPriority)repo.Priority,
|
||||
Metadata = repo.Metadata,
|
||||
CreatedAt = repo.CreatedAt,
|
||||
ExpiresAt = repo.ExpiresAt,
|
||||
ReadAt = repo.ReadAt,
|
||||
SourceChannel = repo.SourceChannel,
|
||||
DeliveryId = repo.DeliveryId
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics;
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
@@ -7,7 +7,7 @@ using System.Text.Json.Serialization;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notify.Storage.Mongo.Repositories;
|
||||
using StellaOps.Notifier.Worker.Storage;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Channels;
|
||||
|
||||
@@ -570,3 +570,4 @@ public sealed class OpsGenieChannelAdapter : IChannelAdapter
|
||||
public string? RequestId { get; init; }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics;
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
@@ -7,7 +7,7 @@ using System.Text.Json.Serialization;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notify.Storage.Mongo.Repositories;
|
||||
using StellaOps.Notifier.Worker.Storage;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Channels;
|
||||
|
||||
@@ -525,3 +525,4 @@ public sealed class PagerDutyChannelAdapter : IChannelAdapter
|
||||
public string? DedupKey { get; init; }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -72,11 +72,11 @@ public sealed class SlackChannelAdapter : INotifyChannelAdapter
|
||||
|
||||
if (response.IsSuccessStatusCode)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Slack delivery to channel {Target} succeeded.",
|
||||
channel.Config?.Target ?? "(default)");
|
||||
return ChannelDispatchResult.Ok(statusCode);
|
||||
}
|
||||
_logger.LogInformation(
|
||||
"Slack delivery to channel {Target} succeeded.",
|
||||
channel.Config?.Target ?? "(default)");
|
||||
return ChannelDispatchResult.Ok(statusCode);
|
||||
}
|
||||
|
||||
var shouldRetry = statusCode >= 500 || statusCode == 429;
|
||||
_logger.LogWarning(
|
||||
@@ -86,8 +86,8 @@ public sealed class SlackChannelAdapter : INotifyChannelAdapter
|
||||
|
||||
return ChannelDispatchResult.Fail(
|
||||
$"HTTP {statusCode}",
|
||||
statusCode,
|
||||
shouldRetry);
|
||||
shouldRetry: shouldRetry,
|
||||
httpStatusCode: statusCode);
|
||||
}
|
||||
catch (HttpRequestException ex)
|
||||
{
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics;
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Security.Cryptography;
|
||||
@@ -7,7 +7,7 @@ using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notify.Storage.Mongo.Repositories;
|
||||
using StellaOps.Notifier.Worker.Storage;
|
||||
using StellaOps.Notifier.Worker.Options;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Channels;
|
||||
@@ -350,3 +350,4 @@ public sealed class WebhookChannelAdapter : IChannelAdapter
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,300 +0,0 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Immutable;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notify.Models;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Correlation;
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation of the correlation engine.
|
||||
/// </summary>
|
||||
public sealed class DefaultCorrelationEngine : ICorrelationEngine
|
||||
{
|
||||
private readonly ICorrelationKeyEvaluator _keyEvaluator;
|
||||
private readonly INotifyThrottler _throttler;
|
||||
private readonly IQuietHoursEvaluator _quietHoursEvaluator;
|
||||
private readonly CorrelationKeyConfig _config;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<DefaultCorrelationEngine> _logger;
|
||||
|
||||
// In-memory incident store (in production, would use a repository)
|
||||
private readonly ConcurrentDictionary<string, NotifyIncident> _incidents = new();
|
||||
|
||||
public DefaultCorrelationEngine(
|
||||
ICorrelationKeyEvaluator keyEvaluator,
|
||||
INotifyThrottler throttler,
|
||||
IQuietHoursEvaluator quietHoursEvaluator,
|
||||
IOptions<CorrelationKeyConfig> config,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<DefaultCorrelationEngine> logger)
|
||||
{
|
||||
_keyEvaluator = keyEvaluator ?? throw new ArgumentNullException(nameof(keyEvaluator));
|
||||
_throttler = throttler ?? throw new ArgumentNullException(nameof(throttler));
|
||||
_quietHoursEvaluator = quietHoursEvaluator ?? throw new ArgumentNullException(nameof(quietHoursEvaluator));
|
||||
_config = config?.Value ?? new CorrelationKeyConfig();
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<CorrelationResult> ProcessAsync(
|
||||
NotifyEvent @event,
|
||||
NotifyRule rule,
|
||||
NotifyRuleAction action,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(@event);
|
||||
ArgumentNullException.ThrowIfNull(rule);
|
||||
ArgumentNullException.ThrowIfNull(action);
|
||||
|
||||
var tenantId = @event.Tenant;
|
||||
|
||||
// 1. Check maintenance window
|
||||
var maintenanceResult = await _quietHoursEvaluator.IsInMaintenanceAsync(tenantId, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (maintenanceResult.IsInMaintenance)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Event {EventId} suppressed due to maintenance window: {Reason}",
|
||||
@event.EventId, maintenanceResult.MaintenanceReason);
|
||||
|
||||
return new CorrelationResult
|
||||
{
|
||||
Decision = CorrelationDecision.Maintenance,
|
||||
Reason = maintenanceResult.MaintenanceReason
|
||||
};
|
||||
}
|
||||
|
||||
// 2. Check quiet hours (per channel if action specifies)
|
||||
var quietHoursResult = await _quietHoursEvaluator.IsInQuietHoursAsync(
|
||||
tenantId, action.Channel, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (quietHoursResult.IsInQuietHours)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Event {EventId} suppressed due to quiet hours: {Reason}",
|
||||
@event.EventId, quietHoursResult.Reason);
|
||||
|
||||
return new CorrelationResult
|
||||
{
|
||||
Decision = CorrelationDecision.QuietHours,
|
||||
Reason = quietHoursResult.Reason,
|
||||
QuietHoursEndsAt = quietHoursResult.QuietHoursEndsAt
|
||||
};
|
||||
}
|
||||
|
||||
// 3. Compute correlation key
|
||||
var correlationKey = _keyEvaluator.EvaluateDefaultKey(@event);
|
||||
|
||||
// 4. Get or create incident
|
||||
var (incident, isNew) = await GetOrCreateIncidentInternalAsync(
|
||||
tenantId, correlationKey, @event.Kind, @event, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// 5. Check if incident is already acknowledged
|
||||
if (incident.Status == NotifyIncidentStatus.Acknowledged)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Event {EventId} suppressed - incident {IncidentId} already acknowledged",
|
||||
@event.EventId, incident.IncidentId);
|
||||
|
||||
return new CorrelationResult
|
||||
{
|
||||
Decision = CorrelationDecision.Acknowledged,
|
||||
Reason = "Incident already acknowledged",
|
||||
CorrelationKey = correlationKey,
|
||||
IncidentId = incident.IncidentId,
|
||||
IsNewIncident = false
|
||||
};
|
||||
}
|
||||
|
||||
// 6. Check throttling (if action has throttle configured)
|
||||
if (action.Throttle is { } throttle && throttle > TimeSpan.Zero)
|
||||
{
|
||||
var throttleKey = $"{rule.RuleId}:{action.ActionId}:{correlationKey}";
|
||||
var isThrottled = await _throttler.IsThrottledAsync(
|
||||
tenantId, throttleKey, throttle, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (isThrottled)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Event {EventId} throttled: key={ThrottleKey}, window={Throttle}",
|
||||
@event.EventId, throttleKey, throttle);
|
||||
|
||||
return new CorrelationResult
|
||||
{
|
||||
Decision = CorrelationDecision.Throttled,
|
||||
Reason = $"Throttled for {throttle}",
|
||||
CorrelationKey = correlationKey,
|
||||
IncidentId = incident.IncidentId,
|
||||
IsNewIncident = isNew,
|
||||
ThrottledUntil = _timeProvider.GetUtcNow().Add(throttle)
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// 7. If this is a new event added to an existing incident within the correlation window,
|
||||
// and it's not the first event, suppress delivery (already notified)
|
||||
if (!isNew && incident.EventCount > 1)
|
||||
{
|
||||
var windowEnd = incident.FirstEventAt.Add(_config.CorrelationWindow);
|
||||
if (_timeProvider.GetUtcNow() < windowEnd)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Event {EventId} correlated to existing incident {IncidentId} within window",
|
||||
@event.EventId, incident.IncidentId);
|
||||
|
||||
return new CorrelationResult
|
||||
{
|
||||
Decision = CorrelationDecision.Correlated,
|
||||
Reason = "Event correlated to existing incident",
|
||||
CorrelationKey = correlationKey,
|
||||
IncidentId = incident.IncidentId,
|
||||
IsNewIncident = false
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// 8. Proceed with delivery
|
||||
_logger.LogDebug(
|
||||
"Event {EventId} approved for delivery: incident={IncidentId}, isNew={IsNew}",
|
||||
@event.EventId, incident.IncidentId, isNew);
|
||||
|
||||
return new CorrelationResult
|
||||
{
|
||||
Decision = CorrelationDecision.Deliver,
|
||||
CorrelationKey = correlationKey,
|
||||
IncidentId = incident.IncidentId,
|
||||
IsNewIncident = isNew
|
||||
};
|
||||
}
|
||||
|
||||
public Task<NotifyIncident> GetOrCreateIncidentAsync(
|
||||
string tenantId,
|
||||
string correlationKey,
|
||||
string kind,
|
||||
NotifyEvent @event,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var (incident, _) = GetOrCreateIncidentInternalAsync(
|
||||
tenantId, correlationKey, kind, @event, cancellationToken).GetAwaiter().GetResult();
|
||||
return Task.FromResult(incident);
|
||||
}
|
||||
|
||||
private Task<(NotifyIncident Incident, bool IsNew)> GetOrCreateIncidentInternalAsync(
|
||||
string tenantId,
|
||||
string correlationKey,
|
||||
string kind,
|
||||
NotifyEvent @event,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var incidentKey = $"{tenantId}:{correlationKey}";
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
// Check if existing incident is within correlation window
|
||||
if (_incidents.TryGetValue(incidentKey, out var existing))
|
||||
{
|
||||
var windowEnd = existing.FirstEventAt.Add(_config.CorrelationWindow);
|
||||
if (now < windowEnd && existing.Status == NotifyIncidentStatus.Open)
|
||||
{
|
||||
// Add event to existing incident
|
||||
var updated = existing with
|
||||
{
|
||||
EventCount = existing.EventCount + 1,
|
||||
LastEventAt = now,
|
||||
EventIds = existing.EventIds.Add(@event.EventId),
|
||||
UpdatedAt = now
|
||||
};
|
||||
_incidents[incidentKey] = updated;
|
||||
return Task.FromResult((updated, false));
|
||||
}
|
||||
}
|
||||
|
||||
// Create new incident
|
||||
var incident = new NotifyIncident
|
||||
{
|
||||
IncidentId = Guid.NewGuid().ToString("N"),
|
||||
TenantId = tenantId,
|
||||
CorrelationKey = correlationKey,
|
||||
Kind = kind,
|
||||
Status = NotifyIncidentStatus.Open,
|
||||
EventCount = 1,
|
||||
FirstEventAt = now,
|
||||
LastEventAt = now,
|
||||
EventIds = [@event.EventId],
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now
|
||||
};
|
||||
|
||||
_incidents[incidentKey] = incident;
|
||||
return Task.FromResult((incident, true));
|
||||
}
|
||||
|
||||
public Task<NotifyIncident> AcknowledgeIncidentAsync(
|
||||
string tenantId,
|
||||
string incidentId,
|
||||
string acknowledgedBy,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var incident = _incidents.Values.FirstOrDefault(i =>
|
||||
i.TenantId == tenantId && i.IncidentId == incidentId);
|
||||
|
||||
if (incident is null)
|
||||
{
|
||||
throw new InvalidOperationException($"Incident {incidentId} not found");
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var updated = incident with
|
||||
{
|
||||
Status = NotifyIncidentStatus.Acknowledged,
|
||||
AcknowledgedAt = now,
|
||||
AcknowledgedBy = acknowledgedBy,
|
||||
UpdatedAt = now
|
||||
};
|
||||
|
||||
var key = $"{tenantId}:{incident.CorrelationKey}";
|
||||
_incidents[key] = updated;
|
||||
|
||||
_logger.LogInformation(
|
||||
"Incident {IncidentId} acknowledged by {AcknowledgedBy}",
|
||||
incidentId, acknowledgedBy);
|
||||
|
||||
return Task.FromResult(updated);
|
||||
}
|
||||
|
||||
public Task<NotifyIncident> ResolveIncidentAsync(
|
||||
string tenantId,
|
||||
string incidentId,
|
||||
string resolvedBy,
|
||||
string? resolutionNote = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var incident = _incidents.Values.FirstOrDefault(i =>
|
||||
i.TenantId == tenantId && i.IncidentId == incidentId);
|
||||
|
||||
if (incident is null)
|
||||
{
|
||||
throw new InvalidOperationException($"Incident {incidentId} not found");
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var updated = incident with
|
||||
{
|
||||
Status = NotifyIncidentStatus.Resolved,
|
||||
ResolvedAt = now,
|
||||
ResolvedBy = resolvedBy,
|
||||
ResolutionNote = resolutionNote,
|
||||
UpdatedAt = now
|
||||
};
|
||||
|
||||
var key = $"{tenantId}:{incident.CorrelationKey}";
|
||||
_incidents[key] = updated;
|
||||
|
||||
_logger.LogInformation(
|
||||
"Incident {IncidentId} resolved by {ResolvedBy}: {ResolutionNote}",
|
||||
incidentId, resolvedBy, resolutionNote);
|
||||
|
||||
return Task.FromResult(updated);
|
||||
}
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Notify.Storage.Mongo.Repositories;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Correlation;
|
||||
|
||||
/// <summary>
|
||||
/// Throttler implementation using the lock repository for distributed throttling.
|
||||
/// </summary>
|
||||
public sealed class LockBasedThrottler : INotifyThrottler
|
||||
{
|
||||
private readonly INotifyLockRepository _lockRepository;
|
||||
private readonly ILogger<LockBasedThrottler> _logger;
|
||||
|
||||
public LockBasedThrottler(
|
||||
INotifyLockRepository lockRepository,
|
||||
ILogger<LockBasedThrottler> logger)
|
||||
{
|
||||
_lockRepository = lockRepository ?? throw new ArgumentNullException(nameof(lockRepository));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<bool> IsThrottledAsync(
|
||||
string tenantId,
|
||||
string throttleKey,
|
||||
TimeSpan window,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(throttleKey);
|
||||
|
||||
if (window <= TimeSpan.Zero)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var lockKey = BuildThrottleKey(throttleKey);
|
||||
|
||||
// Try to acquire the lock - if we can't, it means we're throttled
|
||||
var acquired = await _lockRepository.TryAcquireAsync(
|
||||
tenantId,
|
||||
lockKey,
|
||||
"throttle",
|
||||
window,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (!acquired)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Notification throttled: tenant={TenantId}, key={ThrottleKey}, window={Window}",
|
||||
tenantId, throttleKey, window);
|
||||
return true;
|
||||
}
|
||||
|
||||
// We acquired the lock, so we're not throttled
|
||||
// Note: The lock will automatically expire after the window
|
||||
return false;
|
||||
}
|
||||
|
||||
public Task RecordSentAsync(
|
||||
string tenantId,
|
||||
string throttleKey,
|
||||
TimeSpan window,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
// The lock was already acquired in IsThrottledAsync, which also serves as the marker
|
||||
// This method exists for cases where throttle check and send are separate operations
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private static string BuildThrottleKey(string key)
|
||||
{
|
||||
return $"throttle|{key}";
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Concurrent;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notify.Storage.Mongo.Repositories;
|
||||
using StellaOps.Notifier.Worker.Storage;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Correlation;
|
||||
|
||||
@@ -281,6 +281,7 @@ public sealed class InMemoryQuietHoursCalendarService : IQuietHoursCalendarServi
|
||||
await _auditRepository.AppendAsync(
|
||||
calendar.TenantId,
|
||||
isNew ? "quiet_hours_calendar_created" : "quiet_hours_calendar_updated",
|
||||
actor,
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["calendarId"] = calendar.CalendarId,
|
||||
@@ -288,7 +289,6 @@ public sealed class InMemoryQuietHoursCalendarService : IQuietHoursCalendarServi
|
||||
["enabled"] = calendar.Enabled.ToString(),
|
||||
["scheduleCount"] = calendar.Schedules.Count.ToString()
|
||||
},
|
||||
actor,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
@@ -313,11 +313,11 @@ public sealed class InMemoryQuietHoursCalendarService : IQuietHoursCalendarServi
|
||||
await _auditRepository.AppendAsync(
|
||||
tenantId,
|
||||
"quiet_hours_calendar_deleted",
|
||||
actor,
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["calendarId"] = calendarId
|
||||
},
|
||||
actor,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Concurrent;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Notify.Storage.Mongo.Repositories;
|
||||
using StellaOps.Notifier.Worker.Storage;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Correlation;
|
||||
|
||||
@@ -165,8 +165,8 @@ public sealed class InMemoryThrottleConfigurationService : IThrottleConfiguratio
|
||||
await _auditRepository.AppendAsync(
|
||||
configuration.TenantId,
|
||||
isNew ? "throttle_config_created" : "throttle_config_updated",
|
||||
payload,
|
||||
actor,
|
||||
payload,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
@@ -192,8 +192,8 @@ public sealed class InMemoryThrottleConfigurationService : IThrottleConfiguratio
|
||||
await _auditRepository.AppendAsync(
|
||||
tenantId,
|
||||
"throttle_config_deleted",
|
||||
new Dictionary<string, string>(),
|
||||
actor,
|
||||
new Dictionary<string, string>(),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,186 +0,0 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notify.Storage.Mongo.Repositories;
|
||||
using StellaOps.Notifier.Worker.Processing;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Digest;
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation of the digest generator.
|
||||
/// </summary>
|
||||
public sealed class DefaultDigestGenerator : IDigestGenerator
|
||||
{
|
||||
private readonly INotifyDeliveryRepository _deliveryRepository;
|
||||
private readonly INotifyTemplateRepository _templateRepository;
|
||||
private readonly INotifyTemplateRenderer _templateRenderer;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<DefaultDigestGenerator> _logger;
|
||||
|
||||
public DefaultDigestGenerator(
|
||||
INotifyDeliveryRepository deliveryRepository,
|
||||
INotifyTemplateRepository templateRepository,
|
||||
INotifyTemplateRenderer templateRenderer,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<DefaultDigestGenerator> logger)
|
||||
{
|
||||
_deliveryRepository = deliveryRepository ?? throw new ArgumentNullException(nameof(deliveryRepository));
|
||||
_templateRepository = templateRepository ?? throw new ArgumentNullException(nameof(templateRepository));
|
||||
_templateRenderer = templateRenderer ?? throw new ArgumentNullException(nameof(templateRenderer));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<NotifyDigest> GenerateAsync(
|
||||
DigestSchedule schedule,
|
||||
DateTimeOffset periodStart,
|
||||
DateTimeOffset periodEnd,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(schedule);
|
||||
|
||||
_logger.LogDebug(
|
||||
"Generating digest for schedule {ScheduleId}: period {PeriodStart} to {PeriodEnd}",
|
||||
schedule.ScheduleId, periodStart, periodEnd);
|
||||
|
||||
// Query deliveries for the period
|
||||
var result = await _deliveryRepository.QueryAsync(
|
||||
tenantId: schedule.TenantId,
|
||||
since: periodStart,
|
||||
status: null, // All statuses
|
||||
limit: 1000,
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Filter to relevant event kinds if specified
|
||||
var deliveries = result.Items.AsEnumerable();
|
||||
if (!schedule.EventKinds.IsDefaultOrEmpty)
|
||||
{
|
||||
var kindSet = schedule.EventKinds.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
deliveries = deliveries.Where(d => kindSet.Contains(d.Kind));
|
||||
}
|
||||
|
||||
// Filter to period
|
||||
deliveries = deliveries.Where(d =>
|
||||
d.CreatedAt >= periodStart && d.CreatedAt < periodEnd);
|
||||
|
||||
var deliveryList = deliveries.ToList();
|
||||
|
||||
// Compute event kind counts
|
||||
var kindCounts = deliveryList
|
||||
.GroupBy(d => d.Kind, StringComparer.OrdinalIgnoreCase)
|
||||
.ToImmutableDictionary(
|
||||
g => g.Key,
|
||||
g => g.Count(),
|
||||
StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var eventIds = deliveryList
|
||||
.Select(d => d.EventId)
|
||||
.Distinct()
|
||||
.ToImmutableArray();
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
var digest = new NotifyDigest
|
||||
{
|
||||
DigestId = Guid.NewGuid().ToString("N"),
|
||||
TenantId = schedule.TenantId,
|
||||
DigestKey = schedule.DigestKey,
|
||||
ScheduleId = schedule.ScheduleId,
|
||||
Period = schedule.Period,
|
||||
EventCount = deliveryList.Count,
|
||||
EventIds = eventIds,
|
||||
EventKindCounts = kindCounts,
|
||||
PeriodStart = periodStart,
|
||||
PeriodEnd = periodEnd,
|
||||
GeneratedAt = now,
|
||||
Status = deliveryList.Count > 0 ? NotifyDigestStatus.Ready : NotifyDigestStatus.Skipped,
|
||||
Metadata = schedule.Metadata
|
||||
};
|
||||
|
||||
_logger.LogInformation(
|
||||
"Generated digest {DigestId} for schedule {ScheduleId}: {EventCount} events, {UniqueEvents} unique, {KindCount} kinds",
|
||||
digest.DigestId, schedule.ScheduleId, deliveryList.Count, eventIds.Length, kindCounts.Count);
|
||||
|
||||
return digest;
|
||||
}
|
||||
|
||||
public async Task<string> FormatAsync(
|
||||
NotifyDigest digest,
|
||||
string templateId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(digest);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(templateId);
|
||||
|
||||
var template = await _templateRepository.GetAsync(
|
||||
digest.TenantId, templateId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (template is null)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Digest template {TemplateId} not found for tenant {TenantId}",
|
||||
templateId, digest.TenantId);
|
||||
|
||||
return FormatDefaultDigest(digest);
|
||||
}
|
||||
|
||||
var payload = BuildDigestPayload(digest);
|
||||
return _templateRenderer.Render(template, payload);
|
||||
}
|
||||
|
||||
private static JsonObject BuildDigestPayload(NotifyDigest digest)
|
||||
{
|
||||
var kindCountsArray = new JsonArray();
|
||||
foreach (var (kind, count) in digest.EventKindCounts)
|
||||
{
|
||||
kindCountsArray.Add(new JsonObject
|
||||
{
|
||||
["kind"] = kind,
|
||||
["count"] = count
|
||||
});
|
||||
}
|
||||
|
||||
return new JsonObject
|
||||
{
|
||||
["digestId"] = digest.DigestId,
|
||||
["tenantId"] = digest.TenantId,
|
||||
["digestKey"] = digest.DigestKey,
|
||||
["scheduleId"] = digest.ScheduleId,
|
||||
["period"] = digest.Period.ToString(),
|
||||
["eventCount"] = digest.EventCount,
|
||||
["uniqueEventCount"] = digest.EventIds.Length,
|
||||
["kindCounts"] = kindCountsArray,
|
||||
["periodStart"] = digest.PeriodStart.ToString("o"),
|
||||
["periodEnd"] = digest.PeriodEnd.ToString("o"),
|
||||
["generatedAt"] = digest.GeneratedAt.ToString("o")
|
||||
};
|
||||
}
|
||||
|
||||
private static string FormatDefaultDigest(NotifyDigest digest)
|
||||
{
|
||||
var sb = new System.Text.StringBuilder();
|
||||
sb.AppendLine($"## Notification Digest");
|
||||
sb.AppendLine();
|
||||
sb.AppendLine($"**Period:** {digest.PeriodStart:g} to {digest.PeriodEnd:g}");
|
||||
sb.AppendLine($"**Total Events:** {digest.EventCount}");
|
||||
sb.AppendLine();
|
||||
|
||||
if (digest.EventKindCounts.Count > 0)
|
||||
{
|
||||
sb.AppendLine("### Event Summary");
|
||||
sb.AppendLine();
|
||||
foreach (var (kind, count) in digest.EventKindCounts.OrderByDescending(kv => kv.Value))
|
||||
{
|
||||
sb.AppendLine($"- **{kind}**: {count}");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine("*No events in this period.*");
|
||||
}
|
||||
|
||||
return sb.ToString();
|
||||
}
|
||||
}
|
||||
@@ -1,423 +0,0 @@
|
||||
using System.Net.Http.Json;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Digest;
|
||||
|
||||
/// <summary>
|
||||
/// Distributes generated digests to recipients.
|
||||
/// </summary>
|
||||
public interface IDigestDistributor
|
||||
{
|
||||
/// <summary>
|
||||
/// Distributes a digest to the specified recipients.
|
||||
/// </summary>
|
||||
Task<DigestDistributionResult> DistributeAsync(
|
||||
DigestContent content,
|
||||
string renderedContent,
|
||||
DigestFormat format,
|
||||
IReadOnlyList<DigestRecipient> recipients,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of digest distribution.
|
||||
/// </summary>
|
||||
public sealed record DigestDistributionResult
|
||||
{
|
||||
/// <summary>
|
||||
/// Total recipients attempted.
|
||||
/// </summary>
|
||||
public int TotalRecipients { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Successfully delivered count.
|
||||
/// </summary>
|
||||
public int SuccessCount { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Failed delivery count.
|
||||
/// </summary>
|
||||
public int FailureCount { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Individual delivery results.
|
||||
/// </summary>
|
||||
public IReadOnlyList<RecipientDeliveryResult> Results { get; init; } = [];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of delivery to a single recipient.
|
||||
/// </summary>
|
||||
public sealed record RecipientDeliveryResult
|
||||
{
|
||||
/// <summary>
|
||||
/// Recipient address.
|
||||
/// </summary>
|
||||
public required string Address { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Recipient type.
|
||||
/// </summary>
|
||||
public required string Type { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether delivery succeeded.
|
||||
/// </summary>
|
||||
public required bool Success { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Error message if failed.
|
||||
/// </summary>
|
||||
public string? Error { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// When delivery was attempted.
|
||||
/// </summary>
|
||||
public required DateTimeOffset AttemptedAt { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation of <see cref="IDigestDistributor"/>.
|
||||
/// </summary>
|
||||
public sealed class DigestDistributor : IDigestDistributor
|
||||
{
|
||||
private readonly HttpClient _httpClient;
|
||||
private readonly DigestDistributorOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<DigestDistributor> _logger;
|
||||
|
||||
public DigestDistributor(
|
||||
HttpClient httpClient,
|
||||
IOptions<DigestDistributorOptions> options,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<DigestDistributor> logger)
|
||||
{
|
||||
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<DigestDistributionResult> DistributeAsync(
|
||||
DigestContent content,
|
||||
string renderedContent,
|
||||
DigestFormat format,
|
||||
IReadOnlyList<DigestRecipient> recipients,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(content);
|
||||
ArgumentNullException.ThrowIfNull(renderedContent);
|
||||
ArgumentNullException.ThrowIfNull(recipients);
|
||||
|
||||
var results = new List<RecipientDeliveryResult>();
|
||||
|
||||
foreach (var recipient in recipients)
|
||||
{
|
||||
var result = await DeliverToRecipientAsync(
|
||||
content,
|
||||
renderedContent,
|
||||
format,
|
||||
recipient,
|
||||
cancellationToken);
|
||||
|
||||
results.Add(result);
|
||||
}
|
||||
|
||||
var successCount = results.Count(r => r.Success);
|
||||
var failureCount = results.Count(r => !r.Success);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Distributed digest {DigestId}: {Success}/{Total} successful.",
|
||||
content.DigestId, successCount, recipients.Count);
|
||||
|
||||
return new DigestDistributionResult
|
||||
{
|
||||
TotalRecipients = recipients.Count,
|
||||
SuccessCount = successCount,
|
||||
FailureCount = failureCount,
|
||||
Results = results
|
||||
};
|
||||
}
|
||||
|
||||
private async Task<RecipientDeliveryResult> DeliverToRecipientAsync(
|
||||
DigestContent content,
|
||||
string renderedContent,
|
||||
DigestFormat format,
|
||||
DigestRecipient recipient,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var attemptedAt = _timeProvider.GetUtcNow();
|
||||
|
||||
try
|
||||
{
|
||||
var success = recipient.Type.ToLowerInvariant() switch
|
||||
{
|
||||
"webhook" => await DeliverToWebhookAsync(content, renderedContent, format, recipient, cancellationToken),
|
||||
"slack" => await DeliverToSlackAsync(content, renderedContent, recipient, cancellationToken),
|
||||
"teams" => await DeliverToTeamsAsync(content, renderedContent, recipient, cancellationToken),
|
||||
"email" => await DeliverToEmailAsync(content, renderedContent, format, recipient, cancellationToken),
|
||||
_ => throw new NotSupportedException($"Recipient type '{recipient.Type}' is not supported.")
|
||||
};
|
||||
|
||||
return new RecipientDeliveryResult
|
||||
{
|
||||
Address = recipient.Address,
|
||||
Type = recipient.Type,
|
||||
Success = success,
|
||||
AttemptedAt = attemptedAt
|
||||
};
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"Failed to deliver digest {DigestId} to {Type}:{Address}.",
|
||||
content.DigestId, recipient.Type, recipient.Address);
|
||||
|
||||
return new RecipientDeliveryResult
|
||||
{
|
||||
Address = recipient.Address,
|
||||
Type = recipient.Type,
|
||||
Success = false,
|
||||
Error = ex.Message,
|
||||
AttemptedAt = attemptedAt
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<bool> DeliverToWebhookAsync(
|
||||
DigestContent content,
|
||||
string renderedContent,
|
||||
DigestFormat format,
|
||||
DigestRecipient recipient,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var payload = new
|
||||
{
|
||||
digestId = content.DigestId,
|
||||
tenantId = content.TenantId,
|
||||
title = content.Title,
|
||||
periodStart = content.PeriodStart,
|
||||
periodEnd = content.PeriodEnd,
|
||||
generatedAt = content.GeneratedAt,
|
||||
format = format.ToString().ToLowerInvariant(),
|
||||
content = renderedContent,
|
||||
summary = content.Summary
|
||||
};
|
||||
|
||||
var response = await _httpClient.PostAsJsonAsync(
|
||||
recipient.Address,
|
||||
payload,
|
||||
cancellationToken);
|
||||
|
||||
return response.IsSuccessStatusCode;
|
||||
}
|
||||
|
||||
private async Task<bool> DeliverToSlackAsync(
|
||||
DigestContent content,
|
||||
string renderedContent,
|
||||
DigestRecipient recipient,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
// Build Slack blocks
|
||||
var blocks = new List<object>
|
||||
{
|
||||
new
|
||||
{
|
||||
type = "header",
|
||||
text = new { type = "plain_text", text = content.Title }
|
||||
},
|
||||
new
|
||||
{
|
||||
type = "section",
|
||||
fields = new object[]
|
||||
{
|
||||
new { type = "mrkdwn", text = $"*Total Incidents:*\n{content.Summary.TotalIncidents}" },
|
||||
new { type = "mrkdwn", text = $"*New:*\n{content.Summary.NewIncidents}" },
|
||||
new { type = "mrkdwn", text = $"*Acknowledged:*\n{content.Summary.AcknowledgedIncidents}" },
|
||||
new { type = "mrkdwn", text = $"*Resolved:*\n{content.Summary.ResolvedIncidents}" }
|
||||
}
|
||||
},
|
||||
new
|
||||
{
|
||||
type = "divider"
|
||||
}
|
||||
};
|
||||
|
||||
// Add top incidents
|
||||
foreach (var incident in content.Incidents.Take(5))
|
||||
{
|
||||
var statusEmoji = incident.Status switch
|
||||
{
|
||||
Correlation.IncidentStatus.Open => ":red_circle:",
|
||||
Correlation.IncidentStatus.Acknowledged => ":large_yellow_circle:",
|
||||
Correlation.IncidentStatus.Resolved => ":large_green_circle:",
|
||||
_ => ":white_circle:"
|
||||
};
|
||||
|
||||
blocks.Add(new
|
||||
{
|
||||
type = "section",
|
||||
text = new
|
||||
{
|
||||
type = "mrkdwn",
|
||||
text = $"{statusEmoji} *{incident.Title}*\n_{incident.EventKind}_ • {incident.EventCount} events"
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (content.Incidents.Count > 5)
|
||||
{
|
||||
blocks.Add(new
|
||||
{
|
||||
type = "context",
|
||||
elements = new object[]
|
||||
{
|
||||
new { type = "mrkdwn", text = $"_...and {content.Incidents.Count - 5} more incidents_" }
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
var payload = new { blocks };
|
||||
|
||||
var json = JsonSerializer.Serialize(payload);
|
||||
var httpContent = new StringContent(json, Encoding.UTF8, "application/json");
|
||||
|
||||
var response = await _httpClient.PostAsync(recipient.Address, httpContent, cancellationToken);
|
||||
return response.IsSuccessStatusCode;
|
||||
}
|
||||
|
||||
private async Task<bool> DeliverToTeamsAsync(
|
||||
DigestContent content,
|
||||
string renderedContent,
|
||||
DigestRecipient recipient,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
// Build Teams Adaptive Card
|
||||
var card = new
|
||||
{
|
||||
type = "message",
|
||||
attachments = new object[]
|
||||
{
|
||||
new
|
||||
{
|
||||
contentType = "application/vnd.microsoft.card.adaptive",
|
||||
contentUrl = (string?)null,
|
||||
content = new
|
||||
{
|
||||
type = "AdaptiveCard",
|
||||
version = "1.4",
|
||||
body = new object[]
|
||||
{
|
||||
new
|
||||
{
|
||||
type = "TextBlock",
|
||||
text = content.Title,
|
||||
weight = "Bolder",
|
||||
size = "Large"
|
||||
},
|
||||
new
|
||||
{
|
||||
type = "ColumnSet",
|
||||
columns = new object[]
|
||||
{
|
||||
new
|
||||
{
|
||||
type = "Column",
|
||||
width = "auto",
|
||||
items = new object[]
|
||||
{
|
||||
new { type = "TextBlock", text = "Total", weight = "Bolder" },
|
||||
new { type = "TextBlock", text = content.Summary.TotalIncidents.ToString() }
|
||||
}
|
||||
},
|
||||
new
|
||||
{
|
||||
type = "Column",
|
||||
width = "auto",
|
||||
items = new object[]
|
||||
{
|
||||
new { type = "TextBlock", text = "New", weight = "Bolder" },
|
||||
new { type = "TextBlock", text = content.Summary.NewIncidents.ToString() }
|
||||
}
|
||||
},
|
||||
new
|
||||
{
|
||||
type = "Column",
|
||||
width = "auto",
|
||||
items = new object[]
|
||||
{
|
||||
new { type = "TextBlock", text = "Resolved", weight = "Bolder" },
|
||||
new { type = "TextBlock", text = content.Summary.ResolvedIncidents.ToString() }
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
new
|
||||
{
|
||||
type = "TextBlock",
|
||||
text = $"Period: {content.PeriodStart:yyyy-MM-dd} to {content.PeriodEnd:yyyy-MM-dd}",
|
||||
isSubtle = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
var json = JsonSerializer.Serialize(card);
|
||||
var httpContent = new StringContent(json, Encoding.UTF8, "application/json");
|
||||
|
||||
var response = await _httpClient.PostAsync(recipient.Address, httpContent, cancellationToken);
|
||||
return response.IsSuccessStatusCode;
|
||||
}
|
||||
|
||||
private Task<bool> DeliverToEmailAsync(
|
||||
DigestContent content,
|
||||
string renderedContent,
|
||||
DigestFormat format,
|
||||
DigestRecipient recipient,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
// Email delivery would typically use an email service
|
||||
// For now, log and return success (actual implementation would integrate with email adapter)
|
||||
_logger.LogInformation(
|
||||
"Email delivery for digest {DigestId} to {Address} would be sent here.",
|
||||
content.DigestId, recipient.Address);
|
||||
|
||||
// In a real implementation, this would:
|
||||
// 1. Use an IEmailSender or similar service
|
||||
// 2. Format the content appropriately (HTML for HTML format, etc.)
|
||||
// 3. Send via SMTP or email API
|
||||
|
||||
return Task.FromResult(true);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Configuration options for digest distribution.
|
||||
/// </summary>
|
||||
public sealed class DigestDistributorOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Configuration section name.
|
||||
/// </summary>
|
||||
public const string SectionName = "Notifier:DigestDistributor";
|
||||
|
||||
/// <summary>
|
||||
/// Timeout for HTTP delivery requests.
|
||||
/// </summary>
|
||||
public TimeSpan DeliveryTimeout { get; set; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>
|
||||
/// Maximum retry attempts per recipient.
|
||||
/// </summary>
|
||||
public int MaxRetries { get; set; } = 3;
|
||||
|
||||
/// <summary>
|
||||
/// Whether to continue on individual delivery failures.
|
||||
/// </summary>
|
||||
public bool ContinueOnFailure { get; set; } = true;
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notifier.Worker.Channels;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Digest;
|
||||
@@ -54,7 +55,7 @@ public sealed class DigestScheduleRunner : BackgroundService
|
||||
await Task.WhenAll(scheduleTasks);
|
||||
}
|
||||
|
||||
private async Task RunScheduleAsync(DigestSchedule schedule, CancellationToken stoppingToken)
|
||||
private async Task RunScheduleAsync(DigestScheduleConfig schedule, CancellationToken stoppingToken)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Starting digest schedule '{Name}' with interval {Interval}.",
|
||||
@@ -93,7 +94,7 @@ public sealed class DigestScheduleRunner : BackgroundService
|
||||
_logger.LogInformation("Digest schedule '{Name}' stopped.", schedule.Name);
|
||||
}
|
||||
|
||||
private async Task ExecuteScheduleAsync(DigestSchedule schedule, CancellationToken stoppingToken)
|
||||
private async Task ExecuteScheduleAsync(DigestScheduleConfig schedule, CancellationToken stoppingToken)
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var query = new DigestQuery
|
||||
@@ -150,7 +151,7 @@ public sealed class DigestScheduleRunner : BackgroundService
|
||||
schedule.Name, successCount, errorCount, tenants.Count);
|
||||
}
|
||||
|
||||
private TimeSpan CalculateInitialDelay(DigestSchedule schedule)
|
||||
private TimeSpan CalculateInitialDelay(DigestScheduleConfig schedule)
|
||||
{
|
||||
if (!schedule.AlignToInterval)
|
||||
{
|
||||
@@ -179,7 +180,7 @@ public interface IDigestDistributor
|
||||
/// </summary>
|
||||
Task DistributeAsync(
|
||||
DigestResult digest,
|
||||
DigestSchedule schedule,
|
||||
DigestScheduleConfig schedule,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
@@ -202,48 +203,71 @@ public interface IDigestTenantProvider
|
||||
public sealed class ChannelDigestDistributor : IDigestDistributor
|
||||
{
|
||||
private readonly IChannelAdapterFactory _channelFactory;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<ChannelDigestDistributor> _logger;
|
||||
|
||||
public ChannelDigestDistributor(
|
||||
IChannelAdapterFactory channelFactory,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<ChannelDigestDistributor> logger)
|
||||
{
|
||||
_channelFactory = channelFactory ?? throw new ArgumentNullException(nameof(channelFactory));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task DistributeAsync(
|
||||
DigestResult digest,
|
||||
DigestSchedule schedule,
|
||||
DigestScheduleConfig schedule,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
foreach (var channelConfig in schedule.Channels)
|
||||
{
|
||||
try
|
||||
{
|
||||
var adapter = _channelFactory.Create(channelConfig.Type);
|
||||
if (!Enum.TryParse<NotifyChannelType>(channelConfig.Type, true, out var channelType))
|
||||
{
|
||||
_logger.LogWarning("Unsupported digest channel type {ChannelType}.", channelConfig.Type);
|
||||
continue;
|
||||
}
|
||||
|
||||
var adapter = _channelFactory.GetAdapter(channelType);
|
||||
if (adapter is null)
|
||||
{
|
||||
_logger.LogWarning("No adapter registered for digest channel {ChannelType}.", channelType);
|
||||
continue;
|
||||
}
|
||||
|
||||
var metadata = BuildMetadata(digest, schedule, channelConfig);
|
||||
var channel = BuildChannel(channelType, digest, schedule, channelConfig);
|
||||
var delivery = BuildDelivery(digest, channelType, metadata);
|
||||
var content = SelectContent(digest, channelConfig.Type);
|
||||
|
||||
await adapter.SendAsync(new ChannelMessage
|
||||
{
|
||||
ChannelType = channelConfig.Type,
|
||||
Destination = channelConfig.Destination,
|
||||
Subject = $"Notification Digest - {digest.TenantId}",
|
||||
Body = content,
|
||||
Format = channelConfig.Format ?? GetDefaultFormat(channelConfig.Type),
|
||||
Metadata = new Dictionary<string, string>
|
||||
{
|
||||
["digestId"] = digest.DigestId,
|
||||
["tenantId"] = digest.TenantId,
|
||||
["scheduleName"] = schedule.Name,
|
||||
["from"] = digest.From.ToString("O"),
|
||||
["to"] = digest.To.ToString("O")
|
||||
}
|
||||
}, cancellationToken);
|
||||
var context = new ChannelDispatchContext(
|
||||
delivery.DeliveryId,
|
||||
digest.TenantId,
|
||||
channel,
|
||||
delivery,
|
||||
content,
|
||||
$"Notification Digest - {digest.TenantId}",
|
||||
metadata,
|
||||
_timeProvider.GetUtcNow(),
|
||||
TraceId: $"digest-{digest.DigestId}");
|
||||
|
||||
_logger.LogDebug(
|
||||
"Sent digest {DigestId} to channel {Channel} ({Destination}).",
|
||||
digest.DigestId, channelConfig.Type, channelConfig.Destination);
|
||||
var result = await adapter.DispatchAsync(context, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (result.Success)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Sent digest {DigestId} to channel {Channel} ({Destination}).",
|
||||
digest.DigestId, channelType, channelConfig.Destination);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Digest {DigestId} dispatch to {Channel} failed: {Message}.",
|
||||
digest.DigestId, channelType, result.Message ?? "dispatch failed");
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -254,6 +278,77 @@ public sealed class ChannelDigestDistributor : IDigestDistributor
|
||||
}
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, string> BuildMetadata(
|
||||
DigestResult digest,
|
||||
DigestScheduleConfig schedule,
|
||||
DigestChannelConfig channelConfig)
|
||||
{
|
||||
return new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["digestId"] = digest.DigestId,
|
||||
["tenantId"] = digest.TenantId,
|
||||
["scheduleName"] = schedule.Name,
|
||||
["from"] = digest.From.ToString("O"),
|
||||
["to"] = digest.To.ToString("O"),
|
||||
["destination"] = channelConfig.Destination,
|
||||
["channelType"] = channelConfig.Type
|
||||
};
|
||||
}
|
||||
|
||||
private static NotifyChannel BuildChannel(
|
||||
NotifyChannelType channelType,
|
||||
DigestResult digest,
|
||||
DigestScheduleConfig schedule,
|
||||
DigestChannelConfig channelConfig)
|
||||
{
|
||||
var properties = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["destination"] = channelConfig.Destination
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(channelConfig.Format))
|
||||
{
|
||||
properties["format"] = channelConfig.Format!;
|
||||
}
|
||||
|
||||
var config = NotifyChannelConfig.Create(
|
||||
secretRef: $"digest-{schedule.Name}",
|
||||
target: channelConfig.Destination,
|
||||
endpoint: channelConfig.Destination,
|
||||
properties: properties);
|
||||
|
||||
return NotifyChannel.Create(
|
||||
channelId: $"digest-{schedule.Name}-{channelType}".ToLowerInvariant(),
|
||||
tenantId: digest.TenantId,
|
||||
name: $"{schedule.Name}-{channelType}",
|
||||
type: channelType,
|
||||
config: config,
|
||||
enabled: true,
|
||||
metadata: properties);
|
||||
}
|
||||
|
||||
private static NotifyDelivery BuildDelivery(
|
||||
DigestResult digest,
|
||||
NotifyChannelType channelType,
|
||||
IReadOnlyDictionary<string, string> metadata)
|
||||
{
|
||||
return NotifyDelivery.Create(
|
||||
deliveryId: $"digest-{digest.DigestId}-{channelType}".ToLowerInvariant(),
|
||||
tenantId: digest.TenantId,
|
||||
ruleId: "digest",
|
||||
actionId: channelType.ToString(),
|
||||
eventId: Guid.NewGuid(),
|
||||
kind: "digest",
|
||||
status: NotifyDeliveryStatus.Sending,
|
||||
statusReason: null,
|
||||
rendered: null,
|
||||
attempts: Array.Empty<NotifyDeliveryAttempt>(),
|
||||
metadata: metadata,
|
||||
createdAt: digest.GeneratedAt,
|
||||
sentAt: null,
|
||||
completedAt: null);
|
||||
}
|
||||
|
||||
private static string SelectContent(DigestResult digest, string channelType)
|
||||
{
|
||||
if (digest.Content is null)
|
||||
@@ -269,17 +364,6 @@ public sealed class ChannelDigestDistributor : IDigestDistributor
|
||||
_ => digest.Content.PlainText ?? ""
|
||||
};
|
||||
}
|
||||
|
||||
private static string GetDefaultFormat(string channelType)
|
||||
{
|
||||
return channelType.ToLowerInvariant() switch
|
||||
{
|
||||
"slack" => "blocks",
|
||||
"email" => "html",
|
||||
"webhook" => "json",
|
||||
_ => "text"
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -324,13 +408,13 @@ public sealed class DigestScheduleOptions
|
||||
/// <summary>
|
||||
/// Configured digest schedules.
|
||||
/// </summary>
|
||||
public List<DigestSchedule> Schedules { get; set; } = [];
|
||||
public List<DigestScheduleConfig> Schedules { get; set; } = [];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A single digest schedule configuration.
|
||||
/// </summary>
|
||||
public sealed class DigestSchedule
|
||||
public sealed class DigestScheduleConfig
|
||||
{
|
||||
/// <summary>
|
||||
/// Unique name for this schedule.
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
namespace StellaOps.Notifier.Worker.Digest;
|
||||
|
||||
/// <summary>
|
||||
/// Types of digests supported by the worker.
|
||||
/// </summary>
|
||||
public enum DigestType
|
||||
{
|
||||
Daily,
|
||||
Weekly,
|
||||
Monthly
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Output formats for rendered digests.
|
||||
/// </summary>
|
||||
public enum DigestFormat
|
||||
{
|
||||
Html,
|
||||
PlainText,
|
||||
Markdown,
|
||||
Json,
|
||||
Slack,
|
||||
Teams
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
using System.Collections.Immutable;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Digest;
|
||||
|
||||
/// <summary>
|
||||
/// Represents a compiled digest summarizing multiple events for batch delivery.
|
||||
/// </summary>
|
||||
public sealed record NotifyDigest
|
||||
{
|
||||
public required string DigestId { get; init; }
|
||||
public required string TenantId { get; init; }
|
||||
public required string DigestKey { get; init; }
|
||||
public required string ScheduleId { get; init; }
|
||||
public required DigestPeriod Period { get; init; }
|
||||
public required int EventCount { get; init; }
|
||||
public required ImmutableArray<Guid> EventIds { get; init; }
|
||||
public required ImmutableDictionary<string, int> EventKindCounts { get; init; }
|
||||
public required DateTimeOffset PeriodStart { get; init; }
|
||||
public required DateTimeOffset PeriodEnd { get; init; }
|
||||
public required DateTimeOffset GeneratedAt { get; init; }
|
||||
public NotifyDigestStatus Status { get; init; } = NotifyDigestStatus.Pending;
|
||||
public DateTimeOffset? SentAt { get; init; }
|
||||
public string? RenderedContent { get; init; }
|
||||
public ImmutableDictionary<string, string> Metadata { get; init; } = ImmutableDictionary<string, string>.Empty;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Status of a digest through its lifecycle.
|
||||
/// </summary>
|
||||
public enum NotifyDigestStatus
|
||||
{
|
||||
Pending,
|
||||
Generating,
|
||||
Ready,
|
||||
Sent,
|
||||
Failed,
|
||||
Skipped
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Digest delivery period/frequency.
|
||||
/// </summary>
|
||||
public enum DigestPeriod
|
||||
{
|
||||
Hourly,
|
||||
Daily,
|
||||
Weekly,
|
||||
Custom
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for a digest schedule.
|
||||
/// </summary>
|
||||
public sealed record DigestSchedule
|
||||
{
|
||||
public required string ScheduleId { get; init; }
|
||||
public required string TenantId { get; init; }
|
||||
public required string Name { get; init; }
|
||||
public required string DigestKey { get; init; }
|
||||
public required DigestPeriod Period { get; init; }
|
||||
public string? CronExpression { get; init; }
|
||||
public required string TimeZone { get; init; }
|
||||
public required string ChannelId { get; init; }
|
||||
public required string TemplateId { get; init; }
|
||||
public ImmutableArray<string> EventKinds { get; init; } = [];
|
||||
public bool Enabled { get; init; } = true;
|
||||
public ImmutableDictionary<string, string> Metadata { get; init; } = ImmutableDictionary<string, string>.Empty;
|
||||
}
|
||||
@@ -1,9 +1,10 @@
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notify.Storage.Mongo.Repositories;
|
||||
using StellaOps.Notifier.Worker.Storage;
|
||||
using StellaOps.Notifier.Worker.Options;
|
||||
using System.Collections.Immutable;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Dispatch;
|
||||
|
||||
@@ -205,16 +206,17 @@ public sealed class DeliveryDispatchWorker : BackgroundService
|
||||
// Update delivery status
|
||||
var attempt = new NotifyDeliveryAttempt(
|
||||
timestamp: DateTimeOffset.UtcNow,
|
||||
status: result.Success ? NotifyDeliveryAttemptStatus.Success : NotifyDeliveryAttemptStatus.Failed,
|
||||
status: result.Success ? NotifyDeliveryAttemptStatus.Succeeded : NotifyDeliveryAttemptStatus.Failed,
|
||||
reason: result.ErrorMessage);
|
||||
|
||||
var updatedDelivery = delivery with
|
||||
{
|
||||
Status = result.Status,
|
||||
StatusReason = result.ErrorMessage,
|
||||
CompletedAt = result.Success ? DateTimeOffset.UtcNow : null,
|
||||
Attempts = delivery.Attempts.Add(attempt)
|
||||
};
|
||||
var completedAt = result.Success || !result.IsRetryable ? DateTimeOffset.UtcNow : delivery.CompletedAt;
|
||||
|
||||
var updatedDelivery = CloneDelivery(
|
||||
delivery,
|
||||
result.Status,
|
||||
result.ErrorMessage,
|
||||
delivery.Attempts.Add(attempt),
|
||||
completedAt);
|
||||
|
||||
await deliveryRepository.UpdateAsync(updatedDelivery, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
@@ -250,12 +252,12 @@ public sealed class DeliveryDispatchWorker : BackgroundService
|
||||
status: NotifyDeliveryAttemptStatus.Failed,
|
||||
reason: errorMessage);
|
||||
|
||||
var updated = delivery with
|
||||
{
|
||||
Status = NotifyDeliveryStatus.Failed,
|
||||
StatusReason = errorMessage,
|
||||
Attempts = delivery.Attempts.Add(attempt)
|
||||
};
|
||||
var updated = CloneDelivery(
|
||||
delivery,
|
||||
NotifyDeliveryStatus.Failed,
|
||||
errorMessage,
|
||||
delivery.Attempts.Add(attempt),
|
||||
delivery.CompletedAt ?? DateTimeOffset.UtcNow);
|
||||
|
||||
try
|
||||
{
|
||||
@@ -266,4 +268,28 @@ public sealed class DeliveryDispatchWorker : BackgroundService
|
||||
_logger.LogError(ex, "Failed to update delivery {DeliveryId} status.", delivery.DeliveryId);
|
||||
}
|
||||
}
|
||||
|
||||
private static NotifyDelivery CloneDelivery(
|
||||
NotifyDelivery source,
|
||||
NotifyDeliveryStatus status,
|
||||
string? statusReason,
|
||||
ImmutableArray<NotifyDeliveryAttempt> attempts,
|
||||
DateTimeOffset? completedAt)
|
||||
{
|
||||
return NotifyDelivery.Create(
|
||||
source.DeliveryId,
|
||||
source.TenantId,
|
||||
source.RuleId,
|
||||
source.ActionId,
|
||||
source.EventId,
|
||||
source.Kind,
|
||||
status,
|
||||
statusReason,
|
||||
source.Rendered,
|
||||
attempts,
|
||||
source.Metadata,
|
||||
source.CreatedAt,
|
||||
source.SentAt,
|
||||
completedAt);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,7 +69,7 @@ public sealed partial class SimpleTemplateRenderer : INotifyTemplateRenderer
|
||||
["eventId"] = notifyEvent.EventId.ToString(),
|
||||
["kind"] = notifyEvent.Kind,
|
||||
["tenant"] = notifyEvent.Tenant,
|
||||
["timestamp"] = notifyEvent.Timestamp.ToString("O"),
|
||||
["timestamp"] = notifyEvent.Ts.ToString("O"),
|
||||
["actor"] = notifyEvent.Actor,
|
||||
["version"] = notifyEvent.Version,
|
||||
};
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notify.Storage.Mongo.Repositories;
|
||||
using StellaOps.Notifier.Worker.Storage;
|
||||
using StellaOps.Notifier.Worker.Correlation;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Escalation;
|
||||
@@ -95,11 +95,11 @@ public sealed class AckBridge : IAckBridge
|
||||
cancellationToken);
|
||||
|
||||
// Acknowledge in incident manager
|
||||
await _incidentManager.AcknowledgeAsync(
|
||||
tenantId,
|
||||
incidentId,
|
||||
request.AcknowledgedBy,
|
||||
cancellationToken);
|
||||
await _incidentManager.AcknowledgeAsync(
|
||||
tenantId,
|
||||
incidentId,
|
||||
request.AcknowledgedBy,
|
||||
cancellationToken: cancellationToken);
|
||||
|
||||
// Audit
|
||||
if (_auditRepository is not null)
|
||||
@@ -107,6 +107,7 @@ public sealed class AckBridge : IAckBridge
|
||||
await _auditRepository.AppendAsync(
|
||||
tenantId,
|
||||
"ack_bridge_processed",
|
||||
request.AcknowledgedBy,
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["incidentId"] = incidentId,
|
||||
@@ -115,7 +116,6 @@ public sealed class AckBridge : IAckBridge
|
||||
["externalId"] = request.ExternalId ?? "",
|
||||
["comment"] = request.Comment ?? ""
|
||||
},
|
||||
request.AcknowledgedBy,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,507 +0,0 @@
|
||||
using System.Collections.Immutable;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notify.Storage.Mongo.Repositories;
|
||||
using StellaOps.Notifier.Worker.Channels;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Escalation;
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation of the escalation engine.
|
||||
/// </summary>
|
||||
public sealed class DefaultEscalationEngine : IEscalationEngine
|
||||
{
|
||||
private readonly INotifyEscalationPolicyRepository _policyRepository;
|
||||
private readonly INotifyEscalationStateRepository _stateRepository;
|
||||
private readonly INotifyChannelRepository _channelRepository;
|
||||
private readonly IOnCallResolver _onCallResolver;
|
||||
private readonly IEnumerable<INotifyChannelAdapter> _channelAdapters;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<DefaultEscalationEngine> _logger;
|
||||
|
||||
public DefaultEscalationEngine(
|
||||
INotifyEscalationPolicyRepository policyRepository,
|
||||
INotifyEscalationStateRepository stateRepository,
|
||||
INotifyChannelRepository channelRepository,
|
||||
IOnCallResolver onCallResolver,
|
||||
IEnumerable<INotifyChannelAdapter> channelAdapters,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<DefaultEscalationEngine> logger)
|
||||
{
|
||||
_policyRepository = policyRepository ?? throw new ArgumentNullException(nameof(policyRepository));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_channelRepository = channelRepository ?? throw new ArgumentNullException(nameof(channelRepository));
|
||||
_onCallResolver = onCallResolver ?? throw new ArgumentNullException(nameof(onCallResolver));
|
||||
_channelAdapters = channelAdapters ?? throw new ArgumentNullException(nameof(channelAdapters));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<NotifyEscalationState> StartEscalationAsync(
|
||||
string tenantId,
|
||||
string incidentId,
|
||||
string policyId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(incidentId);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(policyId);
|
||||
|
||||
// Check if escalation already exists for this incident
|
||||
var existingState = await _stateRepository.GetByIncidentAsync(tenantId, incidentId, cancellationToken).ConfigureAwait(false);
|
||||
if (existingState is not null && existingState.Status == NotifyEscalationStatus.Active)
|
||||
{
|
||||
_logger.LogDebug("Escalation already active for incident {IncidentId}", incidentId);
|
||||
return existingState;
|
||||
}
|
||||
|
||||
var policy = await _policyRepository.GetAsync(tenantId, policyId, cancellationToken).ConfigureAwait(false);
|
||||
if (policy is null)
|
||||
{
|
||||
throw new InvalidOperationException($"Escalation policy {policyId} not found.");
|
||||
}
|
||||
|
||||
if (!policy.Enabled)
|
||||
{
|
||||
throw new InvalidOperationException($"Escalation policy {policyId} is disabled.");
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var firstLevel = policy.Levels.FirstOrDefault();
|
||||
var nextEscalationAt = firstLevel is not null ? now.Add(firstLevel.EscalateAfter) : (DateTimeOffset?)null;
|
||||
|
||||
var state = NotifyEscalationState.Create(
|
||||
stateId: Guid.NewGuid().ToString("N"),
|
||||
tenantId: tenantId,
|
||||
incidentId: incidentId,
|
||||
policyId: policyId,
|
||||
currentLevel: 0,
|
||||
repeatIteration: 0,
|
||||
status: NotifyEscalationStatus.Active,
|
||||
nextEscalationAt: nextEscalationAt,
|
||||
createdAt: now);
|
||||
|
||||
await _stateRepository.UpsertAsync(state, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Notify first level immediately
|
||||
if (firstLevel is not null)
|
||||
{
|
||||
await NotifyLevelAsync(tenantId, state, policy, firstLevel, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Started escalation {StateId} for incident {IncidentId} with policy {PolicyId}",
|
||||
state.StateId, incidentId, policyId);
|
||||
|
||||
return state;
|
||||
}
|
||||
|
||||
public async Task<EscalationProcessResult> ProcessPendingEscalationsAsync(
|
||||
string tenantId,
|
||||
int batchSize = 100,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var pendingStates = await _stateRepository.ListDueForEscalationAsync(tenantId, now, batchSize, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var processed = 0;
|
||||
var escalated = 0;
|
||||
var exhausted = 0;
|
||||
var errors = 0;
|
||||
var errorMessages = new List<string>();
|
||||
|
||||
foreach (var state in pendingStates)
|
||||
{
|
||||
try
|
||||
{
|
||||
var policy = await _policyRepository.GetAsync(tenantId, state.PolicyId, cancellationToken).ConfigureAwait(false);
|
||||
if (policy is null || !policy.Enabled)
|
||||
{
|
||||
_logger.LogWarning("Policy {PolicyId} not found or disabled for escalation {StateId}", state.PolicyId, state.StateId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var result = await ProcessEscalationAsync(tenantId, state, policy, now, cancellationToken).ConfigureAwait(false);
|
||||
processed++;
|
||||
|
||||
if (result.Escalated)
|
||||
{
|
||||
escalated++;
|
||||
}
|
||||
else if (result.Exhausted)
|
||||
{
|
||||
exhausted++;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
errors++;
|
||||
errorMessages.Add($"State {state.StateId}: {ex.Message}");
|
||||
_logger.LogError(ex, "Error processing escalation {StateId}", state.StateId);
|
||||
}
|
||||
}
|
||||
|
||||
return new EscalationProcessResult
|
||||
{
|
||||
Processed = processed,
|
||||
Escalated = escalated,
|
||||
Exhausted = exhausted,
|
||||
Errors = errors,
|
||||
ErrorMessages = errorMessages.Count > 0 ? errorMessages : null
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<NotifyEscalationState?> AcknowledgeAsync(
|
||||
string tenantId,
|
||||
string stateIdOrIncidentId,
|
||||
string acknowledgedBy,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var state = await FindStateAsync(tenantId, stateIdOrIncidentId, cancellationToken).ConfigureAwait(false);
|
||||
if (state is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (state.Status != NotifyEscalationStatus.Active)
|
||||
{
|
||||
_logger.LogDebug("Escalation {StateId} is not active, cannot acknowledge", state.StateId);
|
||||
return state;
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
await _stateRepository.AcknowledgeAsync(tenantId, state.StateId, acknowledgedBy, now, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Escalation {StateId} acknowledged by {AcknowledgedBy}",
|
||||
state.StateId, acknowledgedBy);
|
||||
|
||||
return await _stateRepository.GetAsync(tenantId, state.StateId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<NotifyEscalationState?> ResolveAsync(
|
||||
string tenantId,
|
||||
string stateIdOrIncidentId,
|
||||
string resolvedBy,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var state = await FindStateAsync(tenantId, stateIdOrIncidentId, cancellationToken).ConfigureAwait(false);
|
||||
if (state is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (state.Status == NotifyEscalationStatus.Resolved)
|
||||
{
|
||||
return state;
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
await _stateRepository.ResolveAsync(tenantId, state.StateId, resolvedBy, now, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Escalation {StateId} resolved by {ResolvedBy}",
|
||||
state.StateId, resolvedBy);
|
||||
|
||||
return await _stateRepository.GetAsync(tenantId, state.StateId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<NotifyEscalationState?> GetStateForIncidentAsync(
|
||||
string tenantId,
|
||||
string incidentId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
return await _stateRepository.GetByIncidentAsync(tenantId, incidentId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<NotifyEscalationState?> FindStateAsync(
|
||||
string tenantId,
|
||||
string stateIdOrIncidentId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
// Try by state ID first
|
||||
var state = await _stateRepository.GetAsync(tenantId, stateIdOrIncidentId, cancellationToken).ConfigureAwait(false);
|
||||
if (state is not null)
|
||||
{
|
||||
return state;
|
||||
}
|
||||
|
||||
// Try by incident ID
|
||||
return await _stateRepository.GetByIncidentAsync(tenantId, stateIdOrIncidentId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<(bool Escalated, bool Exhausted)> ProcessEscalationAsync(
|
||||
string tenantId,
|
||||
NotifyEscalationState state,
|
||||
NotifyEscalationPolicy policy,
|
||||
DateTimeOffset now,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var nextLevel = state.CurrentLevel + 1;
|
||||
var iteration = state.RepeatIteration;
|
||||
|
||||
if (nextLevel >= policy.Levels.Length)
|
||||
{
|
||||
// Reached end of levels
|
||||
if (policy.RepeatEnabled && (policy.RepeatCount is null || iteration < policy.RepeatCount))
|
||||
{
|
||||
// Repeat from first level
|
||||
nextLevel = 0;
|
||||
iteration++;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Exhausted all levels and repeats
|
||||
await _stateRepository.UpdateLevelAsync(
|
||||
tenantId,
|
||||
state.StateId,
|
||||
state.CurrentLevel,
|
||||
iteration,
|
||||
null, // No next escalation
|
||||
new NotifyEscalationAttempt(state.CurrentLevel, iteration, now, ImmutableArray<string>.Empty, true),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation("Escalation {StateId} exhausted all levels", state.StateId);
|
||||
return (false, true);
|
||||
}
|
||||
}
|
||||
|
||||
var level = policy.Levels[nextLevel];
|
||||
var nextEscalationAt = now.Add(level.EscalateAfter);
|
||||
|
||||
// Notify targets at this level
|
||||
var notifiedTargets = await NotifyLevelAsync(tenantId, state, policy, level, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var attempt = new NotifyEscalationAttempt(
|
||||
nextLevel,
|
||||
iteration,
|
||||
now,
|
||||
notifiedTargets.ToImmutableArray(),
|
||||
notifiedTargets.Count > 0);
|
||||
|
||||
await _stateRepository.UpdateLevelAsync(
|
||||
tenantId,
|
||||
state.StateId,
|
||||
nextLevel,
|
||||
iteration,
|
||||
nextEscalationAt,
|
||||
attempt,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Escalation {StateId} advanced to level {Level} iteration {Iteration}, notified {TargetCount} targets",
|
||||
state.StateId, nextLevel, iteration, notifiedTargets.Count);
|
||||
|
||||
return (true, false);
|
||||
}
|
||||
|
||||
private async Task<List<string>> NotifyLevelAsync(
|
||||
string tenantId,
|
||||
NotifyEscalationState state,
|
||||
NotifyEscalationPolicy policy,
|
||||
NotifyEscalationLevel level,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var notifiedTargets = new List<string>();
|
||||
|
||||
foreach (var target in level.Targets)
|
||||
{
|
||||
try
|
||||
{
|
||||
var notified = await NotifyTargetAsync(tenantId, state, target, cancellationToken).ConfigureAwait(false);
|
||||
if (notified)
|
||||
{
|
||||
notifiedTargets.Add($"{target.Type}:{target.TargetId}");
|
||||
}
|
||||
|
||||
// If NotifyAll is false, stop after first successful notification
|
||||
if (!level.NotifyAll && notified)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to notify target {TargetType}:{TargetId}", target.Type, target.TargetId);
|
||||
}
|
||||
}
|
||||
|
||||
return notifiedTargets;
|
||||
}
|
||||
|
||||
private async Task<bool> NotifyTargetAsync(
|
||||
string tenantId,
|
||||
NotifyEscalationState state,
|
||||
NotifyEscalationTarget target,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
switch (target.Type)
|
||||
{
|
||||
case NotifyEscalationTargetType.OnCallSchedule:
|
||||
var resolution = await _onCallResolver.ResolveAsync(tenantId, target.TargetId, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
if (resolution.OnCallUsers.IsDefaultOrEmpty)
|
||||
{
|
||||
_logger.LogWarning("No on-call user found for schedule {ScheduleId}", target.TargetId);
|
||||
return false;
|
||||
}
|
||||
|
||||
var notifiedAny = false;
|
||||
foreach (var user in resolution.OnCallUsers)
|
||||
{
|
||||
if (await NotifyUserAsync(tenantId, state, user, target.ChannelOverride, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
notifiedAny = true;
|
||||
}
|
||||
}
|
||||
return notifiedAny;
|
||||
|
||||
case NotifyEscalationTargetType.User:
|
||||
// For user targets, we'd need a user repository to get contact info
|
||||
// For now, log and return false
|
||||
_logger.LogDebug("User target notification not yet implemented: {UserId}", target.TargetId);
|
||||
return false;
|
||||
|
||||
case NotifyEscalationTargetType.Channel:
|
||||
// Send directly to a channel
|
||||
return await SendToChannelAsync(tenantId, state, target.TargetId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
case NotifyEscalationTargetType.ExternalService:
|
||||
// Would call PagerDuty/OpsGenie adapters
|
||||
_logger.LogDebug("External service target notification not yet implemented: {ServiceId}", target.TargetId);
|
||||
return false;
|
||||
|
||||
case NotifyEscalationTargetType.InAppInbox:
|
||||
// Would send to in-app inbox
|
||||
_logger.LogDebug("In-app inbox notification not yet implemented");
|
||||
return false;
|
||||
|
||||
default:
|
||||
_logger.LogWarning("Unknown escalation target type: {TargetType}", target.Type);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<bool> NotifyUserAsync(
|
||||
string tenantId,
|
||||
NotifyEscalationState state,
|
||||
NotifyOnCallParticipant user,
|
||||
string? channelOverride,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
// Prefer channel override if specified
|
||||
if (!string.IsNullOrWhiteSpace(channelOverride))
|
||||
{
|
||||
return await SendToChannelAsync(tenantId, state, channelOverride, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// Try contact methods in order
|
||||
foreach (var method in user.ContactMethods.OrderBy(m => m.Priority))
|
||||
{
|
||||
if (!method.Enabled) continue;
|
||||
|
||||
// Map contact method to channel type
|
||||
var channelType = method.Type switch
|
||||
{
|
||||
NotifyContactMethodType.Email => NotifyChannelType.Email,
|
||||
NotifyContactMethodType.Slack => NotifyChannelType.Slack,
|
||||
NotifyContactMethodType.Teams => NotifyChannelType.Teams,
|
||||
NotifyContactMethodType.Webhook => NotifyChannelType.Webhook,
|
||||
_ => NotifyChannelType.Custom
|
||||
};
|
||||
|
||||
var adapter = _channelAdapters.FirstOrDefault(a => a.ChannelType == channelType);
|
||||
if (adapter is not null)
|
||||
{
|
||||
// Create a minimal rendered notification for the escalation
|
||||
var format = channelType switch
|
||||
{
|
||||
NotifyChannelType.Email => NotifyDeliveryFormat.Email,
|
||||
NotifyChannelType.Slack => NotifyDeliveryFormat.Slack,
|
||||
NotifyChannelType.Teams => NotifyDeliveryFormat.Teams,
|
||||
NotifyChannelType.Webhook => NotifyDeliveryFormat.Webhook,
|
||||
NotifyChannelType.PagerDuty => NotifyDeliveryFormat.PagerDuty,
|
||||
NotifyChannelType.OpsGenie => NotifyDeliveryFormat.OpsGenie,
|
||||
NotifyChannelType.Cli => NotifyDeliveryFormat.Cli,
|
||||
NotifyChannelType.InAppInbox => NotifyDeliveryFormat.InAppInbox,
|
||||
_ => NotifyDeliveryFormat.Json
|
||||
};
|
||||
|
||||
var rendered = NotifyDeliveryRendered.Create(
|
||||
channelType,
|
||||
format,
|
||||
method.Address,
|
||||
$"Escalation: Incident {state.IncidentId}",
|
||||
$"Incident {state.IncidentId} requires attention. Escalation level: {state.CurrentLevel + 1}");
|
||||
|
||||
// Get default channel config
|
||||
var channels = await _channelRepository.ListAsync(tenantId, cancellationToken).ConfigureAwait(false);
|
||||
var channel = channels.FirstOrDefault(c => c.Type == channelType);
|
||||
|
||||
if (channel is not null)
|
||||
{
|
||||
var result = await adapter.SendAsync(channel, rendered, cancellationToken).ConfigureAwait(false);
|
||||
if (result.Success)
|
||||
{
|
||||
_logger.LogDebug("Notified user {UserId} via {ContactMethod}", user.UserId, method.Type);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to email if available
|
||||
if (!string.IsNullOrWhiteSpace(user.Email))
|
||||
{
|
||||
_logger.LogDebug("Would send email to {Email} for user {UserId}", user.Email, user.UserId);
|
||||
return true; // Assume success for now
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private async Task<bool> SendToChannelAsync(
|
||||
string tenantId,
|
||||
NotifyEscalationState state,
|
||||
string channelId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var channel = await _channelRepository.GetAsync(tenantId, channelId, cancellationToken).ConfigureAwait(false);
|
||||
if (channel is null)
|
||||
{
|
||||
_logger.LogWarning("Channel {ChannelId} not found for escalation", channelId);
|
||||
return false;
|
||||
}
|
||||
|
||||
var adapter = _channelAdapters.FirstOrDefault(a => a.ChannelType == channel.Type);
|
||||
if (adapter is null)
|
||||
{
|
||||
_logger.LogWarning("No adapter found for channel type {ChannelType}", channel.Type);
|
||||
return false;
|
||||
}
|
||||
|
||||
var channelFormat = channel.Type switch
|
||||
{
|
||||
NotifyChannelType.Email => NotifyDeliveryFormat.Email,
|
||||
NotifyChannelType.Slack => NotifyDeliveryFormat.Slack,
|
||||
NotifyChannelType.Teams => NotifyDeliveryFormat.Teams,
|
||||
NotifyChannelType.Webhook => NotifyDeliveryFormat.Webhook,
|
||||
NotifyChannelType.PagerDuty => NotifyDeliveryFormat.PagerDuty,
|
||||
NotifyChannelType.OpsGenie => NotifyDeliveryFormat.OpsGenie,
|
||||
NotifyChannelType.Cli => NotifyDeliveryFormat.Cli,
|
||||
NotifyChannelType.InAppInbox => NotifyDeliveryFormat.InAppInbox,
|
||||
_ => NotifyDeliveryFormat.Json
|
||||
};
|
||||
|
||||
var rendered = NotifyDeliveryRendered.Create(
|
||||
channel.Type,
|
||||
channelFormat,
|
||||
channel.Config.Target ?? channel.Config.Endpoint ?? string.Empty,
|
||||
$"Escalation: Incident {state.IncidentId}",
|
||||
$"Incident {state.IncidentId} requires attention. Escalation level: {state.CurrentLevel + 1}. Policy: {state.PolicyId}");
|
||||
|
||||
var result = await adapter.SendAsync(channel, rendered, cancellationToken).ConfigureAwait(false);
|
||||
return result.Success;
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Collections.Immutable;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notify.Storage.Mongo.Repositories;
|
||||
using StellaOps.Notifier.Worker.Storage;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Escalation;
|
||||
|
||||
@@ -10,18 +10,18 @@ namespace StellaOps.Notifier.Worker.Escalation;
|
||||
/// </summary>
|
||||
public sealed class DefaultOnCallResolver : IOnCallResolver
|
||||
{
|
||||
private readonly INotifyOnCallScheduleRepository? _scheduleRepository;
|
||||
private readonly IOnCallScheduleService? _scheduleService;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<DefaultOnCallResolver> _logger;
|
||||
|
||||
public DefaultOnCallResolver(
|
||||
TimeProvider timeProvider,
|
||||
ILogger<DefaultOnCallResolver> logger,
|
||||
INotifyOnCallScheduleRepository? scheduleRepository = null)
|
||||
IOnCallScheduleService? scheduleService = null)
|
||||
{
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_scheduleRepository = scheduleRepository;
|
||||
_scheduleService = scheduleService;
|
||||
}
|
||||
|
||||
public async Task<NotifyOnCallResolution> ResolveAsync(
|
||||
@@ -33,13 +33,13 @@ public sealed class DefaultOnCallResolver : IOnCallResolver
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(scheduleId);
|
||||
|
||||
if (_scheduleRepository is null)
|
||||
if (_scheduleService is null)
|
||||
{
|
||||
_logger.LogWarning("On-call schedule repository not available");
|
||||
return new NotifyOnCallResolution(scheduleId, evaluationTime ?? _timeProvider.GetUtcNow(), ImmutableArray<NotifyOnCallParticipant>.Empty);
|
||||
}
|
||||
|
||||
var schedule = await _scheduleRepository.GetAsync(tenantId, scheduleId, cancellationToken).ConfigureAwait(false);
|
||||
var schedule = await _scheduleService.GetScheduleAsync(tenantId, scheduleId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (schedule is null)
|
||||
{
|
||||
@@ -51,171 +51,30 @@ public sealed class DefaultOnCallResolver : IOnCallResolver
|
||||
}
|
||||
|
||||
public NotifyOnCallResolution ResolveAt(
|
||||
NotifyOnCallSchedule schedule,
|
||||
OnCallSchedule schedule,
|
||||
DateTimeOffset evaluationTime)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(schedule);
|
||||
|
||||
// Check for active override first
|
||||
var activeOverride = schedule.Overrides
|
||||
.FirstOrDefault(o => o.IsActiveAt(evaluationTime));
|
||||
var layer = schedule.Layers
|
||||
.Where(l => l.Users is { Count: > 0 })
|
||||
.OrderByDescending(l => l.Priority)
|
||||
.FirstOrDefault();
|
||||
|
||||
if (activeOverride is not null)
|
||||
{
|
||||
// Find the participant matching the override user ID
|
||||
var overrideUser = schedule.Layers
|
||||
.SelectMany(l => l.Participants)
|
||||
.FirstOrDefault(p => p.UserId == activeOverride.UserId);
|
||||
|
||||
if (overrideUser is not null)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"On-call resolved from override {OverrideId} for schedule {ScheduleId}: user={UserId}",
|
||||
activeOverride.OverrideId, schedule.ScheduleId, activeOverride.UserId);
|
||||
|
||||
return new NotifyOnCallResolution(
|
||||
schedule.ScheduleId,
|
||||
evaluationTime,
|
||||
ImmutableArray.Create(overrideUser),
|
||||
sourceOverride: activeOverride.OverrideId);
|
||||
}
|
||||
|
||||
// Override user not in participants - create a minimal participant
|
||||
var minimalParticipant = NotifyOnCallParticipant.Create(activeOverride.UserId);
|
||||
return new NotifyOnCallResolution(
|
||||
schedule.ScheduleId,
|
||||
evaluationTime,
|
||||
ImmutableArray.Create(minimalParticipant),
|
||||
sourceOverride: activeOverride.OverrideId);
|
||||
}
|
||||
|
||||
// No override - find highest priority active layer
|
||||
var activeLayer = FindActiveLayer(schedule, evaluationTime);
|
||||
|
||||
if (activeLayer is null || activeLayer.Participants.IsDefaultOrEmpty)
|
||||
if (layer is null)
|
||||
{
|
||||
_logger.LogDebug("No active on-call layer found for schedule {ScheduleId} at {EvaluationTime}",
|
||||
schedule.ScheduleId, evaluationTime);
|
||||
return new NotifyOnCallResolution(schedule.ScheduleId, evaluationTime, ImmutableArray<NotifyOnCallParticipant>.Empty);
|
||||
}
|
||||
|
||||
// Calculate who is on-call based on rotation
|
||||
var onCallUser = CalculateRotationUser(activeLayer, evaluationTime, schedule.TimeZone);
|
||||
|
||||
if (onCallUser is null)
|
||||
{
|
||||
_logger.LogDebug("No on-call user found in rotation for layer {LayerId}", activeLayer.LayerId);
|
||||
return new NotifyOnCallResolution(schedule.ScheduleId, evaluationTime, ImmutableArray<NotifyOnCallParticipant>.Empty);
|
||||
}
|
||||
|
||||
_logger.LogDebug(
|
||||
"On-call resolved from layer {LayerId} for schedule {ScheduleId}: user={UserId}",
|
||||
activeLayer.LayerId, schedule.ScheduleId, onCallUser.UserId);
|
||||
var user = layer.Users.First();
|
||||
var participant = NotifyOnCallParticipant.Create(user.UserId, user.Name, user.Email, user.Phone);
|
||||
|
||||
return new NotifyOnCallResolution(
|
||||
schedule.ScheduleId,
|
||||
evaluationTime,
|
||||
ImmutableArray.Create(onCallUser),
|
||||
sourceLayer: activeLayer.LayerId);
|
||||
}
|
||||
|
||||
private NotifyOnCallLayer? FindActiveLayer(NotifyOnCallSchedule schedule, DateTimeOffset evaluationTime)
|
||||
{
|
||||
// Order layers by priority (higher priority first)
|
||||
var orderedLayers = schedule.Layers.OrderByDescending(l => l.Priority);
|
||||
|
||||
foreach (var layer in orderedLayers)
|
||||
{
|
||||
if (IsLayerActiveAt(layer, evaluationTime, schedule.TimeZone))
|
||||
{
|
||||
return layer;
|
||||
}
|
||||
}
|
||||
|
||||
// If no layer matches restrictions, return highest priority layer
|
||||
return schedule.Layers.OrderByDescending(l => l.Priority).FirstOrDefault();
|
||||
}
|
||||
|
||||
private bool IsLayerActiveAt(NotifyOnCallLayer layer, DateTimeOffset evaluationTime, string timeZone)
|
||||
{
|
||||
if (layer.Restrictions is null || layer.Restrictions.TimeRanges.IsDefaultOrEmpty)
|
||||
{
|
||||
return true; // No restrictions = always active
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var tz = TimeZoneInfo.FindSystemTimeZoneById(timeZone);
|
||||
var localTime = TimeZoneInfo.ConvertTime(evaluationTime, tz);
|
||||
|
||||
foreach (var range in layer.Restrictions.TimeRanges)
|
||||
{
|
||||
var isTimeInRange = IsTimeInRange(localTime.TimeOfDay, range.StartTime, range.EndTime);
|
||||
|
||||
if (layer.Restrictions.Type == NotifyRestrictionType.DailyRestriction)
|
||||
{
|
||||
if (isTimeInRange) return true;
|
||||
}
|
||||
else if (layer.Restrictions.Type == NotifyRestrictionType.WeeklyRestriction)
|
||||
{
|
||||
if (range.DayOfWeek == localTime.DayOfWeek && isTimeInRange)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to evaluate layer restrictions for layer {LayerId}", layer.LayerId);
|
||||
return true; // On error, assume layer is active
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsTimeInRange(TimeSpan current, TimeOnly start, TimeOnly end)
|
||||
{
|
||||
var currentTimeOnly = TimeOnly.FromTimeSpan(current);
|
||||
|
||||
if (start <= end)
|
||||
{
|
||||
return currentTimeOnly >= start && currentTimeOnly < end;
|
||||
}
|
||||
|
||||
// Handles overnight ranges (e.g., 22:00 - 06:00)
|
||||
return currentTimeOnly >= start || currentTimeOnly < end;
|
||||
}
|
||||
|
||||
private NotifyOnCallParticipant? CalculateRotationUser(
|
||||
NotifyOnCallLayer layer,
|
||||
DateTimeOffset evaluationTime,
|
||||
string timeZone)
|
||||
{
|
||||
if (layer.Participants.IsDefaultOrEmpty)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var participantCount = layer.Participants.Length;
|
||||
if (participantCount == 1)
|
||||
{
|
||||
return layer.Participants[0];
|
||||
}
|
||||
|
||||
// Calculate rotation index based on time since rotation start
|
||||
var rotationStart = layer.RotationStartsAt;
|
||||
var elapsed = evaluationTime - rotationStart;
|
||||
|
||||
if (elapsed < TimeSpan.Zero)
|
||||
{
|
||||
// Evaluation time is before rotation start - return first participant
|
||||
return layer.Participants[0];
|
||||
}
|
||||
|
||||
var rotationCount = (long)(elapsed / layer.RotationInterval);
|
||||
var currentIndex = (int)(rotationCount % participantCount);
|
||||
|
||||
return layer.Participants[currentIndex];
|
||||
ImmutableArray.Create(participant),
|
||||
sourceLayer: layer.Name);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Concurrent;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Notify.Storage.Mongo.Repositories;
|
||||
using StellaOps.Notifier.Worker.Storage;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Escalation;
|
||||
|
||||
@@ -86,6 +86,7 @@ public sealed class EscalationEngine : IEscalationEngine
|
||||
await _auditRepository.AppendAsync(
|
||||
tenantId,
|
||||
"escalation_started",
|
||||
null,
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["stateId"] = stateId,
|
||||
@@ -93,7 +94,6 @@ public sealed class EscalationEngine : IEscalationEngine
|
||||
["policyId"] = policyId,
|
||||
["level"] = firstLevel.Level.ToString()
|
||||
},
|
||||
null,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
@@ -158,6 +158,7 @@ public sealed class EscalationEngine : IEscalationEngine
|
||||
await _auditRepository.AppendAsync(
|
||||
tenantId,
|
||||
"escalation_acknowledged",
|
||||
acknowledgedBy,
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["stateId"] = state.StateId,
|
||||
@@ -165,7 +166,6 @@ public sealed class EscalationEngine : IEscalationEngine
|
||||
["acknowledgedBy"] = acknowledgedBy,
|
||||
["stopped"] = (currentLevel?.StopOnAck == true).ToString()
|
||||
},
|
||||
acknowledgedBy,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
@@ -240,13 +240,13 @@ public sealed class EscalationEngine : IEscalationEngine
|
||||
await _auditRepository.AppendAsync(
|
||||
tenantId,
|
||||
"escalation_stopped",
|
||||
actor,
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["stateId"] = state.StateId,
|
||||
["incidentId"] = incidentId,
|
||||
["reason"] = reason
|
||||
},
|
||||
actor,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
@@ -524,6 +524,7 @@ public sealed class EscalationEngine : IEscalationEngine
|
||||
await _auditRepository.AppendAsync(
|
||||
state.TenantId,
|
||||
"escalation_manual_escalate",
|
||||
actor,
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["stateId"] = state.StateId,
|
||||
@@ -532,7 +533,6 @@ public sealed class EscalationEngine : IEscalationEngine
|
||||
["toLevel"] = action.NewLevel?.ToString() ?? "N/A",
|
||||
["reason"] = reason ?? "Manual escalation"
|
||||
},
|
||||
actor,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Concurrent;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Notify.Storage.Mongo.Repositories;
|
||||
using StellaOps.Notifier.Worker.Storage;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Escalation;
|
||||
|
||||
@@ -87,6 +87,7 @@ public sealed class InMemoryEscalationPolicyService : IEscalationPolicyService
|
||||
await _auditRepository.AppendAsync(
|
||||
policy.TenantId,
|
||||
isNew ? "escalation_policy_created" : "escalation_policy_updated",
|
||||
actor,
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["policyId"] = policy.PolicyId,
|
||||
@@ -95,7 +96,6 @@ public sealed class InMemoryEscalationPolicyService : IEscalationPolicyService
|
||||
["isDefault"] = policy.IsDefault.ToString(),
|
||||
["levelCount"] = policy.Levels.Count.ToString()
|
||||
},
|
||||
actor,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
@@ -120,8 +120,8 @@ public sealed class InMemoryEscalationPolicyService : IEscalationPolicyService
|
||||
await _auditRepository.AppendAsync(
|
||||
tenantId,
|
||||
"escalation_policy_deleted",
|
||||
new Dictionary<string, string> { ["policyId"] = policyId },
|
||||
actor,
|
||||
new Dictionary<string, string> { ["policyId"] = policyId },
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
|
||||
@@ -67,6 +67,18 @@ public interface IEscalationEngine
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of processing an escalation step.
|
||||
/// </summary>
|
||||
public sealed record EscalationProcessResult
|
||||
{
|
||||
public required bool Processed { get; init; }
|
||||
public bool Escalated { get; init; }
|
||||
public bool Exhausted { get; init; }
|
||||
public int Errors { get; init; }
|
||||
public IReadOnlyList<string> ErrorMessages { get; init; } = Array.Empty<string>();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Current state of an escalation.
|
||||
/// </summary>
|
||||
|
||||
@@ -20,6 +20,6 @@ public interface IOnCallResolver
|
||||
/// Resolves the current on-call user(s) for a schedule at a specific time.
|
||||
/// </summary>
|
||||
NotifyOnCallResolution ResolveAt(
|
||||
NotifyOnCallSchedule schedule,
|
||||
OnCallSchedule schedule,
|
||||
DateTimeOffset evaluationTime);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Concurrent;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Notify.Storage.Mongo.Repositories;
|
||||
using StellaOps.Notifier.Worker.Storage;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Escalation;
|
||||
|
||||
@@ -637,10 +637,11 @@ public sealed class CliNotificationChannel : IInboxChannel
|
||||
_ => "[*]"
|
||||
};
|
||||
|
||||
var readMarker = notification.IsRead ? " " : "●";
|
||||
var readMarker = notification.IsRead ? " " : "â—";
|
||||
|
||||
return $"{readMarker} {priorityMarker} {notification.Title}\n {notification.Body}\n [{notification.CreatedAt:yyyy-MM-dd HH:mm}]";
|
||||
}
|
||||
|
||||
private static string BuildKey(string tenantId, string userId) => $"{tenantId}:{userId}";
|
||||
}
|
||||
|
||||
|
||||
@@ -1,537 +0,0 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Escalations;
|
||||
|
||||
/// <summary>
|
||||
/// Extension methods for registering escalation services.
|
||||
/// </summary>
|
||||
public static class EscalationServiceExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Adds escalation, on-call, and integration services to the service collection.
|
||||
/// </summary>
|
||||
public static IServiceCollection AddEscalationServices(
|
||||
this IServiceCollection services,
|
||||
IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
// Register options
|
||||
services.Configure<PagerDutyOptions>(
|
||||
configuration.GetSection(PagerDutyOptions.SectionName));
|
||||
services.Configure<OpsGenieOptions>(
|
||||
configuration.GetSection(OpsGenieOptions.SectionName));
|
||||
|
||||
// Register core services (in-memory implementations)
|
||||
services.AddSingleton<IEscalationPolicyService, InMemoryEscalationPolicyService>();
|
||||
services.AddSingleton<IOnCallScheduleService, InMemoryOnCallScheduleService>();
|
||||
services.AddSingleton<IInboxService, InMemoryInboxService>();
|
||||
|
||||
// Register integration adapters
|
||||
services.AddHttpClient<PagerDutyAdapter>();
|
||||
services.AddHttpClient<OpsGenieAdapter>();
|
||||
services.AddSingleton<IIntegrationAdapterFactory, IntegrationAdapterFactory>();
|
||||
|
||||
// Register CLI inbox adapter
|
||||
services.AddSingleton<CliInboxChannelAdapter>();
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds escalation services with custom implementations.
|
||||
/// </summary>
|
||||
public static IServiceCollection AddEscalationServices(
|
||||
this IServiceCollection services,
|
||||
IConfiguration configuration,
|
||||
Action<EscalationServiceBuilder> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
// Register options
|
||||
services.Configure<PagerDutyOptions>(
|
||||
configuration.GetSection(PagerDutyOptions.SectionName));
|
||||
services.Configure<OpsGenieOptions>(
|
||||
configuration.GetSection(OpsGenieOptions.SectionName));
|
||||
|
||||
// Apply custom configuration
|
||||
var builder = new EscalationServiceBuilder(services);
|
||||
configure(builder);
|
||||
|
||||
// Register defaults for any services not configured
|
||||
services.TryAddSingleton<IEscalationPolicyService, InMemoryEscalationPolicyService>();
|
||||
services.TryAddSingleton<IOnCallScheduleService, InMemoryOnCallScheduleService>();
|
||||
services.TryAddSingleton<IInboxService, InMemoryInboxService>();
|
||||
|
||||
// Register integration adapters
|
||||
services.AddHttpClient<PagerDutyAdapter>();
|
||||
services.AddHttpClient<OpsGenieAdapter>();
|
||||
services.TryAddSingleton<IIntegrationAdapterFactory, IntegrationAdapterFactory>();
|
||||
|
||||
// Register CLI inbox adapter
|
||||
services.TryAddSingleton<CliInboxChannelAdapter>();
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void TryAddSingleton<TService, TImplementation>(this IServiceCollection services)
|
||||
where TService : class
|
||||
where TImplementation : class, TService
|
||||
{
|
||||
if (!services.Any(d => d.ServiceType == typeof(TService)))
|
||||
{
|
||||
services.AddSingleton<TService, TImplementation>();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builder for customizing escalation service registrations.
|
||||
/// </summary>
|
||||
public sealed class EscalationServiceBuilder
|
||||
{
|
||||
private readonly IServiceCollection _services;
|
||||
|
||||
internal EscalationServiceBuilder(IServiceCollection services)
|
||||
{
|
||||
_services = services;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Registers a custom escalation policy service.
|
||||
/// </summary>
|
||||
public EscalationServiceBuilder UseEscalationPolicyService<TService>()
|
||||
where TService : class, IEscalationPolicyService
|
||||
{
|
||||
_services.AddSingleton<IEscalationPolicyService, TService>();
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Registers a custom on-call schedule service.
|
||||
/// </summary>
|
||||
public EscalationServiceBuilder UseOnCallScheduleService<TService>()
|
||||
where TService : class, IOnCallScheduleService
|
||||
{
|
||||
_services.AddSingleton<IOnCallScheduleService, TService>();
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Registers a custom inbox service.
|
||||
/// </summary>
|
||||
public EscalationServiceBuilder UseInboxService<TService>()
|
||||
where TService : class, IInboxService
|
||||
{
|
||||
_services.AddSingleton<IInboxService, TService>();
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Registers a custom integration adapter.
|
||||
/// </summary>
|
||||
public EscalationServiceBuilder AddIntegrationAdapter<TAdapter>(string integrationType)
|
||||
where TAdapter : class, IIncidentIntegrationAdapter
|
||||
{
|
||||
_services.AddSingleton<TAdapter>();
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// In-memory implementation of escalation policy service.
|
||||
/// </summary>
|
||||
public sealed class InMemoryEscalationPolicyService : IEscalationPolicyService
|
||||
{
|
||||
private readonly Dictionary<string, EscalationPolicy> _policies = new();
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public InMemoryEscalationPolicyService(TimeProvider timeProvider)
|
||||
{
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
}
|
||||
|
||||
public Task<EscalationPolicy?> GetAsync(string tenantId, string policyId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var key = BuildKey(tenantId, policyId);
|
||||
_policies.TryGetValue(key, out var policy);
|
||||
return Task.FromResult(policy);
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<EscalationPolicy>> ListAsync(string tenantId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var policies = _policies.Values
|
||||
.Where(p => p.TenantId == tenantId)
|
||||
.OrderBy(p => p.Name)
|
||||
.ToList();
|
||||
return Task.FromResult<IReadOnlyList<EscalationPolicy>>(policies);
|
||||
}
|
||||
|
||||
public Task<EscalationPolicy> UpsertAsync(EscalationPolicy policy, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var key = BuildKey(policy.TenantId, policy.PolicyId);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
var updated = policy with
|
||||
{
|
||||
CreatedAt = _policies.ContainsKey(key) ? _policies[key].CreatedAt : now,
|
||||
UpdatedAt = now
|
||||
};
|
||||
|
||||
_policies[key] = updated;
|
||||
return Task.FromResult(updated);
|
||||
}
|
||||
|
||||
public Task<bool> DeleteAsync(string tenantId, string policyId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var key = BuildKey(tenantId, policyId);
|
||||
return Task.FromResult(_policies.Remove(key));
|
||||
}
|
||||
|
||||
public Task<EscalationPolicy?> GetDefaultAsync(string tenantId, string? eventKind = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var policy = _policies.Values
|
||||
.Where(p => p.TenantId == tenantId && p.IsDefault && p.Enabled)
|
||||
.Where(p => eventKind is null || p.EventKinds.Count == 0 || p.EventKinds.Contains(eventKind, StringComparer.OrdinalIgnoreCase))
|
||||
.OrderByDescending(p => p.EventKinds.Count) // Prefer more specific policies
|
||||
.FirstOrDefault();
|
||||
|
||||
return Task.FromResult(policy);
|
||||
}
|
||||
|
||||
public Task<EscalationStepResult> EvaluateAsync(
|
||||
string tenantId,
|
||||
string policyId,
|
||||
EscalationContext context,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var key = BuildKey(tenantId, policyId);
|
||||
if (!_policies.TryGetValue(key, out var policy) || !policy.Enabled)
|
||||
{
|
||||
return Task.FromResult(EscalationStepResult.NoEscalation("Policy not found or disabled"));
|
||||
}
|
||||
|
||||
if (policy.Steps.Count == 0)
|
||||
{
|
||||
return Task.FromResult(EscalationStepResult.NoEscalation("Policy has no steps"));
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var incidentAge = now - context.IncidentCreatedAt;
|
||||
|
||||
// Find the next step to execute
|
||||
var cumulativeDelay = TimeSpan.Zero;
|
||||
for (var i = 0; i < policy.Steps.Count; i++)
|
||||
{
|
||||
var step = policy.Steps[i];
|
||||
cumulativeDelay += step.DelayFromPrevious;
|
||||
|
||||
if (incidentAge >= cumulativeDelay && !context.NotifiedSteps.Contains(step.StepNumber))
|
||||
{
|
||||
// Check if acknowledged and step should skip
|
||||
if (context.IsAcknowledged && !step.NotifyEvenIfAcknowledged)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var nextStepDelay = i + 1 < policy.Steps.Count
|
||||
? cumulativeDelay + policy.Steps[i + 1].DelayFromPrevious
|
||||
: (TimeSpan?)null;
|
||||
|
||||
var nextEvaluation = nextStepDelay.HasValue
|
||||
? context.IncidentCreatedAt + nextStepDelay.Value
|
||||
: null;
|
||||
|
||||
return Task.FromResult(EscalationStepResult.Escalate(step, context.CompletedCycles, nextEvaluation));
|
||||
}
|
||||
}
|
||||
|
||||
// All steps executed, check repeat behavior
|
||||
if (context.NotifiedSteps.Count >= policy.Steps.Count)
|
||||
{
|
||||
if (policy.RepeatBehavior == EscalationRepeatBehavior.Repeat &&
|
||||
context.CompletedCycles < policy.MaxRepeats)
|
||||
{
|
||||
// Start next cycle
|
||||
return Task.FromResult(EscalationStepResult.Escalate(
|
||||
policy.Steps[0],
|
||||
context.CompletedCycles + 1,
|
||||
context.IncidentCreatedAt + policy.Steps[0].DelayFromPrevious));
|
||||
}
|
||||
|
||||
return Task.FromResult(EscalationStepResult.Exhausted(context.CompletedCycles));
|
||||
}
|
||||
|
||||
// Not yet time for next step
|
||||
var nextStep = policy.Steps.FirstOrDefault(s => !context.NotifiedSteps.Contains(s.StepNumber));
|
||||
if (nextStep is not null)
|
||||
{
|
||||
var stepDelay = policy.Steps.TakeWhile(s => s.StepNumber <= nextStep.StepNumber)
|
||||
.Aggregate(TimeSpan.Zero, (acc, s) => acc + s.DelayFromPrevious);
|
||||
return Task.FromResult(EscalationStepResult.NoEscalation(
|
||||
"Waiting for next step",
|
||||
context.IncidentCreatedAt + stepDelay));
|
||||
}
|
||||
|
||||
return Task.FromResult(EscalationStepResult.NoEscalation("No steps pending"));
|
||||
}
|
||||
|
||||
private static string BuildKey(string tenantId, string policyId) => $"{tenantId}:{policyId}";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// In-memory implementation of on-call schedule service.
|
||||
/// </summary>
|
||||
public sealed class InMemoryOnCallScheduleService : IOnCallScheduleService
|
||||
{
|
||||
private readonly Dictionary<string, OnCallSchedule> _schedules = new();
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public InMemoryOnCallScheduleService(TimeProvider timeProvider)
|
||||
{
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
}
|
||||
|
||||
public Task<OnCallSchedule?> GetAsync(string tenantId, string scheduleId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var key = BuildKey(tenantId, scheduleId);
|
||||
_schedules.TryGetValue(key, out var schedule);
|
||||
return Task.FromResult(schedule);
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<OnCallSchedule>> ListAsync(string tenantId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var schedules = _schedules.Values
|
||||
.Where(s => s.TenantId == tenantId)
|
||||
.OrderBy(s => s.Name)
|
||||
.ToList();
|
||||
return Task.FromResult<IReadOnlyList<OnCallSchedule>>(schedules);
|
||||
}
|
||||
|
||||
public Task<OnCallSchedule> UpsertAsync(OnCallSchedule schedule, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var key = BuildKey(schedule.TenantId, schedule.ScheduleId);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
var updated = schedule with
|
||||
{
|
||||
CreatedAt = _schedules.ContainsKey(key) ? _schedules[key].CreatedAt : now,
|
||||
UpdatedAt = now
|
||||
};
|
||||
|
||||
_schedules[key] = updated;
|
||||
return Task.FromResult(updated);
|
||||
}
|
||||
|
||||
public Task<bool> DeleteAsync(string tenantId, string scheduleId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var key = BuildKey(tenantId, scheduleId);
|
||||
return Task.FromResult(_schedules.Remove(key));
|
||||
}
|
||||
|
||||
public Task<OnCallResolution> GetCurrentOnCallAsync(
|
||||
string tenantId,
|
||||
string scheduleId,
|
||||
DateTimeOffset? asOf = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var key = BuildKey(tenantId, scheduleId);
|
||||
if (!_schedules.TryGetValue(key, out var schedule) || !schedule.Enabled)
|
||||
{
|
||||
return Task.FromResult(OnCallResolution.NoOneOnCall(asOf ?? _timeProvider.GetUtcNow()));
|
||||
}
|
||||
|
||||
var now = asOf ?? _timeProvider.GetUtcNow();
|
||||
|
||||
// Check overrides first
|
||||
var activeOverride = schedule.Overrides
|
||||
.FirstOrDefault(o => o.StartTime <= now && o.EndTime > now);
|
||||
|
||||
if (activeOverride is not null)
|
||||
{
|
||||
var overrideUser = new OnCallUser
|
||||
{
|
||||
UserId = activeOverride.UserId,
|
||||
DisplayName = activeOverride.UserDisplayName
|
||||
};
|
||||
return Task.FromResult(OnCallResolution.FromOverride(overrideUser, activeOverride, now));
|
||||
}
|
||||
|
||||
// Check layers in priority order
|
||||
foreach (var layer in schedule.Layers.OrderBy(l => l.Priority))
|
||||
{
|
||||
if (!IsLayerActive(layer, now))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var onCallUser = GetOnCallUserForLayer(layer, now);
|
||||
if (onCallUser is not null)
|
||||
{
|
||||
var shiftEnds = CalculateShiftEnd(layer, now);
|
||||
return Task.FromResult(OnCallResolution.FromUser(onCallUser, layer.Name, now, shiftEnds));
|
||||
}
|
||||
}
|
||||
|
||||
// Check fallback
|
||||
if (!string.IsNullOrEmpty(schedule.FallbackUserId))
|
||||
{
|
||||
var fallbackUser = new OnCallUser { UserId = schedule.FallbackUserId };
|
||||
return Task.FromResult(OnCallResolution.FromFallback(fallbackUser, now));
|
||||
}
|
||||
|
||||
return Task.FromResult(OnCallResolution.NoOneOnCall(now));
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<OnCallCoverage>> GetCoverageAsync(
|
||||
string tenantId,
|
||||
string scheduleId,
|
||||
DateTimeOffset from,
|
||||
DateTimeOffset to,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
// Simplified implementation - just get current on-call
|
||||
var coverage = new List<OnCallCoverage>();
|
||||
|
||||
var current = from;
|
||||
while (current < to)
|
||||
{
|
||||
var resolution = GetCurrentOnCallAsync(tenantId, scheduleId, current, cancellationToken).Result;
|
||||
if (resolution.HasOnCall && resolution.OnCallUser is not null)
|
||||
{
|
||||
var end = resolution.ShiftEndsAt ?? to;
|
||||
if (end > to) end = to;
|
||||
|
||||
coverage.Add(new OnCallCoverage
|
||||
{
|
||||
From = current,
|
||||
To = end,
|
||||
User = resolution.OnCallUser,
|
||||
Layer = resolution.ResolvedFromLayer,
|
||||
IsOverride = resolution.IsOverride
|
||||
});
|
||||
|
||||
current = end;
|
||||
}
|
||||
else
|
||||
{
|
||||
current = current.AddHours(1); // Move forward if no coverage
|
||||
}
|
||||
}
|
||||
|
||||
return Task.FromResult<IReadOnlyList<OnCallCoverage>>(coverage);
|
||||
}
|
||||
|
||||
public Task<OnCallOverride> AddOverrideAsync(
|
||||
string tenantId,
|
||||
string scheduleId,
|
||||
OnCallOverride @override,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var key = BuildKey(tenantId, scheduleId);
|
||||
if (!_schedules.TryGetValue(key, out var schedule))
|
||||
{
|
||||
throw new InvalidOperationException($"Schedule {scheduleId} not found.");
|
||||
}
|
||||
|
||||
var newOverride = @override with
|
||||
{
|
||||
OverrideId = @override.OverrideId ?? $"ovr-{Guid.NewGuid():N}"[..16],
|
||||
CreatedAt = _timeProvider.GetUtcNow()
|
||||
};
|
||||
|
||||
var overrides = schedule.Overrides.ToList();
|
||||
overrides.Add(newOverride);
|
||||
|
||||
_schedules[key] = schedule with { Overrides = overrides };
|
||||
|
||||
return Task.FromResult(newOverride);
|
||||
}
|
||||
|
||||
public Task<bool> RemoveOverrideAsync(
|
||||
string tenantId,
|
||||
string scheduleId,
|
||||
string overrideId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var key = BuildKey(tenantId, scheduleId);
|
||||
if (!_schedules.TryGetValue(key, out var schedule))
|
||||
{
|
||||
return Task.FromResult(false);
|
||||
}
|
||||
|
||||
var overrides = schedule.Overrides.ToList();
|
||||
var removed = overrides.RemoveAll(o => o.OverrideId == overrideId) > 0;
|
||||
|
||||
if (removed)
|
||||
{
|
||||
_schedules[key] = schedule with { Overrides = overrides };
|
||||
}
|
||||
|
||||
return Task.FromResult(removed);
|
||||
}
|
||||
|
||||
private static bool IsLayerActive(RotationLayer layer, DateTimeOffset now)
|
||||
{
|
||||
// Check day of week
|
||||
if (layer.ActiveDays is { Count: > 0 } && !layer.ActiveDays.Contains(now.DayOfWeek))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check time restriction
|
||||
if (layer.TimeRestriction is not null)
|
||||
{
|
||||
var time = TimeOnly.FromDateTime(now.DateTime);
|
||||
var start = layer.TimeRestriction.StartTime;
|
||||
var end = layer.TimeRestriction.EndTime;
|
||||
|
||||
if (layer.TimeRestriction.SpansMidnight)
|
||||
{
|
||||
if (time < start && time >= end)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (time < start || time >= end)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private static OnCallUser? GetOnCallUserForLayer(RotationLayer layer, DateTimeOffset now)
|
||||
{
|
||||
if (layer.Users.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
// Calculate which user is on-call based on rotation
|
||||
var elapsed = now - layer.StartTime;
|
||||
var rotations = (int)(elapsed.Ticks / layer.RotationInterval.Ticks);
|
||||
var userIndex = rotations % layer.Users.Count;
|
||||
|
||||
return layer.Users[userIndex];
|
||||
}
|
||||
|
||||
private static DateTimeOffset? CalculateShiftEnd(RotationLayer layer, DateTimeOffset now)
|
||||
{
|
||||
var elapsed = now - layer.StartTime;
|
||||
var currentRotation = (int)(elapsed.Ticks / layer.RotationInterval.Ticks);
|
||||
var nextRotationStart = layer.StartTime + TimeSpan.FromTicks((currentRotation + 1) * layer.RotationInterval.Ticks);
|
||||
|
||||
return nextRotationStart;
|
||||
}
|
||||
|
||||
private static string BuildKey(string tenantId, string scheduleId) => $"{tenantId}:{scheduleId}";
|
||||
}
|
||||
@@ -1,355 +0,0 @@
|
||||
namespace StellaOps.Notifier.Worker.Escalations;
|
||||
|
||||
/// <summary>
|
||||
/// Manages escalation policies for incidents.
|
||||
/// </summary>
|
||||
public interface IEscalationPolicyService
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets an escalation policy by ID.
|
||||
/// </summary>
|
||||
Task<EscalationPolicy?> GetAsync(string tenantId, string policyId, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Lists escalation policies for a tenant.
|
||||
/// </summary>
|
||||
Task<IReadOnlyList<EscalationPolicy>> ListAsync(string tenantId, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Creates or updates an escalation policy.
|
||||
/// </summary>
|
||||
Task<EscalationPolicy> UpsertAsync(EscalationPolicy policy, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Deletes an escalation policy.
|
||||
/// </summary>
|
||||
Task<bool> DeleteAsync(string tenantId, string policyId, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the default policy for a tenant/event kind.
|
||||
/// </summary>
|
||||
Task<EscalationPolicy?> GetDefaultAsync(string tenantId, string? eventKind = null, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates which escalation step should be active for an incident.
|
||||
/// </summary>
|
||||
Task<EscalationStepResult> EvaluateAsync(
|
||||
string tenantId,
|
||||
string policyId,
|
||||
EscalationContext context,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Escalation policy defining how incidents escalate over time.
|
||||
/// </summary>
|
||||
public sealed record EscalationPolicy
|
||||
{
|
||||
/// <summary>
|
||||
/// Unique policy identifier.
|
||||
/// </summary>
|
||||
public required string PolicyId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Tenant this policy belongs to.
|
||||
/// </summary>
|
||||
public required string TenantId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Human-readable name.
|
||||
/// </summary>
|
||||
public required string Name { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Description of the policy.
|
||||
/// </summary>
|
||||
public string? Description { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether this is the default policy for the tenant.
|
||||
/// </summary>
|
||||
public bool IsDefault { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Event kinds this policy applies to (empty = all).
|
||||
/// </summary>
|
||||
public IReadOnlyList<string> EventKinds { get; init; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Severity threshold for this policy (only events >= this severity use this policy).
|
||||
/// </summary>
|
||||
public string? MinimumSeverity { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Ordered escalation steps.
|
||||
/// </summary>
|
||||
public required IReadOnlyList<EscalationStep> Steps { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// What happens after all steps are exhausted.
|
||||
/// </summary>
|
||||
public EscalationRepeatBehavior RepeatBehavior { get; init; } = EscalationRepeatBehavior.StopAtLast;
|
||||
|
||||
/// <summary>
|
||||
/// Number of times to repeat the escalation cycle (only if RepeatBehavior is Repeat).
|
||||
/// </summary>
|
||||
public int MaxRepeats { get; init; } = 3;
|
||||
|
||||
/// <summary>
|
||||
/// When the policy was created.
|
||||
/// </summary>
|
||||
public DateTimeOffset CreatedAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// When the policy was last updated.
|
||||
/// </summary>
|
||||
public DateTimeOffset UpdatedAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether the policy is enabled.
|
||||
/// </summary>
|
||||
public bool Enabled { get; init; } = true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A single step in an escalation policy.
|
||||
/// </summary>
|
||||
public sealed record EscalationStep
|
||||
{
|
||||
/// <summary>
|
||||
/// Step number (1-based).
|
||||
/// </summary>
|
||||
public required int StepNumber { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Delay before this step activates (from incident creation or previous step).
|
||||
/// </summary>
|
||||
public required TimeSpan DelayFromPrevious { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Targets to notify at this step.
|
||||
/// </summary>
|
||||
public required IReadOnlyList<EscalationTarget> Targets { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether to notify targets in sequence or parallel.
|
||||
/// </summary>
|
||||
public EscalationTargetMode TargetMode { get; init; } = EscalationTargetMode.Parallel;
|
||||
|
||||
/// <summary>
|
||||
/// Delay between sequential targets (only if TargetMode is Sequential).
|
||||
/// </summary>
|
||||
public TimeSpan SequentialDelay { get; init; } = TimeSpan.FromMinutes(5);
|
||||
|
||||
/// <summary>
|
||||
/// Whether this step should notify even if incident is acknowledged.
|
||||
/// </summary>
|
||||
public bool NotifyEvenIfAcknowledged { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Custom message template for this step.
|
||||
/// </summary>
|
||||
public string? MessageTemplate { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A target to notify during escalation.
|
||||
/// </summary>
|
||||
public sealed record EscalationTarget
|
||||
{
|
||||
/// <summary>
|
||||
/// Target type (user, schedule, channel, integration).
|
||||
/// </summary>
|
||||
public required EscalationTargetType Type { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Target identifier (user ID, schedule ID, channel ID, etc.).
|
||||
/// </summary>
|
||||
public required string TargetId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Display name for the target.
|
||||
/// </summary>
|
||||
public string? DisplayName { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Channels to use for this target (if not specified, uses target's preferences).
|
||||
/// </summary>
|
||||
public IReadOnlyList<string>? Channels { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Type of escalation target.
|
||||
/// </summary>
|
||||
public enum EscalationTargetType
|
||||
{
|
||||
/// <summary>
|
||||
/// Specific user.
|
||||
/// </summary>
|
||||
User,
|
||||
|
||||
/// <summary>
|
||||
/// On-call schedule (notifies whoever is currently on-call).
|
||||
/// </summary>
|
||||
Schedule,
|
||||
|
||||
/// <summary>
|
||||
/// Notification channel (Slack channel, email group, etc.).
|
||||
/// </summary>
|
||||
Channel,
|
||||
|
||||
/// <summary>
|
||||
/// External integration (PagerDuty, OpsGenie, etc.).
|
||||
/// </summary>
|
||||
Integration
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// How targets are notified within a step.
|
||||
/// </summary>
|
||||
public enum EscalationTargetMode
|
||||
{
|
||||
/// <summary>
|
||||
/// Notify all targets at once.
|
||||
/// </summary>
|
||||
Parallel,
|
||||
|
||||
/// <summary>
|
||||
/// Notify targets one by one with delays.
|
||||
/// </summary>
|
||||
Sequential
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// What happens after all escalation steps complete.
|
||||
/// </summary>
|
||||
public enum EscalationRepeatBehavior
|
||||
{
|
||||
/// <summary>
|
||||
/// Stop at the last step, continue notifying that step.
|
||||
/// </summary>
|
||||
StopAtLast,
|
||||
|
||||
/// <summary>
|
||||
/// Repeat the entire escalation cycle.
|
||||
/// </summary>
|
||||
Repeat,
|
||||
|
||||
/// <summary>
|
||||
/// Stop escalating entirely.
|
||||
/// </summary>
|
||||
Stop
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Context for evaluating escalation.
|
||||
/// </summary>
|
||||
public sealed record EscalationContext
|
||||
{
|
||||
/// <summary>
|
||||
/// Incident ID.
|
||||
/// </summary>
|
||||
public required string IncidentId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// When the incident was created.
|
||||
/// </summary>
|
||||
public required DateTimeOffset IncidentCreatedAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Current incident status.
|
||||
/// </summary>
|
||||
public required string Status { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether the incident is acknowledged.
|
||||
/// </summary>
|
||||
public bool IsAcknowledged { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// When the incident was acknowledged (if applicable).
|
||||
/// </summary>
|
||||
public DateTimeOffset? AcknowledgedAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Number of escalation cycles completed.
|
||||
/// </summary>
|
||||
public int CompletedCycles { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Last escalation step that was executed.
|
||||
/// </summary>
|
||||
public int LastExecutedStep { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// When the last step was executed.
|
||||
/// </summary>
|
||||
public DateTimeOffset? LastStepExecutedAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Steps that have been notified in the current cycle.
|
||||
/// </summary>
|
||||
public IReadOnlySet<int> NotifiedSteps { get; init; } = new HashSet<int>();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of escalation evaluation.
|
||||
/// </summary>
|
||||
public sealed record EscalationStepResult
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether escalation should proceed.
|
||||
/// </summary>
|
||||
public required bool ShouldEscalate { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The step to execute (if ShouldEscalate is true).
|
||||
/// </summary>
|
||||
public EscalationStep? NextStep { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Reason if not escalating.
|
||||
/// </summary>
|
||||
public string? Reason { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// When the next evaluation should occur.
|
||||
/// </summary>
|
||||
public DateTimeOffset? NextEvaluationAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether all steps have been exhausted.
|
||||
/// </summary>
|
||||
public bool AllStepsExhausted { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Current cycle number.
|
||||
/// </summary>
|
||||
public int CurrentCycle { get; init; }
|
||||
|
||||
public static EscalationStepResult NoEscalation(string reason, DateTimeOffset? nextEvaluation = null) =>
|
||||
new()
|
||||
{
|
||||
ShouldEscalate = false,
|
||||
Reason = reason,
|
||||
NextEvaluationAt = nextEvaluation
|
||||
};
|
||||
|
||||
public static EscalationStepResult Escalate(EscalationStep step, int cycle, DateTimeOffset? nextEvaluation = null) =>
|
||||
new()
|
||||
{
|
||||
ShouldEscalate = true,
|
||||
NextStep = step,
|
||||
CurrentCycle = cycle,
|
||||
NextEvaluationAt = nextEvaluation
|
||||
};
|
||||
|
||||
public static EscalationStepResult Exhausted(int cycles) =>
|
||||
new()
|
||||
{
|
||||
ShouldEscalate = false,
|
||||
AllStepsExhausted = true,
|
||||
CurrentCycle = cycles,
|
||||
Reason = "All escalation steps exhausted"
|
||||
};
|
||||
}
|
||||
@@ -1,431 +0,0 @@
|
||||
namespace StellaOps.Notifier.Worker.Escalations;
|
||||
|
||||
/// <summary>
|
||||
/// Manages on-call schedules and determines who is currently on-call.
|
||||
/// </summary>
|
||||
public interface IOnCallScheduleService
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets a schedule by ID.
|
||||
/// </summary>
|
||||
Task<OnCallSchedule?> GetAsync(string tenantId, string scheduleId, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Lists all schedules for a tenant.
|
||||
/// </summary>
|
||||
Task<IReadOnlyList<OnCallSchedule>> ListAsync(string tenantId, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Creates or updates a schedule.
|
||||
/// </summary>
|
||||
Task<OnCallSchedule> UpsertAsync(OnCallSchedule schedule, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Deletes a schedule.
|
||||
/// </summary>
|
||||
Task<bool> DeleteAsync(string tenantId, string scheduleId, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Gets who is currently on-call for a schedule.
|
||||
/// </summary>
|
||||
Task<OnCallResolution> GetCurrentOnCallAsync(
|
||||
string tenantId,
|
||||
string scheduleId,
|
||||
DateTimeOffset? asOf = null,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Gets on-call coverage for a time range.
|
||||
/// </summary>
|
||||
Task<IReadOnlyList<OnCallCoverage>> GetCoverageAsync(
|
||||
string tenantId,
|
||||
string scheduleId,
|
||||
DateTimeOffset from,
|
||||
DateTimeOffset to,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Adds an override to a schedule.
|
||||
/// </summary>
|
||||
Task<OnCallOverride> AddOverrideAsync(
|
||||
string tenantId,
|
||||
string scheduleId,
|
||||
OnCallOverride @override,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Removes an override from a schedule.
|
||||
/// </summary>
|
||||
Task<bool> RemoveOverrideAsync(
|
||||
string tenantId,
|
||||
string scheduleId,
|
||||
string overrideId,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// On-call schedule defining rotation of responders.
|
||||
/// </summary>
|
||||
public sealed record OnCallSchedule
|
||||
{
|
||||
/// <summary>
|
||||
/// Unique schedule identifier.
|
||||
/// </summary>
|
||||
public required string ScheduleId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Tenant this schedule belongs to.
|
||||
/// </summary>
|
||||
public required string TenantId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Human-readable name.
|
||||
/// </summary>
|
||||
public required string Name { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Description of the schedule.
|
||||
/// </summary>
|
||||
public string? Description { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Timezone for the schedule (IANA format).
|
||||
/// </summary>
|
||||
public string Timezone { get; init; } = "UTC";
|
||||
|
||||
/// <summary>
|
||||
/// Rotation layers (evaluated in order, first match wins).
|
||||
/// </summary>
|
||||
public required IReadOnlyList<RotationLayer> Layers { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Current overrides to the schedule.
|
||||
/// </summary>
|
||||
public IReadOnlyList<OnCallOverride> Overrides { get; init; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Fallback user if no one is on-call.
|
||||
/// </summary>
|
||||
public string? FallbackUserId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// When the schedule was created.
|
||||
/// </summary>
|
||||
public DateTimeOffset CreatedAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// When the schedule was last updated.
|
||||
/// </summary>
|
||||
public DateTimeOffset UpdatedAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether the schedule is enabled.
|
||||
/// </summary>
|
||||
public bool Enabled { get; init; } = true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A rotation layer within an on-call schedule.
|
||||
/// </summary>
|
||||
public sealed record RotationLayer
|
||||
{
|
||||
/// <summary>
|
||||
/// Layer name.
|
||||
/// </summary>
|
||||
public required string Name { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Rotation type.
|
||||
/// </summary>
|
||||
public required RotationType Type { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Users in the rotation (in order).
|
||||
/// </summary>
|
||||
public required IReadOnlyList<OnCallUser> Users { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// When this rotation starts.
|
||||
/// </summary>
|
||||
public required DateTimeOffset StartTime { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Rotation interval (e.g., 1 week for weekly rotation).
|
||||
/// </summary>
|
||||
public required TimeSpan RotationInterval { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Handoff time of day (in schedule timezone).
|
||||
/// </summary>
|
||||
public TimeOnly HandoffTime { get; init; } = new(9, 0);
|
||||
|
||||
/// <summary>
|
||||
/// Days of week this layer is active (empty = all days).
|
||||
/// </summary>
|
||||
public IReadOnlyList<DayOfWeek>? ActiveDays { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Time restrictions (e.g., only active 9am-5pm).
|
||||
/// </summary>
|
||||
public OnCallTimeRestriction? TimeRestriction { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Layer priority (lower = higher priority).
|
||||
/// </summary>
|
||||
public int Priority { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Type of rotation.
|
||||
/// </summary>
|
||||
public enum RotationType
|
||||
{
|
||||
/// <summary>
|
||||
/// Users rotate on a regular interval.
|
||||
/// </summary>
|
||||
Daily,
|
||||
|
||||
/// <summary>
|
||||
/// Users rotate weekly.
|
||||
/// </summary>
|
||||
Weekly,
|
||||
|
||||
/// <summary>
|
||||
/// Custom rotation interval.
|
||||
/// </summary>
|
||||
Custom
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A user in an on-call rotation.
|
||||
/// </summary>
|
||||
public sealed record OnCallUser
|
||||
{
|
||||
/// <summary>
|
||||
/// User identifier.
|
||||
/// </summary>
|
||||
public required string UserId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Display name.
|
||||
/// </summary>
|
||||
public string? DisplayName { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Email address.
|
||||
/// </summary>
|
||||
public string? Email { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Preferred notification channels.
|
||||
/// </summary>
|
||||
public IReadOnlyList<string> PreferredChannels { get; init; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Contact methods in priority order.
|
||||
/// </summary>
|
||||
public IReadOnlyList<ContactMethod> ContactMethods { get; init; } = [];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Contact method for a user.
|
||||
/// </summary>
|
||||
public sealed record ContactMethod
|
||||
{
|
||||
/// <summary>
|
||||
/// Contact type (email, sms, phone, slack, etc.).
|
||||
/// </summary>
|
||||
public required string Type { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Contact address/number.
|
||||
/// </summary>
|
||||
public required string Address { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Label for this contact method.
|
||||
/// </summary>
|
||||
public string? Label { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether this is verified.
|
||||
/// </summary>
|
||||
public bool Verified { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Time restriction for a rotation layer.
|
||||
/// </summary>
|
||||
public sealed record OnCallTimeRestriction
|
||||
{
|
||||
/// <summary>
|
||||
/// Start time of active period.
|
||||
/// </summary>
|
||||
public required TimeOnly StartTime { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// End time of active period.
|
||||
/// </summary>
|
||||
public required TimeOnly EndTime { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether the restriction spans midnight (e.g., 10pm-6am).
|
||||
/// </summary>
|
||||
public bool SpansMidnight => EndTime < StartTime;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Override to the normal on-call schedule.
|
||||
/// </summary>
|
||||
public sealed record OnCallOverride
|
||||
{
|
||||
/// <summary>
|
||||
/// Override identifier.
|
||||
/// </summary>
|
||||
public required string OverrideId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// User who will be on-call during this override.
|
||||
/// </summary>
|
||||
public required string UserId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Display name of the override user.
|
||||
/// </summary>
|
||||
public string? UserDisplayName { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// When the override starts.
|
||||
/// </summary>
|
||||
public required DateTimeOffset StartTime { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// When the override ends.
|
||||
/// </summary>
|
||||
public required DateTimeOffset EndTime { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Reason for the override.
|
||||
/// </summary>
|
||||
public string? Reason { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Who created the override.
|
||||
/// </summary>
|
||||
public string? CreatedBy { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// When the override was created.
|
||||
/// </summary>
|
||||
public DateTimeOffset CreatedAt { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of on-call resolution.
|
||||
/// </summary>
|
||||
public sealed record OnCallResolution
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether someone is on-call.
|
||||
/// </summary>
|
||||
public required bool HasOnCall { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The on-call user (if any).
|
||||
/// </summary>
|
||||
public OnCallUser? OnCallUser { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Which layer resolved the on-call.
|
||||
/// </summary>
|
||||
public string? ResolvedFromLayer { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether this is from an override.
|
||||
/// </summary>
|
||||
public bool IsOverride { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Override details if applicable.
|
||||
/// </summary>
|
||||
public OnCallOverride? Override { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether this is the fallback user.
|
||||
/// </summary>
|
||||
public bool IsFallback { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// When the current on-call shift ends.
|
||||
/// </summary>
|
||||
public DateTimeOffset? ShiftEndsAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The time this resolution was calculated for.
|
||||
/// </summary>
|
||||
public DateTimeOffset AsOf { get; init; }
|
||||
|
||||
public static OnCallResolution NoOneOnCall(DateTimeOffset asOf) =>
|
||||
new() { HasOnCall = false, AsOf = asOf };
|
||||
|
||||
public static OnCallResolution FromUser(OnCallUser user, string layer, DateTimeOffset asOf, DateTimeOffset? shiftEnds = null) =>
|
||||
new()
|
||||
{
|
||||
HasOnCall = true,
|
||||
OnCallUser = user,
|
||||
ResolvedFromLayer = layer,
|
||||
AsOf = asOf,
|
||||
ShiftEndsAt = shiftEnds
|
||||
};
|
||||
|
||||
public static OnCallResolution FromOverride(OnCallUser user, OnCallOverride @override, DateTimeOffset asOf) =>
|
||||
new()
|
||||
{
|
||||
HasOnCall = true,
|
||||
OnCallUser = user,
|
||||
IsOverride = true,
|
||||
Override = @override,
|
||||
AsOf = asOf,
|
||||
ShiftEndsAt = @override.EndTime
|
||||
};
|
||||
|
||||
public static OnCallResolution FromFallback(OnCallUser user, DateTimeOffset asOf) =>
|
||||
new()
|
||||
{
|
||||
HasOnCall = true,
|
||||
OnCallUser = user,
|
||||
IsFallback = true,
|
||||
AsOf = asOf
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// On-call coverage for a time period.
|
||||
/// </summary>
|
||||
public sealed record OnCallCoverage
|
||||
{
|
||||
/// <summary>
|
||||
/// Start of this coverage period.
|
||||
/// </summary>
|
||||
public required DateTimeOffset From { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// End of this coverage period.
|
||||
/// </summary>
|
||||
public required DateTimeOffset To { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// User on-call during this period.
|
||||
/// </summary>
|
||||
public required OnCallUser User { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Layer providing coverage.
|
||||
/// </summary>
|
||||
public string? Layer { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether this is from an override.
|
||||
/// </summary>
|
||||
public bool IsOverride { get; init; }
|
||||
}
|
||||
@@ -1,597 +0,0 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Escalations;
|
||||
|
||||
/// <summary>
|
||||
/// In-app inbox channel for notifications that users can view in the UI/CLI.
|
||||
/// </summary>
|
||||
public interface IInboxService
|
||||
{
|
||||
/// <summary>
|
||||
/// Adds a notification to a user's inbox.
|
||||
/// </summary>
|
||||
Task<InboxNotification> AddAsync(
|
||||
string tenantId,
|
||||
string userId,
|
||||
InboxNotificationRequest request,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Gets notifications for a user.
|
||||
/// </summary>
|
||||
Task<IReadOnlyList<InboxNotification>> GetAsync(
|
||||
string tenantId,
|
||||
string userId,
|
||||
InboxQuery? query = null,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Marks notifications as read.
|
||||
/// </summary>
|
||||
Task<int> MarkAsReadAsync(
|
||||
string tenantId,
|
||||
string userId,
|
||||
IEnumerable<string> notificationIds,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Marks all notifications as read for a user.
|
||||
/// </summary>
|
||||
Task<int> MarkAllAsReadAsync(
|
||||
string tenantId,
|
||||
string userId,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Deletes notifications.
|
||||
/// </summary>
|
||||
Task<int> DeleteAsync(
|
||||
string tenantId,
|
||||
string userId,
|
||||
IEnumerable<string> notificationIds,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the unread count for a user.
|
||||
/// </summary>
|
||||
Task<int> GetUnreadCountAsync(
|
||||
string tenantId,
|
||||
string userId,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Archives old notifications.
|
||||
/// </summary>
|
||||
Task<int> ArchiveOldAsync(
|
||||
string tenantId,
|
||||
TimeSpan olderThan,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Request to add an inbox notification.
|
||||
/// </summary>
|
||||
public sealed record InboxNotificationRequest
|
||||
{
|
||||
/// <summary>
|
||||
/// Notification title.
|
||||
/// </summary>
|
||||
public required string Title { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Notification body.
|
||||
/// </summary>
|
||||
public required string Body { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Type of notification (incident, digest, approval, etc.).
|
||||
/// </summary>
|
||||
public required string Type { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Severity level.
|
||||
/// </summary>
|
||||
public string Severity { get; init; } = "info";
|
||||
|
||||
/// <summary>
|
||||
/// Related incident ID (if applicable).
|
||||
/// </summary>
|
||||
public string? IncidentId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Link to view more details.
|
||||
/// </summary>
|
||||
public string? ActionUrl { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Action button text.
|
||||
/// </summary>
|
||||
public string? ActionText { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Additional metadata.
|
||||
/// </summary>
|
||||
public IReadOnlyDictionary<string, string> Metadata { get; init; } = new Dictionary<string, string>();
|
||||
|
||||
/// <summary>
|
||||
/// Whether this notification requires acknowledgement.
|
||||
/// </summary>
|
||||
public bool RequiresAck { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Expiration time for the notification.
|
||||
/// </summary>
|
||||
public DateTimeOffset? ExpiresAt { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// An inbox notification.
|
||||
/// </summary>
|
||||
public sealed record InboxNotification
|
||||
{
|
||||
/// <summary>
|
||||
/// Unique notification ID.
|
||||
/// </summary>
|
||||
public required string NotificationId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Tenant ID.
|
||||
/// </summary>
|
||||
public required string TenantId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// User ID this notification is for.
|
||||
/// </summary>
|
||||
public required string UserId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Notification title.
|
||||
/// </summary>
|
||||
public required string Title { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Notification body.
|
||||
/// </summary>
|
||||
public required string Body { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Type of notification.
|
||||
/// </summary>
|
||||
public required string Type { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Severity level.
|
||||
/// </summary>
|
||||
public required string Severity { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Related incident ID.
|
||||
/// </summary>
|
||||
public string? IncidentId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Link to view more details.
|
||||
/// </summary>
|
||||
public string? ActionUrl { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Action button text.
|
||||
/// </summary>
|
||||
public string? ActionText { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Additional metadata.
|
||||
/// </summary>
|
||||
public IReadOnlyDictionary<string, string> Metadata { get; init; } = new Dictionary<string, string>();
|
||||
|
||||
/// <summary>
|
||||
/// Whether this has been read.
|
||||
/// </summary>
|
||||
public bool IsRead { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// When the notification was read.
|
||||
/// </summary>
|
||||
public DateTimeOffset? ReadAt { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether this requires acknowledgement.
|
||||
/// </summary>
|
||||
public bool RequiresAck { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether this has been acknowledged.
|
||||
/// </summary>
|
||||
public bool IsAcknowledged { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// When the notification was created.
|
||||
/// </summary>
|
||||
public required DateTimeOffset CreatedAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// When the notification expires.
|
||||
/// </summary>
|
||||
public DateTimeOffset? ExpiresAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether the notification is archived.
|
||||
/// </summary>
|
||||
public bool IsArchived { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Query parameters for inbox notifications.
|
||||
/// </summary>
|
||||
public sealed record InboxQuery
|
||||
{
|
||||
/// <summary>
|
||||
/// Filter by read status.
|
||||
/// </summary>
|
||||
public bool? IsRead { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Filter by notification type.
|
||||
/// </summary>
|
||||
public IReadOnlyList<string>? Types { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Filter by severity.
|
||||
/// </summary>
|
||||
public IReadOnlyList<string>? Severities { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Filter by incident ID.
|
||||
/// </summary>
|
||||
public string? IncidentId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Include archived notifications.
|
||||
/// </summary>
|
||||
public bool IncludeArchived { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Only include notifications after this time.
|
||||
/// </summary>
|
||||
public DateTimeOffset? After { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Maximum notifications to return.
|
||||
/// </summary>
|
||||
public int Limit { get; init; } = 50;
|
||||
|
||||
/// <summary>
|
||||
/// Offset for pagination.
|
||||
/// </summary>
|
||||
public int Offset { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// In-memory implementation of inbox service.
|
||||
/// </summary>
|
||||
public sealed class InMemoryInboxService : IInboxService
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, List<InboxNotification>> _notifications = new();
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<InMemoryInboxService> _logger;
|
||||
|
||||
public InMemoryInboxService(
|
||||
TimeProvider timeProvider,
|
||||
ILogger<InMemoryInboxService> logger)
|
||||
{
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public Task<InboxNotification> AddAsync(
|
||||
string tenantId,
|
||||
string userId,
|
||||
InboxNotificationRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var notification = new InboxNotification
|
||||
{
|
||||
NotificationId = $"inbox-{Guid.NewGuid():N}"[..20],
|
||||
TenantId = tenantId,
|
||||
UserId = userId,
|
||||
Title = request.Title,
|
||||
Body = request.Body,
|
||||
Type = request.Type,
|
||||
Severity = request.Severity,
|
||||
IncidentId = request.IncidentId,
|
||||
ActionUrl = request.ActionUrl,
|
||||
ActionText = request.ActionText,
|
||||
Metadata = request.Metadata,
|
||||
RequiresAck = request.RequiresAck,
|
||||
CreatedAt = _timeProvider.GetUtcNow(),
|
||||
ExpiresAt = request.ExpiresAt
|
||||
};
|
||||
|
||||
var key = BuildKey(tenantId, userId);
|
||||
_notifications.AddOrUpdate(
|
||||
key,
|
||||
_ => [notification],
|
||||
(_, list) =>
|
||||
{
|
||||
list.Add(notification);
|
||||
return list;
|
||||
});
|
||||
|
||||
_logger.LogInformation(
|
||||
"Added inbox notification {NotificationId} for user {UserId} in tenant {TenantId}.",
|
||||
notification.NotificationId, userId, tenantId);
|
||||
|
||||
return Task.FromResult(notification);
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<InboxNotification>> GetAsync(
|
||||
string tenantId,
|
||||
string userId,
|
||||
InboxQuery? query = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var key = BuildKey(tenantId, userId);
|
||||
if (!_notifications.TryGetValue(key, out var notifications))
|
||||
{
|
||||
return Task.FromResult<IReadOnlyList<InboxNotification>>([]);
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
IEnumerable<InboxNotification> filtered = notifications
|
||||
.Where(n => !n.ExpiresAt.HasValue || n.ExpiresAt > now);
|
||||
|
||||
if (query is not null)
|
||||
{
|
||||
if (query.IsRead.HasValue)
|
||||
{
|
||||
filtered = filtered.Where(n => n.IsRead == query.IsRead.Value);
|
||||
}
|
||||
|
||||
if (query.Types is { Count: > 0 })
|
||||
{
|
||||
filtered = filtered.Where(n => query.Types.Contains(n.Type, StringComparer.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
if (query.Severities is { Count: > 0 })
|
||||
{
|
||||
filtered = filtered.Where(n => query.Severities.Contains(n.Severity, StringComparer.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(query.IncidentId))
|
||||
{
|
||||
filtered = filtered.Where(n => n.IncidentId == query.IncidentId);
|
||||
}
|
||||
|
||||
if (!query.IncludeArchived)
|
||||
{
|
||||
filtered = filtered.Where(n => !n.IsArchived);
|
||||
}
|
||||
|
||||
if (query.After.HasValue)
|
||||
{
|
||||
filtered = filtered.Where(n => n.CreatedAt > query.After.Value);
|
||||
}
|
||||
}
|
||||
|
||||
var result = filtered
|
||||
.OrderByDescending(n => n.CreatedAt)
|
||||
.Skip(query?.Offset ?? 0)
|
||||
.Take(query?.Limit ?? 50)
|
||||
.ToList();
|
||||
|
||||
return Task.FromResult<IReadOnlyList<InboxNotification>>(result);
|
||||
}
|
||||
|
||||
public Task<int> MarkAsReadAsync(
|
||||
string tenantId,
|
||||
string userId,
|
||||
IEnumerable<string> notificationIds,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var key = BuildKey(tenantId, userId);
|
||||
if (!_notifications.TryGetValue(key, out var notifications))
|
||||
{
|
||||
return Task.FromResult(0);
|
||||
}
|
||||
|
||||
var ids = notificationIds.ToHashSet();
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var count = 0;
|
||||
|
||||
foreach (var notification in notifications.Where(n => ids.Contains(n.NotificationId) && !n.IsRead))
|
||||
{
|
||||
notification.IsRead = true;
|
||||
notification.ReadAt = now;
|
||||
count++;
|
||||
}
|
||||
|
||||
return Task.FromResult(count);
|
||||
}
|
||||
|
||||
public Task<int> MarkAllAsReadAsync(
|
||||
string tenantId,
|
||||
string userId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var key = BuildKey(tenantId, userId);
|
||||
if (!_notifications.TryGetValue(key, out var notifications))
|
||||
{
|
||||
return Task.FromResult(0);
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var count = 0;
|
||||
|
||||
foreach (var notification in notifications.Where(n => !n.IsRead))
|
||||
{
|
||||
notification.IsRead = true;
|
||||
notification.ReadAt = now;
|
||||
count++;
|
||||
}
|
||||
|
||||
return Task.FromResult(count);
|
||||
}
|
||||
|
||||
public Task<int> DeleteAsync(
|
||||
string tenantId,
|
||||
string userId,
|
||||
IEnumerable<string> notificationIds,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var key = BuildKey(tenantId, userId);
|
||||
if (!_notifications.TryGetValue(key, out var notifications))
|
||||
{
|
||||
return Task.FromResult(0);
|
||||
}
|
||||
|
||||
var ids = notificationIds.ToHashSet();
|
||||
var count = notifications.RemoveAll(n => ids.Contains(n.NotificationId));
|
||||
|
||||
return Task.FromResult(count);
|
||||
}
|
||||
|
||||
public Task<int> GetUnreadCountAsync(
|
||||
string tenantId,
|
||||
string userId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var key = BuildKey(tenantId, userId);
|
||||
if (!_notifications.TryGetValue(key, out var notifications))
|
||||
{
|
||||
return Task.FromResult(0);
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var count = notifications.Count(n =>
|
||||
!n.IsRead &&
|
||||
!n.IsArchived &&
|
||||
(!n.ExpiresAt.HasValue || n.ExpiresAt > now));
|
||||
|
||||
return Task.FromResult(count);
|
||||
}
|
||||
|
||||
public Task<int> ArchiveOldAsync(
|
||||
string tenantId,
|
||||
TimeSpan olderThan,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var cutoff = _timeProvider.GetUtcNow() - olderThan;
|
||||
var count = 0;
|
||||
|
||||
foreach (var (key, notifications) in _notifications)
|
||||
{
|
||||
if (!key.StartsWith(tenantId + ":"))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var notification in notifications.Where(n => n.CreatedAt < cutoff && !n.IsArchived))
|
||||
{
|
||||
notification.IsArchived = true;
|
||||
count++;
|
||||
}
|
||||
}
|
||||
|
||||
return Task.FromResult(count);
|
||||
}
|
||||
|
||||
private static string BuildKey(string tenantId, string userId) => $"{tenantId}:{userId}";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// CLI channel adapter for inbox notifications.
|
||||
/// </summary>
|
||||
public sealed class CliInboxChannelAdapter
|
||||
{
|
||||
private readonly IInboxService _inboxService;
|
||||
private readonly ILogger<CliInboxChannelAdapter> _logger;
|
||||
|
||||
public CliInboxChannelAdapter(
|
||||
IInboxService inboxService,
|
||||
ILogger<CliInboxChannelAdapter> logger)
|
||||
{
|
||||
_inboxService = inboxService ?? throw new ArgumentNullException(nameof(inboxService));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sends a notification to a user's CLI inbox.
|
||||
/// </summary>
|
||||
public async Task<InboxNotification> SendAsync(
|
||||
string tenantId,
|
||||
string userId,
|
||||
string title,
|
||||
string body,
|
||||
string type = "notification",
|
||||
string severity = "info",
|
||||
string? incidentId = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var request = new InboxNotificationRequest
|
||||
{
|
||||
Title = title,
|
||||
Body = body,
|
||||
Type = type,
|
||||
Severity = severity,
|
||||
IncidentId = incidentId
|
||||
};
|
||||
|
||||
var notification = await _inboxService.AddAsync(tenantId, userId, request, cancellationToken);
|
||||
|
||||
_logger.LogDebug(
|
||||
"Sent CLI inbox notification {NotificationId} to {UserId}.",
|
||||
notification.NotificationId, userId);
|
||||
|
||||
return notification;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Formats notifications for CLI display.
|
||||
/// </summary>
|
||||
public string FormatForCli(IReadOnlyList<InboxNotification> notifications, bool verbose = false)
|
||||
{
|
||||
if (notifications.Count == 0)
|
||||
{
|
||||
return "No notifications.";
|
||||
}
|
||||
|
||||
var sb = new System.Text.StringBuilder();
|
||||
sb.AppendLine($"Notifications ({notifications.Count}):");
|
||||
sb.AppendLine(new string('-', 60));
|
||||
|
||||
foreach (var n in notifications)
|
||||
{
|
||||
var readMarker = n.IsRead ? " " : "*";
|
||||
var severityMarker = n.Severity.ToUpperInvariant() switch
|
||||
{
|
||||
"CRITICAL" => "[!!]",
|
||||
"HIGH" => "[! ]",
|
||||
"MEDIUM" or "WARNING" => "[~ ]",
|
||||
_ => "[ ]"
|
||||
};
|
||||
|
||||
sb.AppendLine($"{readMarker}{severityMarker} [{n.CreatedAt:MM-dd HH:mm}] {n.Title}");
|
||||
|
||||
if (verbose)
|
||||
{
|
||||
sb.AppendLine($" ID: {n.NotificationId}");
|
||||
sb.AppendLine($" Type: {n.Type}");
|
||||
if (!string.IsNullOrEmpty(n.Body))
|
||||
{
|
||||
var body = n.Body.Length > 100 ? n.Body[..100] + "..." : n.Body;
|
||||
sb.AppendLine($" {body}");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(n.ActionUrl))
|
||||
{
|
||||
sb.AppendLine($" Link: {n.ActionUrl}");
|
||||
}
|
||||
|
||||
sb.AppendLine();
|
||||
}
|
||||
}
|
||||
|
||||
return sb.ToString();
|
||||
}
|
||||
}
|
||||
@@ -1,609 +0,0 @@
|
||||
using System.Net.Http.Json;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Escalations;
|
||||
|
||||
/// <summary>
|
||||
/// Adapter for external incident management integrations.
|
||||
/// </summary>
|
||||
public interface IIncidentIntegrationAdapter
|
||||
{
|
||||
/// <summary>
|
||||
/// Integration type identifier.
|
||||
/// </summary>
|
||||
string IntegrationType { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Creates an incident in the external system.
|
||||
/// </summary>
|
||||
Task<IntegrationIncidentResult> CreateIncidentAsync(
|
||||
IntegrationIncidentRequest request,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Acknowledges an incident in the external system.
|
||||
/// </summary>
|
||||
Task<IntegrationAckResult> AcknowledgeAsync(
|
||||
string externalIncidentId,
|
||||
string? actor = null,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Resolves an incident in the external system.
|
||||
/// </summary>
|
||||
Task<IntegrationResolveResult> ResolveAsync(
|
||||
string externalIncidentId,
|
||||
string? resolution = null,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the current status of an incident.
|
||||
/// </summary>
|
||||
Task<IntegrationIncidentStatus?> GetStatusAsync(
|
||||
string externalIncidentId,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Tests connectivity to the integration.
|
||||
/// </summary>
|
||||
Task<IntegrationHealthResult> HealthCheckAsync(CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Factory for creating integration adapters.
|
||||
/// </summary>
|
||||
public interface IIntegrationAdapterFactory
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets an adapter for the specified integration type.
|
||||
/// </summary>
|
||||
IIncidentIntegrationAdapter? GetAdapter(string integrationType);
|
||||
|
||||
/// <summary>
|
||||
/// Gets all available integration types.
|
||||
/// </summary>
|
||||
IReadOnlyList<string> GetAvailableIntegrations();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Request to create an incident in an external system.
|
||||
/// </summary>
|
||||
public sealed record IntegrationIncidentRequest
|
||||
{
|
||||
public required string TenantId { get; init; }
|
||||
public required string IncidentId { get; init; }
|
||||
public required string Title { get; init; }
|
||||
public string? Description { get; init; }
|
||||
public string Severity { get; init; } = "high";
|
||||
public string? ServiceKey { get; init; }
|
||||
public string? RoutingKey { get; init; }
|
||||
public IReadOnlyDictionary<string, string> CustomDetails { get; init; } = new Dictionary<string, string>();
|
||||
public string? DeduplicationKey { get; init; }
|
||||
public string? Source { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of creating an incident.
|
||||
/// </summary>
|
||||
public sealed record IntegrationIncidentResult
|
||||
{
|
||||
public required bool Success { get; init; }
|
||||
public string? ExternalIncidentId { get; init; }
|
||||
public string? ExternalUrl { get; init; }
|
||||
public string? ErrorMessage { get; init; }
|
||||
public string? ErrorCode { get; init; }
|
||||
|
||||
public static IntegrationIncidentResult Succeeded(string externalId, string? url = null) =>
|
||||
new() { Success = true, ExternalIncidentId = externalId, ExternalUrl = url };
|
||||
|
||||
public static IntegrationIncidentResult Failed(string message, string? code = null) =>
|
||||
new() { Success = false, ErrorMessage = message, ErrorCode = code };
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of acknowledging an incident.
|
||||
/// </summary>
|
||||
public sealed record IntegrationAckResult
|
||||
{
|
||||
public required bool Success { get; init; }
|
||||
public string? ErrorMessage { get; init; }
|
||||
|
||||
public static IntegrationAckResult Succeeded() => new() { Success = true };
|
||||
public static IntegrationAckResult Failed(string message) => new() { Success = false, ErrorMessage = message };
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of resolving an incident.
|
||||
/// </summary>
|
||||
public sealed record IntegrationResolveResult
|
||||
{
|
||||
public required bool Success { get; init; }
|
||||
public string? ErrorMessage { get; init; }
|
||||
|
||||
public static IntegrationResolveResult Succeeded() => new() { Success = true };
|
||||
public static IntegrationResolveResult Failed(string message) => new() { Success = false, ErrorMessage = message };
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Status of an incident in the external system.
|
||||
/// </summary>
|
||||
public sealed record IntegrationIncidentStatus
|
||||
{
|
||||
public required string ExternalIncidentId { get; init; }
|
||||
public required string Status { get; init; }
|
||||
public bool IsAcknowledged { get; init; }
|
||||
public bool IsResolved { get; init; }
|
||||
public DateTimeOffset? AcknowledgedAt { get; init; }
|
||||
public DateTimeOffset? ResolvedAt { get; init; }
|
||||
public string? AssignedTo { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of integration health check.
|
||||
/// </summary>
|
||||
public sealed record IntegrationHealthResult
|
||||
{
|
||||
public required bool Healthy { get; init; }
|
||||
public string? Message { get; init; }
|
||||
public TimeSpan? Latency { get; init; }
|
||||
|
||||
public static IntegrationHealthResult Ok(TimeSpan? latency = null) =>
|
||||
new() { Healthy = true, Latency = latency };
|
||||
|
||||
public static IntegrationHealthResult Unhealthy(string message) =>
|
||||
new() { Healthy = false, Message = message };
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// PagerDuty integration adapter.
|
||||
/// </summary>
|
||||
public sealed class PagerDutyAdapter : IIncidentIntegrationAdapter
|
||||
{
|
||||
private readonly HttpClient _httpClient;
|
||||
private readonly PagerDutyOptions _options;
|
||||
private readonly ILogger<PagerDutyAdapter> _logger;
|
||||
|
||||
public PagerDutyAdapter(
|
||||
HttpClient httpClient,
|
||||
IOptions<PagerDutyOptions> options,
|
||||
ILogger<PagerDutyAdapter> logger)
|
||||
{
|
||||
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
|
||||
_httpClient.BaseAddress = new Uri(_options.ApiBaseUrl);
|
||||
if (!string.IsNullOrEmpty(_options.ApiKey))
|
||||
{
|
||||
_httpClient.DefaultRequestHeaders.Add("Authorization", $"Token token={_options.ApiKey}");
|
||||
}
|
||||
}
|
||||
|
||||
public string IntegrationType => "pagerduty";
|
||||
|
||||
public async Task<IntegrationIncidentResult> CreateIncidentAsync(
|
||||
IntegrationIncidentRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
var payload = new
|
||||
{
|
||||
routing_key = request.RoutingKey ?? _options.DefaultRoutingKey,
|
||||
event_action = "trigger",
|
||||
dedup_key = request.DeduplicationKey ?? request.IncidentId,
|
||||
payload = new
|
||||
{
|
||||
summary = request.Title,
|
||||
source = request.Source ?? "stellaops",
|
||||
severity = MapSeverity(request.Severity),
|
||||
custom_details = request.CustomDetails
|
||||
},
|
||||
client = "StellaOps",
|
||||
client_url = _options.ClientUrl
|
||||
};
|
||||
|
||||
var response = await _httpClient.PostAsJsonAsync(
|
||||
"/v2/enqueue",
|
||||
payload,
|
||||
cancellationToken);
|
||||
|
||||
if (response.IsSuccessStatusCode)
|
||||
{
|
||||
var result = await response.Content.ReadFromJsonAsync<PagerDutyEventResponse>(cancellationToken);
|
||||
_logger.LogInformation(
|
||||
"Created PagerDuty incident {DedupKey} with status {Status}.",
|
||||
result?.DedupKey, result?.Status);
|
||||
|
||||
return IntegrationIncidentResult.Succeeded(
|
||||
result?.DedupKey ?? request.IncidentId,
|
||||
$"https://app.pagerduty.com/incidents/{result?.DedupKey}");
|
||||
}
|
||||
|
||||
var error = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
_logger.LogError("PagerDuty create incident failed: {Error}", error);
|
||||
return IntegrationIncidentResult.Failed(error, response.StatusCode.ToString());
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "PagerDuty create incident exception");
|
||||
return IntegrationIncidentResult.Failed(ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<IntegrationAckResult> AcknowledgeAsync(
|
||||
string externalIncidentId,
|
||||
string? actor = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
var payload = new
|
||||
{
|
||||
routing_key = _options.DefaultRoutingKey,
|
||||
event_action = "acknowledge",
|
||||
dedup_key = externalIncidentId
|
||||
};
|
||||
|
||||
var response = await _httpClient.PostAsJsonAsync("/v2/enqueue", payload, cancellationToken);
|
||||
|
||||
if (response.IsSuccessStatusCode)
|
||||
{
|
||||
_logger.LogInformation("Acknowledged PagerDuty incident {IncidentId}.", externalIncidentId);
|
||||
return IntegrationAckResult.Succeeded();
|
||||
}
|
||||
|
||||
var error = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
return IntegrationAckResult.Failed(error);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "PagerDuty acknowledge exception");
|
||||
return IntegrationAckResult.Failed(ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<IntegrationResolveResult> ResolveAsync(
|
||||
string externalIncidentId,
|
||||
string? resolution = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
var payload = new
|
||||
{
|
||||
routing_key = _options.DefaultRoutingKey,
|
||||
event_action = "resolve",
|
||||
dedup_key = externalIncidentId
|
||||
};
|
||||
|
||||
var response = await _httpClient.PostAsJsonAsync("/v2/enqueue", payload, cancellationToken);
|
||||
|
||||
if (response.IsSuccessStatusCode)
|
||||
{
|
||||
_logger.LogInformation("Resolved PagerDuty incident {IncidentId}.", externalIncidentId);
|
||||
return IntegrationResolveResult.Succeeded();
|
||||
}
|
||||
|
||||
var error = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
return IntegrationResolveResult.Failed(error);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "PagerDuty resolve exception");
|
||||
return IntegrationResolveResult.Failed(ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
public Task<IntegrationIncidentStatus?> GetStatusAsync(
|
||||
string externalIncidentId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
// PagerDuty Events API v2 doesn't provide status lookup
|
||||
// Would need to use REST API with incident ID
|
||||
return Task.FromResult<IntegrationIncidentStatus?>(null);
|
||||
}
|
||||
|
||||
public async Task<IntegrationHealthResult> HealthCheckAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
var sw = System.Diagnostics.Stopwatch.StartNew();
|
||||
var response = await _httpClient.GetAsync("/", cancellationToken);
|
||||
sw.Stop();
|
||||
|
||||
return response.IsSuccessStatusCode
|
||||
? IntegrationHealthResult.Ok(sw.Elapsed)
|
||||
: IntegrationHealthResult.Unhealthy($"Status: {response.StatusCode}");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return IntegrationHealthResult.Unhealthy(ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
private static string MapSeverity(string severity) => severity.ToLowerInvariant() switch
|
||||
{
|
||||
"critical" => "critical",
|
||||
"high" => "error",
|
||||
"medium" or "warning" => "warning",
|
||||
"low" or "info" => "info",
|
||||
_ => "error"
|
||||
};
|
||||
|
||||
private sealed record PagerDutyEventResponse(string Status, string Message, string DedupKey);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// OpsGenie integration adapter.
|
||||
/// </summary>
|
||||
public sealed class OpsGenieAdapter : IIncidentIntegrationAdapter
|
||||
{
|
||||
private readonly HttpClient _httpClient;
|
||||
private readonly OpsGenieOptions _options;
|
||||
private readonly ILogger<OpsGenieAdapter> _logger;
|
||||
|
||||
public OpsGenieAdapter(
|
||||
HttpClient httpClient,
|
||||
IOptions<OpsGenieOptions> options,
|
||||
ILogger<OpsGenieAdapter> logger)
|
||||
{
|
||||
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
|
||||
_httpClient.BaseAddress = new Uri(_options.ApiBaseUrl);
|
||||
if (!string.IsNullOrEmpty(_options.ApiKey))
|
||||
{
|
||||
_httpClient.DefaultRequestHeaders.Add("Authorization", $"GenieKey {_options.ApiKey}");
|
||||
}
|
||||
}
|
||||
|
||||
public string IntegrationType => "opsgenie";
|
||||
|
||||
public async Task<IntegrationIncidentResult> CreateIncidentAsync(
|
||||
IntegrationIncidentRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
var payload = new
|
||||
{
|
||||
message = request.Title,
|
||||
description = request.Description,
|
||||
alias = request.DeduplicationKey ?? request.IncidentId,
|
||||
priority = MapPriority(request.Severity),
|
||||
source = request.Source ?? "StellaOps",
|
||||
details = request.CustomDetails,
|
||||
tags = new[] { "stellaops", request.TenantId }
|
||||
};
|
||||
|
||||
var response = await _httpClient.PostAsJsonAsync("/v2/alerts", payload, cancellationToken);
|
||||
|
||||
if (response.IsSuccessStatusCode)
|
||||
{
|
||||
var result = await response.Content.ReadFromJsonAsync<OpsGenieAlertResponse>(cancellationToken);
|
||||
_logger.LogInformation(
|
||||
"Created OpsGenie alert {AlertId} with request {RequestId}.",
|
||||
result?.Data?.AlertId, result?.RequestId);
|
||||
|
||||
return IntegrationIncidentResult.Succeeded(
|
||||
result?.Data?.AlertId ?? request.IncidentId,
|
||||
$"https://app.opsgenie.com/alert/detail/{result?.Data?.AlertId}");
|
||||
}
|
||||
|
||||
var error = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
_logger.LogError("OpsGenie create alert failed: {Error}", error);
|
||||
return IntegrationIncidentResult.Failed(error, response.StatusCode.ToString());
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "OpsGenie create alert exception");
|
||||
return IntegrationIncidentResult.Failed(ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<IntegrationAckResult> AcknowledgeAsync(
|
||||
string externalIncidentId,
|
||||
string? actor = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
var payload = new
|
||||
{
|
||||
user = actor ?? "StellaOps",
|
||||
source = "StellaOps"
|
||||
};
|
||||
|
||||
var response = await _httpClient.PostAsJsonAsync(
|
||||
$"/v2/alerts/{externalIncidentId}/acknowledge",
|
||||
payload,
|
||||
cancellationToken);
|
||||
|
||||
if (response.IsSuccessStatusCode)
|
||||
{
|
||||
_logger.LogInformation("Acknowledged OpsGenie alert {AlertId}.", externalIncidentId);
|
||||
return IntegrationAckResult.Succeeded();
|
||||
}
|
||||
|
||||
var error = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
return IntegrationAckResult.Failed(error);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "OpsGenie acknowledge exception");
|
||||
return IntegrationAckResult.Failed(ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<IntegrationResolveResult> ResolveAsync(
|
||||
string externalIncidentId,
|
||||
string? resolution = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
var payload = new
|
||||
{
|
||||
user = "StellaOps",
|
||||
source = "StellaOps",
|
||||
note = resolution
|
||||
};
|
||||
|
||||
var response = await _httpClient.PostAsJsonAsync(
|
||||
$"/v2/alerts/{externalIncidentId}/close",
|
||||
payload,
|
||||
cancellationToken);
|
||||
|
||||
if (response.IsSuccessStatusCode)
|
||||
{
|
||||
_logger.LogInformation("Resolved OpsGenie alert {AlertId}.", externalIncidentId);
|
||||
return IntegrationResolveResult.Succeeded();
|
||||
}
|
||||
|
||||
var error = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
return IntegrationResolveResult.Failed(error);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "OpsGenie resolve exception");
|
||||
return IntegrationResolveResult.Failed(ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<IntegrationIncidentStatus?> GetStatusAsync(
|
||||
string externalIncidentId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
var response = await _httpClient.GetAsync($"/v2/alerts/{externalIncidentId}", cancellationToken);
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var result = await response.Content.ReadFromJsonAsync<OpsGenieAlertDetailResponse>(cancellationToken);
|
||||
var alert = result?.Data;
|
||||
|
||||
if (alert is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return new IntegrationIncidentStatus
|
||||
{
|
||||
ExternalIncidentId = externalIncidentId,
|
||||
Status = alert.Status ?? "unknown",
|
||||
IsAcknowledged = alert.Acknowledged,
|
||||
IsResolved = string.Equals(alert.Status, "closed", StringComparison.OrdinalIgnoreCase),
|
||||
AcknowledgedAt = alert.AcknowledgedAt,
|
||||
ResolvedAt = alert.ClosedAt
|
||||
};
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "OpsGenie get status exception");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<IntegrationHealthResult> HealthCheckAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
var sw = System.Diagnostics.Stopwatch.StartNew();
|
||||
var response = await _httpClient.GetAsync("/v2/heartbeats", cancellationToken);
|
||||
sw.Stop();
|
||||
|
||||
return response.IsSuccessStatusCode
|
||||
? IntegrationHealthResult.Ok(sw.Elapsed)
|
||||
: IntegrationHealthResult.Unhealthy($"Status: {response.StatusCode}");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return IntegrationHealthResult.Unhealthy(ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
private static string MapPriority(string severity) => severity.ToLowerInvariant() switch
|
||||
{
|
||||
"critical" => "P1",
|
||||
"high" => "P2",
|
||||
"medium" or "warning" => "P3",
|
||||
"low" => "P4",
|
||||
"info" => "P5",
|
||||
_ => "P3"
|
||||
};
|
||||
|
||||
private sealed record OpsGenieAlertResponse(string RequestId, OpsGenieAlertData? Data);
|
||||
private sealed record OpsGenieAlertData(string AlertId);
|
||||
private sealed record OpsGenieAlertDetailResponse(OpsGenieAlertDetail? Data);
|
||||
private sealed record OpsGenieAlertDetail(
|
||||
string? Status,
|
||||
bool Acknowledged,
|
||||
DateTimeOffset? AcknowledgedAt,
|
||||
DateTimeOffset? ClosedAt);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// PagerDuty integration options.
|
||||
/// </summary>
|
||||
public sealed class PagerDutyOptions
|
||||
{
|
||||
public const string SectionName = "Notifier:Integrations:PagerDuty";
|
||||
|
||||
public bool Enabled { get; set; }
|
||||
public string ApiBaseUrl { get; set; } = "https://events.pagerduty.com";
|
||||
public string? ApiKey { get; set; }
|
||||
public string? DefaultRoutingKey { get; set; }
|
||||
public string? ClientUrl { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// OpsGenie integration options.
|
||||
/// </summary>
|
||||
public sealed class OpsGenieOptions
|
||||
{
|
||||
public const string SectionName = "Notifier:Integrations:OpsGenie";
|
||||
|
||||
public bool Enabled { get; set; }
|
||||
public string ApiBaseUrl { get; set; } = "https://api.opsgenie.com";
|
||||
public string? ApiKey { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation of integration adapter factory.
|
||||
/// </summary>
|
||||
public sealed class IntegrationAdapterFactory : IIntegrationAdapterFactory
|
||||
{
|
||||
private readonly IServiceProvider _serviceProvider;
|
||||
private readonly Dictionary<string, Type> _adapterTypes;
|
||||
|
||||
public IntegrationAdapterFactory(IServiceProvider serviceProvider)
|
||||
{
|
||||
_serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
|
||||
_adapterTypes = new Dictionary<string, Type>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["pagerduty"] = typeof(PagerDutyAdapter),
|
||||
["opsgenie"] = typeof(OpsGenieAdapter)
|
||||
};
|
||||
}
|
||||
|
||||
public IIncidentIntegrationAdapter? GetAdapter(string integrationType)
|
||||
{
|
||||
if (_adapterTypes.TryGetValue(integrationType, out var type))
|
||||
{
|
||||
return _serviceProvider.GetService(type) as IIncidentIntegrationAdapter;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
public IReadOnlyList<string> GetAvailableIntegrations() => _adapterTypes.Keys.ToList();
|
||||
}
|
||||
@@ -72,7 +72,9 @@ public enum ChaosFaultType
|
||||
AuthFailure,
|
||||
Timeout,
|
||||
PartialFailure,
|
||||
Intermittent
|
||||
Intermittent,
|
||||
ErrorResponse,
|
||||
CorruptResponse
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -115,52 +115,6 @@ public sealed record ChaosExperimentConfig
|
||||
public required string InitiatedBy { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Types of faults that can be injected.
|
||||
/// </summary>
|
||||
public enum ChaosFaultType
|
||||
{
|
||||
/// <summary>
|
||||
/// Complete outage - all requests fail.
|
||||
/// </summary>
|
||||
Outage,
|
||||
|
||||
/// <summary>
|
||||
/// Partial failure - percentage of requests fail.
|
||||
/// </summary>
|
||||
PartialFailure,
|
||||
|
||||
/// <summary>
|
||||
/// Latency injection - requests are delayed.
|
||||
/// </summary>
|
||||
Latency,
|
||||
|
||||
/// <summary>
|
||||
/// Intermittent failures - random failures.
|
||||
/// </summary>
|
||||
Intermittent,
|
||||
|
||||
/// <summary>
|
||||
/// Rate limiting - throttle requests.
|
||||
/// </summary>
|
||||
RateLimit,
|
||||
|
||||
/// <summary>
|
||||
/// Timeout - requests timeout after delay.
|
||||
/// </summary>
|
||||
Timeout,
|
||||
|
||||
/// <summary>
|
||||
/// Error response - return specific error codes.
|
||||
/// </summary>
|
||||
ErrorResponse,
|
||||
|
||||
/// <summary>
|
||||
/// Corrupt response - return malformed data.
|
||||
/// </summary>
|
||||
CorruptResponse
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for fault behavior.
|
||||
/// </summary>
|
||||
|
||||
@@ -124,6 +124,7 @@ public enum DeadLetterStatus
|
||||
/// </summary>
|
||||
public sealed record DeadLetterQuery
|
||||
{
|
||||
public string? Id { get; init; }
|
||||
public DeadLetterReason? Reason { get; init; }
|
||||
public string? ChannelType { get; init; }
|
||||
public DeadLetterStatus? Status { get; init; }
|
||||
@@ -260,6 +261,7 @@ public sealed class InMemoryDeadLetterHandler : IDeadLetterHandler
|
||||
|
||||
if (query is not null)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(query.Id)) filtered = filtered.Where(d => d.DeadLetterId == query.Id);
|
||||
if (query.Reason.HasValue) filtered = filtered.Where(d => d.Reason == query.Reason.Value);
|
||||
if (!string.IsNullOrEmpty(query.ChannelType)) filtered = filtered.Where(d => d.ChannelType == query.ChannelType);
|
||||
if (query.Status.HasValue) filtered = filtered.Where(d => d.Status == query.Status.Value);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Notifier.Worker.Retention;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Observability;
|
||||
|
||||
@@ -93,8 +94,7 @@ public static class ObservabilityServiceExtensions
|
||||
services.Configure<RetentionOptions>(
|
||||
configuration.GetSection(RetentionOptions.SectionName));
|
||||
|
||||
services.AddSingleton<IRetentionPolicyService, InMemoryRetentionPolicyService>();
|
||||
services.AddHostedService<RetentionPolicyRunner>();
|
||||
services.AddSingleton<IRetentionPolicyService, DefaultRetentionPolicyService>();
|
||||
|
||||
return services;
|
||||
}
|
||||
@@ -220,8 +220,7 @@ public sealed class ObservabilityServiceBuilder
|
||||
_services.TryAddSingleton<INotifierTracing, DefaultNotifierTracing>();
|
||||
_services.TryAddSingleton<IDeadLetterHandler, InMemoryDeadLetterHandler>();
|
||||
_services.TryAddSingleton<IChaosEngine, DefaultChaosEngine>();
|
||||
_services.TryAddSingleton<IRetentionPolicyService, InMemoryRetentionPolicyService>();
|
||||
_services.AddHostedService<RetentionPolicyRunner>();
|
||||
_services.TryAddSingleton<IRetentionPolicyService, DefaultRetentionPolicyService>();
|
||||
|
||||
return _services;
|
||||
}
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
using System.Text.Json.Nodes;
|
||||
using StellaOps.Notify.Models;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Processing;
|
||||
|
||||
/// <summary>
|
||||
/// Renders notification templates with event payload data.
|
||||
/// </summary>
|
||||
public interface INotifyTemplateRenderer
|
||||
{
|
||||
/// <summary>
|
||||
/// Renders a template body using the provided data context.
|
||||
/// </summary>
|
||||
/// <param name="template">The template containing the body pattern.</param>
|
||||
/// <param name="payload">The event payload data to interpolate.</param>
|
||||
/// <returns>The rendered string.</returns>
|
||||
string Render(NotifyTemplate template, JsonNode? payload);
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Processing;
|
||||
|
||||
internal sealed class MongoInitializationHostedService : IHostedService
|
||||
{
|
||||
private const string InitializerTypeName = "StellaOps.Notify.Storage.Mongo.Internal.NotifyMongoInitializer, StellaOps.Notify.Storage.Mongo";
|
||||
|
||||
private readonly IServiceProvider _serviceProvider;
|
||||
private readonly ILogger<MongoInitializationHostedService> _logger;
|
||||
|
||||
public MongoInitializationHostedService(IServiceProvider serviceProvider, ILogger<MongoInitializationHostedService> logger)
|
||||
{
|
||||
_serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var initializerType = Type.GetType(InitializerTypeName, throwOnError: false, ignoreCase: false);
|
||||
if (initializerType is null)
|
||||
{
|
||||
_logger.LogWarning("Notify Mongo initializer type {TypeName} was not found; skipping migration run.", InitializerTypeName);
|
||||
return;
|
||||
}
|
||||
|
||||
using var scope = _serviceProvider.CreateScope();
|
||||
var initializer = scope.ServiceProvider.GetService(initializerType);
|
||||
if (initializer is null)
|
||||
{
|
||||
_logger.LogWarning("Notify Mongo initializer could not be resolved from the service provider.");
|
||||
return;
|
||||
}
|
||||
|
||||
var method = initializerType.GetMethod("EnsureIndexesAsync");
|
||||
if (method is null)
|
||||
{
|
||||
_logger.LogWarning("Notify Mongo initializer does not expose EnsureIndexesAsync; skipping migration run.");
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var task = method.Invoke(initializer, new object?[] { cancellationToken }) as Task;
|
||||
if (task is not null)
|
||||
{
|
||||
await task.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to run Notify Mongo migrations.");
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
}
|
||||
@@ -1,11 +1,12 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Nodes;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notify.Storage.Mongo.Repositories;
|
||||
using StellaOps.Notifier.Worker.Storage;
|
||||
using StellaOps.Notifier.Worker.Channels;
|
||||
using StellaOps.Notifier.Worker.Dispatch;
|
||||
using StellaOps.Notifier.Worker.Options;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Processing;
|
||||
@@ -50,8 +51,8 @@ public sealed class NotifierDispatchWorker : BackgroundService
|
||||
{
|
||||
_logger.LogInformation("Notifier dispatch worker {WorkerId} started.", _workerId);
|
||||
|
||||
var pollInterval = _options.DispatchPollInterval > TimeSpan.Zero
|
||||
? _options.DispatchPollInterval
|
||||
var pollInterval = _options.DispatchInterval > TimeSpan.Zero
|
||||
? _options.DispatchInterval
|
||||
: TimeSpan.FromSeconds(5);
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
@@ -149,29 +150,21 @@ public sealed class NotifierDispatchWorker : BackgroundService
|
||||
NotifyDeliveryRendered rendered;
|
||||
if (template is not null)
|
||||
{
|
||||
// Create a payload from the delivery kind and metadata
|
||||
var payload = BuildPayloadFromDelivery(delivery);
|
||||
var renderedBody = _templateRenderer.Render(template, payload);
|
||||
var notifyEvent = BuildEventFromDelivery(delivery);
|
||||
var renderedContent = await _templateRenderer
|
||||
.RenderAsync(template, notifyEvent, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var subject = template.Metadata.TryGetValue("subject", out var subj)
|
||||
? _templateRenderer.Render(
|
||||
NotifyTemplate.Create(
|
||||
templateId: "subject-inline",
|
||||
tenantId: tenantId,
|
||||
channelType: template.ChannelType,
|
||||
key: "subject",
|
||||
locale: locale,
|
||||
body: subj),
|
||||
payload)
|
||||
: $"Notification: {delivery.Kind}";
|
||||
var subject = renderedContent.Subject ?? $"Notification: {delivery.Kind}";
|
||||
|
||||
rendered = NotifyDeliveryRendered.Create(
|
||||
channelType: channel.Type,
|
||||
format: template.Format,
|
||||
format: renderedContent.Format,
|
||||
target: channel.Config?.Target ?? string.Empty,
|
||||
title: subject,
|
||||
body: renderedBody,
|
||||
locale: locale);
|
||||
body: renderedContent.Body,
|
||||
locale: locale,
|
||||
bodyHash: renderedContent.BodyHash);
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -199,12 +192,16 @@ public sealed class NotifierDispatchWorker : BackgroundService
|
||||
var attempt = new NotifyDeliveryAttempt(
|
||||
timestamp: _timeProvider.GetUtcNow(),
|
||||
status: dispatchResult.Success ? NotifyDeliveryAttemptStatus.Succeeded : NotifyDeliveryAttemptStatus.Failed,
|
||||
statusCode: dispatchResult.StatusCode,
|
||||
reason: dispatchResult.Reason);
|
||||
statusCode: dispatchResult.HttpStatusCode,
|
||||
reason: dispatchResult.Message);
|
||||
|
||||
var shouldRetry = !dispatchResult.Success && (dispatchResult.Status == ChannelDispatchStatus.Throttled
|
||||
|| dispatchResult.Status == ChannelDispatchStatus.Timeout
|
||||
|| dispatchResult.Status == ChannelDispatchStatus.NetworkError);
|
||||
|
||||
var newStatus = dispatchResult.Success
|
||||
? NotifyDeliveryStatus.Sent
|
||||
: (dispatchResult.ShouldRetry ? NotifyDeliveryStatus.Pending : NotifyDeliveryStatus.Failed);
|
||||
? NotifyDeliveryStatus.Delivered
|
||||
: (shouldRetry ? NotifyDeliveryStatus.Pending : NotifyDeliveryStatus.Failed);
|
||||
|
||||
var updatedDelivery = NotifyDelivery.Create(
|
||||
deliveryId: delivery.DeliveryId,
|
||||
@@ -214,13 +211,13 @@ public sealed class NotifierDispatchWorker : BackgroundService
|
||||
eventId: delivery.EventId,
|
||||
kind: delivery.Kind,
|
||||
status: newStatus,
|
||||
statusReason: dispatchResult.Reason,
|
||||
statusReason: dispatchResult.Message,
|
||||
rendered: rendered,
|
||||
attempts: delivery.Attempts.Add(attempt),
|
||||
metadata: delivery.Metadata,
|
||||
createdAt: delivery.CreatedAt,
|
||||
sentAt: dispatchResult.Success ? _timeProvider.GetUtcNow() : delivery.SentAt,
|
||||
completedAt: newStatus == NotifyDeliveryStatus.Sent || newStatus == NotifyDeliveryStatus.Failed
|
||||
completedAt: newStatus == NotifyDeliveryStatus.Delivered || newStatus == NotifyDeliveryStatus.Failed
|
||||
? _timeProvider.GetUtcNow()
|
||||
: null);
|
||||
|
||||
@@ -257,7 +254,7 @@ public sealed class NotifierDispatchWorker : BackgroundService
|
||||
_logger.LogWarning("Delivery {DeliveryId} marked failed: {Reason}", delivery.DeliveryId, reason);
|
||||
}
|
||||
|
||||
private static JsonObject BuildPayloadFromDelivery(NotifyDelivery delivery)
|
||||
private static NotifyEvent BuildEventFromDelivery(NotifyDelivery delivery)
|
||||
{
|
||||
var payload = new JsonObject
|
||||
{
|
||||
@@ -272,7 +269,18 @@ public sealed class NotifierDispatchWorker : BackgroundService
|
||||
payload[key] = value;
|
||||
}
|
||||
|
||||
return payload;
|
||||
delivery.Metadata.TryGetValue("version", out var version);
|
||||
delivery.Metadata.TryGetValue("actor", out var actor);
|
||||
|
||||
return NotifyEvent.Create(
|
||||
eventId: delivery.EventId,
|
||||
kind: delivery.Kind,
|
||||
tenant: delivery.TenantId,
|
||||
ts: delivery.CreatedAt,
|
||||
payload: payload,
|
||||
version: version,
|
||||
actor: actor,
|
||||
attributes: delivery.Metadata);
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<NotifyChannelType, INotifyChannelAdapter> BuildAdapterMap(
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Generic;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.AirGap.Policy;
|
||||
using StellaOps.Notify.Engine;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notify.Storage.Mongo.Repositories;
|
||||
using StellaOps.Notifier.Worker.Storage;
|
||||
using StellaOps.Notifier.Worker.Options;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Processing;
|
||||
@@ -331,3 +331,4 @@ internal sealed class NotifierEventProcessor
|
||||
return metadata;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,100 +0,0 @@
|
||||
using System.Text.Json.Nodes;
|
||||
using System.Text.RegularExpressions;
|
||||
using StellaOps.Notify.Models;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Processing;
|
||||
|
||||
/// <summary>
|
||||
/// Simple Handlebars-like template renderer supporting {{property}} and {{#each}} blocks.
|
||||
/// </summary>
|
||||
public sealed partial class SimpleTemplateRenderer : INotifyTemplateRenderer
|
||||
{
|
||||
private static readonly Regex PlaceholderPattern = PlaceholderRegex();
|
||||
private static readonly Regex EachBlockPattern = EachBlockRegex();
|
||||
|
||||
public string Render(NotifyTemplate template, JsonNode? payload)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(template);
|
||||
|
||||
var body = template.Body;
|
||||
if (string.IsNullOrWhiteSpace(body))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
// Process {{#each}} blocks first
|
||||
body = ProcessEachBlocks(body, payload);
|
||||
|
||||
// Then substitute simple placeholders
|
||||
body = SubstitutePlaceholders(body, payload);
|
||||
|
||||
return body;
|
||||
}
|
||||
|
||||
private static string ProcessEachBlocks(string body, JsonNode? payload)
|
||||
{
|
||||
return EachBlockPattern.Replace(body, match =>
|
||||
{
|
||||
var collectionPath = match.Groups[1].Value.Trim();
|
||||
var innerTemplate = match.Groups[2].Value;
|
||||
|
||||
var collection = ResolvePath(payload, collectionPath);
|
||||
if (collection is not JsonObject obj)
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var results = new List<string>();
|
||||
foreach (var (key, value) in obj)
|
||||
{
|
||||
var itemResult = innerTemplate
|
||||
.Replace("{{@key}}", key)
|
||||
.Replace("{{this}}", value?.ToString() ?? string.Empty);
|
||||
results.Add(itemResult);
|
||||
}
|
||||
|
||||
return string.Join(string.Empty, results);
|
||||
});
|
||||
}
|
||||
|
||||
private static string SubstitutePlaceholders(string body, JsonNode? payload)
|
||||
{
|
||||
return PlaceholderPattern.Replace(body, match =>
|
||||
{
|
||||
var path = match.Groups[1].Value.Trim();
|
||||
var resolved = ResolvePath(payload, path);
|
||||
return resolved?.ToString() ?? string.Empty;
|
||||
});
|
||||
}
|
||||
|
||||
private static JsonNode? ResolvePath(JsonNode? root, string path)
|
||||
{
|
||||
if (root is null || string.IsNullOrWhiteSpace(path))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var segments = path.Split('.');
|
||||
var current = root;
|
||||
|
||||
foreach (var segment in segments)
|
||||
{
|
||||
if (current is JsonObject obj && obj.TryGetPropertyValue(segment, out var next))
|
||||
{
|
||||
current = next;
|
||||
}
|
||||
else
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
return current;
|
||||
}
|
||||
|
||||
[GeneratedRegex(@"\{\{([^#/}]+)\}\}", RegexOptions.Compiled)]
|
||||
private static partial Regex PlaceholderRegex();
|
||||
|
||||
[GeneratedRegex(@"\{\{#each\s+([^}]+)\}\}(.*?)\{\{/each\}\}", RegexOptions.Compiled | RegexOptions.Singleline)]
|
||||
private static partial Regex EachBlockRegex();
|
||||
}
|
||||
@@ -1,11 +1,13 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.AirGap.Policy;
|
||||
using StellaOps.Notifier.Worker.Channels;
|
||||
using StellaOps.Notify.Engine;
|
||||
using StellaOps.Notify.Queue;
|
||||
using StellaOps.Notify.Storage.Mongo;
|
||||
using StellaOps.Notify.Storage.Postgres;
|
||||
using StellaOps.Notifier.Worker.Storage;
|
||||
using StellaOps.Notifier.Worker.Dispatch;
|
||||
using StellaOps.Notifier.Worker.Options;
|
||||
using StellaOps.Notifier.Worker.Processing;
|
||||
@@ -27,17 +29,25 @@ builder.Logging.AddSimpleConsole(options =>
|
||||
builder.Services.Configure<NotifierWorkerOptions>(builder.Configuration.GetSection("notifier:worker"));
|
||||
builder.Services.AddSingleton(TimeProvider.System);
|
||||
|
||||
var mongoSection = builder.Configuration.GetSection("notifier:storage:mongo");
|
||||
builder.Services.AddNotifyMongoStorage(mongoSection);
|
||||
var postgresSection = builder.Configuration.GetSection("notifier:storage:postgres");
|
||||
builder.Services.AddNotifyPostgresStorage(builder.Configuration, postgresSection.Path);
|
||||
|
||||
builder.Services.AddAirGapEgressPolicy(builder.Configuration);
|
||||
|
||||
builder.Services.AddNotifyEventQueue(builder.Configuration, "notifier:queue");
|
||||
builder.Services.AddHealthChecks().AddNotifyQueueHealthCheck();
|
||||
|
||||
// In-memory storage replacements (document store removed)
|
||||
builder.Services.AddSingleton<INotifyChannelRepository, InMemoryNotifyRepositories>();
|
||||
builder.Services.AddSingleton<INotifyRuleRepository, InMemoryNotifyRepositories>();
|
||||
builder.Services.AddSingleton<INotifyTemplateRepository, InMemoryNotifyRepositories>();
|
||||
builder.Services.AddSingleton<INotifyDeliveryRepository, InMemoryNotifyRepositories>();
|
||||
builder.Services.AddSingleton<INotifyAuditRepository, InMemoryNotifyRepositories>();
|
||||
builder.Services.AddSingleton<INotifyLockRepository, InMemoryNotifyRepositories>();
|
||||
builder.Services.AddSingleton<IInAppInboxStore, InMemoryInboxStore>();
|
||||
|
||||
builder.Services.AddSingleton<INotifyRuleEvaluator, DefaultNotifyRuleEvaluator>();
|
||||
builder.Services.AddSingleton<NotifierEventProcessor>();
|
||||
builder.Services.AddHostedService<MongoInitializationHostedService>();
|
||||
builder.Services.AddHostedService<NotifierEventWorker>();
|
||||
|
||||
// Template service (versioning, localization, redaction)
|
||||
|
||||
@@ -2,6 +2,7 @@ using System.Collections.Concurrent;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Notifier.Worker.DeadLetter;
|
||||
using StellaOps.Notifier.Worker.Observability;
|
||||
using DeadLetterStats = StellaOps.Notifier.Worker.DeadLetter.DeadLetterStats;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Retention;
|
||||
|
||||
|
||||
@@ -53,6 +53,11 @@ public interface IRetentionPolicyService
|
||||
/// </summary>
|
||||
public sealed record RetentionPolicy
|
||||
{
|
||||
/// <summary>
|
||||
/// Identifier for the policy (defaults to tenant-specific "default").
|
||||
/// </summary>
|
||||
public string Id { get; init; } = "default";
|
||||
|
||||
/// <summary>
|
||||
/// Retention period for delivery records.
|
||||
/// </summary>
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
namespace StellaOps.Notifier.Worker.Retention;
|
||||
|
||||
/// <summary>
|
||||
/// Options for retention policy configuration.
|
||||
/// </summary>
|
||||
public sealed class RetentionOptions
|
||||
{
|
||||
public const string SectionName = "Notifier:Observability:Retention";
|
||||
|
||||
/// <summary>
|
||||
/// Default policy values applied when no tenant-specific policy is set.
|
||||
/// </summary>
|
||||
public RetentionPolicy DefaultPolicy { get; set; } = RetentionPolicy.Default;
|
||||
}
|
||||
@@ -1,509 +0,0 @@
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Security;
|
||||
|
||||
/// <summary>
|
||||
/// Default HTML sanitizer implementation using regex-based filtering.
|
||||
/// For production, consider using a dedicated library like HtmlSanitizer or AngleSharp.
|
||||
/// </summary>
|
||||
public sealed partial class DefaultHtmlSanitizer : IHtmlSanitizer
|
||||
{
|
||||
private readonly ILogger<DefaultHtmlSanitizer> _logger;
|
||||
|
||||
// Safe elements (whitelist approach)
|
||||
private static readonly HashSet<string> SafeElements = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"p", "div", "span", "br", "hr",
|
||||
"h1", "h2", "h3", "h4", "h5", "h6",
|
||||
"strong", "b", "em", "i", "u", "s", "strike",
|
||||
"ul", "ol", "li", "dl", "dt", "dd",
|
||||
"table", "thead", "tbody", "tfoot", "tr", "th", "td",
|
||||
"a", "img",
|
||||
"blockquote", "pre", "code",
|
||||
"sub", "sup", "small", "mark",
|
||||
"caption", "figure", "figcaption"
|
||||
};
|
||||
|
||||
// Safe attributes
|
||||
private static readonly HashSet<string> SafeAttributes = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"href", "src", "alt", "title", "class", "id",
|
||||
"width", "height", "style",
|
||||
"colspan", "rowspan", "scope",
|
||||
"target", "rel"
|
||||
};
|
||||
|
||||
// Dangerous URL schemes
|
||||
private static readonly HashSet<string> DangerousSchemes = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"javascript", "vbscript", "data", "file"
|
||||
};
|
||||
|
||||
// Event handler attributes (all start with "on")
|
||||
private static readonly Regex EventHandlerRegex = EventHandlerPattern();
|
||||
|
||||
// Style-based attacks
|
||||
private static readonly Regex DangerousStyleRegex = DangerousStylePattern();
|
||||
|
||||
public DefaultHtmlSanitizer(ILogger<DefaultHtmlSanitizer> logger)
|
||||
{
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public string Sanitize(string html, HtmlSanitizeOptions? options = null)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(html))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
options ??= new HtmlSanitizeOptions();
|
||||
|
||||
if (html.Length > options.MaxContentLength)
|
||||
{
|
||||
_logger.LogWarning("HTML content exceeds max length {MaxLength}, truncating", options.MaxContentLength);
|
||||
html = html[..options.MaxContentLength];
|
||||
}
|
||||
|
||||
var allowedTags = new HashSet<string>(SafeElements, StringComparer.OrdinalIgnoreCase);
|
||||
if (options.AdditionalAllowedTags is not null)
|
||||
{
|
||||
foreach (var tag in options.AdditionalAllowedTags)
|
||||
{
|
||||
allowedTags.Add(tag);
|
||||
}
|
||||
}
|
||||
|
||||
var allowedAttrs = new HashSet<string>(SafeAttributes, StringComparer.OrdinalIgnoreCase);
|
||||
if (options.AdditionalAllowedAttributes is not null)
|
||||
{
|
||||
foreach (var attr in options.AdditionalAllowedAttributes)
|
||||
{
|
||||
allowedAttrs.Add(attr);
|
||||
}
|
||||
}
|
||||
|
||||
// Process HTML
|
||||
var result = new StringBuilder();
|
||||
var depth = 0;
|
||||
var pos = 0;
|
||||
|
||||
while (pos < html.Length)
|
||||
{
|
||||
var tagStart = html.IndexOf('<', pos);
|
||||
if (tagStart < 0)
|
||||
{
|
||||
// No more tags, append rest
|
||||
result.Append(EncodeText(html[pos..]));
|
||||
break;
|
||||
}
|
||||
|
||||
// Append text before tag
|
||||
if (tagStart > pos)
|
||||
{
|
||||
result.Append(EncodeText(html[pos..tagStart]));
|
||||
}
|
||||
|
||||
var tagEnd = html.IndexOf('>', tagStart);
|
||||
if (tagEnd < 0)
|
||||
{
|
||||
// Malformed, skip rest
|
||||
break;
|
||||
}
|
||||
|
||||
var tagContent = html[(tagStart + 1)..tagEnd];
|
||||
var isClosing = tagContent.StartsWith('/');
|
||||
var tagName = ExtractTagName(tagContent);
|
||||
|
||||
if (isClosing)
|
||||
{
|
||||
depth--;
|
||||
}
|
||||
|
||||
if (allowedTags.Contains(tagName))
|
||||
{
|
||||
if (isClosing)
|
||||
{
|
||||
result.Append($"</{tagName}>");
|
||||
}
|
||||
else
|
||||
{
|
||||
// Process attributes
|
||||
var sanitizedTag = SanitizeTag(tagContent, tagName, allowedAttrs, options);
|
||||
result.Append($"<{sanitizedTag}>");
|
||||
|
||||
if (!IsSelfClosing(tagName) && !tagContent.EndsWith('/'))
|
||||
{
|
||||
depth++;
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogDebug("Stripped disallowed tag: {TagName}", tagName);
|
||||
}
|
||||
|
||||
if (depth > options.MaxNestingDepth)
|
||||
{
|
||||
_logger.LogWarning("HTML nesting depth exceeds max {MaxDepth}, truncating", options.MaxNestingDepth);
|
||||
break;
|
||||
}
|
||||
|
||||
pos = tagEnd + 1;
|
||||
}
|
||||
|
||||
return result.ToString();
|
||||
}
|
||||
|
||||
public HtmlValidationResult Validate(string html)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(html))
|
||||
{
|
||||
return HtmlValidationResult.Safe(new HtmlContentStats());
|
||||
}
|
||||
|
||||
var issues = new List<HtmlSecurityIssue>();
|
||||
var stats = new HtmlContentStats
|
||||
{
|
||||
CharacterCount = html.Length
|
||||
};
|
||||
|
||||
var pos = 0;
|
||||
var depth = 0;
|
||||
var maxDepth = 0;
|
||||
var elementCount = 0;
|
||||
var linkCount = 0;
|
||||
var imageCount = 0;
|
||||
|
||||
// Check for script tags
|
||||
if (ScriptTagRegex().IsMatch(html))
|
||||
{
|
||||
issues.Add(new HtmlSecurityIssue
|
||||
{
|
||||
Type = HtmlSecurityIssueType.ScriptInjection,
|
||||
Description = "Script tags are not allowed"
|
||||
});
|
||||
}
|
||||
|
||||
// Check for event handlers
|
||||
var eventMatches = EventHandlerRegex.Matches(html);
|
||||
foreach (Match match in eventMatches)
|
||||
{
|
||||
issues.Add(new HtmlSecurityIssue
|
||||
{
|
||||
Type = HtmlSecurityIssueType.EventHandler,
|
||||
Description = "Event handler attributes are not allowed",
|
||||
AttributeName = match.Value,
|
||||
Position = match.Index
|
||||
});
|
||||
}
|
||||
|
||||
// Check for dangerous URLs
|
||||
var hrefMatches = DangerousUrlRegex().Matches(html);
|
||||
foreach (Match match in hrefMatches)
|
||||
{
|
||||
issues.Add(new HtmlSecurityIssue
|
||||
{
|
||||
Type = HtmlSecurityIssueType.DangerousUrl,
|
||||
Description = "Dangerous URL scheme detected",
|
||||
Position = match.Index
|
||||
});
|
||||
}
|
||||
|
||||
// Check for dangerous style content
|
||||
var styleMatches = DangerousStyleRegex.Matches(html);
|
||||
foreach (Match match in styleMatches)
|
||||
{
|
||||
issues.Add(new HtmlSecurityIssue
|
||||
{
|
||||
Type = HtmlSecurityIssueType.StyleInjection,
|
||||
Description = "Dangerous style content detected",
|
||||
Position = match.Index
|
||||
});
|
||||
}
|
||||
|
||||
// Check for dangerous elements
|
||||
var dangerousElements = new[] { "iframe", "object", "embed", "form", "input", "button", "meta", "link", "base" };
|
||||
foreach (var element in dangerousElements)
|
||||
{
|
||||
var elementRegex = new Regex($@"<{element}\b", RegexOptions.IgnoreCase);
|
||||
if (elementRegex.IsMatch(html))
|
||||
{
|
||||
issues.Add(new HtmlSecurityIssue
|
||||
{
|
||||
Type = HtmlSecurityIssueType.DangerousElement,
|
||||
Description = $"Dangerous element '{element}' is not allowed",
|
||||
ElementName = element
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Count elements and check nesting
|
||||
while (pos < html.Length)
|
||||
{
|
||||
var tagStart = html.IndexOf('<', pos);
|
||||
if (tagStart < 0) break;
|
||||
|
||||
var tagEnd = html.IndexOf('>', tagStart);
|
||||
if (tagEnd < 0) break;
|
||||
|
||||
var tagContent = html[(tagStart + 1)..tagEnd];
|
||||
var isClosing = tagContent.StartsWith('/');
|
||||
var tagName = ExtractTagName(tagContent);
|
||||
|
||||
if (!isClosing && !string.IsNullOrEmpty(tagName) && !tagContent.EndsWith('/'))
|
||||
{
|
||||
if (!IsSelfClosing(tagName))
|
||||
{
|
||||
depth++;
|
||||
maxDepth = Math.Max(maxDepth, depth);
|
||||
}
|
||||
elementCount++;
|
||||
|
||||
if (tagName.Equals("a", StringComparison.OrdinalIgnoreCase)) linkCount++;
|
||||
if (tagName.Equals("img", StringComparison.OrdinalIgnoreCase)) imageCount++;
|
||||
}
|
||||
else if (isClosing)
|
||||
{
|
||||
depth--;
|
||||
}
|
||||
|
||||
pos = tagEnd + 1;
|
||||
}
|
||||
|
||||
stats = stats with
|
||||
{
|
||||
ElementCount = elementCount,
|
||||
MaxDepth = maxDepth,
|
||||
LinkCount = linkCount,
|
||||
ImageCount = imageCount
|
||||
};
|
||||
|
||||
return issues.Count == 0
|
||||
? HtmlValidationResult.Safe(stats)
|
||||
: HtmlValidationResult.Unsafe(issues, stats);
|
||||
}
|
||||
|
||||
public string StripHtml(string html)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(html))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
// Remove all tags
|
||||
var text = HtmlTagRegex().Replace(html, " ");
|
||||
|
||||
// Decode entities
|
||||
text = System.Net.WebUtility.HtmlDecode(text);
|
||||
|
||||
// Normalize whitespace
|
||||
text = WhitespaceRegex().Replace(text, " ").Trim();
|
||||
|
||||
return text;
|
||||
}
|
||||
|
||||
private static string SanitizeTag(
|
||||
string tagContent,
|
||||
string tagName,
|
||||
HashSet<string> allowedAttrs,
|
||||
HtmlSanitizeOptions options)
|
||||
{
|
||||
var result = new StringBuilder(tagName);
|
||||
|
||||
// Extract and sanitize attributes
|
||||
var attrMatches = AttributeRegex().Matches(tagContent);
|
||||
foreach (Match match in attrMatches)
|
||||
{
|
||||
var attrName = match.Groups[1].Value;
|
||||
var attrValue = match.Groups[2].Value;
|
||||
|
||||
if (!allowedAttrs.Contains(attrName))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip event handlers
|
||||
if (EventHandlerRegex.IsMatch(attrName))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
// Sanitize href/src values
|
||||
if (attrName.Equals("href", StringComparison.OrdinalIgnoreCase) ||
|
||||
attrName.Equals("src", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
attrValue = SanitizeUrl(attrValue, options);
|
||||
if (string.IsNullOrEmpty(attrValue))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Sanitize style values
|
||||
if (attrName.Equals("style", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
attrValue = SanitizeStyle(attrValue);
|
||||
if (string.IsNullOrEmpty(attrValue))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
result.Append($" {attrName}=\"{EncodeAttributeValue(attrValue)}\"");
|
||||
}
|
||||
|
||||
// Add rel="noopener noreferrer" to links with target
|
||||
if (tagName.Equals("a", StringComparison.OrdinalIgnoreCase) &&
|
||||
tagContent.Contains("target=", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (!tagContent.Contains("rel=", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
result.Append(" rel=\"noopener noreferrer\"");
|
||||
}
|
||||
}
|
||||
|
||||
if (tagContent.TrimEnd().EndsWith('/'))
|
||||
{
|
||||
result.Append(" /");
|
||||
}
|
||||
|
||||
return result.ToString();
|
||||
}
|
||||
|
||||
private static string SanitizeUrl(string url, HtmlSanitizeOptions options)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(url))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
url = url.Trim();
|
||||
|
||||
// Check for dangerous schemes
|
||||
var colonIndex = url.IndexOf(':');
|
||||
if (colonIndex > 0 && colonIndex < 10)
|
||||
{
|
||||
var scheme = url[..colonIndex].ToLowerInvariant();
|
||||
if (DangerousSchemes.Contains(scheme))
|
||||
{
|
||||
if (scheme == "data" && options.AllowDataUrls)
|
||||
{
|
||||
// Allow data URLs if explicitly enabled
|
||||
return url;
|
||||
}
|
||||
return string.Empty;
|
||||
}
|
||||
}
|
||||
|
||||
// Allow relative URLs and safe absolute URLs
|
||||
if (url.StartsWith("http://", StringComparison.OrdinalIgnoreCase) ||
|
||||
url.StartsWith("https://", StringComparison.OrdinalIgnoreCase) ||
|
||||
url.StartsWith("mailto:", StringComparison.OrdinalIgnoreCase) ||
|
||||
url.StartsWith("tel:", StringComparison.OrdinalIgnoreCase) ||
|
||||
url.StartsWith('/') ||
|
||||
url.StartsWith('#') ||
|
||||
!url.Contains(':'))
|
||||
{
|
||||
return url;
|
||||
}
|
||||
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
private static string SanitizeStyle(string style)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(style))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
// Remove dangerous CSS
|
||||
if (DangerousStyleRegex.IsMatch(style))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
// Only allow simple property:value pairs
|
||||
var safeProperties = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"color", "background-color", "font-size", "font-weight", "font-style",
|
||||
"text-align", "text-decoration", "margin", "padding", "border",
|
||||
"width", "height", "max-width", "max-height", "display"
|
||||
};
|
||||
|
||||
var result = new StringBuilder();
|
||||
var pairs = style.Split(';', StringSplitOptions.RemoveEmptyEntries);
|
||||
|
||||
foreach (var pair in pairs)
|
||||
{
|
||||
var colonIndex = pair.IndexOf(':');
|
||||
if (colonIndex <= 0) continue;
|
||||
|
||||
var property = pair[..colonIndex].Trim().ToLowerInvariant();
|
||||
var value = pair[(colonIndex + 1)..].Trim();
|
||||
|
||||
if (safeProperties.Contains(property) && !value.Contains("url(", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (result.Length > 0) result.Append("; ");
|
||||
result.Append($"{property}: {value}");
|
||||
}
|
||||
}
|
||||
|
||||
return result.ToString();
|
||||
}
|
||||
|
||||
private static string ExtractTagName(string tagContent)
|
||||
{
|
||||
var content = tagContent.TrimStart('/').Trim();
|
||||
var spaceIndex = content.IndexOfAny([' ', '\t', '\n', '\r', '/']);
|
||||
return spaceIndex > 0 ? content[..spaceIndex] : content;
|
||||
}
|
||||
|
||||
private static bool IsSelfClosing(string tagName)
|
||||
{
|
||||
return tagName.Equals("br", StringComparison.OrdinalIgnoreCase) ||
|
||||
tagName.Equals("hr", StringComparison.OrdinalIgnoreCase) ||
|
||||
tagName.Equals("img", StringComparison.OrdinalIgnoreCase) ||
|
||||
tagName.Equals("input", StringComparison.OrdinalIgnoreCase) ||
|
||||
tagName.Equals("meta", StringComparison.OrdinalIgnoreCase) ||
|
||||
tagName.Equals("link", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static string EncodeText(string text)
|
||||
{
|
||||
return System.Net.WebUtility.HtmlEncode(text);
|
||||
}
|
||||
|
||||
private static string EncodeAttributeValue(string value)
|
||||
{
|
||||
return value
|
||||
.Replace("&", "&")
|
||||
.Replace("\"", """)
|
||||
.Replace("<", "<")
|
||||
.Replace(">", ">");
|
||||
}
|
||||
|
||||
[GeneratedRegex(@"\bon\w+\s*=", RegexOptions.IgnoreCase)]
|
||||
private static partial Regex EventHandlerPattern();
|
||||
|
||||
[GeneratedRegex(@"expression\s*\(|behavior\s*:|@import|@charset|binding\s*:", RegexOptions.IgnoreCase)]
|
||||
private static partial Regex DangerousStylePattern();
|
||||
|
||||
[GeneratedRegex(@"<script\b", RegexOptions.IgnoreCase)]
|
||||
private static partial Regex ScriptTagRegex();
|
||||
|
||||
[GeneratedRegex(@"(javascript|vbscript|data)\s*:", RegexOptions.IgnoreCase)]
|
||||
private static partial Regex DangerousUrlRegex();
|
||||
|
||||
[GeneratedRegex(@"<[^>]*>")]
|
||||
private static partial Regex HtmlTagRegex();
|
||||
|
||||
[GeneratedRegex(@"\s+")]
|
||||
private static partial Regex WhitespaceRegex();
|
||||
|
||||
[GeneratedRegex(@"(\w+)\s*=\s*""([^""]*)""", RegexOptions.Compiled)]
|
||||
private static partial Regex AttributeRegex();
|
||||
}
|
||||
@@ -1,221 +0,0 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Text.RegularExpressions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Security;
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation of tenant isolation validation.
|
||||
/// </summary>
|
||||
public sealed partial class DefaultTenantIsolationValidator : ITenantIsolationValidator
|
||||
{
|
||||
private readonly TenantIsolationOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<DefaultTenantIsolationValidator> _logger;
|
||||
private readonly ConcurrentQueue<TenantIsolationViolation> _violations = new();
|
||||
|
||||
// Valid tenant ID pattern: alphanumeric, hyphens, underscores, 3-64 chars
|
||||
private static readonly Regex TenantIdPattern = TenantIdRegex();
|
||||
|
||||
public DefaultTenantIsolationValidator(
|
||||
IOptions<TenantIsolationOptions> options,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<DefaultTenantIsolationValidator> logger)
|
||||
{
|
||||
_options = options?.Value ?? new TenantIsolationOptions();
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public TenantIsolationResult ValidateAccess(
|
||||
string requestTenantId,
|
||||
string resourceTenantId,
|
||||
string resourceType,
|
||||
string resourceId)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(requestTenantId);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(resourceTenantId);
|
||||
|
||||
// Normalize tenant IDs
|
||||
var normalizedRequest = NormalizeTenantId(requestTenantId);
|
||||
var normalizedResource = NormalizeTenantId(resourceTenantId);
|
||||
|
||||
// Check for exact match
|
||||
if (string.Equals(normalizedRequest, normalizedResource, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return TenantIsolationResult.Allow(requestTenantId, resourceTenantId);
|
||||
}
|
||||
|
||||
// Check for cross-tenant access exceptions (admin tenants, shared resources)
|
||||
if (_options.AllowCrossTenantAccess &&
|
||||
_options.CrossTenantAllowedPairs.Contains($"{normalizedRequest}:{normalizedResource}"))
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Cross-tenant access allowed: {RequestTenant} -> {ResourceTenant} for {ResourceType}",
|
||||
requestTenantId, resourceTenantId, resourceType);
|
||||
return TenantIsolationResult.Allow(requestTenantId, resourceTenantId);
|
||||
}
|
||||
|
||||
// Check if request tenant is an admin tenant
|
||||
if (_options.AdminTenants.Contains(normalizedRequest))
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Admin tenant {AdminTenant} accessing resource from {ResourceTenant}",
|
||||
requestTenantId, resourceTenantId);
|
||||
return TenantIsolationResult.Allow(requestTenantId, resourceTenantId);
|
||||
}
|
||||
|
||||
// Violation detected
|
||||
var violation = new TenantIsolationViolation
|
||||
{
|
||||
OccurredAt = _timeProvider.GetUtcNow(),
|
||||
RequestTenantId = requestTenantId,
|
||||
ResourceTenantId = resourceTenantId,
|
||||
ResourceType = resourceType,
|
||||
ResourceId = resourceId,
|
||||
Operation = "access"
|
||||
};
|
||||
|
||||
RecordViolation(violation);
|
||||
|
||||
_logger.LogWarning(
|
||||
"Tenant isolation violation: {RequestTenant} attempted to access {ResourceType}/{ResourceId} belonging to {ResourceTenant}",
|
||||
requestTenantId, resourceType, resourceId, resourceTenantId);
|
||||
|
||||
return TenantIsolationResult.Deny(
|
||||
requestTenantId,
|
||||
resourceTenantId,
|
||||
"Cross-tenant access denied",
|
||||
resourceType,
|
||||
resourceId);
|
||||
}
|
||||
|
||||
public IReadOnlyList<TenantIsolationResult> ValidateBatch(
|
||||
string requestTenantId,
|
||||
IEnumerable<TenantResource> resources)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(requestTenantId);
|
||||
ArgumentNullException.ThrowIfNull(resources);
|
||||
|
||||
return resources
|
||||
.Select(r => ValidateAccess(requestTenantId, r.TenantId, r.ResourceType, r.ResourceId))
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
public string? SanitizeTenantId(string? tenantId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tenantId))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var sanitized = tenantId.Trim();
|
||||
|
||||
// Remove any control characters
|
||||
sanitized = ControlCharsRegex().Replace(sanitized, "");
|
||||
|
||||
// Check format
|
||||
if (!TenantIdPattern.IsMatch(sanitized))
|
||||
{
|
||||
_logger.LogWarning("Invalid tenant ID format: {TenantId}", tenantId);
|
||||
return null;
|
||||
}
|
||||
|
||||
return sanitized;
|
||||
}
|
||||
|
||||
public bool IsValidTenantIdFormat(string? tenantId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tenantId))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return TenantIdPattern.IsMatch(tenantId.Trim());
|
||||
}
|
||||
|
||||
public void RecordViolation(TenantIsolationViolation violation)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(violation);
|
||||
|
||||
_violations.Enqueue(violation);
|
||||
|
||||
// Keep only recent violations
|
||||
while (_violations.Count > _options.MaxStoredViolations)
|
||||
{
|
||||
_violations.TryDequeue(out _);
|
||||
}
|
||||
|
||||
// Emit metrics
|
||||
TenantIsolationMetrics.RecordViolation(
|
||||
violation.RequestTenantId,
|
||||
violation.ResourceTenantId,
|
||||
violation.ResourceType);
|
||||
}
|
||||
|
||||
public IReadOnlyList<TenantIsolationViolation> GetRecentViolations(int limit = 100)
|
||||
{
|
||||
return _violations.TakeLast(Math.Min(limit, _options.MaxStoredViolations)).ToArray();
|
||||
}
|
||||
|
||||
private static string NormalizeTenantId(string tenantId)
|
||||
{
|
||||
return tenantId.Trim().ToLowerInvariant();
|
||||
}
|
||||
|
||||
[GeneratedRegex(@"^[a-zA-Z0-9][a-zA-Z0-9_-]{2,63}$")]
|
||||
private static partial Regex TenantIdRegex();
|
||||
|
||||
[GeneratedRegex(@"[\x00-\x1F\x7F]")]
|
||||
private static partial Regex ControlCharsRegex();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Configuration options for tenant isolation.
|
||||
/// </summary>
|
||||
public sealed class TenantIsolationOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether to allow any cross-tenant access.
|
||||
/// </summary>
|
||||
public bool AllowCrossTenantAccess { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Pairs of tenants allowed to access each other's resources.
|
||||
/// Format: "tenant1:tenant2" means tenant1 can access tenant2's resources.
|
||||
/// </summary>
|
||||
public HashSet<string> CrossTenantAllowedPairs { get; set; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Tenants with admin access to all resources.
|
||||
/// </summary>
|
||||
public HashSet<string> AdminTenants { get; set; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of violations to store in memory.
|
||||
/// </summary>
|
||||
public int MaxStoredViolations { get; set; } = 1000;
|
||||
|
||||
/// <summary>
|
||||
/// Whether to throw exceptions on violations (vs returning result).
|
||||
/// </summary>
|
||||
public bool ThrowOnViolation { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Metrics for tenant isolation.
|
||||
/// </summary>
|
||||
internal static class TenantIsolationMetrics
|
||||
{
|
||||
// In a real implementation, these would emit to metrics system
|
||||
private static long _violationCount;
|
||||
|
||||
public static void RecordViolation(string requestTenant, string resourceTenant, string resourceType)
|
||||
{
|
||||
Interlocked.Increment(ref _violationCount);
|
||||
// In production: emit to Prometheus/StatsD/etc.
|
||||
}
|
||||
|
||||
public static long GetViolationCount() => _violationCount;
|
||||
}
|
||||
@@ -1,329 +0,0 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Net;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Cryptography;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Security;
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation of webhook security service using HMAC.
|
||||
/// Note: External webhooks always use HMAC-SHA256 for interoperability via HmacPurpose.WebhookInterop.
|
||||
/// </summary>
|
||||
public sealed class DefaultWebhookSecurityService : IWebhookSecurityService
|
||||
{
|
||||
private const string SignaturePrefix = "v1";
|
||||
private const int TimestampToleranceSeconds = 300; // 5 minutes
|
||||
|
||||
private readonly WebhookSecurityOptions _options;
|
||||
private readonly ICryptoHmac _cryptoHmac;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<DefaultWebhookSecurityService> _logger;
|
||||
|
||||
// In-memory storage for channel secrets (in production, use persistent storage)
|
||||
private readonly ConcurrentDictionary<string, ChannelSecurityConfig> _channelConfigs = new();
|
||||
|
||||
public DefaultWebhookSecurityService(
|
||||
IOptions<WebhookSecurityOptions> options,
|
||||
ICryptoHmac cryptoHmac,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<DefaultWebhookSecurityService> logger)
|
||||
{
|
||||
_options = options?.Value ?? new WebhookSecurityOptions();
|
||||
_cryptoHmac = cryptoHmac ?? throw new ArgumentNullException(nameof(cryptoHmac));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public string SignPayload(string tenantId, string channelId, ReadOnlySpan<byte> payload, DateTimeOffset timestamp)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(channelId);
|
||||
|
||||
var config = GetOrCreateConfig(tenantId, channelId);
|
||||
var timestampUnix = timestamp.ToUnixTimeSeconds();
|
||||
|
||||
// Create signed payload: timestamp.payload
|
||||
var signedData = CreateSignedData(timestampUnix, payload);
|
||||
|
||||
// WebhookInterop always uses HMAC-SHA256 for external webhook compatibility
|
||||
var signatureHex = _cryptoHmac.ComputeHmacHexForPurpose(config.SecretBytes, signedData, HmacPurpose.WebhookInterop);
|
||||
|
||||
// Format: v1=timestamp,signature
|
||||
return $"{SignaturePrefix}={timestampUnix},{signatureHex}";
|
||||
}
|
||||
|
||||
public bool VerifySignature(string tenantId, string channelId, ReadOnlySpan<byte> payload, string signatureHeader)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(channelId);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(signatureHeader))
|
||||
{
|
||||
_logger.LogWarning("Missing signature header for webhook callback");
|
||||
return false;
|
||||
}
|
||||
|
||||
// Parse header: v1=timestamp,signature
|
||||
if (!signatureHeader.StartsWith($"{SignaturePrefix}=", StringComparison.Ordinal))
|
||||
{
|
||||
_logger.LogWarning("Invalid signature prefix in header");
|
||||
return false;
|
||||
}
|
||||
|
||||
var parts = signatureHeader[(SignaturePrefix.Length + 1)..].Split(',');
|
||||
if (parts.Length != 2)
|
||||
{
|
||||
_logger.LogWarning("Invalid signature format in header");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!long.TryParse(parts[0], out var timestampUnix))
|
||||
{
|
||||
_logger.LogWarning("Invalid timestamp in signature header");
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check timestamp is within tolerance
|
||||
var now = _timeProvider.GetUtcNow().ToUnixTimeSeconds();
|
||||
if (Math.Abs(now - timestampUnix) > TimestampToleranceSeconds)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Signature timestamp {Timestamp} is outside tolerance window (now: {Now})",
|
||||
timestampUnix, now);
|
||||
return false;
|
||||
}
|
||||
|
||||
byte[] providedSignature;
|
||||
try
|
||||
{
|
||||
providedSignature = Convert.FromHexString(parts[1]);
|
||||
}
|
||||
catch (FormatException)
|
||||
{
|
||||
_logger.LogWarning("Invalid signature hex encoding");
|
||||
return false;
|
||||
}
|
||||
|
||||
var config = GetOrCreateConfig(tenantId, channelId);
|
||||
var signedData = CreateSignedData(timestampUnix, payload);
|
||||
|
||||
// WebhookInterop always uses HMAC-SHA256 for external webhook compatibility
|
||||
if (_cryptoHmac.VerifyHmacForPurpose(config.SecretBytes, signedData, providedSignature, HmacPurpose.WebhookInterop))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
// Also check previous secret if within rotation window
|
||||
if (config.PreviousSecretBytes is not null &&
|
||||
config.PreviousSecretExpiresAt.HasValue &&
|
||||
_timeProvider.GetUtcNow() < config.PreviousSecretExpiresAt.Value)
|
||||
{
|
||||
return _cryptoHmac.VerifyHmacForPurpose(config.PreviousSecretBytes, signedData, providedSignature, HmacPurpose.WebhookInterop);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public IpValidationResult ValidateIp(string tenantId, string channelId, IPAddress ipAddress)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(channelId);
|
||||
ArgumentNullException.ThrowIfNull(ipAddress);
|
||||
|
||||
var config = GetOrCreateConfig(tenantId, channelId);
|
||||
|
||||
if (config.IpAllowlist.Count == 0)
|
||||
{
|
||||
// No allowlist configured - allow all
|
||||
return IpValidationResult.Allow(hasAllowlist: false);
|
||||
}
|
||||
|
||||
foreach (var entry in config.IpAllowlist)
|
||||
{
|
||||
if (IsIpInRange(ipAddress, entry.CidrOrIp))
|
||||
{
|
||||
return IpValidationResult.Allow(entry.CidrOrIp, hasAllowlist: true);
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogWarning(
|
||||
"IP {IpAddress} not in allowlist for channel {ChannelId}",
|
||||
ipAddress, channelId);
|
||||
|
||||
return IpValidationResult.Deny($"IP {ipAddress} not in allowlist");
|
||||
}
|
||||
|
||||
public string GetMaskedSecret(string tenantId, string channelId)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(channelId);
|
||||
|
||||
var config = GetOrCreateConfig(tenantId, channelId);
|
||||
var secret = config.Secret;
|
||||
|
||||
if (secret.Length <= 8)
|
||||
{
|
||||
return "****";
|
||||
}
|
||||
|
||||
return $"{secret[..4]}...{secret[^4..]}";
|
||||
}
|
||||
|
||||
public Task<WebhookSecretRotationResult> RotateSecretAsync(
|
||||
string tenantId,
|
||||
string channelId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(channelId);
|
||||
|
||||
var key = GetConfigKey(tenantId, channelId);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var newSecret = GenerateSecret();
|
||||
|
||||
var result = _channelConfigs.AddOrUpdate(
|
||||
key,
|
||||
_ => new ChannelSecurityConfig(newSecret),
|
||||
(_, existing) =>
|
||||
{
|
||||
return new ChannelSecurityConfig(newSecret)
|
||||
{
|
||||
PreviousSecret = existing.Secret,
|
||||
PreviousSecretBytes = existing.SecretBytes,
|
||||
PreviousSecretExpiresAt = now.Add(_options.SecretRotationGracePeriod),
|
||||
IpAllowlist = existing.IpAllowlist
|
||||
};
|
||||
});
|
||||
|
||||
_logger.LogInformation(
|
||||
"Rotated webhook secret for channel {ChannelId}, old secret valid until {ExpiresAt}",
|
||||
channelId, result.PreviousSecretExpiresAt);
|
||||
|
||||
return Task.FromResult(new WebhookSecretRotationResult
|
||||
{
|
||||
Success = true,
|
||||
NewSecret = newSecret,
|
||||
ActiveAt = now,
|
||||
OldSecretExpiresAt = result.PreviousSecretExpiresAt
|
||||
});
|
||||
}
|
||||
|
||||
private ChannelSecurityConfig GetOrCreateConfig(string tenantId, string channelId)
|
||||
{
|
||||
var key = GetConfigKey(tenantId, channelId);
|
||||
return _channelConfigs.GetOrAdd(key, _ => new ChannelSecurityConfig(GenerateSecret()));
|
||||
}
|
||||
|
||||
private static string GetConfigKey(string tenantId, string channelId)
|
||||
=> $"{tenantId}:{channelId}";
|
||||
|
||||
private static string GenerateSecret()
|
||||
{
|
||||
var bytes = RandomNumberGenerator.GetBytes(32);
|
||||
return Convert.ToBase64String(bytes);
|
||||
}
|
||||
|
||||
private static byte[] CreateSignedData(long timestamp, ReadOnlySpan<byte> payload)
|
||||
{
|
||||
var timestampBytes = Encoding.UTF8.GetBytes(timestamp.ToString());
|
||||
var result = new byte[timestampBytes.Length + 1 + payload.Length];
|
||||
timestampBytes.CopyTo(result, 0);
|
||||
result[timestampBytes.Length] = (byte)'.';
|
||||
payload.CopyTo(result.AsSpan(timestampBytes.Length + 1));
|
||||
return result;
|
||||
}
|
||||
|
||||
private static bool IsIpInRange(IPAddress ip, string cidrOrIp)
|
||||
{
|
||||
if (cidrOrIp.Contains('/'))
|
||||
{
|
||||
// CIDR notation
|
||||
var parts = cidrOrIp.Split('/');
|
||||
if (!IPAddress.TryParse(parts[0], out var networkAddress) ||
|
||||
!int.TryParse(parts[1], out var prefixLength))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return IsInSubnet(ip, networkAddress, prefixLength);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Single IP
|
||||
return IPAddress.TryParse(cidrOrIp, out var singleIp) && ip.Equals(singleIp);
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsInSubnet(IPAddress ip, IPAddress network, int prefixLength)
|
||||
{
|
||||
var ipBytes = ip.GetAddressBytes();
|
||||
var networkBytes = network.GetAddressBytes();
|
||||
|
||||
if (ipBytes.Length != networkBytes.Length)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var fullBytes = prefixLength / 8;
|
||||
var remainingBits = prefixLength % 8;
|
||||
|
||||
for (var i = 0; i < fullBytes; i++)
|
||||
{
|
||||
if (ipBytes[i] != networkBytes[i])
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (remainingBits > 0 && fullBytes < ipBytes.Length)
|
||||
{
|
||||
var mask = (byte)(0xFF << (8 - remainingBits));
|
||||
if ((ipBytes[fullBytes] & mask) != (networkBytes[fullBytes] & mask))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private sealed class ChannelSecurityConfig
|
||||
{
|
||||
public ChannelSecurityConfig(string secret)
|
||||
{
|
||||
Secret = secret;
|
||||
SecretBytes = Encoding.UTF8.GetBytes(secret);
|
||||
}
|
||||
|
||||
public string Secret { get; }
|
||||
public byte[] SecretBytes { get; }
|
||||
public string? PreviousSecret { get; init; }
|
||||
public byte[]? PreviousSecretBytes { get; init; }
|
||||
public DateTimeOffset? PreviousSecretExpiresAt { get; init; }
|
||||
public List<IpAllowlistEntry> IpAllowlist { get; init; } = [];
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Configuration options for webhook security.
|
||||
/// </summary>
|
||||
public sealed class WebhookSecurityOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Grace period during which both old and new secrets are valid after rotation.
|
||||
/// </summary>
|
||||
public TimeSpan SecretRotationGracePeriod { get; set; } = TimeSpan.FromHours(24);
|
||||
|
||||
/// <summary>
|
||||
/// Whether to enforce IP allowlists when configured.
|
||||
/// </summary>
|
||||
public bool EnforceIpAllowlist { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Timestamp tolerance for signature verification (in seconds).
|
||||
/// </summary>
|
||||
public int TimestampToleranceSeconds { get; set; } = 300;
|
||||
}
|
||||
@@ -466,6 +466,11 @@ public sealed class TenantIsolationOptions
|
||||
/// </summary>
|
||||
public bool EnforceStrict { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Whether to allow configured cross-tenant access without grants.
|
||||
/// </summary>
|
||||
public bool AllowCrossTenantAccess { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether to log violations.
|
||||
/// </summary>
|
||||
@@ -496,15 +501,35 @@ public sealed class TenantIsolationOptions
|
||||
/// </summary>
|
||||
public List<string> AdminTenantPatterns { get; set; } = ["^admin$", "^system$", "^\\*$"];
|
||||
|
||||
/// <summary>
|
||||
/// Tenants with admin access to all resources.
|
||||
/// </summary>
|
||||
public HashSet<string> AdminTenants { get; set; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Whether to allow cross-tenant grants.
|
||||
/// </summary>
|
||||
public bool AllowCrossTenantGrants { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Pairs of tenants allowed to access each other's resources (format: tenant1:tenant2).
|
||||
/// </summary>
|
||||
public HashSet<string> CrossTenantAllowedPairs { get; set; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Maximum grant duration.
|
||||
/// </summary>
|
||||
public TimeSpan MaxGrantDuration { get; set; } = TimeSpan.FromDays(365);
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of violations to retain in memory.
|
||||
/// </summary>
|
||||
public int MaxStoredViolations { get; set; } = 1000;
|
||||
|
||||
/// <summary>
|
||||
/// Whether to throw exceptions on violations instead of returning results.
|
||||
/// </summary>
|
||||
public bool ThrowOnViolation { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -2,6 +2,7 @@ using System.Collections.Concurrent;
|
||||
using System.Net;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
@@ -57,6 +58,19 @@ public interface IWebhookSecurityService
|
||||
string channelId,
|
||||
string ipAddress,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Rotates the secret for a webhook configuration.
|
||||
/// </summary>
|
||||
Task<WebhookSecretRotationResult> RotateSecretAsync(
|
||||
string tenantId,
|
||||
string channelId,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Returns a masked representation of the secret.
|
||||
/// </summary>
|
||||
string? GetMaskedSecret(string tenantId, string channelId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -254,6 +268,18 @@ public sealed record WebhookSecurityConfig
|
||||
public DateTimeOffset UpdatedAt { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result returned when rotating a webhook secret.
|
||||
/// </summary>
|
||||
public sealed record WebhookSecretRotationResult
|
||||
{
|
||||
public bool Success { get; init; }
|
||||
public string? NewSecret { get; init; }
|
||||
public DateTimeOffset? ActiveAt { get; init; }
|
||||
public DateTimeOffset? OldSecretExpiresAt { get; init; }
|
||||
public string? Error { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Options for webhook security service.
|
||||
/// </summary>
|
||||
@@ -276,6 +302,11 @@ public sealed class WebhookSecurityOptions
|
||||
/// </summary>
|
||||
public TimeSpan DefaultMaxRequestAge { get; set; } = TimeSpan.FromMinutes(5);
|
||||
|
||||
/// <summary>
|
||||
/// Grace period during which both old and new secrets are valid after rotation.
|
||||
/// </summary>
|
||||
public TimeSpan SecretRotationGracePeriod { get; set; } = TimeSpan.FromHours(24);
|
||||
|
||||
/// <summary>
|
||||
/// Whether to enable replay protection by default.
|
||||
/// </summary>
|
||||
@@ -286,6 +317,16 @@ public sealed class WebhookSecurityOptions
|
||||
/// </summary>
|
||||
public TimeSpan NonceCacheExpiry { get; set; } = TimeSpan.FromMinutes(10);
|
||||
|
||||
/// <summary>
|
||||
/// Whether to enforce IP allowlists when configured.
|
||||
/// </summary>
|
||||
public bool EnforceIpAllowlist { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Timestamp tolerance for signature verification (seconds).
|
||||
/// </summary>
|
||||
public int TimestampToleranceSeconds { get; set; } = 300;
|
||||
|
||||
/// <summary>
|
||||
/// Global IP allowlist (in addition to per-webhook allowlists).
|
||||
/// </summary>
|
||||
@@ -573,6 +614,59 @@ public sealed class InMemoryWebhookSecurityService : IWebhookSecurityService
|
||||
return Task.FromResult(IsIpAllowedInternal(ipAddress, config.AllowedIps));
|
||||
}
|
||||
|
||||
public async Task<WebhookSecretRotationResult> RotateSecretAsync(
|
||||
string tenantId,
|
||||
string channelId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var existing = await GetConfigAsync(tenantId, channelId, cancellationToken).ConfigureAwait(false);
|
||||
var newSecret = Convert.ToHexString(Guid.NewGuid().ToByteArray());
|
||||
|
||||
var updatedConfig = existing is null
|
||||
? new WebhookSecurityConfig
|
||||
{
|
||||
ConfigId = $"wh-{Guid.NewGuid():N}"[..16],
|
||||
TenantId = tenantId,
|
||||
ChannelId = channelId,
|
||||
SecretKey = newSecret,
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now
|
||||
}
|
||||
: existing with
|
||||
{
|
||||
SecretKey = newSecret,
|
||||
UpdatedAt = now
|
||||
};
|
||||
|
||||
await RegisterWebhookAsync(updatedConfig, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return new WebhookSecretRotationResult
|
||||
{
|
||||
Success = true,
|
||||
NewSecret = newSecret,
|
||||
ActiveAt = now,
|
||||
OldSecretExpiresAt = null
|
||||
};
|
||||
}
|
||||
|
||||
public string? GetMaskedSecret(string tenantId, string channelId)
|
||||
{
|
||||
var key = BuildConfigKey(tenantId, channelId);
|
||||
if (!_configs.TryGetValue(key, out var config) || string.IsNullOrWhiteSpace(config.SecretKey))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var secret = config.SecretKey;
|
||||
if (secret.Length <= 4)
|
||||
{
|
||||
return "****";
|
||||
}
|
||||
|
||||
return $"{secret[..2]}****{secret[^2..]}";
|
||||
}
|
||||
|
||||
private bool IsIpAllowedInternal(string ipAddress, IReadOnlyList<string> allowedIps)
|
||||
{
|
||||
if (!IPAddress.TryParse(ipAddress, out var ip))
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Collections.Immutable;
|
||||
using System.Diagnostics;
|
||||
using System.Text.Json.Nodes;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Notify.Engine;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notify.Storage.Mongo.Documents;
|
||||
using StellaOps.Notify.Storage.Mongo.Repositories;
|
||||
using StellaOps.Notifier.Worker.Storage;
|
||||
using StellaOps.Notifier.Worker.Correlation;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Simulation;
|
||||
@@ -284,10 +283,14 @@ public sealed class DefaultNotifySimulationEngine : INotifySimulationEngine
|
||||
{
|
||||
var throttleKey = $"{rule.RuleId}:{action.ActionId}:{@event.Kind}";
|
||||
var throttleWindow = action.Throttle is { Ticks: > 0 } ? action.Throttle.Value : DefaultThrottleWindow;
|
||||
var isThrottled = await _throttler.IsThrottledAsync(
|
||||
@event.Tenant, throttleKey, throttleWindow, cancellationToken).ConfigureAwait(false);
|
||||
var throttleResult = await _throttler.CheckAsync(
|
||||
@event.Tenant,
|
||||
throttleKey,
|
||||
throttleWindow,
|
||||
null,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (isThrottled)
|
||||
if (throttleResult.IsThrottled)
|
||||
{
|
||||
wouldDeliver = false;
|
||||
throttleReason = $"Would be throttled (key: {throttleKey})";
|
||||
@@ -298,10 +301,10 @@ public sealed class DefaultNotifySimulationEngine : INotifySimulationEngine
|
||||
// Check quiet hours
|
||||
if (wouldDeliver && request.EvaluateQuietHours && _quietHoursEvaluator is not null)
|
||||
{
|
||||
var quietHoursResult = await _quietHoursEvaluator.IsInQuietHoursAsync(
|
||||
@event.Tenant, channelId, cancellationToken).ConfigureAwait(false);
|
||||
var quietHoursResult = await _quietHoursEvaluator.EvaluateAsync(
|
||||
@event.Tenant, @event.Kind, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (quietHoursResult.IsInQuietHours)
|
||||
if (quietHoursResult.IsSuppressed)
|
||||
{
|
||||
wouldDeliver = false;
|
||||
quietHoursReason = quietHoursResult.Reason ?? "In quiet hours period";
|
||||
@@ -431,7 +434,7 @@ public sealed class DefaultNotifySimulationEngine : INotifySimulationEngine
|
||||
}
|
||||
|
||||
private static IReadOnlyList<NotifyEvent> ConvertAuditEntriesToEvents(
|
||||
IReadOnlyList<NotifyAuditEntryDocument> auditEntries,
|
||||
IReadOnlyList<NotifyAuditEntry> auditEntries,
|
||||
DateTimeOffset periodStart,
|
||||
DateTimeOffset periodEnd,
|
||||
ImmutableArray<string> eventKinds)
|
||||
@@ -444,34 +447,31 @@ public sealed class DefaultNotifySimulationEngine : INotifySimulationEngine
|
||||
|
||||
foreach (var entry in auditEntries)
|
||||
{
|
||||
// Skip entries outside the period
|
||||
if (entry.Timestamp < periodStart || entry.Timestamp >= periodEnd)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
// Try to extract event info from the audit entry's action or payload
|
||||
// Audit entries may not contain full event data, so we reconstruct what we can
|
||||
var eventKind = ExtractEventKindFromAuditEntry(entry);
|
||||
if (string.IsNullOrWhiteSpace(eventKind))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
// Filter by event kind if specified
|
||||
if (kindSet is not null && !kindSet.Contains(eventKind))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var eventId = ExtractEventIdFromAuditEntry(entry);
|
||||
var payload = ToPayload(entry.Data);
|
||||
|
||||
var @event = NotifyEvent.Create(
|
||||
eventId: eventId,
|
||||
kind: eventKind,
|
||||
tenant: entry.TenantId,
|
||||
ts: entry.Timestamp,
|
||||
payload: TryParsePayloadFromBson(entry.Payload));
|
||||
payload: payload);
|
||||
|
||||
events.Add(@event);
|
||||
}
|
||||
@@ -479,7 +479,7 @@ public sealed class DefaultNotifySimulationEngine : INotifySimulationEngine
|
||||
return events;
|
||||
}
|
||||
|
||||
private static string? ExtractEventKindFromAuditEntry(NotifyAuditEntryDocument entry)
|
||||
private static string? ExtractEventKindFromAuditEntry(NotifyAuditEntry entry)
|
||||
{
|
||||
// The event kind might be encoded in the action field or payload
|
||||
// Action format is typically "event.kind.action" or we look in payload
|
||||
@@ -496,41 +496,35 @@ public sealed class DefaultNotifySimulationEngine : INotifySimulationEngine
|
||||
}
|
||||
|
||||
// Try to extract from payload
|
||||
if (entry.Payload is { } payload)
|
||||
if (entry.Data.TryGetValue("Kind", out var kind) ||
|
||||
entry.Data.TryGetValue("kind", out kind))
|
||||
{
|
||||
if (payload.TryGetValue("Kind", out var kindValue) || payload.TryGetValue("kind", out kindValue))
|
||||
{
|
||||
return kindValue.AsString;
|
||||
}
|
||||
return kind;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static Guid ExtractEventIdFromAuditEntry(NotifyAuditEntryDocument entry)
|
||||
private static Guid ExtractEventIdFromAuditEntry(NotifyAuditEntry entry)
|
||||
{
|
||||
// Try to extract event ID from payload
|
||||
if (entry.Payload is { } payload)
|
||||
if (entry.Data.TryGetValue("eventId", out var eventId) &&
|
||||
Guid.TryParse(eventId, out var parsed))
|
||||
{
|
||||
if (payload.TryGetValue("EventId", out var eventIdValue) || payload.TryGetValue("eventId", out eventIdValue))
|
||||
{
|
||||
if (Guid.TryParse(eventIdValue.ToString(), out var id))
|
||||
{
|
||||
return id;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Try entity ID
|
||||
if (Guid.TryParse(entry.EntityId, out var entityId))
|
||||
{
|
||||
return entityId;
|
||||
return parsed;
|
||||
}
|
||||
|
||||
return Guid.NewGuid();
|
||||
}
|
||||
|
||||
private static JsonNode? TryParsePayloadFromBson(JsonObject? payload) => payload;
|
||||
private static JsonNode ToPayload(IReadOnlyDictionary<string, string> data)
|
||||
{
|
||||
var obj = new JsonObject();
|
||||
foreach (var (key, value) in data)
|
||||
{
|
||||
obj[key] = value;
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
private static NotifyEvent ParseEventFromPayload(string tenantId, JsonObject payload)
|
||||
{
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notify.Engine;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notify.Storage.Mongo.Repositories;
|
||||
using StellaOps.Notifier.Worker.Storage;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Simulation;
|
||||
|
||||
@@ -533,3 +533,4 @@ public sealed class SimulationOptions
|
||||
/// </summary>
|
||||
public bool AllowAllRulesSimulation { get; set; } = true;
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Cronos" Version="0.9.0" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Http.Abstractions" Version="2.2.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="10.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Http" Version="10.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting" Version="10.0.0" />
|
||||
@@ -21,7 +22,6 @@
|
||||
<ProjectReference Include="../../../Notify/__Libraries/StellaOps.Notify.Models/StellaOps.Notify.Models.csproj" />
|
||||
<ProjectReference Include="../../../Notify/__Libraries/StellaOps.Notify.Queue/StellaOps.Notify.Queue.csproj" />
|
||||
<ProjectReference Include="../../../Notify/__Libraries/StellaOps.Notify.Storage.Postgres/StellaOps.Notify.Storage.Postgres.csproj" />
|
||||
<ProjectReference Include="../../../Notify/__Libraries/StellaOps.Notify.Storage.Mongo/StellaOps.Notify.Storage.Mongo.csproj" />
|
||||
<ProjectReference Include="../../../Notify/__Libraries/StellaOps.Notify.Engine/StellaOps.Notify.Engine.csproj" />
|
||||
<ProjectReference Include="../../../AirGap/StellaOps.AirGap.Policy/StellaOps.AirGap.Policy/StellaOps.AirGap.Policy.csproj" />
|
||||
<ProjectReference Include="../../../__Libraries/StellaOps.Cryptography/StellaOps.Cryptography.csproj" />
|
||||
|
||||
@@ -0,0 +1,762 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Text.Json.Nodes;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notifier.Worker.Channels;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Storage;
|
||||
|
||||
public interface INotifyChannelRepository
|
||||
{
|
||||
Task<NotifyChannel?> GetAsync(string tenantId, string channelId, CancellationToken cancellationToken = default);
|
||||
Task<IReadOnlyList<NotifyChannel>> ListAsync(
|
||||
string tenantId,
|
||||
bool? enabled = null,
|
||||
NotifyChannelType? channelType = null,
|
||||
int limit = 100,
|
||||
int offset = 0,
|
||||
CancellationToken cancellationToken = default);
|
||||
Task<NotifyChannel> UpsertAsync(NotifyChannel channel, CancellationToken cancellationToken = default);
|
||||
Task<bool> DeleteAsync(string tenantId, string channelId, CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
public interface INotifyRuleRepository
|
||||
{
|
||||
Task<IReadOnlyList<NotifyRule>> ListAsync(string tenantId, CancellationToken cancellationToken = default);
|
||||
Task<NotifyRule?> GetAsync(string tenantId, string ruleId, CancellationToken cancellationToken = default);
|
||||
Task<NotifyRule> UpsertAsync(NotifyRule rule, CancellationToken cancellationToken = default);
|
||||
Task<bool> DeleteAsync(string tenantId, string ruleId, CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
public interface INotifyTemplateRepository
|
||||
{
|
||||
Task<IReadOnlyList<NotifyTemplate>> ListAsync(string tenantId, CancellationToken cancellationToken = default);
|
||||
Task<NotifyTemplate?> GetAsync(string tenantId, string templateId, CancellationToken cancellationToken = default);
|
||||
Task<NotifyTemplate> UpsertAsync(NotifyTemplate template, CancellationToken cancellationToken = default);
|
||||
Task<bool> DeleteAsync(string tenantId, string templateId, CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
public interface INotifyDeliveryRepository
|
||||
{
|
||||
Task<NotifyDelivery?> GetAsync(string tenantId, string deliveryId, CancellationToken cancellationToken = default);
|
||||
Task<IReadOnlyList<NotifyDelivery>> ListAsync(string tenantId, CancellationToken cancellationToken = default);
|
||||
Task AppendAsync(NotifyDelivery delivery, CancellationToken cancellationToken = default);
|
||||
Task<IReadOnlyList<NotifyDelivery>> ListPendingAsync(int limit = 100, CancellationToken cancellationToken = default);
|
||||
Task<NotifyDeliveryQueryResult> QueryAsync(
|
||||
string tenantId,
|
||||
DateTimeOffset? since,
|
||||
string? status,
|
||||
int limit,
|
||||
string? continuationToken = null,
|
||||
CancellationToken cancellationToken = default);
|
||||
Task UpdateAsync(NotifyDelivery delivery, CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
public sealed record NotifyDeliveryQueryResult(
|
||||
IReadOnlyList<NotifyDelivery> Items,
|
||||
string? ContinuationToken);
|
||||
|
||||
public interface INotifyAuditRepository
|
||||
{
|
||||
Task AppendAsync(
|
||||
string tenantId,
|
||||
string action,
|
||||
string? actor,
|
||||
IReadOnlyDictionary<string, string> data,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
Task AppendAsync(
|
||||
NotifyAuditEntryDocument entry,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
Task<IReadOnlyList<NotifyAuditEntry>> QueryAsync(
|
||||
string tenantId,
|
||||
DateTimeOffset since,
|
||||
int limit,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
public sealed record NotifyAuditEntryDocument
|
||||
{
|
||||
public required string TenantId { get; init; }
|
||||
public required string Action { get; init; }
|
||||
public string? Actor { get; init; }
|
||||
public string? EntityId { get; init; }
|
||||
public string? EntityType { get; init; }
|
||||
public DateTimeOffset Timestamp { get; init; }
|
||||
public JsonObject? Payload { get; init; }
|
||||
}
|
||||
|
||||
public sealed record NotifyAuditEntry(
|
||||
string TenantId,
|
||||
string Action,
|
||||
string? Actor,
|
||||
DateTimeOffset Timestamp,
|
||||
IReadOnlyDictionary<string, string> Data);
|
||||
|
||||
public interface INotifyLockRepository
|
||||
{
|
||||
Task<bool> TryAcquireAsync(
|
||||
string tenantId,
|
||||
string lockKey,
|
||||
string owner,
|
||||
TimeSpan ttl,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
Task<bool> ReleaseAsync(
|
||||
string tenantId,
|
||||
string lockKey,
|
||||
string owner,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
Task<bool> ExtendAsync(
|
||||
string tenantId,
|
||||
string lockKey,
|
||||
string owner,
|
||||
TimeSpan ttl,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
public interface INotifyLocalizationRepository
|
||||
{
|
||||
Task<IReadOnlyList<NotifyLocalizationBundle>> ListAsync(
|
||||
string tenantId,
|
||||
string? bundleKey,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
Task<IReadOnlyList<string>> ListLocalesAsync(
|
||||
string tenantId,
|
||||
string bundleKey,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
Task<NotifyLocalizationBundle?> GetAsync(
|
||||
string tenantId,
|
||||
string bundleId,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
Task<NotifyLocalizationBundle?> GetByKeyAndLocaleAsync(
|
||||
string tenantId,
|
||||
string bundleKey,
|
||||
string locale,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
Task<NotifyLocalizationBundle?> GetDefaultAsync(
|
||||
string tenantId,
|
||||
string bundleKey,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
Task<NotifyLocalizationBundle> UpsertAsync(
|
||||
NotifyLocalizationBundle bundle,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
Task DeleteAsync(
|
||||
string tenantId,
|
||||
string bundleId,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
public interface INotifyInboxRepository
|
||||
{
|
||||
Task<IReadOnlyList<InAppInboxMessage>> GetForUserAsync(
|
||||
string tenantId,
|
||||
string userId,
|
||||
int limit = 50,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
Task<InAppInboxMessage?> GetAsync(
|
||||
string tenantId,
|
||||
string messageId,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
Task MarkReadAsync(
|
||||
string tenantId,
|
||||
string messageId,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
Task MarkAllReadAsync(
|
||||
string tenantId,
|
||||
string userId,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
Task<int> GetUnreadCountAsync(
|
||||
string tenantId,
|
||||
string userId,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
Task DeleteAsync(
|
||||
string tenantId,
|
||||
string messageId,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// In-memory repository implementations that replace the legacy document store.
|
||||
/// </summary>
|
||||
public sealed class InMemoryNotifyRepositories :
|
||||
INotifyChannelRepository,
|
||||
INotifyRuleRepository,
|
||||
INotifyTemplateRepository,
|
||||
INotifyDeliveryRepository,
|
||||
INotifyAuditRepository,
|
||||
INotifyLockRepository,
|
||||
INotifyLocalizationRepository
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, ConcurrentDictionary<string, NotifyChannel>> _channels = new();
|
||||
private readonly ConcurrentDictionary<string, ConcurrentDictionary<string, NotifyRule>> _rules = new();
|
||||
private readonly ConcurrentDictionary<string, ConcurrentDictionary<string, NotifyTemplate>> _templates = new();
|
||||
private readonly ConcurrentDictionary<string, ConcurrentDictionary<string, NotifyDelivery>> _deliveries = new();
|
||||
private readonly ConcurrentDictionary<string, List<NotifyAuditEntry>> _audits = new();
|
||||
private readonly ConcurrentDictionary<string, ConcurrentDictionary<string, NotifyLocalizationBundle>> _localizations = new();
|
||||
private readonly ConcurrentDictionary<string, LockState> _locks = new();
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public InMemoryNotifyRepositories(TimeProvider timeProvider)
|
||||
{
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
#region Channel
|
||||
Task<NotifyChannel?> INotifyChannelRepository.GetAsync(string tenantId, string channelId, CancellationToken cancellationToken)
|
||||
{
|
||||
var items = ForTenant(_channels, tenantId);
|
||||
items.TryGetValue(channelId, out var channel);
|
||||
return Task.FromResult(channel);
|
||||
}
|
||||
|
||||
Task<IReadOnlyList<NotifyChannel>> INotifyChannelRepository.ListAsync(
|
||||
string tenantId,
|
||||
bool? enabled,
|
||||
NotifyChannelType? channelType,
|
||||
int limit,
|
||||
int offset,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var items = ForTenant(_channels, tenantId).Values.AsEnumerable();
|
||||
|
||||
if (enabled.HasValue)
|
||||
{
|
||||
items = items.Where(c => c.Enabled == enabled.Value);
|
||||
}
|
||||
|
||||
if (channelType.HasValue)
|
||||
{
|
||||
items = items.Where(c => c.Type == channelType.Value);
|
||||
}
|
||||
|
||||
var result = items
|
||||
.OrderBy(c => c.Name, StringComparer.Ordinal)
|
||||
.Skip(Math.Max(offset, 0))
|
||||
.Take(Math.Max(limit, 0))
|
||||
.ToList();
|
||||
|
||||
return Task.FromResult<IReadOnlyList<NotifyChannel>>(result);
|
||||
}
|
||||
|
||||
Task<NotifyChannel> INotifyChannelRepository.UpsertAsync(NotifyChannel channel, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(channel);
|
||||
var items = ForTenant(_channels, channel.TenantId);
|
||||
items[channel.ChannelId] = channel;
|
||||
return Task.FromResult(channel);
|
||||
}
|
||||
|
||||
Task<bool> INotifyChannelRepository.DeleteAsync(string tenantId, string channelId, CancellationToken cancellationToken)
|
||||
{
|
||||
var items = ForTenant(_channels, tenantId);
|
||||
return Task.FromResult(items.TryRemove(channelId, out _));
|
||||
}
|
||||
#endregion
|
||||
|
||||
#region Rule
|
||||
Task<IReadOnlyList<NotifyRule>> INotifyRuleRepository.ListAsync(string tenantId, CancellationToken cancellationToken)
|
||||
{
|
||||
var items = ForTenant(_rules, tenantId).Values
|
||||
.OrderBy(r => r.Name, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
return Task.FromResult<IReadOnlyList<NotifyRule>>(items);
|
||||
}
|
||||
|
||||
Task<NotifyRule?> INotifyRuleRepository.GetAsync(string tenantId, string ruleId, CancellationToken cancellationToken)
|
||||
{
|
||||
var items = ForTenant(_rules, tenantId);
|
||||
items.TryGetValue(ruleId, out var rule);
|
||||
return Task.FromResult(rule);
|
||||
}
|
||||
|
||||
Task<NotifyRule> INotifyRuleRepository.UpsertAsync(NotifyRule rule, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(rule);
|
||||
var items = ForTenant(_rules, rule.TenantId);
|
||||
items[rule.RuleId] = rule;
|
||||
return Task.FromResult(rule);
|
||||
}
|
||||
|
||||
Task<bool> INotifyRuleRepository.DeleteAsync(string tenantId, string ruleId, CancellationToken cancellationToken)
|
||||
{
|
||||
var items = ForTenant(_rules, tenantId);
|
||||
return Task.FromResult(items.TryRemove(ruleId, out _));
|
||||
}
|
||||
#endregion
|
||||
|
||||
#region Template
|
||||
Task<IReadOnlyList<NotifyTemplate>> INotifyTemplateRepository.ListAsync(string tenantId, CancellationToken cancellationToken)
|
||||
{
|
||||
var items = ForTenant(_templates, tenantId).Values
|
||||
.OrderBy(t => t.Key, StringComparer.Ordinal)
|
||||
.ThenBy(t => t.Locale, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
return Task.FromResult<IReadOnlyList<NotifyTemplate>>(items);
|
||||
}
|
||||
|
||||
Task<NotifyTemplate?> INotifyTemplateRepository.GetAsync(string tenantId, string templateId, CancellationToken cancellationToken)
|
||||
{
|
||||
var items = ForTenant(_templates, tenantId);
|
||||
items.TryGetValue(templateId, out var template);
|
||||
return Task.FromResult(template);
|
||||
}
|
||||
|
||||
Task<NotifyTemplate> INotifyTemplateRepository.UpsertAsync(NotifyTemplate template, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(template);
|
||||
var items = ForTenant(_templates, template.TenantId);
|
||||
items[template.TemplateId] = template;
|
||||
return Task.FromResult(template);
|
||||
}
|
||||
|
||||
Task<bool> INotifyTemplateRepository.DeleteAsync(string tenantId, string templateId, CancellationToken cancellationToken)
|
||||
{
|
||||
var items = ForTenant(_templates, tenantId);
|
||||
return Task.FromResult(items.TryRemove(templateId, out _));
|
||||
}
|
||||
#endregion
|
||||
|
||||
#region Delivery
|
||||
Task<NotifyDelivery?> INotifyDeliveryRepository.GetAsync(
|
||||
string tenantId,
|
||||
string deliveryId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var items = ForTenant(_deliveries, tenantId);
|
||||
items.TryGetValue(deliveryId, out var delivery);
|
||||
return Task.FromResult(delivery);
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<NotifyDelivery>> ListAsync(
|
||||
string tenantId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var items = ForTenant(_deliveries, tenantId).Values
|
||||
.OrderBy(d => d.CreatedAt)
|
||||
.ToList();
|
||||
return Task.FromResult<IReadOnlyList<NotifyDelivery>>(items);
|
||||
}
|
||||
|
||||
public Task AppendAsync(NotifyDelivery delivery, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(delivery);
|
||||
var items = ForTenant(_deliveries, delivery.TenantId);
|
||||
items[delivery.DeliveryId] = delivery;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<NotifyDelivery>> ListPendingAsync(int limit = 100, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var pending = _deliveries.Values
|
||||
.SelectMany(dict => dict.Values)
|
||||
.Where(d => d.Status == NotifyDeliveryStatus.Pending)
|
||||
.OrderBy(d => d.CreatedAt)
|
||||
.Take(Math.Max(limit, 0))
|
||||
.ToList();
|
||||
|
||||
return Task.FromResult<IReadOnlyList<NotifyDelivery>>(pending);
|
||||
}
|
||||
|
||||
public Task<NotifyDeliveryQueryResult> QueryAsync(
|
||||
string tenantId,
|
||||
DateTimeOffset? since,
|
||||
string? status,
|
||||
int limit,
|
||||
string? continuationToken,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var items = ForTenant(_deliveries, tenantId).Values.AsEnumerable();
|
||||
|
||||
if (since.HasValue)
|
||||
{
|
||||
items = items.Where(d => d.CreatedAt >= since.Value);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(status) &&
|
||||
Enum.TryParse<NotifyDeliveryStatus>(status, true, out var parsedStatus))
|
||||
{
|
||||
items = items.Where(d => d.Status == parsedStatus);
|
||||
}
|
||||
|
||||
var result = items
|
||||
.OrderBy(d => d.CreatedAt)
|
||||
.Take(Math.Max(limit, 0))
|
||||
.ToList();
|
||||
|
||||
return Task.FromResult(new NotifyDeliveryQueryResult(result, null));
|
||||
}
|
||||
|
||||
public Task UpdateAsync(NotifyDelivery delivery, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(delivery);
|
||||
var items = ForTenant(_deliveries, delivery.TenantId);
|
||||
items[delivery.DeliveryId] = delivery;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
#endregion
|
||||
|
||||
#region Audit
|
||||
public Task AppendAsync(
|
||||
string tenantId,
|
||||
string action,
|
||||
string? actor,
|
||||
IReadOnlyDictionary<string, string> data,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var entry = new NotifyAuditEntry(
|
||||
tenantId,
|
||||
action,
|
||||
actor,
|
||||
_timeProvider.GetUtcNow(),
|
||||
data.ToImmutableDictionary(StringComparer.Ordinal));
|
||||
|
||||
var list = _audits.GetOrAdd(tenantId, _ => new List<NotifyAuditEntry>());
|
||||
lock (list)
|
||||
{
|
||||
list.Add(entry);
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task AppendAsync(
|
||||
NotifyAuditEntryDocument entry,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(entry);
|
||||
|
||||
var data = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(entry.EntityId))
|
||||
{
|
||||
data["entityId"] = entry.EntityId!;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(entry.EntityType))
|
||||
{
|
||||
data["entityType"] = entry.EntityType!;
|
||||
}
|
||||
|
||||
if (entry.Payload is not null)
|
||||
{
|
||||
data["payload"] = entry.Payload.ToJsonString();
|
||||
}
|
||||
|
||||
return ((INotifyAuditRepository)this).AppendAsync(
|
||||
entry.TenantId,
|
||||
entry.Action,
|
||||
entry.Actor,
|
||||
data,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<NotifyAuditEntry>> QueryAsync(
|
||||
string tenantId,
|
||||
DateTimeOffset since,
|
||||
int limit,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (!_audits.TryGetValue(tenantId, out var list))
|
||||
{
|
||||
return Task.FromResult<IReadOnlyList<NotifyAuditEntry>>(Array.Empty<NotifyAuditEntry>());
|
||||
}
|
||||
|
||||
List<NotifyAuditEntry> snapshot;
|
||||
lock (list)
|
||||
{
|
||||
snapshot = list
|
||||
.Where(e => e.Timestamp >= since)
|
||||
.OrderByDescending(e => e.Timestamp)
|
||||
.Take(Math.Max(limit, 0))
|
||||
.ToList();
|
||||
}
|
||||
|
||||
return Task.FromResult<IReadOnlyList<NotifyAuditEntry>>(snapshot);
|
||||
}
|
||||
#endregion
|
||||
|
||||
#region Localization
|
||||
public Task<IReadOnlyList<NotifyLocalizationBundle>> ListAsync(
|
||||
string tenantId,
|
||||
string? bundleKey,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var items = ForTenant(_localizations, tenantId).Values.AsEnumerable();
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(bundleKey))
|
||||
{
|
||||
items = items.Where(b => string.Equals(b.BundleKey, bundleKey, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
var result = items
|
||||
.OrderBy(b => b.BundleKey, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(b => b.Locale, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
return Task.FromResult<IReadOnlyList<NotifyLocalizationBundle>>(result);
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<string>> ListLocalesAsync(
|
||||
string tenantId,
|
||||
string bundleKey,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var locales = ForTenant(_localizations, tenantId).Values
|
||||
.Where(b => string.Equals(b.BundleKey, bundleKey, StringComparison.OrdinalIgnoreCase))
|
||||
.Select(b => b.Locale)
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(l => l, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
return Task.FromResult<IReadOnlyList<string>>(locales);
|
||||
}
|
||||
|
||||
public Task<NotifyLocalizationBundle?> GetAsync(
|
||||
string tenantId,
|
||||
string bundleId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var items = ForTenant(_localizations, tenantId);
|
||||
items.TryGetValue(bundleId, out var bundle);
|
||||
return Task.FromResult(bundle);
|
||||
}
|
||||
|
||||
public Task<NotifyLocalizationBundle?> GetByKeyAndLocaleAsync(
|
||||
string tenantId,
|
||||
string bundleKey,
|
||||
string locale,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var match = ForTenant(_localizations, tenantId).Values
|
||||
.FirstOrDefault(b =>
|
||||
string.Equals(b.BundleKey, bundleKey, StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(b.Locale, locale, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
return Task.FromResult(match);
|
||||
}
|
||||
|
||||
public Task<NotifyLocalizationBundle?> GetDefaultAsync(
|
||||
string tenantId,
|
||||
string bundleKey,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var match = ForTenant(_localizations, tenantId).Values
|
||||
.FirstOrDefault(b =>
|
||||
string.Equals(b.BundleKey, bundleKey, StringComparison.OrdinalIgnoreCase) &&
|
||||
b.IsDefault);
|
||||
|
||||
return Task.FromResult(match);
|
||||
}
|
||||
|
||||
public Task<NotifyLocalizationBundle> UpsertAsync(
|
||||
NotifyLocalizationBundle bundle,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(bundle);
|
||||
|
||||
var items = ForTenant(_localizations, bundle.TenantId);
|
||||
items[bundle.BundleId] = bundle;
|
||||
|
||||
return Task.FromResult(bundle);
|
||||
}
|
||||
|
||||
public Task DeleteAsync(
|
||||
string tenantId,
|
||||
string bundleId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var items = ForTenant(_localizations, tenantId);
|
||||
items.TryRemove(bundleId, out _);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
#endregion
|
||||
|
||||
#region Locks
|
||||
public Task<bool> TryAcquireAsync(
|
||||
string tenantId,
|
||||
string lockKey,
|
||||
string owner,
|
||||
TimeSpan ttl,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var key = BuildLockKey(tenantId, lockKey);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var expiresAt = now + ttl;
|
||||
|
||||
while (true)
|
||||
{
|
||||
var current = _locks.GetOrAdd(key, _ => new LockState(owner, expiresAt));
|
||||
|
||||
if (current.ExpiresAt <= now || string.Equals(current.Owner, owner, StringComparison.Ordinal))
|
||||
{
|
||||
_locks[key] = new LockState(owner, expiresAt);
|
||||
return Task.FromResult(true);
|
||||
}
|
||||
|
||||
return Task.FromResult(false);
|
||||
}
|
||||
}
|
||||
|
||||
public Task<bool> ReleaseAsync(
|
||||
string tenantId,
|
||||
string lockKey,
|
||||
string owner,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var key = BuildLockKey(tenantId, lockKey);
|
||||
if (_locks.TryGetValue(key, out var state) &&
|
||||
string.Equals(state.Owner, owner, StringComparison.Ordinal))
|
||||
{
|
||||
return Task.FromResult(_locks.TryRemove(key, out _));
|
||||
}
|
||||
|
||||
return Task.FromResult(false);
|
||||
}
|
||||
|
||||
public Task<bool> ExtendAsync(
|
||||
string tenantId,
|
||||
string lockKey,
|
||||
string owner,
|
||||
TimeSpan ttl,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var key = BuildLockKey(tenantId, lockKey);
|
||||
if (!_locks.TryGetValue(key, out var state) ||
|
||||
!string.Equals(state.Owner, owner, StringComparison.Ordinal))
|
||||
{
|
||||
return Task.FromResult(false);
|
||||
}
|
||||
|
||||
var newState = state with { ExpiresAt = _timeProvider.GetUtcNow() + ttl };
|
||||
_locks[key] = newState;
|
||||
return Task.FromResult(true);
|
||||
}
|
||||
#endregion
|
||||
|
||||
private static ConcurrentDictionary<string, T> ForTenant<T>(
|
||||
ConcurrentDictionary<string, ConcurrentDictionary<string, T>> map,
|
||||
string tenant) => map.GetOrAdd(tenant, _ => new ConcurrentDictionary<string, T>());
|
||||
|
||||
private static string BuildLockKey(string tenantId, string lockKey) => $"{tenantId}:{lockKey}";
|
||||
|
||||
private sealed record LockState(string Owner, DateTimeOffset ExpiresAt);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// In-memory implementation of in-app inbox storage.
|
||||
/// </summary>
|
||||
public sealed class InMemoryInboxStore : IInAppInboxStore, INotifyInboxRepository
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, ConcurrentDictionary<string, InAppInboxMessage>> _messages = new();
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public InMemoryInboxStore(TimeProvider timeProvider)
|
||||
{
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public Task StoreAsync(InAppInboxMessage message, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(message);
|
||||
var tenantBox = _messages.GetOrAdd(message.TenantId, _ => new ConcurrentDictionary<string, InAppInboxMessage>());
|
||||
tenantBox[message.MessageId] = message with { CreatedAt = message.CreatedAt == default ? _timeProvider.GetUtcNow() : message.CreatedAt };
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<InAppInboxMessage>> GetForUserAsync(
|
||||
string tenantId,
|
||||
string userId,
|
||||
int limit = 50,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var items = _messages.GetValueOrDefault(tenantId)?.Values
|
||||
.Where(m => string.Equals(m.UserId, userId, StringComparison.Ordinal))
|
||||
.OrderByDescending(m => m.CreatedAt)
|
||||
.Take(Math.Max(limit, 0))
|
||||
.ToList() ?? new List<InAppInboxMessage>();
|
||||
|
||||
return Task.FromResult<IReadOnlyList<InAppInboxMessage>>(items);
|
||||
}
|
||||
|
||||
public Task<InAppInboxMessage?> GetAsync(
|
||||
string tenantId,
|
||||
string messageId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (_messages.TryGetValue(tenantId, out var inbox) &&
|
||||
inbox.TryGetValue(messageId, out var message))
|
||||
{
|
||||
return Task.FromResult<InAppInboxMessage?>(message);
|
||||
}
|
||||
|
||||
return Task.FromResult<InAppInboxMessage?>(null);
|
||||
}
|
||||
|
||||
public Task MarkReadAsync(string tenantId, string messageId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (_messages.TryGetValue(tenantId, out var inbox) &&
|
||||
inbox.TryGetValue(messageId, out var message))
|
||||
{
|
||||
inbox[messageId] = message with { ReadAt = _timeProvider.GetUtcNow() };
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task MarkAllReadAsync(string tenantId, string userId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (_messages.TryGetValue(tenantId, out var inbox))
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
foreach (var (key, value) in inbox)
|
||||
{
|
||||
if (string.Equals(value.UserId, userId, StringComparison.Ordinal) && value.ReadAt is null)
|
||||
{
|
||||
inbox[key] = value with { ReadAt = now };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task DeleteAsync(string tenantId, string messageId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (_messages.TryGetValue(tenantId, out var inbox))
|
||||
{
|
||||
inbox.TryRemove(messageId, out _);
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<int> GetUnreadCountAsync(string tenantId, string userId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (_messages.TryGetValue(tenantId, out var inbox))
|
||||
{
|
||||
var count = inbox.Values.Count(m =>
|
||||
string.Equals(m.UserId, userId, StringComparison.Ordinal) &&
|
||||
m.ReadAt is null &&
|
||||
(!m.ExpiresAt.HasValue || m.ExpiresAt > _timeProvider.GetUtcNow()));
|
||||
return Task.FromResult(count);
|
||||
}
|
||||
|
||||
return Task.FromResult(0);
|
||||
}
|
||||
}
|
||||
@@ -1,294 +0,0 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notify.Models;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.StormBreaker;
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation of storm breaker using in-memory tracking.
|
||||
/// </summary>
|
||||
public sealed class DefaultStormBreaker : IStormBreaker
|
||||
{
|
||||
private readonly StormBreakerConfig _config;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<DefaultStormBreaker> _logger;
|
||||
|
||||
// In-memory storm tracking (keyed by storm key)
|
||||
private readonly ConcurrentDictionary<string, StormTracker> _storms = new();
|
||||
|
||||
public DefaultStormBreaker(
|
||||
IOptions<StormBreakerConfig> config,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<DefaultStormBreaker> logger)
|
||||
{
|
||||
_config = config?.Value ?? new StormBreakerConfig();
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public Task<StormDetectionResult> DetectAsync(
|
||||
string tenantId,
|
||||
NotifyEvent @event,
|
||||
NotifyRule rule,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
ArgumentNullException.ThrowIfNull(@event);
|
||||
ArgumentNullException.ThrowIfNull(rule);
|
||||
|
||||
if (!_config.Enabled)
|
||||
{
|
||||
return Task.FromResult(new StormDetectionResult
|
||||
{
|
||||
Decision = StormDecision.DeliverNormally,
|
||||
Reason = "Storm breaking disabled"
|
||||
});
|
||||
}
|
||||
|
||||
var stormKey = ComputeStormKey(tenantId, @event.Kind, rule.RuleId);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
var tracker = _storms.GetOrAdd(stormKey, _ => new StormTracker
|
||||
{
|
||||
StormKey = stormKey,
|
||||
TenantId = tenantId,
|
||||
EventKind = @event.Kind,
|
||||
RuleId = rule.RuleId,
|
||||
WindowStart = now
|
||||
});
|
||||
|
||||
// Clean up old events outside the detection window
|
||||
CleanupOldEvents(tracker, now);
|
||||
|
||||
var eventCount = tracker.EventTimestamps.Count;
|
||||
|
||||
// Check if we're in storm mode
|
||||
if (eventCount >= _config.StormThreshold)
|
||||
{
|
||||
// Check if we should send a summary
|
||||
var shouldSendSummary = tracker.LastSummaryAt is null ||
|
||||
(now - tracker.LastSummaryAt.Value) >= _config.SummaryInterval;
|
||||
|
||||
if (shouldSendSummary)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Storm detected for {StormKey}: {EventCount} events in window, triggering summary",
|
||||
stormKey, eventCount);
|
||||
|
||||
return Task.FromResult(new StormDetectionResult
|
||||
{
|
||||
Decision = StormDecision.SendSummary,
|
||||
StormKey = stormKey,
|
||||
Reason = $"Storm threshold ({_config.StormThreshold}) reached with {eventCount} events",
|
||||
AccumulatedCount = eventCount,
|
||||
Threshold = _config.StormThreshold,
|
||||
WindowStart = tracker.WindowStart
|
||||
});
|
||||
}
|
||||
|
||||
_logger.LogDebug(
|
||||
"Storm active for {StormKey}: {EventCount} events, summary sent at {LastSummaryAt}",
|
||||
stormKey, eventCount, tracker.LastSummaryAt);
|
||||
|
||||
return Task.FromResult(new StormDetectionResult
|
||||
{
|
||||
Decision = StormDecision.SuppressedBySummary,
|
||||
StormKey = stormKey,
|
||||
Reason = $"Storm active, summary already sent at {tracker.LastSummaryAt}",
|
||||
AccumulatedCount = eventCount,
|
||||
Threshold = _config.StormThreshold,
|
||||
WindowStart = tracker.WindowStart,
|
||||
NextSummaryAt = tracker.LastSummaryAt?.Add(_config.SummaryInterval)
|
||||
});
|
||||
}
|
||||
|
||||
// Check if we're approaching storm threshold
|
||||
if (eventCount >= _config.StormThreshold - 1)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Storm threshold approaching for {StormKey}: {EventCount} events",
|
||||
stormKey, eventCount);
|
||||
|
||||
return Task.FromResult(new StormDetectionResult
|
||||
{
|
||||
Decision = StormDecision.SuppressAndAccumulate,
|
||||
StormKey = stormKey,
|
||||
Reason = $"Approaching storm threshold ({eventCount + 1}/{_config.StormThreshold})",
|
||||
AccumulatedCount = eventCount,
|
||||
Threshold = _config.StormThreshold,
|
||||
WindowStart = tracker.WindowStart
|
||||
});
|
||||
}
|
||||
|
||||
// Normal delivery
|
||||
return Task.FromResult(new StormDetectionResult
|
||||
{
|
||||
Decision = StormDecision.DeliverNormally,
|
||||
StormKey = stormKey,
|
||||
AccumulatedCount = eventCount,
|
||||
Threshold = _config.StormThreshold,
|
||||
WindowStart = tracker.WindowStart
|
||||
});
|
||||
}
|
||||
|
||||
public Task RecordEventAsync(
|
||||
string tenantId,
|
||||
NotifyEvent @event,
|
||||
NotifyRule rule,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
ArgumentNullException.ThrowIfNull(@event);
|
||||
ArgumentNullException.ThrowIfNull(rule);
|
||||
|
||||
var stormKey = ComputeStormKey(tenantId, @event.Kind, rule.RuleId);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
var tracker = _storms.GetOrAdd(stormKey, _ => new StormTracker
|
||||
{
|
||||
StormKey = stormKey,
|
||||
TenantId = tenantId,
|
||||
EventKind = @event.Kind,
|
||||
RuleId = rule.RuleId,
|
||||
WindowStart = now
|
||||
});
|
||||
|
||||
// Add event timestamp
|
||||
tracker.EventTimestamps.Add(now);
|
||||
tracker.LastEventAt = now;
|
||||
|
||||
// Track sample event IDs
|
||||
if (tracker.SampleEventIds.Count < _config.MaxSampleEvents)
|
||||
{
|
||||
tracker.SampleEventIds.Add(@event.EventId.ToString("N"));
|
||||
}
|
||||
|
||||
_logger.LogDebug(
|
||||
"Recorded event {EventId} for storm {StormKey}, count: {Count}",
|
||||
@event.EventId, stormKey, tracker.EventTimestamps.Count);
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<StormSummary?> TriggerSummaryAsync(
|
||||
string tenantId,
|
||||
string stormKey,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(stormKey);
|
||||
|
||||
if (!_storms.TryGetValue(stormKey, out var tracker))
|
||||
{
|
||||
return Task.FromResult<StormSummary?>(null);
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
CleanupOldEvents(tracker, now);
|
||||
|
||||
var summary = new StormSummary
|
||||
{
|
||||
SummaryId = Guid.NewGuid().ToString("N"),
|
||||
StormKey = stormKey,
|
||||
TenantId = tenantId,
|
||||
EventCount = tracker.EventTimestamps.Count,
|
||||
EventKind = tracker.EventKind,
|
||||
RuleId = tracker.RuleId,
|
||||
WindowStart = tracker.WindowStart,
|
||||
WindowEnd = now,
|
||||
SampleEventIds = tracker.SampleEventIds.ToArray(),
|
||||
GeneratedAt = now
|
||||
};
|
||||
|
||||
// Update tracker state
|
||||
tracker.LastSummaryAt = now;
|
||||
tracker.SummaryCount++;
|
||||
|
||||
// Reset window for next batch
|
||||
tracker.WindowStart = now;
|
||||
tracker.EventTimestamps.Clear();
|
||||
tracker.SampleEventIds.Clear();
|
||||
|
||||
_logger.LogInformation(
|
||||
"Generated storm summary {SummaryId} for {StormKey}: {EventCount} events",
|
||||
summary.SummaryId, stormKey, summary.EventCount);
|
||||
|
||||
return Task.FromResult<StormSummary?>(summary);
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<StormState>> GetActiveStormsAsync(
|
||||
string tenantId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var activeStorms = new List<StormState>();
|
||||
|
||||
foreach (var tracker in _storms.Values)
|
||||
{
|
||||
if (tracker.TenantId != tenantId)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
CleanupOldEvents(tracker, now);
|
||||
|
||||
if (tracker.EventTimestamps.Count == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
activeStorms.Add(new StormState
|
||||
{
|
||||
StormKey = tracker.StormKey,
|
||||
TenantId = tracker.TenantId,
|
||||
EventKind = tracker.EventKind,
|
||||
RuleId = tracker.RuleId,
|
||||
EventCount = tracker.EventTimestamps.Count,
|
||||
WindowStart = tracker.WindowStart,
|
||||
LastEventAt = tracker.LastEventAt,
|
||||
LastSummaryAt = tracker.LastSummaryAt,
|
||||
SummaryCount = tracker.SummaryCount
|
||||
});
|
||||
}
|
||||
|
||||
return Task.FromResult<IReadOnlyList<StormState>>(activeStorms);
|
||||
}
|
||||
|
||||
private void CleanupOldEvents(StormTracker tracker, DateTimeOffset now)
|
||||
{
|
||||
var cutoff = now - _config.DetectionWindow;
|
||||
tracker.EventTimestamps.RemoveAll(t => t < cutoff);
|
||||
|
||||
// Reset window if all events expired
|
||||
if (tracker.EventTimestamps.Count == 0)
|
||||
{
|
||||
tracker.WindowStart = now;
|
||||
tracker.SampleEventIds.Clear();
|
||||
}
|
||||
}
|
||||
|
||||
private static string ComputeStormKey(string tenantId, string eventKind, string ruleId)
|
||||
{
|
||||
return $"{tenantId}:{eventKind}:{ruleId}";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Internal tracker for storm state.
|
||||
/// </summary>
|
||||
private sealed class StormTracker
|
||||
{
|
||||
public required string StormKey { get; init; }
|
||||
public required string TenantId { get; init; }
|
||||
public required string EventKind { get; init; }
|
||||
public required string RuleId { get; init; }
|
||||
public DateTimeOffset WindowStart { get; set; }
|
||||
public DateTimeOffset LastEventAt { get; set; }
|
||||
public DateTimeOffset? LastSummaryAt { get; set; }
|
||||
public int SummaryCount { get; set; }
|
||||
public List<DateTimeOffset> EventTimestamps { get; } = [];
|
||||
public List<string> SampleEventIds { get; } = [];
|
||||
}
|
||||
}
|
||||
@@ -86,7 +86,7 @@ public sealed partial class EnhancedTemplateRenderer : INotifyTemplateRenderer
|
||||
["eventId"] = notifyEvent.EventId.ToString(),
|
||||
["kind"] = notifyEvent.Kind,
|
||||
["tenant"] = notifyEvent.Tenant,
|
||||
["timestamp"] = notifyEvent.Timestamp.ToString("O"),
|
||||
["timestamp"] = notifyEvent.Ts.ToString("O"),
|
||||
["actor"] = notifyEvent.Actor,
|
||||
["version"] = notifyEvent.Version,
|
||||
};
|
||||
@@ -305,7 +305,7 @@ public sealed partial class EnhancedTemplateRenderer : INotifyTemplateRenderer
|
||||
return format.ToLowerInvariant() switch
|
||||
{
|
||||
"json" => JsonSerializer.Serialize(value),
|
||||
"html" => HttpUtility.HtmlEncode(value.ToString()),
|
||||
"html" => HttpUtility.HtmlEncode(value.ToString() ?? string.Empty) ?? string.Empty,
|
||||
"url" => Uri.EscapeDataString(value.ToString() ?? string.Empty),
|
||||
"upper" => value.ToString()?.ToUpperInvariant() ?? string.Empty,
|
||||
"lower" => value.ToString()?.ToLowerInvariant() ?? string.Empty,
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using System.Text.RegularExpressions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notify.Storage.Mongo.Repositories;
|
||||
using StellaOps.Notifier.Worker.Storage;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Templates;
|
||||
|
||||
@@ -383,3 +383,4 @@ public sealed partial class NotifyTemplateService : INotifyTemplateService
|
||||
[GeneratedRegex(@"\{\{([^#/}][^}]*)\}\}")]
|
||||
private static partial Regex VariableRegex();
|
||||
}
|
||||
|
||||
|
||||
@@ -1,129 +0,0 @@
|
||||
namespace StellaOps.Notifier.Worker.Tenancy;
|
||||
|
||||
/// <summary>
|
||||
/// Provides tenant context for the current async scope.
|
||||
/// Uses AsyncLocal to flow tenant information through async operations.
|
||||
/// </summary>
|
||||
public interface ITenantContext
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the current tenant ID.
|
||||
/// </summary>
|
||||
string? TenantId { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the current actor (user or service).
|
||||
/// </summary>
|
||||
string? Actor { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Sets the tenant context for the current async scope.
|
||||
/// </summary>
|
||||
IDisposable SetContext(string tenantId, string? actor = null);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the current context as a snapshot.
|
||||
/// </summary>
|
||||
TenantContextSnapshot GetSnapshot();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Snapshot of tenant context for serialization.
|
||||
/// </summary>
|
||||
public sealed record TenantContextSnapshot(string TenantId, string? Actor);
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation using AsyncLocal for context propagation.
|
||||
/// </summary>
|
||||
public sealed class TenantContext : ITenantContext
|
||||
{
|
||||
private static readonly AsyncLocal<TenantContextHolder> _current = new();
|
||||
|
||||
public string? TenantId => _current.Value?.TenantId;
|
||||
public string? Actor => _current.Value?.Actor;
|
||||
|
||||
public IDisposable SetContext(string tenantId, string? actor = null)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
|
||||
var previous = _current.Value;
|
||||
_current.Value = new TenantContextHolder(tenantId, actor ?? "system");
|
||||
|
||||
return new ContextScope(previous);
|
||||
}
|
||||
|
||||
public TenantContextSnapshot GetSnapshot()
|
||||
{
|
||||
var holder = _current.Value;
|
||||
if (holder is null)
|
||||
{
|
||||
throw new InvalidOperationException("No tenant context is set for the current scope.");
|
||||
}
|
||||
|
||||
return new TenantContextSnapshot(holder.TenantId, holder.Actor);
|
||||
}
|
||||
|
||||
private sealed record TenantContextHolder(string TenantId, string Actor);
|
||||
|
||||
private sealed class ContextScope : IDisposable
|
||||
{
|
||||
private readonly TenantContextHolder? _previous;
|
||||
|
||||
public ContextScope(TenantContextHolder? previous)
|
||||
{
|
||||
_previous = previous;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_current.Value = _previous;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extension methods for tenant context.
|
||||
/// </summary>
|
||||
public static class TenantContextExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Requires a tenant context to be set, throwing if missing.
|
||||
/// </summary>
|
||||
public static string RequireTenantId(this ITenantContext context)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
return context.TenantId ?? throw new InvalidOperationException("Tenant context is required but not set.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes an action within a tenant context scope.
|
||||
/// </summary>
|
||||
public static async Task<T> WithTenantAsync<T>(
|
||||
this ITenantContext context,
|
||||
string tenantId,
|
||||
string? actor,
|
||||
Func<Task<T>> action)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
ArgumentNullException.ThrowIfNull(action);
|
||||
|
||||
using var scope = context.SetContext(tenantId, actor);
|
||||
return await action().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes an action within a tenant context scope.
|
||||
/// </summary>
|
||||
public static async Task WithTenantAsync(
|
||||
this ITenantContext context,
|
||||
string tenantId,
|
||||
string? actor,
|
||||
Func<Task> action)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
ArgumentNullException.ThrowIfNull(action);
|
||||
|
||||
using var scope = context.SetContext(tenantId, actor);
|
||||
await action().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,7 @@
|
||||
using System.Diagnostics;
|
||||
using System.Text.Json;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
@@ -49,7 +51,7 @@ public sealed class TenantMiddleware
|
||||
context.Request.Path);
|
||||
|
||||
context.Response.StatusCode = StatusCodes.Status400BadRequest;
|
||||
await context.Response.WriteAsJsonAsync(new
|
||||
await WriteJsonAsync(context.Response, new
|
||||
{
|
||||
error = new
|
||||
{
|
||||
@@ -73,7 +75,7 @@ public sealed class TenantMiddleware
|
||||
tenantContext.TenantId);
|
||||
|
||||
context.Response.StatusCode = StatusCodes.Status400BadRequest;
|
||||
await context.Response.WriteAsJsonAsync(new
|
||||
await WriteJsonAsync(context.Response, new
|
||||
{
|
||||
error = new
|
||||
{
|
||||
@@ -185,6 +187,13 @@ public sealed class TenantMiddleware
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private static Task WriteJsonAsync(HttpResponse response, object payload)
|
||||
{
|
||||
response.ContentType = "application/json";
|
||||
var json = JsonSerializer.Serialize(payload);
|
||||
return response.WriteAsync(json);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
Reference in New Issue
Block a user