up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
This commit is contained in:
@@ -1,407 +1,407 @@
|
||||
using System.Diagnostics;
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notifier.Worker.Storage;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Channels;
|
||||
|
||||
/// <summary>
|
||||
/// Channel adapter for Slack and Teams webhooks with retry policies.
|
||||
/// Handles Slack incoming webhooks and Teams connectors.
|
||||
/// </summary>
|
||||
public sealed class ChatWebhookChannelAdapter : IChannelAdapter
|
||||
{
|
||||
private readonly HttpClient _httpClient;
|
||||
private readonly INotifyAuditRepository _auditRepository;
|
||||
private readonly ChannelAdapterOptions _options;
|
||||
private readonly ILogger<ChatWebhookChannelAdapter> _logger;
|
||||
|
||||
public ChatWebhookChannelAdapter(
|
||||
HttpClient httpClient,
|
||||
INotifyAuditRepository auditRepository,
|
||||
IOptions<ChannelAdapterOptions> options,
|
||||
ILogger<ChatWebhookChannelAdapter> logger)
|
||||
{
|
||||
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
|
||||
_auditRepository = auditRepository ?? throw new ArgumentNullException(nameof(auditRepository));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
// Routes Slack type to this adapter; Teams uses Custom type
|
||||
public NotifyChannelType ChannelType => NotifyChannelType.Slack;
|
||||
|
||||
/// <summary>
|
||||
/// Determines if this adapter can handle the specified channel.
|
||||
/// </summary>
|
||||
public bool CanHandle(NotifyChannel channel)
|
||||
{
|
||||
return channel.Type is NotifyChannelType.Slack or NotifyChannelType.Teams;
|
||||
}
|
||||
|
||||
public async Task<ChannelDispatchResult> DispatchAsync(
|
||||
ChannelDispatchContext context,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
|
||||
var endpoint = context.Channel.Config.Endpoint;
|
||||
if (string.IsNullOrWhiteSpace(endpoint) || !Uri.TryCreate(endpoint, UriKind.Absolute, out var uri))
|
||||
{
|
||||
await AuditDispatchAsync(context, false, "Invalid webhook URL.", null, cancellationToken);
|
||||
return ChannelDispatchResult.Failed(
|
||||
"Chat webhook URL is not configured or invalid.",
|
||||
ChannelDispatchStatus.InvalidConfiguration);
|
||||
}
|
||||
|
||||
var isSlack = context.Channel.Type == NotifyChannelType.Slack || IsSlackWebhook(uri);
|
||||
var payload = isSlack
|
||||
? BuildSlackPayload(context)
|
||||
: BuildTeamsPayload(context);
|
||||
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
var attempt = 0;
|
||||
var maxRetries = _options.MaxRetries;
|
||||
Exception? lastException = null;
|
||||
int? lastStatusCode = null;
|
||||
|
||||
while (attempt <= maxRetries)
|
||||
{
|
||||
attempt++;
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
try
|
||||
{
|
||||
using var request = new HttpRequestMessage(HttpMethod.Post, uri);
|
||||
request.Content = new StringContent(payload, Encoding.UTF8, "application/json");
|
||||
request.Headers.UserAgent.Add(new ProductInfoHeaderValue("StellaOps-Notifier", "1.0"));
|
||||
|
||||
using var response = await _httpClient
|
||||
.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
lastStatusCode = (int)response.StatusCode;
|
||||
|
||||
if (response.IsSuccessStatusCode)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
var metadata = BuildSuccessMetadata(context, isSlack, attempt);
|
||||
await AuditDispatchAsync(context, true, null, metadata, cancellationToken);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Chat webhook delivery {DeliveryId} to {Platform} succeeded on attempt {Attempt}.",
|
||||
context.DeliveryId, isSlack ? "Slack" : "Teams", attempt);
|
||||
|
||||
return ChannelDispatchResult.Succeeded(
|
||||
message: $"Delivered to {(isSlack ? "Slack" : "Teams")}.",
|
||||
duration: stopwatch.Elapsed,
|
||||
metadata: metadata);
|
||||
}
|
||||
|
||||
if (response.StatusCode == HttpStatusCode.TooManyRequests)
|
||||
{
|
||||
var retryAfter = ParseRetryAfter(response.Headers);
|
||||
stopwatch.Stop();
|
||||
|
||||
await AuditDispatchAsync(context, false, "Rate limited.", null, cancellationToken);
|
||||
|
||||
_logger.LogWarning(
|
||||
"Chat webhook delivery {DeliveryId} throttled. Retry after: {RetryAfter}.",
|
||||
context.DeliveryId, retryAfter);
|
||||
|
||||
return ChannelDispatchResult.Throttled(
|
||||
$"Rate limited by {(isSlack ? "Slack" : "Teams")}.",
|
||||
retryAfter);
|
||||
}
|
||||
|
||||
if (!IsRetryable(response.StatusCode))
|
||||
{
|
||||
stopwatch.Stop();
|
||||
var responseBody = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
var errorMessage = $"Chat webhook returned {response.StatusCode}: {TruncateError(responseBody)}";
|
||||
await AuditDispatchAsync(context, false, errorMessage, null, cancellationToken);
|
||||
|
||||
_logger.LogWarning(
|
||||
"Chat webhook delivery {DeliveryId} failed: {StatusCode}.",
|
||||
context.DeliveryId, response.StatusCode);
|
||||
|
||||
return ChannelDispatchResult.Failed(
|
||||
errorMessage,
|
||||
httpStatusCode: lastStatusCode,
|
||||
duration: stopwatch.Elapsed);
|
||||
}
|
||||
|
||||
_logger.LogDebug(
|
||||
"Chat webhook delivery {DeliveryId} attempt {Attempt} returned {StatusCode}, will retry.",
|
||||
context.DeliveryId, attempt, response.StatusCode);
|
||||
}
|
||||
catch (HttpRequestException ex)
|
||||
{
|
||||
lastException = ex;
|
||||
_logger.LogDebug(
|
||||
ex,
|
||||
"Chat webhook delivery {DeliveryId} attempt {Attempt} failed with network error.",
|
||||
context.DeliveryId, attempt);
|
||||
}
|
||||
catch (TaskCanceledException ex) when (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
lastException = ex;
|
||||
_logger.LogDebug(
|
||||
"Chat webhook delivery {DeliveryId} attempt {Attempt} timed out.",
|
||||
context.DeliveryId, attempt);
|
||||
}
|
||||
|
||||
if (attempt <= maxRetries)
|
||||
{
|
||||
var delay = CalculateBackoff(attempt);
|
||||
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
stopwatch.Stop();
|
||||
var finalMessage = lastException?.Message ?? $"Failed after {maxRetries + 1} attempts.";
|
||||
await AuditDispatchAsync(context, false, finalMessage, null, cancellationToken);
|
||||
|
||||
_logger.LogError(
|
||||
lastException,
|
||||
"Chat webhook delivery {DeliveryId} exhausted all retries.",
|
||||
context.DeliveryId);
|
||||
|
||||
return ChannelDispatchResult.Failed(
|
||||
finalMessage,
|
||||
lastException is TaskCanceledException ? ChannelDispatchStatus.Timeout : ChannelDispatchStatus.NetworkError,
|
||||
httpStatusCode: lastStatusCode,
|
||||
exception: lastException,
|
||||
duration: stopwatch.Elapsed);
|
||||
}
|
||||
|
||||
public async Task<ChannelHealthCheckResult> CheckHealthAsync(
|
||||
NotifyChannel channel,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(channel);
|
||||
|
||||
var endpoint = channel.Config.Endpoint;
|
||||
if (string.IsNullOrWhiteSpace(endpoint) || !Uri.TryCreate(endpoint, UriKind.Absolute, out var uri))
|
||||
{
|
||||
return ChannelHealthCheckResult.Unhealthy("Webhook URL is not configured or invalid.");
|
||||
}
|
||||
|
||||
if (!channel.Enabled)
|
||||
{
|
||||
return ChannelHealthCheckResult.Degraded("Channel is disabled.");
|
||||
}
|
||||
|
||||
var isSlack = channel.Type == NotifyChannelType.Slack || IsSlackWebhook(uri);
|
||||
|
||||
// Slack/Teams webhooks don't support HEAD, so we just validate the URL format
|
||||
if (isSlack && !uri.Host.Contains("slack.com", StringComparison.OrdinalIgnoreCase) &&
|
||||
!uri.Host.Contains("hooks.slack.com", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return ChannelHealthCheckResult.Degraded(
|
||||
"Webhook URL doesn't appear to be a Slack webhook.");
|
||||
}
|
||||
|
||||
if (!isSlack && !uri.Host.Contains("webhook.office.com", StringComparison.OrdinalIgnoreCase) &&
|
||||
!uri.Host.Contains("outlook.office.com", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return ChannelHealthCheckResult.Degraded(
|
||||
"Webhook URL doesn't appear to be a Teams connector.");
|
||||
}
|
||||
|
||||
return ChannelHealthCheckResult.Ok(
|
||||
$"{(isSlack ? "Slack" : "Teams")} webhook URL validated.");
|
||||
}
|
||||
|
||||
private static bool IsSlackWebhook(Uri uri)
|
||||
{
|
||||
return uri.Host.Contains("slack.com", StringComparison.OrdinalIgnoreCase) ||
|
||||
uri.Host.Contains("hooks.slack.com", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static string BuildSlackPayload(ChannelDispatchContext context)
|
||||
{
|
||||
var message = new
|
||||
{
|
||||
text = context.Subject ?? "StellaOps Notification",
|
||||
blocks = new object[]
|
||||
{
|
||||
new
|
||||
{
|
||||
type = "section",
|
||||
text = new
|
||||
{
|
||||
type = "mrkdwn",
|
||||
text = context.RenderedBody
|
||||
}
|
||||
},
|
||||
new
|
||||
{
|
||||
type = "context",
|
||||
elements = new object[]
|
||||
{
|
||||
new
|
||||
{
|
||||
type = "mrkdwn",
|
||||
text = $"*Delivery ID:* {context.DeliveryId} | *Trace:* {context.TraceId}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
return JsonSerializer.Serialize(message, new JsonSerializerOptions { WriteIndented = false });
|
||||
}
|
||||
|
||||
private static string BuildTeamsPayload(ChannelDispatchContext context)
|
||||
{
|
||||
var card = new
|
||||
{
|
||||
type = "message",
|
||||
attachments = new object[]
|
||||
{
|
||||
new
|
||||
{
|
||||
contentType = "application/vnd.microsoft.card.adaptive",
|
||||
content = new
|
||||
{
|
||||
type = "AdaptiveCard",
|
||||
version = "1.4",
|
||||
body = new object[]
|
||||
{
|
||||
new
|
||||
{
|
||||
type = "TextBlock",
|
||||
text = context.Subject ?? "StellaOps Notification",
|
||||
weight = "bolder",
|
||||
size = "medium"
|
||||
},
|
||||
new
|
||||
{
|
||||
type = "TextBlock",
|
||||
text = context.RenderedBody,
|
||||
wrap = true
|
||||
},
|
||||
new
|
||||
{
|
||||
type = "FactSet",
|
||||
facts = new object[]
|
||||
{
|
||||
new { title = "Delivery ID", value = context.DeliveryId },
|
||||
new { title = "Trace ID", value = context.TraceId }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
return JsonSerializer.Serialize(card, new JsonSerializerOptions { WriteIndented = false });
|
||||
}
|
||||
|
||||
private static TimeSpan? ParseRetryAfter(HttpResponseHeaders headers)
|
||||
{
|
||||
if (headers.RetryAfter?.Delta is { } delta)
|
||||
{
|
||||
return delta;
|
||||
}
|
||||
|
||||
if (headers.RetryAfter?.Date is { } date)
|
||||
{
|
||||
var delay = date - DateTimeOffset.UtcNow;
|
||||
return delay > TimeSpan.Zero ? delay : null;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool IsRetryable(HttpStatusCode statusCode)
|
||||
{
|
||||
return statusCode switch
|
||||
{
|
||||
HttpStatusCode.RequestTimeout => true,
|
||||
HttpStatusCode.BadGateway => true,
|
||||
HttpStatusCode.ServiceUnavailable => true,
|
||||
HttpStatusCode.GatewayTimeout => true,
|
||||
_ => false
|
||||
};
|
||||
}
|
||||
|
||||
private TimeSpan CalculateBackoff(int attempt)
|
||||
{
|
||||
var baseDelay = _options.RetryBaseDelay;
|
||||
var maxDelay = _options.RetryMaxDelay;
|
||||
var jitter = Random.Shared.NextDouble() * 0.3 + 0.85;
|
||||
var delay = TimeSpan.FromMilliseconds(baseDelay.TotalMilliseconds * Math.Pow(2, attempt - 1) * jitter);
|
||||
return delay > maxDelay ? maxDelay : delay;
|
||||
}
|
||||
|
||||
private static string TruncateError(string error)
|
||||
{
|
||||
const int maxLength = 200;
|
||||
return error.Length > maxLength ? error[..maxLength] + "..." : error;
|
||||
}
|
||||
|
||||
private static Dictionary<string, string> BuildSuccessMetadata(
|
||||
ChannelDispatchContext context,
|
||||
bool isSlack,
|
||||
int attempt)
|
||||
{
|
||||
return new Dictionary<string, string>
|
||||
{
|
||||
["platform"] = isSlack ? "Slack" : "Teams",
|
||||
["attempt"] = attempt.ToString()
|
||||
};
|
||||
}
|
||||
|
||||
private async Task AuditDispatchAsync(
|
||||
ChannelDispatchContext context,
|
||||
bool success,
|
||||
string? errorMessage,
|
||||
IReadOnlyDictionary<string, string>? metadata,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var auditMetadata = new Dictionary<string, string>
|
||||
{
|
||||
["deliveryId"] = context.DeliveryId,
|
||||
["channelId"] = context.Channel.ChannelId,
|
||||
["channelType"] = context.Channel.Type.ToString(),
|
||||
["success"] = success.ToString().ToLowerInvariant(),
|
||||
["traceId"] = context.TraceId
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(errorMessage))
|
||||
{
|
||||
auditMetadata["error"] = errorMessage;
|
||||
}
|
||||
|
||||
if (metadata is not null)
|
||||
{
|
||||
foreach (var (key, value) in metadata)
|
||||
{
|
||||
auditMetadata[$"dispatch.{key}"] = value;
|
||||
}
|
||||
}
|
||||
|
||||
await _auditRepository.AppendAsync(
|
||||
context.TenantId,
|
||||
success ? "channel.dispatch.success" : "channel.dispatch.failure",
|
||||
"notifier-worker",
|
||||
auditMetadata,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to write dispatch audit for delivery {DeliveryId}.", context.DeliveryId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
using System.Diagnostics;
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notifier.Worker.Storage;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Channels;
|
||||
|
||||
/// <summary>
|
||||
/// Channel adapter for Slack and Teams webhooks with retry policies.
|
||||
/// Handles Slack incoming webhooks and Teams connectors.
|
||||
/// </summary>
|
||||
public sealed class ChatWebhookChannelAdapter : IChannelAdapter
|
||||
{
|
||||
private readonly HttpClient _httpClient;
|
||||
private readonly INotifyAuditRepository _auditRepository;
|
||||
private readonly ChannelAdapterOptions _options;
|
||||
private readonly ILogger<ChatWebhookChannelAdapter> _logger;
|
||||
|
||||
public ChatWebhookChannelAdapter(
|
||||
HttpClient httpClient,
|
||||
INotifyAuditRepository auditRepository,
|
||||
IOptions<ChannelAdapterOptions> options,
|
||||
ILogger<ChatWebhookChannelAdapter> logger)
|
||||
{
|
||||
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
|
||||
_auditRepository = auditRepository ?? throw new ArgumentNullException(nameof(auditRepository));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
// Routes Slack type to this adapter; Teams uses Custom type
|
||||
public NotifyChannelType ChannelType => NotifyChannelType.Slack;
|
||||
|
||||
/// <summary>
|
||||
/// Determines if this adapter can handle the specified channel.
|
||||
/// </summary>
|
||||
public bool CanHandle(NotifyChannel channel)
|
||||
{
|
||||
return channel.Type is NotifyChannelType.Slack or NotifyChannelType.Teams;
|
||||
}
|
||||
|
||||
public async Task<ChannelDispatchResult> DispatchAsync(
|
||||
ChannelDispatchContext context,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
|
||||
var endpoint = context.Channel.Config.Endpoint;
|
||||
if (string.IsNullOrWhiteSpace(endpoint) || !Uri.TryCreate(endpoint, UriKind.Absolute, out var uri))
|
||||
{
|
||||
await AuditDispatchAsync(context, false, "Invalid webhook URL.", null, cancellationToken);
|
||||
return ChannelDispatchResult.Failed(
|
||||
"Chat webhook URL is not configured or invalid.",
|
||||
ChannelDispatchStatus.InvalidConfiguration);
|
||||
}
|
||||
|
||||
var isSlack = context.Channel.Type == NotifyChannelType.Slack || IsSlackWebhook(uri);
|
||||
var payload = isSlack
|
||||
? BuildSlackPayload(context)
|
||||
: BuildTeamsPayload(context);
|
||||
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
var attempt = 0;
|
||||
var maxRetries = _options.MaxRetries;
|
||||
Exception? lastException = null;
|
||||
int? lastStatusCode = null;
|
||||
|
||||
while (attempt <= maxRetries)
|
||||
{
|
||||
attempt++;
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
try
|
||||
{
|
||||
using var request = new HttpRequestMessage(HttpMethod.Post, uri);
|
||||
request.Content = new StringContent(payload, Encoding.UTF8, "application/json");
|
||||
request.Headers.UserAgent.Add(new ProductInfoHeaderValue("StellaOps-Notifier", "1.0"));
|
||||
|
||||
using var response = await _httpClient
|
||||
.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
lastStatusCode = (int)response.StatusCode;
|
||||
|
||||
if (response.IsSuccessStatusCode)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
var metadata = BuildSuccessMetadata(context, isSlack, attempt);
|
||||
await AuditDispatchAsync(context, true, null, metadata, cancellationToken);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Chat webhook delivery {DeliveryId} to {Platform} succeeded on attempt {Attempt}.",
|
||||
context.DeliveryId, isSlack ? "Slack" : "Teams", attempt);
|
||||
|
||||
return ChannelDispatchResult.Succeeded(
|
||||
message: $"Delivered to {(isSlack ? "Slack" : "Teams")}.",
|
||||
duration: stopwatch.Elapsed,
|
||||
metadata: metadata);
|
||||
}
|
||||
|
||||
if (response.StatusCode == HttpStatusCode.TooManyRequests)
|
||||
{
|
||||
var retryAfter = ParseRetryAfter(response.Headers);
|
||||
stopwatch.Stop();
|
||||
|
||||
await AuditDispatchAsync(context, false, "Rate limited.", null, cancellationToken);
|
||||
|
||||
_logger.LogWarning(
|
||||
"Chat webhook delivery {DeliveryId} throttled. Retry after: {RetryAfter}.",
|
||||
context.DeliveryId, retryAfter);
|
||||
|
||||
return ChannelDispatchResult.Throttled(
|
||||
$"Rate limited by {(isSlack ? "Slack" : "Teams")}.",
|
||||
retryAfter);
|
||||
}
|
||||
|
||||
if (!IsRetryable(response.StatusCode))
|
||||
{
|
||||
stopwatch.Stop();
|
||||
var responseBody = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
var errorMessage = $"Chat webhook returned {response.StatusCode}: {TruncateError(responseBody)}";
|
||||
await AuditDispatchAsync(context, false, errorMessage, null, cancellationToken);
|
||||
|
||||
_logger.LogWarning(
|
||||
"Chat webhook delivery {DeliveryId} failed: {StatusCode}.",
|
||||
context.DeliveryId, response.StatusCode);
|
||||
|
||||
return ChannelDispatchResult.Failed(
|
||||
errorMessage,
|
||||
httpStatusCode: lastStatusCode,
|
||||
duration: stopwatch.Elapsed);
|
||||
}
|
||||
|
||||
_logger.LogDebug(
|
||||
"Chat webhook delivery {DeliveryId} attempt {Attempt} returned {StatusCode}, will retry.",
|
||||
context.DeliveryId, attempt, response.StatusCode);
|
||||
}
|
||||
catch (HttpRequestException ex)
|
||||
{
|
||||
lastException = ex;
|
||||
_logger.LogDebug(
|
||||
ex,
|
||||
"Chat webhook delivery {DeliveryId} attempt {Attempt} failed with network error.",
|
||||
context.DeliveryId, attempt);
|
||||
}
|
||||
catch (TaskCanceledException ex) when (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
lastException = ex;
|
||||
_logger.LogDebug(
|
||||
"Chat webhook delivery {DeliveryId} attempt {Attempt} timed out.",
|
||||
context.DeliveryId, attempt);
|
||||
}
|
||||
|
||||
if (attempt <= maxRetries)
|
||||
{
|
||||
var delay = CalculateBackoff(attempt);
|
||||
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
stopwatch.Stop();
|
||||
var finalMessage = lastException?.Message ?? $"Failed after {maxRetries + 1} attempts.";
|
||||
await AuditDispatchAsync(context, false, finalMessage, null, cancellationToken);
|
||||
|
||||
_logger.LogError(
|
||||
lastException,
|
||||
"Chat webhook delivery {DeliveryId} exhausted all retries.",
|
||||
context.DeliveryId);
|
||||
|
||||
return ChannelDispatchResult.Failed(
|
||||
finalMessage,
|
||||
lastException is TaskCanceledException ? ChannelDispatchStatus.Timeout : ChannelDispatchStatus.NetworkError,
|
||||
httpStatusCode: lastStatusCode,
|
||||
exception: lastException,
|
||||
duration: stopwatch.Elapsed);
|
||||
}
|
||||
|
||||
public async Task<ChannelHealthCheckResult> CheckHealthAsync(
|
||||
NotifyChannel channel,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(channel);
|
||||
|
||||
var endpoint = channel.Config.Endpoint;
|
||||
if (string.IsNullOrWhiteSpace(endpoint) || !Uri.TryCreate(endpoint, UriKind.Absolute, out var uri))
|
||||
{
|
||||
return ChannelHealthCheckResult.Unhealthy("Webhook URL is not configured or invalid.");
|
||||
}
|
||||
|
||||
if (!channel.Enabled)
|
||||
{
|
||||
return ChannelHealthCheckResult.Degraded("Channel is disabled.");
|
||||
}
|
||||
|
||||
var isSlack = channel.Type == NotifyChannelType.Slack || IsSlackWebhook(uri);
|
||||
|
||||
// Slack/Teams webhooks don't support HEAD, so we just validate the URL format
|
||||
if (isSlack && !uri.Host.Contains("slack.com", StringComparison.OrdinalIgnoreCase) &&
|
||||
!uri.Host.Contains("hooks.slack.com", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return ChannelHealthCheckResult.Degraded(
|
||||
"Webhook URL doesn't appear to be a Slack webhook.");
|
||||
}
|
||||
|
||||
if (!isSlack && !uri.Host.Contains("webhook.office.com", StringComparison.OrdinalIgnoreCase) &&
|
||||
!uri.Host.Contains("outlook.office.com", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return ChannelHealthCheckResult.Degraded(
|
||||
"Webhook URL doesn't appear to be a Teams connector.");
|
||||
}
|
||||
|
||||
return ChannelHealthCheckResult.Ok(
|
||||
$"{(isSlack ? "Slack" : "Teams")} webhook URL validated.");
|
||||
}
|
||||
|
||||
private static bool IsSlackWebhook(Uri uri)
|
||||
{
|
||||
return uri.Host.Contains("slack.com", StringComparison.OrdinalIgnoreCase) ||
|
||||
uri.Host.Contains("hooks.slack.com", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static string BuildSlackPayload(ChannelDispatchContext context)
|
||||
{
|
||||
var message = new
|
||||
{
|
||||
text = context.Subject ?? "StellaOps Notification",
|
||||
blocks = new object[]
|
||||
{
|
||||
new
|
||||
{
|
||||
type = "section",
|
||||
text = new
|
||||
{
|
||||
type = "mrkdwn",
|
||||
text = context.RenderedBody
|
||||
}
|
||||
},
|
||||
new
|
||||
{
|
||||
type = "context",
|
||||
elements = new object[]
|
||||
{
|
||||
new
|
||||
{
|
||||
type = "mrkdwn",
|
||||
text = $"*Delivery ID:* {context.DeliveryId} | *Trace:* {context.TraceId}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
return JsonSerializer.Serialize(message, new JsonSerializerOptions { WriteIndented = false });
|
||||
}
|
||||
|
||||
private static string BuildTeamsPayload(ChannelDispatchContext context)
|
||||
{
|
||||
var card = new
|
||||
{
|
||||
type = "message",
|
||||
attachments = new object[]
|
||||
{
|
||||
new
|
||||
{
|
||||
contentType = "application/vnd.microsoft.card.adaptive",
|
||||
content = new
|
||||
{
|
||||
type = "AdaptiveCard",
|
||||
version = "1.4",
|
||||
body = new object[]
|
||||
{
|
||||
new
|
||||
{
|
||||
type = "TextBlock",
|
||||
text = context.Subject ?? "StellaOps Notification",
|
||||
weight = "bolder",
|
||||
size = "medium"
|
||||
},
|
||||
new
|
||||
{
|
||||
type = "TextBlock",
|
||||
text = context.RenderedBody,
|
||||
wrap = true
|
||||
},
|
||||
new
|
||||
{
|
||||
type = "FactSet",
|
||||
facts = new object[]
|
||||
{
|
||||
new { title = "Delivery ID", value = context.DeliveryId },
|
||||
new { title = "Trace ID", value = context.TraceId }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
return JsonSerializer.Serialize(card, new JsonSerializerOptions { WriteIndented = false });
|
||||
}
|
||||
|
||||
private static TimeSpan? ParseRetryAfter(HttpResponseHeaders headers)
|
||||
{
|
||||
if (headers.RetryAfter?.Delta is { } delta)
|
||||
{
|
||||
return delta;
|
||||
}
|
||||
|
||||
if (headers.RetryAfter?.Date is { } date)
|
||||
{
|
||||
var delay = date - DateTimeOffset.UtcNow;
|
||||
return delay > TimeSpan.Zero ? delay : null;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool IsRetryable(HttpStatusCode statusCode)
|
||||
{
|
||||
return statusCode switch
|
||||
{
|
||||
HttpStatusCode.RequestTimeout => true,
|
||||
HttpStatusCode.BadGateway => true,
|
||||
HttpStatusCode.ServiceUnavailable => true,
|
||||
HttpStatusCode.GatewayTimeout => true,
|
||||
_ => false
|
||||
};
|
||||
}
|
||||
|
||||
private TimeSpan CalculateBackoff(int attempt)
|
||||
{
|
||||
var baseDelay = _options.RetryBaseDelay;
|
||||
var maxDelay = _options.RetryMaxDelay;
|
||||
var jitter = Random.Shared.NextDouble() * 0.3 + 0.85;
|
||||
var delay = TimeSpan.FromMilliseconds(baseDelay.TotalMilliseconds * Math.Pow(2, attempt - 1) * jitter);
|
||||
return delay > maxDelay ? maxDelay : delay;
|
||||
}
|
||||
|
||||
private static string TruncateError(string error)
|
||||
{
|
||||
const int maxLength = 200;
|
||||
return error.Length > maxLength ? error[..maxLength] + "..." : error;
|
||||
}
|
||||
|
||||
private static Dictionary<string, string> BuildSuccessMetadata(
|
||||
ChannelDispatchContext context,
|
||||
bool isSlack,
|
||||
int attempt)
|
||||
{
|
||||
return new Dictionary<string, string>
|
||||
{
|
||||
["platform"] = isSlack ? "Slack" : "Teams",
|
||||
["attempt"] = attempt.ToString()
|
||||
};
|
||||
}
|
||||
|
||||
private async Task AuditDispatchAsync(
|
||||
ChannelDispatchContext context,
|
||||
bool success,
|
||||
string? errorMessage,
|
||||
IReadOnlyDictionary<string, string>? metadata,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var auditMetadata = new Dictionary<string, string>
|
||||
{
|
||||
["deliveryId"] = context.DeliveryId,
|
||||
["channelId"] = context.Channel.ChannelId,
|
||||
["channelType"] = context.Channel.Type.ToString(),
|
||||
["success"] = success.ToString().ToLowerInvariant(),
|
||||
["traceId"] = context.TraceId
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(errorMessage))
|
||||
{
|
||||
auditMetadata["error"] = errorMessage;
|
||||
}
|
||||
|
||||
if (metadata is not null)
|
||||
{
|
||||
foreach (var (key, value) in metadata)
|
||||
{
|
||||
auditMetadata[$"dispatch.{key}"] = value;
|
||||
}
|
||||
}
|
||||
|
||||
await _auditRepository.AppendAsync(
|
||||
context.TenantId,
|
||||
success ? "channel.dispatch.success" : "channel.dispatch.failure",
|
||||
"notifier-worker",
|
||||
auditMetadata,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to write dispatch audit for delivery {DeliveryId}.", context.DeliveryId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user