411 lines
15 KiB
C#
411 lines
15 KiB
C#
using System.Diagnostics;
|
|
using System.Net;
|
|
using System.Net.Http.Headers;
|
|
using System.Text;
|
|
using System.Text.Json;
|
|
using Microsoft.Extensions.Logging;
|
|
using Microsoft.Extensions.Options;
|
|
using StellaOps.Notify.Models;
|
|
using StellaOps.Notifier.Worker.Storage;
|
|
|
|
namespace StellaOps.Notifier.Worker.Channels;
|
|
|
|
/// <summary>
|
|
/// Channel adapter for Slack and Teams webhooks with retry policies.
|
|
/// Handles Slack incoming webhooks and Teams connectors.
|
|
/// </summary>
|
|
public sealed class ChatWebhookChannelAdapter : IChannelAdapter
|
|
{
|
|
private readonly HttpClient _httpClient;
|
|
private readonly INotifyAuditRepository _auditRepository;
|
|
private readonly ChannelAdapterOptions _options;
|
|
private readonly ILogger<ChatWebhookChannelAdapter> _logger;
|
|
private readonly Func<double> _jitterSource;
|
|
|
|
public ChatWebhookChannelAdapter(
|
|
HttpClient httpClient,
|
|
INotifyAuditRepository auditRepository,
|
|
IOptions<ChannelAdapterOptions> options,
|
|
ILogger<ChatWebhookChannelAdapter> logger,
|
|
Func<double>? jitterSource = null)
|
|
{
|
|
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
|
|
_auditRepository = auditRepository ?? throw new ArgumentNullException(nameof(auditRepository));
|
|
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
|
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
|
_jitterSource = jitterSource ?? Random.Shared.NextDouble;
|
|
}
|
|
|
|
// Routes Slack type to this adapter; Teams uses Custom type
|
|
public NotifyChannelType ChannelType => NotifyChannelType.Slack;
|
|
|
|
/// <summary>
|
|
/// Determines if this adapter can handle the specified channel.
|
|
/// </summary>
|
|
public bool CanHandle(NotifyChannel channel)
|
|
{
|
|
return channel.Type is NotifyChannelType.Slack or NotifyChannelType.Teams;
|
|
}
|
|
|
|
public async Task<ChannelDispatchResult> DispatchAsync(
|
|
ChannelDispatchContext context,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(context);
|
|
|
|
var endpoint = context.Channel.Config.Endpoint;
|
|
if (string.IsNullOrWhiteSpace(endpoint) || !Uri.TryCreate(endpoint, UriKind.Absolute, out var uri))
|
|
{
|
|
await AuditDispatchAsync(context, false, "Invalid webhook URL.", null, cancellationToken);
|
|
return ChannelDispatchResult.Failed(
|
|
"Chat webhook URL is not configured or invalid.",
|
|
ChannelDispatchStatus.InvalidConfiguration);
|
|
}
|
|
|
|
var isSlack = context.Channel.Type == NotifyChannelType.Slack || IsSlackWebhook(uri);
|
|
var payload = isSlack
|
|
? BuildSlackPayload(context)
|
|
: BuildTeamsPayload(context);
|
|
|
|
var stopwatch = Stopwatch.StartNew();
|
|
var attempt = 0;
|
|
var maxRetries = _options.MaxRetries;
|
|
Exception? lastException = null;
|
|
int? lastStatusCode = null;
|
|
|
|
while (attempt <= maxRetries)
|
|
{
|
|
attempt++;
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
|
|
try
|
|
{
|
|
using var request = new HttpRequestMessage(HttpMethod.Post, uri);
|
|
request.Content = new StringContent(payload, Encoding.UTF8, "application/json");
|
|
request.Headers.UserAgent.Add(new ProductInfoHeaderValue("StellaOps-Notifier", "1.0"));
|
|
|
|
using var response = await _httpClient
|
|
.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
|
|
lastStatusCode = (int)response.StatusCode;
|
|
|
|
if (response.IsSuccessStatusCode)
|
|
{
|
|
stopwatch.Stop();
|
|
var metadata = BuildSuccessMetadata(context, isSlack, attempt);
|
|
await AuditDispatchAsync(context, true, null, metadata, cancellationToken);
|
|
|
|
_logger.LogInformation(
|
|
"Chat webhook delivery {DeliveryId} to {Platform} succeeded on attempt {Attempt}.",
|
|
context.DeliveryId, isSlack ? "Slack" : "Teams", attempt);
|
|
|
|
return ChannelDispatchResult.Succeeded(
|
|
message: $"Delivered to {(isSlack ? "Slack" : "Teams")}.",
|
|
duration: stopwatch.Elapsed,
|
|
metadata: metadata);
|
|
}
|
|
|
|
if (response.StatusCode == HttpStatusCode.TooManyRequests)
|
|
{
|
|
var retryAfter = ParseRetryAfter(response.Headers);
|
|
stopwatch.Stop();
|
|
|
|
await AuditDispatchAsync(context, false, "Rate limited.", null, cancellationToken);
|
|
|
|
_logger.LogWarning(
|
|
"Chat webhook delivery {DeliveryId} throttled. Retry after: {RetryAfter}.",
|
|
context.DeliveryId, retryAfter);
|
|
|
|
return ChannelDispatchResult.Throttled(
|
|
$"Rate limited by {(isSlack ? "Slack" : "Teams")}.",
|
|
retryAfter);
|
|
}
|
|
|
|
if (!IsRetryable(response.StatusCode))
|
|
{
|
|
stopwatch.Stop();
|
|
var responseBody = await response.Content.ReadAsStringAsync(cancellationToken);
|
|
var errorMessage = $"Chat webhook returned {response.StatusCode}: {TruncateError(responseBody)}";
|
|
await AuditDispatchAsync(context, false, errorMessage, null, cancellationToken);
|
|
|
|
_logger.LogWarning(
|
|
"Chat webhook delivery {DeliveryId} failed: {StatusCode}.",
|
|
context.DeliveryId, response.StatusCode);
|
|
|
|
return ChannelDispatchResult.Failed(
|
|
errorMessage,
|
|
httpStatusCode: lastStatusCode,
|
|
duration: stopwatch.Elapsed);
|
|
}
|
|
|
|
_logger.LogDebug(
|
|
"Chat webhook delivery {DeliveryId} attempt {Attempt} returned {StatusCode}, will retry.",
|
|
context.DeliveryId, attempt, response.StatusCode);
|
|
}
|
|
catch (HttpRequestException ex)
|
|
{
|
|
lastException = ex;
|
|
_logger.LogDebug(
|
|
ex,
|
|
"Chat webhook delivery {DeliveryId} attempt {Attempt} failed with network error.",
|
|
context.DeliveryId, attempt);
|
|
}
|
|
catch (TaskCanceledException ex) when (!cancellationToken.IsCancellationRequested)
|
|
{
|
|
lastException = ex;
|
|
_logger.LogDebug(
|
|
"Chat webhook delivery {DeliveryId} attempt {Attempt} timed out.",
|
|
context.DeliveryId, attempt);
|
|
}
|
|
|
|
if (attempt <= maxRetries)
|
|
{
|
|
var delay = CalculateBackoff(attempt);
|
|
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
}
|
|
|
|
stopwatch.Stop();
|
|
var finalMessage = lastException?.Message ?? $"Failed after {maxRetries + 1} attempts.";
|
|
await AuditDispatchAsync(context, false, finalMessage, null, cancellationToken);
|
|
|
|
_logger.LogError(
|
|
lastException,
|
|
"Chat webhook delivery {DeliveryId} exhausted all retries.",
|
|
context.DeliveryId);
|
|
|
|
return ChannelDispatchResult.Failed(
|
|
finalMessage,
|
|
lastException is TaskCanceledException ? ChannelDispatchStatus.Timeout : ChannelDispatchStatus.NetworkError,
|
|
httpStatusCode: lastStatusCode,
|
|
exception: lastException,
|
|
duration: stopwatch.Elapsed);
|
|
}
|
|
|
|
public async Task<ChannelHealthCheckResult> CheckHealthAsync(
|
|
NotifyChannel channel,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(channel);
|
|
|
|
var endpoint = channel.Config.Endpoint;
|
|
if (string.IsNullOrWhiteSpace(endpoint) || !Uri.TryCreate(endpoint, UriKind.Absolute, out var uri))
|
|
{
|
|
return ChannelHealthCheckResult.Unhealthy("Webhook URL is not configured or invalid.");
|
|
}
|
|
|
|
if (!channel.Enabled)
|
|
{
|
|
return ChannelHealthCheckResult.Degraded("Channel is disabled.");
|
|
}
|
|
|
|
var isSlack = channel.Type == NotifyChannelType.Slack || IsSlackWebhook(uri);
|
|
|
|
// Slack/Teams webhooks don't support HEAD, so we just validate the URL format
|
|
if (isSlack && !uri.Host.Contains("slack.com", StringComparison.OrdinalIgnoreCase) &&
|
|
!uri.Host.Contains("hooks.slack.com", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
return ChannelHealthCheckResult.Degraded(
|
|
"Webhook URL doesn't appear to be a Slack webhook.");
|
|
}
|
|
|
|
if (!isSlack && !uri.Host.Contains("webhook.office.com", StringComparison.OrdinalIgnoreCase) &&
|
|
!uri.Host.Contains("outlook.office.com", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
return ChannelHealthCheckResult.Degraded(
|
|
"Webhook URL doesn't appear to be a Teams connector.");
|
|
}
|
|
|
|
return ChannelHealthCheckResult.Ok(
|
|
$"{(isSlack ? "Slack" : "Teams")} webhook URL validated.");
|
|
}
|
|
|
|
private static bool IsSlackWebhook(Uri uri)
|
|
{
|
|
return uri.Host.Contains("slack.com", StringComparison.OrdinalIgnoreCase) ||
|
|
uri.Host.Contains("hooks.slack.com", StringComparison.OrdinalIgnoreCase);
|
|
}
|
|
|
|
private static string BuildSlackPayload(ChannelDispatchContext context)
|
|
{
|
|
var message = new
|
|
{
|
|
text = context.Subject ?? "StellaOps Notification",
|
|
blocks = new object[]
|
|
{
|
|
new
|
|
{
|
|
type = "section",
|
|
text = new
|
|
{
|
|
type = "mrkdwn",
|
|
text = context.RenderedBody
|
|
}
|
|
},
|
|
new
|
|
{
|
|
type = "context",
|
|
elements = new object[]
|
|
{
|
|
new
|
|
{
|
|
type = "mrkdwn",
|
|
text = $"*Delivery ID:* {context.DeliveryId} | *Trace:* {context.TraceId}"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
return JsonSerializer.Serialize(message, new JsonSerializerOptions { WriteIndented = false });
|
|
}
|
|
|
|
private static string BuildTeamsPayload(ChannelDispatchContext context)
|
|
{
|
|
var card = new
|
|
{
|
|
type = "message",
|
|
attachments = new object[]
|
|
{
|
|
new
|
|
{
|
|
contentType = "application/vnd.microsoft.card.adaptive",
|
|
content = new
|
|
{
|
|
type = "AdaptiveCard",
|
|
version = "1.4",
|
|
body = new object[]
|
|
{
|
|
new
|
|
{
|
|
type = "TextBlock",
|
|
text = context.Subject ?? "StellaOps Notification",
|
|
weight = "bolder",
|
|
size = "medium"
|
|
},
|
|
new
|
|
{
|
|
type = "TextBlock",
|
|
text = context.RenderedBody,
|
|
wrap = true
|
|
},
|
|
new
|
|
{
|
|
type = "FactSet",
|
|
facts = new object[]
|
|
{
|
|
new { title = "Delivery ID", value = context.DeliveryId },
|
|
new { title = "Trace ID", value = context.TraceId }
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
return JsonSerializer.Serialize(card, new JsonSerializerOptions { WriteIndented = false });
|
|
}
|
|
|
|
private static TimeSpan? ParseRetryAfter(HttpResponseHeaders headers)
|
|
{
|
|
if (headers.RetryAfter?.Delta is { } delta)
|
|
{
|
|
return delta;
|
|
}
|
|
|
|
if (headers.RetryAfter?.Date is { } date)
|
|
{
|
|
var delay = date - DateTimeOffset.UtcNow;
|
|
return delay > TimeSpan.Zero ? delay : null;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
private static bool IsRetryable(HttpStatusCode statusCode)
|
|
{
|
|
return statusCode switch
|
|
{
|
|
HttpStatusCode.RequestTimeout => true,
|
|
HttpStatusCode.BadGateway => true,
|
|
HttpStatusCode.ServiceUnavailable => true,
|
|
HttpStatusCode.GatewayTimeout => true,
|
|
_ => false
|
|
};
|
|
}
|
|
|
|
private TimeSpan CalculateBackoff(int attempt)
|
|
{
|
|
var baseDelay = _options.RetryBaseDelay;
|
|
var maxDelay = _options.RetryMaxDelay;
|
|
var jitter = _jitterSource() * 0.3 + 0.85;
|
|
var delay = TimeSpan.FromMilliseconds(baseDelay.TotalMilliseconds * Math.Pow(2, attempt - 1) * jitter);
|
|
return delay > maxDelay ? maxDelay : delay;
|
|
}
|
|
|
|
private static string TruncateError(string error)
|
|
{
|
|
const int maxLength = 200;
|
|
return error.Length > maxLength ? error[..maxLength] + "..." : error;
|
|
}
|
|
|
|
private static Dictionary<string, string> BuildSuccessMetadata(
|
|
ChannelDispatchContext context,
|
|
bool isSlack,
|
|
int attempt)
|
|
{
|
|
return new Dictionary<string, string>
|
|
{
|
|
["platform"] = isSlack ? "Slack" : "Teams",
|
|
["attempt"] = attempt.ToString()
|
|
};
|
|
}
|
|
|
|
private async Task AuditDispatchAsync(
|
|
ChannelDispatchContext context,
|
|
bool success,
|
|
string? errorMessage,
|
|
IReadOnlyDictionary<string, string>? metadata,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
try
|
|
{
|
|
var auditMetadata = new Dictionary<string, string>
|
|
{
|
|
["deliveryId"] = context.DeliveryId,
|
|
["channelId"] = context.Channel.ChannelId,
|
|
["channelType"] = context.Channel.Type.ToString(),
|
|
["success"] = success.ToString().ToLowerInvariant(),
|
|
["traceId"] = context.TraceId
|
|
};
|
|
|
|
if (!string.IsNullOrWhiteSpace(errorMessage))
|
|
{
|
|
auditMetadata["error"] = errorMessage;
|
|
}
|
|
|
|
if (metadata is not null)
|
|
{
|
|
foreach (var (key, value) in metadata)
|
|
{
|
|
auditMetadata[$"dispatch.{key}"] = value;
|
|
}
|
|
}
|
|
|
|
await _auditRepository.AppendAsync(
|
|
context.TenantId,
|
|
success ? "channel.dispatch.success" : "channel.dispatch.failure",
|
|
"notifier-worker",
|
|
auditMetadata,
|
|
cancellationToken).ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogWarning(ex, "Failed to write dispatch audit for delivery {DeliveryId}.", context.DeliveryId);
|
|
}
|
|
}
|
|
}
|
|
|