Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
api-governance / spectral-lint (push) Has been cancelled
oas-ci / oas-validate (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Policy Simulation / policy-simulate (push) Has been cancelled
SDK Publish & Sign / sdk-publish (push) Has been cancelled
492 lines
15 KiB
C#
492 lines
15 KiB
C#
using System.Net.Http.Json;
|
|
using System.Text.Json;
|
|
using System.Text.Json.Nodes;
|
|
using Microsoft.Extensions.Logging;
|
|
using Microsoft.Extensions.Options;
|
|
|
|
namespace StellaOps.Notifier.Worker.Escalation;
|
|
|
|
/// <summary>
|
|
/// Interface for external incident management integrations.
|
|
/// </summary>
|
|
public interface IExternalIntegrationAdapter
|
|
{
|
|
/// <summary>
|
|
/// Creates an incident/alert in the external system.
|
|
/// </summary>
|
|
Task<ExternalIncidentResult> CreateIncidentAsync(
|
|
ExternalIncidentRequest request,
|
|
CancellationToken cancellationToken = default);
|
|
|
|
/// <summary>
|
|
/// Acknowledges an incident/alert in the external system.
|
|
/// </summary>
|
|
Task<bool> AcknowledgeAsync(
|
|
string externalId,
|
|
string acknowledgedBy,
|
|
CancellationToken cancellationToken = default);
|
|
|
|
/// <summary>
|
|
/// Resolves an incident/alert in the external system.
|
|
/// </summary>
|
|
Task<bool> ResolveAsync(
|
|
string externalId,
|
|
string resolvedBy,
|
|
CancellationToken cancellationToken = default);
|
|
|
|
/// <summary>
|
|
/// Parses a webhook payload from the external system.
|
|
/// </summary>
|
|
AckBridgeRequest? ParseWebhook(string payload, IDictionary<string, string>? headers = null);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Request to create an external incident.
|
|
/// </summary>
|
|
public sealed record ExternalIncidentRequest
|
|
{
|
|
/// <summary>
|
|
/// Internal incident ID.
|
|
/// </summary>
|
|
public required string IncidentId { get; init; }
|
|
|
|
/// <summary>
|
|
/// Tenant ID.
|
|
/// </summary>
|
|
public required string TenantId { get; init; }
|
|
|
|
/// <summary>
|
|
/// Incident title/summary.
|
|
/// </summary>
|
|
public required string Title { get; init; }
|
|
|
|
/// <summary>
|
|
/// Detailed description.
|
|
/// </summary>
|
|
public string? Description { get; init; }
|
|
|
|
/// <summary>
|
|
/// Severity level.
|
|
/// </summary>
|
|
public string? Severity { get; init; }
|
|
|
|
/// <summary>
|
|
/// Source/service name.
|
|
/// </summary>
|
|
public string? Source { get; init; }
|
|
|
|
/// <summary>
|
|
/// Additional details.
|
|
/// </summary>
|
|
public IDictionary<string, string>? Details { get; init; }
|
|
|
|
/// <summary>
|
|
/// Integration-specific routing key/service.
|
|
/// </summary>
|
|
public string? RoutingKey { get; init; }
|
|
}
|
|
|
|
/// <summary>
|
|
/// Result of creating an external incident.
|
|
/// </summary>
|
|
public sealed record ExternalIncidentResult
|
|
{
|
|
/// <summary>
|
|
/// Whether creation was successful.
|
|
/// </summary>
|
|
public required bool Success { get; init; }
|
|
|
|
/// <summary>
|
|
/// External incident/alert ID.
|
|
/// </summary>
|
|
public string? ExternalId { get; init; }
|
|
|
|
/// <summary>
|
|
/// URL to view the incident.
|
|
/// </summary>
|
|
public string? Url { get; init; }
|
|
|
|
/// <summary>
|
|
/// Error message if not successful.
|
|
/// </summary>
|
|
public string? Error { get; init; }
|
|
|
|
public static ExternalIncidentResult Succeeded(string externalId, string? url = null) => new()
|
|
{
|
|
Success = true,
|
|
ExternalId = externalId,
|
|
Url = url
|
|
};
|
|
|
|
public static ExternalIncidentResult Failed(string error) => new()
|
|
{
|
|
Success = false,
|
|
Error = error
|
|
};
|
|
}
|
|
|
|
/// <summary>
|
|
/// PagerDuty integration adapter.
|
|
/// </summary>
|
|
public sealed class PagerDutyAdapter : IExternalIntegrationAdapter
|
|
{
|
|
private readonly HttpClient _httpClient;
|
|
private readonly PagerDutyOptions _options;
|
|
private readonly ILogger<PagerDutyAdapter> _logger;
|
|
|
|
public PagerDutyAdapter(
|
|
HttpClient httpClient,
|
|
IOptions<PagerDutyOptions> options,
|
|
ILogger<PagerDutyAdapter> logger)
|
|
{
|
|
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
|
|
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
|
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
|
|
|
_httpClient.BaseAddress = new Uri("https://events.pagerduty.com");
|
|
}
|
|
|
|
public async Task<ExternalIncidentResult> CreateIncidentAsync(
|
|
ExternalIncidentRequest request,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var routingKey = request.RoutingKey ?? _options.DefaultRoutingKey;
|
|
if (string.IsNullOrEmpty(routingKey))
|
|
{
|
|
return ExternalIncidentResult.Failed("PagerDuty routing key not configured");
|
|
}
|
|
|
|
var payload = new
|
|
{
|
|
routing_key = routingKey,
|
|
event_action = "trigger",
|
|
dedup_key = $"stellaops-{request.TenantId}-{request.IncidentId}",
|
|
payload = new
|
|
{
|
|
summary = request.Title,
|
|
source = request.Source ?? "StellaOps",
|
|
severity = MapSeverity(request.Severity),
|
|
custom_details = request.Details
|
|
},
|
|
links = new[]
|
|
{
|
|
new
|
|
{
|
|
href = $"{_options.StellaOpsBaseUrl}/incidents/{request.IncidentId}",
|
|
text = "View in StellaOps"
|
|
}
|
|
}
|
|
};
|
|
|
|
try
|
|
{
|
|
var response = await _httpClient.PostAsJsonAsync("/v2/enqueue", payload, cancellationToken);
|
|
|
|
if (response.IsSuccessStatusCode)
|
|
{
|
|
var result = await response.Content.ReadFromJsonAsync<JsonElement>(cancellationToken);
|
|
var dedupKey = result.GetProperty("dedup_key").GetString();
|
|
|
|
_logger.LogInformation(
|
|
"Created PagerDuty alert for incident {IncidentId}: {DedupKey}",
|
|
request.IncidentId, dedupKey);
|
|
|
|
return ExternalIncidentResult.Succeeded(dedupKey ?? request.IncidentId);
|
|
}
|
|
|
|
var error = await response.Content.ReadAsStringAsync(cancellationToken);
|
|
_logger.LogWarning("PagerDuty API error: {Error}", error);
|
|
return ExternalIncidentResult.Failed($"PagerDuty API error: {response.StatusCode}");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Failed to create PagerDuty alert for incident {IncidentId}", request.IncidentId);
|
|
return ExternalIncidentResult.Failed(ex.Message);
|
|
}
|
|
}
|
|
|
|
public async Task<bool> AcknowledgeAsync(
|
|
string externalId,
|
|
string acknowledgedBy,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var payload = new
|
|
{
|
|
routing_key = _options.DefaultRoutingKey,
|
|
event_action = "acknowledge",
|
|
dedup_key = externalId
|
|
};
|
|
|
|
try
|
|
{
|
|
var response = await _httpClient.PostAsJsonAsync("/v2/enqueue", payload, cancellationToken);
|
|
return response.IsSuccessStatusCode;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Failed to acknowledge PagerDuty alert {ExternalId}", externalId);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
public async Task<bool> ResolveAsync(
|
|
string externalId,
|
|
string resolvedBy,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var payload = new
|
|
{
|
|
routing_key = _options.DefaultRoutingKey,
|
|
event_action = "resolve",
|
|
dedup_key = externalId
|
|
};
|
|
|
|
try
|
|
{
|
|
var response = await _httpClient.PostAsJsonAsync("/v2/enqueue", payload, cancellationToken);
|
|
return response.IsSuccessStatusCode;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Failed to resolve PagerDuty alert {ExternalId}", externalId);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
public AckBridgeRequest? ParseWebhook(string payload, IDictionary<string, string>? headers = null)
|
|
{
|
|
try
|
|
{
|
|
var json = JsonDocument.Parse(payload);
|
|
var messages = json.RootElement.GetProperty("messages");
|
|
|
|
foreach (var message in messages.EnumerateArray())
|
|
{
|
|
var eventType = message.GetProperty("event").GetString();
|
|
if (eventType != "incident.acknowledge")
|
|
{
|
|
continue;
|
|
}
|
|
|
|
var incident = message.GetProperty("incident");
|
|
var dedupKey = incident.GetProperty("incident_key").GetString();
|
|
var acknowledgedBy = "pagerduty";
|
|
|
|
if (incident.TryGetProperty("last_status_change_by", out var changedBy))
|
|
{
|
|
acknowledgedBy = changedBy.GetProperty("summary").GetString() ?? "pagerduty";
|
|
}
|
|
|
|
return new AckBridgeRequest
|
|
{
|
|
Source = AckSource.PagerDuty,
|
|
ExternalId = dedupKey,
|
|
AcknowledgedBy = acknowledgedBy,
|
|
RawPayload = payload
|
|
};
|
|
}
|
|
|
|
return null;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogWarning(ex, "Failed to parse PagerDuty webhook");
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private static string MapSeverity(string? severity) => severity?.ToLowerInvariant() switch
|
|
{
|
|
"critical" => "critical",
|
|
"high" => "error",
|
|
"medium" => "warning",
|
|
"low" => "info",
|
|
_ => "warning"
|
|
};
|
|
}
|
|
|
|
/// <summary>
|
|
/// OpsGenie integration adapter.
|
|
/// </summary>
|
|
public sealed class OpsGenieAdapter : IExternalIntegrationAdapter
|
|
{
|
|
private readonly HttpClient _httpClient;
|
|
private readonly OpsGenieOptions _options;
|
|
private readonly ILogger<OpsGenieAdapter> _logger;
|
|
|
|
public OpsGenieAdapter(
|
|
HttpClient httpClient,
|
|
IOptions<OpsGenieOptions> options,
|
|
ILogger<OpsGenieAdapter> logger)
|
|
{
|
|
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
|
|
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
|
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
|
|
|
_httpClient.BaseAddress = new Uri(_options.ApiBaseUrl);
|
|
_httpClient.DefaultRequestHeaders.Add("Authorization", $"GenieKey {_options.ApiKey}");
|
|
}
|
|
|
|
public async Task<ExternalIncidentResult> CreateIncidentAsync(
|
|
ExternalIncidentRequest request,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var payload = new
|
|
{
|
|
message = request.Title,
|
|
description = request.Description,
|
|
alias = $"stellaops-{request.TenantId}-{request.IncidentId}",
|
|
source = request.Source ?? "StellaOps",
|
|
priority = MapPriority(request.Severity),
|
|
details = request.Details
|
|
};
|
|
|
|
try
|
|
{
|
|
var response = await _httpClient.PostAsJsonAsync("/v2/alerts", payload, cancellationToken);
|
|
|
|
if (response.IsSuccessStatusCode)
|
|
{
|
|
var result = await response.Content.ReadFromJsonAsync<JsonElement>(cancellationToken);
|
|
var alertId = result.GetProperty("data").GetProperty("alertId").GetString();
|
|
|
|
_logger.LogInformation(
|
|
"Created OpsGenie alert for incident {IncidentId}: {AlertId}",
|
|
request.IncidentId, alertId);
|
|
|
|
return ExternalIncidentResult.Succeeded(
|
|
alertId ?? request.IncidentId,
|
|
$"https://app.opsgenie.com/alert/detail/{alertId}");
|
|
}
|
|
|
|
var error = await response.Content.ReadAsStringAsync(cancellationToken);
|
|
_logger.LogWarning("OpsGenie API error: {Error}", error);
|
|
return ExternalIncidentResult.Failed($"OpsGenie API error: {response.StatusCode}");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Failed to create OpsGenie alert for incident {IncidentId}", request.IncidentId);
|
|
return ExternalIncidentResult.Failed(ex.Message);
|
|
}
|
|
}
|
|
|
|
public async Task<bool> AcknowledgeAsync(
|
|
string externalId,
|
|
string acknowledgedBy,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var payload = new
|
|
{
|
|
user = acknowledgedBy,
|
|
source = "StellaOps"
|
|
};
|
|
|
|
try
|
|
{
|
|
var response = await _httpClient.PostAsJsonAsync(
|
|
$"/v2/alerts/{externalId}/acknowledge",
|
|
payload,
|
|
cancellationToken);
|
|
return response.IsSuccessStatusCode;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Failed to acknowledge OpsGenie alert {ExternalId}", externalId);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
public async Task<bool> ResolveAsync(
|
|
string externalId,
|
|
string resolvedBy,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var payload = new
|
|
{
|
|
user = resolvedBy,
|
|
source = "StellaOps"
|
|
};
|
|
|
|
try
|
|
{
|
|
var response = await _httpClient.PostAsJsonAsync(
|
|
$"/v2/alerts/{externalId}/close",
|
|
payload,
|
|
cancellationToken);
|
|
return response.IsSuccessStatusCode;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Failed to resolve OpsGenie alert {ExternalId}", externalId);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
public AckBridgeRequest? ParseWebhook(string payload, IDictionary<string, string>? headers = null)
|
|
{
|
|
try
|
|
{
|
|
var json = JsonDocument.Parse(payload);
|
|
var action = json.RootElement.GetProperty("action").GetString();
|
|
|
|
if (action != "Acknowledge")
|
|
{
|
|
return null;
|
|
}
|
|
|
|
var alert = json.RootElement.GetProperty("alert");
|
|
var alertId = alert.GetProperty("alertId").GetString();
|
|
var acknowledgedBy = "opsgenie";
|
|
|
|
if (json.RootElement.TryGetProperty("source", out var source))
|
|
{
|
|
acknowledgedBy = source.GetProperty("name").GetString() ?? "opsgenie";
|
|
}
|
|
|
|
return new AckBridgeRequest
|
|
{
|
|
Source = AckSource.OpsGenie,
|
|
ExternalId = alertId,
|
|
AcknowledgedBy = acknowledgedBy,
|
|
RawPayload = payload
|
|
};
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogWarning(ex, "Failed to parse OpsGenie webhook");
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private static string MapPriority(string? severity) => severity?.ToLowerInvariant() switch
|
|
{
|
|
"critical" => "P1",
|
|
"high" => "P2",
|
|
"medium" => "P3",
|
|
"low" => "P4",
|
|
_ => "P3"
|
|
};
|
|
}
|
|
|
|
/// <summary>
|
|
/// PagerDuty configuration options.
|
|
/// </summary>
|
|
public sealed class PagerDutyOptions
|
|
{
|
|
public const string SectionName = "Notifier:Integrations:PagerDuty";
|
|
|
|
public string? DefaultRoutingKey { get; set; }
|
|
public string StellaOpsBaseUrl { get; set; } = "https://stellaops.example.com";
|
|
}
|
|
|
|
/// <summary>
|
|
/// OpsGenie configuration options.
|
|
/// </summary>
|
|
public sealed class OpsGenieOptions
|
|
{
|
|
public const string SectionName = "Notifier:Integrations:OpsGenie";
|
|
|
|
public string ApiBaseUrl { get; set; } = "https://api.opsgenie.com";
|
|
public string? ApiKey { get; set; }
|
|
}
|