git.stella-ops.org/src/__Libraries/StellaOps.Doctor.Plugins.Observability/Checks/AlertingConfigurationCheck.cs

using Microsoft.Extensions.Configuration;
using StellaOps.Doctor.Models;
using StellaOps.Doctor.Plugins;

namespace StellaOps.Doctor.Plugins.Observability.Checks;

/// <summary>
/// Validates alerting configuration.
/// </summary>
public sealed class AlertingConfigurationCheck : IDoctorCheck
{
    /// <inheritdoc />
    public string CheckId => "check.observability.alerting";

    /// <inheritdoc />
    public string Name => "Alerting Configuration";

    /// <inheritdoc />
    public string Description => "Validates alerting rules and notification destinations";

    /// <inheritdoc />
    public DoctorSeverity DefaultSeverity => DoctorSeverity.Info;

    /// <inheritdoc />
    public IReadOnlyList<string> Tags => ["observability", "alerting", "notifications"];

    /// <inheritdoc />
    public TimeSpan EstimatedDuration => TimeSpan.FromMilliseconds(50);

    /// <inheritdoc />
    public bool CanRun(DoctorPluginContext context) => true;

    /// <inheritdoc />
    public Task<DoctorCheckResult> RunAsync(DoctorPluginContext context, CancellationToken ct)
    {
        var result = context.CreateResult(CheckId, "stellaops.doctor.observability", DoctorCategory.Observability.ToString());

        var alertingEnabled = context.Configuration.GetValue<bool?>("Alerting:Enabled")
            ?? context.Configuration.GetValue<bool?>("Notifications:Alerts:Enabled");

        var alertManagerUrl = context.Configuration.GetValue<string>("Alerting:AlertManagerUrl")
            ?? context.Configuration.GetValue<string>("Prometheus:AlertManager:Url");

        var slackWebhook = context.Configuration.GetValue<string>("Alerting:Slack:WebhookUrl")
            ?? context.Configuration.GetValue<string>("Notifications:Slack:WebhookUrl");

        var emailRecipients = context.Configuration.GetSection("Alerting:Email:Recipients").Get<string[]>()
            ?? context.Configuration.GetSection("Notifications:Email:Recipients").Get<string[]>();

        var pagerDutyKey = context.Configuration.GetValue<string>("Alerting:PagerDuty:RoutingKey")
            ?? context.Configuration.GetValue<string>("Notifications:PagerDuty:IntegrationKey");

        var hasAnyDestination = !string.IsNullOrWhiteSpace(alertManagerUrl)
            || !string.IsNullOrWhiteSpace(slackWebhook)
            || (emailRecipients?.Length > 0)
            || !string.IsNullOrWhiteSpace(pagerDutyKey);

        if (alertingEnabled == false)
        {
            return Task.FromResult(result
                .Info("Alerting is explicitly disabled")
                .WithEvidence("Alerting configuration", e =>
                {
                    e.Add("Enabled", "false");
                })
                .Build());
        }

        if (!hasAnyDestination)
        {
            return Task.FromResult(result
                .Info("No alerting destinations configured")
                .WithEvidence("Alerting configuration", e =>
                {
                    e.Add("AlertManagerConfigured", "false");
                    e.Add("SlackConfigured", "false");
                    e.Add("EmailConfigured", "false");
                    e.Add("PagerDutyConfigured", "false");
                    e.Add("Recommendation", "Configure at least one alert destination for production");
                })
                .Build());
        }

        var issues = new List<string>();

        if (emailRecipients?.Length > 0 && emailRecipients.Any(e => !e.Contains('@')))
        {
            issues.Add("Some email recipients appear to be invalid");
        }

        var destinations = new List<string>();
        if (!string.IsNullOrWhiteSpace(alertManagerUrl)) destinations.Add("AlertManager");
        if (!string.IsNullOrWhiteSpace(slackWebhook)) destinations.Add("Slack");
        if (emailRecipients?.Length > 0) destinations.Add("Email");
        if (!string.IsNullOrWhiteSpace(pagerDutyKey)) destinations.Add("PagerDuty");

        if (issues.Count > 0)
        {
            return Task.FromResult(result
                .Warn($"{issues.Count} alerting configuration issue(s)")
                .WithEvidence("Alerting configuration", e =>
                {
                    e.Add("Enabled", alertingEnabled?.ToString() ?? "default");
                    e.Add("ConfiguredDestinations", string.Join(", ", destinations));
                    e.Add("AlertManagerUrl", !string.IsNullOrWhiteSpace(alertManagerUrl) ? "configured" : "(not set)");
                    e.Add("SlackWebhook", !string.IsNullOrWhiteSpace(slackWebhook) ? "configured" : "(not set)");
                    e.Add("EmailRecipients", emailRecipients?.Length.ToString() ?? "0");
                    e.Add("PagerDuty", !string.IsNullOrWhiteSpace(pagerDutyKey) ? "configured" : "(not set)");
                })
                .WithCauses(issues.ToArray())
                .Build());
        }

        return Task.FromResult(result
            .Pass($"Alerting configured with {destinations.Count} destination(s)")
            .WithEvidence("Alerting configuration", e =>
            {
                e.Add("Enabled", alertingEnabled?.ToString() ?? "default");
                e.Add("ConfiguredDestinations", string.Join(", ", destinations));
                e.Add("DestinationCount", destinations.Count.ToString());
            })
            .Build());
    }
}