git.stella-ops.org/src/__Libraries/StellaOps.Doctor.Plugins.Integration/Checks/SecretsManagerConnectivityCheck.cs

// -----------------------------------------------------------------------------
// SecretsManagerConnectivityCheck.cs
// Sprint: SPRINT_20260118_018_Doctor_integration_health_expansion
// Task: INTH-004 - Implement SecretsManagerConnectivityCheck
// Description: Verify connectivity to secrets managers (Vault, AWS Secrets Manager, Azure Key Vault, etc.)
// -----------------------------------------------------------------------------


using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Doctor.Models;
using StellaOps.Doctor.Plugins;
using StellaOps.Doctor.Plugins.Builders;
using System.Globalization;
using System.Net.Http;
using System.Text.Json;

namespace StellaOps.Doctor.Plugins.Integration.Checks;

/// <summary>
/// Verifies connectivity to configured secrets managers.
/// Checks authentication, seal status (Vault), and API accessibility.
/// </summary>
public sealed class SecretsManagerConnectivityCheck : IDoctorCheck
{
    private const string PluginId = "stellaops.doctor.integration";
    private const string CategoryName = "Integration";

    /// <inheritdoc />
    public string CheckId => "check.integration.secrets.manager";

    /// <inheritdoc />
    public string Name => "Secrets Manager Connectivity";

    /// <inheritdoc />
    public string Description => "Verify connectivity to secrets managers (Vault, AWS Secrets Manager, Azure Key Vault)";

    /// <inheritdoc />
    public DoctorSeverity DefaultSeverity => DoctorSeverity.Fail;

    /// <inheritdoc />
    public IReadOnlyList<string> Tags => ["integration", "secrets", "vault", "security", "keyvault"];

    /// <inheritdoc />
    public TimeSpan EstimatedDuration => TimeSpan.FromSeconds(10);

    /// <inheritdoc />
    public bool CanRun(DoctorPluginContext context)
    {
        var secretsConfig = context.Configuration.GetSection("Secrets");
        return secretsConfig.Exists() && secretsConfig.GetChildren().Any();
    }

    /// <inheritdoc />
    public async Task<DoctorCheckResult> RunAsync(DoctorPluginContext context, CancellationToken ct)
    {
        var builder = context.CreateResult(CheckId, PluginId, CategoryName);

        var managers = GetConfiguredSecretsManagers(context.Configuration);

        if (managers.Count == 0)
        {
            return builder
                .Skip("No secrets managers configured")
                .WithEvidence("Secrets Managers", eb => eb.Add("configured_managers", "0"))
                .Build();
        }

        var httpClientFactory = context.Services.GetService<IHttpClientFactory>();
        if (httpClientFactory == null)
        {
            return builder
                .Skip("IHttpClientFactory not available")
                .Build();
        }

        var httpClient = httpClientFactory.CreateClient("DoctorHealthCheck");
        httpClient.Timeout = TimeSpan.FromSeconds(10);

        var results = new List<SecretsManagerResult>();
        var unhealthy = new List<string>();
        var sealed_ = new List<string>(); // 'sealed' is a keyword

        foreach (var mgr in managers)
        {
            var result = await CheckSecretsManagerAsync(httpClient, mgr, ct);
            results.Add(result);

            if (!result.Reachable || !result.AuthSuccess)
            {
                unhealthy.Add(mgr.Name);
            }
            else if (result.IsSealed)
            {
                sealed_.Add(mgr.Name);
            }
        }

        // Secrets manager issues are critical - blocks deployments
        if (unhealthy.Count > 0)
        {
            return builder
                .Fail($"{unhealthy.Count} secrets manager(s) unreachable")
                .WithEvidence("Secrets Managers", eb =>
                {
                    eb.Add("total_managers", managers.Count.ToString(CultureInfo.InvariantCulture));
                    eb.Add("healthy_managers", (managers.Count - unhealthy.Count).ToString(CultureInfo.InvariantCulture));
                    eb.Add("unhealthy_managers", string.Join(", ", unhealthy));
                    AddSecretsEvidence(eb, results);
                })
                .WithCauses(
                    "Secrets manager is down",
                    "Network connectivity issue",
                    "Authentication token expired",
                    "TLS certificate issue")
                .WithRemediation(rb =>
                {
                    rb.AddStep(1, "Test secrets manager connectivity",
                        $"stella secrets ping {unhealthy[0]}",
                        CommandType.Shell);
                    rb.AddStep(2, "Refresh authentication",
                        $"stella secrets auth refresh {unhealthy[0]}",
                        CommandType.Manual);
                    rb.WithRunbookUrl("docs/doctor/articles/integration/integration-secrets-manager.md");
                })
                .WithVerification($"stella doctor --check {CheckId}")
                .Build();
        }

        if (sealed_.Count > 0)
        {
            return builder
                .Fail($"{sealed_.Count} Vault instance(s) are sealed")
                .WithEvidence("Secrets Managers", eb =>
                {
                    eb.Add("total_managers", managers.Count.ToString(CultureInfo.InvariantCulture));
                    eb.Add("healthy_managers", (managers.Count - sealed_.Count).ToString(CultureInfo.InvariantCulture));
                    eb.Add("sealed_vaults", string.Join(", ", sealed_));
                    AddSecretsEvidence(eb, results);
                })
                .WithCauses(
                    "Vault was restarted and needs unseal",
                    "Vault auto-seal triggered",
                    "HSM connectivity lost")
                .WithRemediation(rb =>
                {
                    rb.AddStep(1, "Unseal Vault",
                        $"vault operator unseal",
                        CommandType.Manual);
                    rb.AddStep(2, "Check seal status",
                        $"stella secrets status {sealed_[0]}",
                        CommandType.Shell);
                    rb.WithRunbookUrl("docs/doctor/articles/integration/integration-secrets-manager.md");
                })
                .WithVerification($"stella doctor --check {CheckId}")
                .Build();
        }

        return builder
            .Pass($"{managers.Count} secrets manager(s) healthy")
            .WithEvidence("Secrets Managers", eb =>
            {
                eb.Add("total_managers", managers.Count.ToString(CultureInfo.InvariantCulture));
                eb.Add("healthy_managers", managers.Count.ToString(CultureInfo.InvariantCulture));
                AddSecretsEvidence(eb, results);
            })
            .Build();
    }

    private static async Task<SecretsManagerResult> CheckSecretsManagerAsync(
        HttpClient client, SecretsManagerConfig mgr, CancellationToken ct)
    {
        var result = new SecretsManagerResult { Name = mgr.Name, Type = mgr.Type };

        try
        {
            var healthEndpoint = GetHealthEndpoint(mgr);
            var request = new HttpRequestMessage(HttpMethod.Get, healthEndpoint);

            // Add auth headers based on type
            if (!string.IsNullOrEmpty(mgr.Token))
            {
                if (mgr.Type.Equals("vault", StringComparison.OrdinalIgnoreCase))
                {
                    request.Headers.Add("X-Vault-Token", mgr.Token);
                }
                else
                {
                    request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", mgr.Token);
                }
            }

            var sw = System.Diagnostics.Stopwatch.StartNew();
            var response = await client.SendAsync(request, ct);
            sw.Stop();

            result.LatencyMs = (int)sw.ElapsedMilliseconds;
            result.Reachable = true;
            result.AuthSuccess = response.StatusCode != System.Net.HttpStatusCode.Unauthorized &&
                                 response.StatusCode != System.Net.HttpStatusCode.Forbidden;

            // Parse response for type-specific info
            if (response.IsSuccessStatusCode)
            {
                var json = await response.Content.ReadAsStringAsync(ct);
                ParseSecretsManagerResponse(json, mgr.Type, result);
            }
        }
        catch (HttpRequestException ex)
        {
            result.Reachable = false;
            result.ErrorMessage = ex.Message;
        }
        catch (TaskCanceledException)
        {
            result.Reachable = false;
            result.ErrorMessage = "Timeout";
        }

        return result;
    }

    private static void ParseSecretsManagerResponse(string json, string type, SecretsManagerResult result)
    {
        try
        {
            using var doc = JsonDocument.Parse(json);

            if (type.Equals("vault", StringComparison.OrdinalIgnoreCase))
            {
                // Vault health endpoint returns sealed status
                if (doc.RootElement.TryGetProperty("sealed", out var sealedEl))
                {
                    result.IsSealed = sealedEl.GetBoolean();
                }
                if (doc.RootElement.TryGetProperty("initialized", out var initEl))
                {
                    result.IsInitialized = initEl.GetBoolean();
                }
                if (doc.RootElement.TryGetProperty("version", out var verEl))
                {
                    result.Version = verEl.GetString();
                }
            }
        }
        catch { }
    }

    private static string GetHealthEndpoint(SecretsManagerConfig mgr)
    {
        return mgr.Type.ToLowerInvariant() switch
        {
            "vault" => $"{mgr.Url.TrimEnd('/')}/v1/sys/health?standbyok=true&sealedcode=200&uninitcode=200",
            "aws" => mgr.Url, // AWS uses SDK, URL is just for config reference
            "azure" => $"{mgr.Url.TrimEnd('/')}/healthstatus",
            "gcp" => mgr.Url, // GCP uses SDK
            _ => $"{mgr.Url.TrimEnd('/')}/health"
        };
    }

    private static void AddSecretsEvidence(EvidenceBuilder eb, List<SecretsManagerResult> results)
    {
        foreach (var r in results)
        {
            var prefix = $"secrets_{r.Name.ToLowerInvariant().Replace(" ", "_").Replace("-", "_")}";
            eb.Add($"{prefix}_type", r.Type);
            eb.Add($"{prefix}_reachable", r.Reachable.ToString().ToLowerInvariant());
            eb.Add($"{prefix}_auth_success", r.AuthSuccess.ToString().ToLowerInvariant());
            eb.Add($"{prefix}_latency_ms", r.LatencyMs.ToString(CultureInfo.InvariantCulture));
            if (r.Type.Equals("vault", StringComparison.OrdinalIgnoreCase))
            {
                eb.Add($"{prefix}_sealed", r.IsSealed.ToString().ToLowerInvariant());
                eb.Add($"{prefix}_initialized", r.IsInitialized.ToString().ToLowerInvariant());
            }
        }
    }

    private static List<SecretsManagerConfig> GetConfiguredSecretsManagers(IConfiguration config)
    {
        var managers = new List<SecretsManagerConfig>();

        var secretsSection = config.GetSection("Secrets:Managers");
        if (secretsSection.Exists())
        {
            foreach (var child in secretsSection.GetChildren())
            {
                var name = child.GetValue<string>("Name") ?? child.Key;
                var url = child.GetValue<string>("Url");
                var type = child.GetValue<string>("Type") ?? "vault";
                var token = child.GetValue<string>("Token");

                if (!string.IsNullOrEmpty(url))
                {
                    managers.Add(new SecretsManagerConfig
                    {
                        Name = name,
                        Url = url,
                        Type = type,
                        Token = token
                    });
                }
            }
        }

        // Check legacy single-manager config
        var legacyUrl = config.GetValue<string>("Secrets:Vault:Url") 
            ?? config.GetValue<string>("Vault:Url");
        if (!string.IsNullOrEmpty(legacyUrl) && managers.Count == 0)
        {
            managers.Add(new SecretsManagerConfig
            {
                Name = "vault",
                Url = legacyUrl,
                Type = "vault",
                Token = config.GetValue<string>("Secrets:Vault:Token") ?? config.GetValue<string>("Vault:Token")
            });
        }

        return managers;
    }

    private sealed class SecretsManagerConfig
    {
        public required string Name { get; init; }
        public required string Url { get; init; }
        public required string Type { get; init; }
        public string? Token { get; init; }
    }

    private sealed class SecretsManagerResult
    {
        public required string Name { get; init; }
        public required string Type { get; init; }
        public bool Reachable { get; set; }
        public bool AuthSuccess { get; set; }
        public int LatencyMs { get; set; }
        public bool IsSealed { get; set; }
        public bool IsInitialized { get; set; } = true;
        public string? Version { get; set; }
        public string? ErrorMessage { get; set; }
    }
}