// ----------------------------------------------------------------------------- // SecretsManagerConnectivityCheck.cs // Sprint: SPRINT_20260118_018_Doctor_integration_health_expansion // Task: INTH-004 - Implement SecretsManagerConnectivityCheck // Description: Verify connectivity to secrets managers (Vault, AWS Secrets Manager, Azure Key Vault, etc.) // ----------------------------------------------------------------------------- using System.Globalization; using System.Net.Http; using System.Text.Json; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using StellaOps.Doctor.Models; using StellaOps.Doctor.Plugins; using StellaOps.Doctor.Plugins.Builders; namespace StellaOps.Doctor.Plugins.Integration.Checks; /// /// Verifies connectivity to configured secrets managers. /// Checks authentication, seal status (Vault), and API accessibility. /// public sealed class SecretsManagerConnectivityCheck : IDoctorCheck { private const string PluginId = "stellaops.doctor.integration"; private const string CategoryName = "Integration"; /// public string CheckId => "check.integration.secrets.manager"; /// public string Name => "Secrets Manager Connectivity"; /// public string Description => "Verify connectivity to secrets managers (Vault, AWS Secrets Manager, Azure Key Vault)"; /// public DoctorSeverity DefaultSeverity => DoctorSeverity.Fail; /// public IReadOnlyList Tags => ["integration", "secrets", "vault", "security", "keyvault"]; /// public TimeSpan EstimatedDuration => TimeSpan.FromSeconds(10); /// public bool CanRun(DoctorPluginContext context) { var secretsConfig = context.Configuration.GetSection("Secrets"); return secretsConfig.Exists() && secretsConfig.GetChildren().Any(); } /// public async Task RunAsync(DoctorPluginContext context, CancellationToken ct) { var builder = context.CreateResult(CheckId, PluginId, CategoryName); var managers = GetConfiguredSecretsManagers(context.Configuration); if (managers.Count == 0) { return builder .Skip("No secrets managers configured") .WithEvidence("Secrets Managers", eb => eb.Add("configured_managers", "0")) .Build(); } var httpClientFactory = context.Services.GetService(); if (httpClientFactory == null) { return builder .Skip("IHttpClientFactory not available") .Build(); } var httpClient = httpClientFactory.CreateClient("DoctorHealthCheck"); httpClient.Timeout = TimeSpan.FromSeconds(10); var results = new List(); var unhealthy = new List(); var sealed_ = new List(); // 'sealed' is a keyword foreach (var mgr in managers) { var result = await CheckSecretsManagerAsync(httpClient, mgr, ct); results.Add(result); if (!result.Reachable || !result.AuthSuccess) { unhealthy.Add(mgr.Name); } else if (result.IsSealed) { sealed_.Add(mgr.Name); } } // Secrets manager issues are critical - blocks deployments if (unhealthy.Count > 0) { return builder .Fail($"{unhealthy.Count} secrets manager(s) unreachable") .WithEvidence("Secrets Managers", eb => { eb.Add("total_managers", managers.Count.ToString(CultureInfo.InvariantCulture)); eb.Add("healthy_managers", (managers.Count - unhealthy.Count).ToString(CultureInfo.InvariantCulture)); eb.Add("unhealthy_managers", string.Join(", ", unhealthy)); AddSecretsEvidence(eb, results); }) .WithCauses( "Secrets manager is down", "Network connectivity issue", "Authentication token expired", "TLS certificate issue") .WithRemediation(rb => { rb.AddStep(1, "Test secrets manager connectivity", $"stella secrets ping {unhealthy[0]}", CommandType.Shell); rb.AddStep(2, "Refresh authentication", $"stella secrets auth refresh {unhealthy[0]}", CommandType.Manual); }) .WithVerification($"stella doctor --check {CheckId}") .Build(); } if (sealed_.Count > 0) { return builder .Fail($"{sealed_.Count} Vault instance(s) are sealed") .WithEvidence("Secrets Managers", eb => { eb.Add("total_managers", managers.Count.ToString(CultureInfo.InvariantCulture)); eb.Add("healthy_managers", (managers.Count - sealed_.Count).ToString(CultureInfo.InvariantCulture)); eb.Add("sealed_vaults", string.Join(", ", sealed_)); AddSecretsEvidence(eb, results); }) .WithCauses( "Vault was restarted and needs unseal", "Vault auto-seal triggered", "HSM connectivity lost") .WithRemediation(rb => { rb.AddStep(1, "Unseal Vault", $"vault operator unseal", CommandType.Manual); rb.AddStep(2, "Check seal status", $"stella secrets status {sealed_[0]}", CommandType.Shell); }) .WithVerification($"stella doctor --check {CheckId}") .Build(); } return builder .Pass($"{managers.Count} secrets manager(s) healthy") .WithEvidence("Secrets Managers", eb => { eb.Add("total_managers", managers.Count.ToString(CultureInfo.InvariantCulture)); eb.Add("healthy_managers", managers.Count.ToString(CultureInfo.InvariantCulture)); AddSecretsEvidence(eb, results); }) .Build(); } private static async Task CheckSecretsManagerAsync( HttpClient client, SecretsManagerConfig mgr, CancellationToken ct) { var result = new SecretsManagerResult { Name = mgr.Name, Type = mgr.Type }; try { var healthEndpoint = GetHealthEndpoint(mgr); var request = new HttpRequestMessage(HttpMethod.Get, healthEndpoint); // Add auth headers based on type if (!string.IsNullOrEmpty(mgr.Token)) { if (mgr.Type.Equals("vault", StringComparison.OrdinalIgnoreCase)) { request.Headers.Add("X-Vault-Token", mgr.Token); } else { request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", mgr.Token); } } var sw = System.Diagnostics.Stopwatch.StartNew(); var response = await client.SendAsync(request, ct); sw.Stop(); result.LatencyMs = (int)sw.ElapsedMilliseconds; result.Reachable = true; result.AuthSuccess = response.StatusCode != System.Net.HttpStatusCode.Unauthorized && response.StatusCode != System.Net.HttpStatusCode.Forbidden; // Parse response for type-specific info if (response.IsSuccessStatusCode) { var json = await response.Content.ReadAsStringAsync(ct); ParseSecretsManagerResponse(json, mgr.Type, result); } } catch (HttpRequestException ex) { result.Reachable = false; result.ErrorMessage = ex.Message; } catch (TaskCanceledException) { result.Reachable = false; result.ErrorMessage = "Timeout"; } return result; } private static void ParseSecretsManagerResponse(string json, string type, SecretsManagerResult result) { try { using var doc = JsonDocument.Parse(json); if (type.Equals("vault", StringComparison.OrdinalIgnoreCase)) { // Vault health endpoint returns sealed status if (doc.RootElement.TryGetProperty("sealed", out var sealedEl)) { result.IsSealed = sealedEl.GetBoolean(); } if (doc.RootElement.TryGetProperty("initialized", out var initEl)) { result.IsInitialized = initEl.GetBoolean(); } if (doc.RootElement.TryGetProperty("version", out var verEl)) { result.Version = verEl.GetString(); } } } catch { } } private static string GetHealthEndpoint(SecretsManagerConfig mgr) { return mgr.Type.ToLowerInvariant() switch { "vault" => $"{mgr.Url.TrimEnd('/')}/v1/sys/health?standbyok=true&sealedcode=200&uninitcode=200", "aws" => mgr.Url, // AWS uses SDK, URL is just for config reference "azure" => $"{mgr.Url.TrimEnd('/')}/healthstatus", "gcp" => mgr.Url, // GCP uses SDK _ => $"{mgr.Url.TrimEnd('/')}/health" }; } private static void AddSecretsEvidence(EvidenceBuilder eb, List results) { foreach (var r in results) { var prefix = $"secrets_{r.Name.ToLowerInvariant().Replace(" ", "_").Replace("-", "_")}"; eb.Add($"{prefix}_type", r.Type); eb.Add($"{prefix}_reachable", r.Reachable.ToString().ToLowerInvariant()); eb.Add($"{prefix}_auth_success", r.AuthSuccess.ToString().ToLowerInvariant()); eb.Add($"{prefix}_latency_ms", r.LatencyMs.ToString(CultureInfo.InvariantCulture)); if (r.Type.Equals("vault", StringComparison.OrdinalIgnoreCase)) { eb.Add($"{prefix}_sealed", r.IsSealed.ToString().ToLowerInvariant()); eb.Add($"{prefix}_initialized", r.IsInitialized.ToString().ToLowerInvariant()); } } } private static List GetConfiguredSecretsManagers(IConfiguration config) { var managers = new List(); var secretsSection = config.GetSection("Secrets:Managers"); if (secretsSection.Exists()) { foreach (var child in secretsSection.GetChildren()) { var name = child.GetValue("Name") ?? child.Key; var url = child.GetValue("Url"); var type = child.GetValue("Type") ?? "vault"; var token = child.GetValue("Token"); if (!string.IsNullOrEmpty(url)) { managers.Add(new SecretsManagerConfig { Name = name, Url = url, Type = type, Token = token }); } } } // Check legacy single-manager config var legacyUrl = config.GetValue("Secrets:Vault:Url") ?? config.GetValue("Vault:Url"); if (!string.IsNullOrEmpty(legacyUrl) && managers.Count == 0) { managers.Add(new SecretsManagerConfig { Name = "vault", Url = legacyUrl, Type = "vault", Token = config.GetValue("Secrets:Vault:Token") ?? config.GetValue("Vault:Token") }); } return managers; } private sealed class SecretsManagerConfig { public required string Name { get; init; } public required string Url { get; init; } public required string Type { get; init; } public string? Token { get; init; } } private sealed class SecretsManagerResult { public required string Name { get; init; } public required string Type { get; init; } public bool Reachable { get; set; } public bool AuthSuccess { get; set; } public int LatencyMs { get; set; } public bool IsSealed { get; set; } public bool IsInitialized { get; set; } = true; public string? Version { get; set; } public string? ErrorMessage { get; set; } } }