340 lines
13 KiB
C#
340 lines
13 KiB
C#
// -----------------------------------------------------------------------------
|
|
// SecretsManagerConnectivityCheck.cs
|
|
// Sprint: SPRINT_20260118_018_Doctor_integration_health_expansion
|
|
// Task: INTH-004 - Implement SecretsManagerConnectivityCheck
|
|
// Description: Verify connectivity to secrets managers (Vault, AWS Secrets Manager, Azure Key Vault, etc.)
|
|
// -----------------------------------------------------------------------------
|
|
|
|
using System.Globalization;
|
|
using System.Net.Http;
|
|
using System.Text.Json;
|
|
using Microsoft.Extensions.Configuration;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using StellaOps.Doctor.Models;
|
|
using StellaOps.Doctor.Plugins;
|
|
using StellaOps.Doctor.Plugins.Builders;
|
|
|
|
namespace StellaOps.Doctor.Plugins.Integration.Checks;
|
|
|
|
/// <summary>
|
|
/// Verifies connectivity to configured secrets managers.
|
|
/// Checks authentication, seal status (Vault), and API accessibility.
|
|
/// </summary>
|
|
public sealed class SecretsManagerConnectivityCheck : IDoctorCheck
|
|
{
|
|
private const string PluginId = "stellaops.doctor.integration";
|
|
private const string CategoryName = "Integration";
|
|
|
|
/// <inheritdoc />
|
|
public string CheckId => "check.integration.secrets.manager";
|
|
|
|
/// <inheritdoc />
|
|
public string Name => "Secrets Manager Connectivity";
|
|
|
|
/// <inheritdoc />
|
|
public string Description => "Verify connectivity to secrets managers (Vault, AWS Secrets Manager, Azure Key Vault)";
|
|
|
|
/// <inheritdoc />
|
|
public DoctorSeverity DefaultSeverity => DoctorSeverity.Fail;
|
|
|
|
/// <inheritdoc />
|
|
public IReadOnlyList<string> Tags => ["integration", "secrets", "vault", "security", "keyvault"];
|
|
|
|
/// <inheritdoc />
|
|
public TimeSpan EstimatedDuration => TimeSpan.FromSeconds(10);
|
|
|
|
/// <inheritdoc />
|
|
public bool CanRun(DoctorPluginContext context)
|
|
{
|
|
var secretsConfig = context.Configuration.GetSection("Secrets");
|
|
return secretsConfig.Exists() && secretsConfig.GetChildren().Any();
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<DoctorCheckResult> RunAsync(DoctorPluginContext context, CancellationToken ct)
|
|
{
|
|
var builder = context.CreateResult(CheckId, PluginId, CategoryName);
|
|
|
|
var managers = GetConfiguredSecretsManagers(context.Configuration);
|
|
|
|
if (managers.Count == 0)
|
|
{
|
|
return builder
|
|
.Skip("No secrets managers configured")
|
|
.WithEvidence("Secrets Managers", eb => eb.Add("configured_managers", "0"))
|
|
.Build();
|
|
}
|
|
|
|
var httpClientFactory = context.Services.GetService<IHttpClientFactory>();
|
|
if (httpClientFactory == null)
|
|
{
|
|
return builder
|
|
.Skip("IHttpClientFactory not available")
|
|
.Build();
|
|
}
|
|
|
|
var httpClient = httpClientFactory.CreateClient("DoctorHealthCheck");
|
|
httpClient.Timeout = TimeSpan.FromSeconds(10);
|
|
|
|
var results = new List<SecretsManagerResult>();
|
|
var unhealthy = new List<string>();
|
|
var sealed_ = new List<string>(); // 'sealed' is a keyword
|
|
|
|
foreach (var mgr in managers)
|
|
{
|
|
var result = await CheckSecretsManagerAsync(httpClient, mgr, ct);
|
|
results.Add(result);
|
|
|
|
if (!result.Reachable || !result.AuthSuccess)
|
|
{
|
|
unhealthy.Add(mgr.Name);
|
|
}
|
|
else if (result.IsSealed)
|
|
{
|
|
sealed_.Add(mgr.Name);
|
|
}
|
|
}
|
|
|
|
// Secrets manager issues are critical - blocks deployments
|
|
if (unhealthy.Count > 0)
|
|
{
|
|
return builder
|
|
.Fail($"{unhealthy.Count} secrets manager(s) unreachable")
|
|
.WithEvidence("Secrets Managers", eb =>
|
|
{
|
|
eb.Add("total_managers", managers.Count.ToString(CultureInfo.InvariantCulture));
|
|
eb.Add("healthy_managers", (managers.Count - unhealthy.Count).ToString(CultureInfo.InvariantCulture));
|
|
eb.Add("unhealthy_managers", string.Join(", ", unhealthy));
|
|
AddSecretsEvidence(eb, results);
|
|
})
|
|
.WithCauses(
|
|
"Secrets manager is down",
|
|
"Network connectivity issue",
|
|
"Authentication token expired",
|
|
"TLS certificate issue")
|
|
.WithRemediation(rb =>
|
|
{
|
|
rb.AddStep(1, "Test secrets manager connectivity",
|
|
$"stella secrets ping {unhealthy[0]}",
|
|
CommandType.Shell);
|
|
rb.AddStep(2, "Refresh authentication",
|
|
$"stella secrets auth refresh {unhealthy[0]}",
|
|
CommandType.Manual);
|
|
})
|
|
.WithVerification($"stella doctor --check {CheckId}")
|
|
.Build();
|
|
}
|
|
|
|
if (sealed_.Count > 0)
|
|
{
|
|
return builder
|
|
.Fail($"{sealed_.Count} Vault instance(s) are sealed")
|
|
.WithEvidence("Secrets Managers", eb =>
|
|
{
|
|
eb.Add("total_managers", managers.Count.ToString(CultureInfo.InvariantCulture));
|
|
eb.Add("healthy_managers", (managers.Count - sealed_.Count).ToString(CultureInfo.InvariantCulture));
|
|
eb.Add("sealed_vaults", string.Join(", ", sealed_));
|
|
AddSecretsEvidence(eb, results);
|
|
})
|
|
.WithCauses(
|
|
"Vault was restarted and needs unseal",
|
|
"Vault auto-seal triggered",
|
|
"HSM connectivity lost")
|
|
.WithRemediation(rb =>
|
|
{
|
|
rb.AddStep(1, "Unseal Vault",
|
|
$"vault operator unseal",
|
|
CommandType.Manual);
|
|
rb.AddStep(2, "Check seal status",
|
|
$"stella secrets status {sealed_[0]}",
|
|
CommandType.Shell);
|
|
})
|
|
.WithVerification($"stella doctor --check {CheckId}")
|
|
.Build();
|
|
}
|
|
|
|
return builder
|
|
.Pass($"{managers.Count} secrets manager(s) healthy")
|
|
.WithEvidence("Secrets Managers", eb =>
|
|
{
|
|
eb.Add("total_managers", managers.Count.ToString(CultureInfo.InvariantCulture));
|
|
eb.Add("healthy_managers", managers.Count.ToString(CultureInfo.InvariantCulture));
|
|
AddSecretsEvidence(eb, results);
|
|
})
|
|
.Build();
|
|
}
|
|
|
|
private static async Task<SecretsManagerResult> CheckSecretsManagerAsync(
|
|
HttpClient client, SecretsManagerConfig mgr, CancellationToken ct)
|
|
{
|
|
var result = new SecretsManagerResult { Name = mgr.Name, Type = mgr.Type };
|
|
|
|
try
|
|
{
|
|
var healthEndpoint = GetHealthEndpoint(mgr);
|
|
var request = new HttpRequestMessage(HttpMethod.Get, healthEndpoint);
|
|
|
|
// Add auth headers based on type
|
|
if (!string.IsNullOrEmpty(mgr.Token))
|
|
{
|
|
if (mgr.Type.Equals("vault", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
request.Headers.Add("X-Vault-Token", mgr.Token);
|
|
}
|
|
else
|
|
{
|
|
request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", mgr.Token);
|
|
}
|
|
}
|
|
|
|
var sw = System.Diagnostics.Stopwatch.StartNew();
|
|
var response = await client.SendAsync(request, ct);
|
|
sw.Stop();
|
|
|
|
result.LatencyMs = (int)sw.ElapsedMilliseconds;
|
|
result.Reachable = true;
|
|
result.AuthSuccess = response.StatusCode != System.Net.HttpStatusCode.Unauthorized &&
|
|
response.StatusCode != System.Net.HttpStatusCode.Forbidden;
|
|
|
|
// Parse response for type-specific info
|
|
if (response.IsSuccessStatusCode)
|
|
{
|
|
var json = await response.Content.ReadAsStringAsync(ct);
|
|
ParseSecretsManagerResponse(json, mgr.Type, result);
|
|
}
|
|
}
|
|
catch (HttpRequestException ex)
|
|
{
|
|
result.Reachable = false;
|
|
result.ErrorMessage = ex.Message;
|
|
}
|
|
catch (TaskCanceledException)
|
|
{
|
|
result.Reachable = false;
|
|
result.ErrorMessage = "Timeout";
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
private static void ParseSecretsManagerResponse(string json, string type, SecretsManagerResult result)
|
|
{
|
|
try
|
|
{
|
|
using var doc = JsonDocument.Parse(json);
|
|
|
|
if (type.Equals("vault", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
// Vault health endpoint returns sealed status
|
|
if (doc.RootElement.TryGetProperty("sealed", out var sealedEl))
|
|
{
|
|
result.IsSealed = sealedEl.GetBoolean();
|
|
}
|
|
if (doc.RootElement.TryGetProperty("initialized", out var initEl))
|
|
{
|
|
result.IsInitialized = initEl.GetBoolean();
|
|
}
|
|
if (doc.RootElement.TryGetProperty("version", out var verEl))
|
|
{
|
|
result.Version = verEl.GetString();
|
|
}
|
|
}
|
|
}
|
|
catch { }
|
|
}
|
|
|
|
private static string GetHealthEndpoint(SecretsManagerConfig mgr)
|
|
{
|
|
return mgr.Type.ToLowerInvariant() switch
|
|
{
|
|
"vault" => $"{mgr.Url.TrimEnd('/')}/v1/sys/health?standbyok=true&sealedcode=200&uninitcode=200",
|
|
"aws" => mgr.Url, // AWS uses SDK, URL is just for config reference
|
|
"azure" => $"{mgr.Url.TrimEnd('/')}/healthstatus",
|
|
"gcp" => mgr.Url, // GCP uses SDK
|
|
_ => $"{mgr.Url.TrimEnd('/')}/health"
|
|
};
|
|
}
|
|
|
|
private static void AddSecretsEvidence(EvidenceBuilder eb, List<SecretsManagerResult> results)
|
|
{
|
|
foreach (var r in results)
|
|
{
|
|
var prefix = $"secrets_{r.Name.ToLowerInvariant().Replace(" ", "_").Replace("-", "_")}";
|
|
eb.Add($"{prefix}_type", r.Type);
|
|
eb.Add($"{prefix}_reachable", r.Reachable.ToString().ToLowerInvariant());
|
|
eb.Add($"{prefix}_auth_success", r.AuthSuccess.ToString().ToLowerInvariant());
|
|
eb.Add($"{prefix}_latency_ms", r.LatencyMs.ToString(CultureInfo.InvariantCulture));
|
|
if (r.Type.Equals("vault", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
eb.Add($"{prefix}_sealed", r.IsSealed.ToString().ToLowerInvariant());
|
|
eb.Add($"{prefix}_initialized", r.IsInitialized.ToString().ToLowerInvariant());
|
|
}
|
|
}
|
|
}
|
|
|
|
private static List<SecretsManagerConfig> GetConfiguredSecretsManagers(IConfiguration config)
|
|
{
|
|
var managers = new List<SecretsManagerConfig>();
|
|
|
|
var secretsSection = config.GetSection("Secrets:Managers");
|
|
if (secretsSection.Exists())
|
|
{
|
|
foreach (var child in secretsSection.GetChildren())
|
|
{
|
|
var name = child.GetValue<string>("Name") ?? child.Key;
|
|
var url = child.GetValue<string>("Url");
|
|
var type = child.GetValue<string>("Type") ?? "vault";
|
|
var token = child.GetValue<string>("Token");
|
|
|
|
if (!string.IsNullOrEmpty(url))
|
|
{
|
|
managers.Add(new SecretsManagerConfig
|
|
{
|
|
Name = name,
|
|
Url = url,
|
|
Type = type,
|
|
Token = token
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
// Check legacy single-manager config
|
|
var legacyUrl = config.GetValue<string>("Secrets:Vault:Url")
|
|
?? config.GetValue<string>("Vault:Url");
|
|
if (!string.IsNullOrEmpty(legacyUrl) && managers.Count == 0)
|
|
{
|
|
managers.Add(new SecretsManagerConfig
|
|
{
|
|
Name = "vault",
|
|
Url = legacyUrl,
|
|
Type = "vault",
|
|
Token = config.GetValue<string>("Secrets:Vault:Token") ?? config.GetValue<string>("Vault:Token")
|
|
});
|
|
}
|
|
|
|
return managers;
|
|
}
|
|
|
|
private sealed class SecretsManagerConfig
|
|
{
|
|
public required string Name { get; init; }
|
|
public required string Url { get; init; }
|
|
public required string Type { get; init; }
|
|
public string? Token { get; init; }
|
|
}
|
|
|
|
private sealed class SecretsManagerResult
|
|
{
|
|
public required string Name { get; init; }
|
|
public required string Type { get; init; }
|
|
public bool Reachable { get; set; }
|
|
public bool AuthSuccess { get; set; }
|
|
public int LatencyMs { get; set; }
|
|
public bool IsSealed { get; set; }
|
|
public bool IsInitialized { get; set; } = true;
|
|
public string? Version { get; set; }
|
|
public string? ErrorMessage { get; set; }
|
|
}
|
|
}
|