doctor enhancements, setup, enhancements, ui functionality and design consolidation and , test projects fixes , product advisory attestation/rekor and delta verfications enhancements
This commit is contained in:
master
@@ -0,0 +1,338 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// SecretsManagerConnectivityCheck.cs
|
||||
// Sprint: SPRINT_20260118_018_Doctor_integration_health_expansion
|
||||
// Task: INTH-004 - Implement SecretsManagerConnectivityCheck
|
||||
// Description: Verify connectivity to secrets managers (Vault, AWS Secrets Manager, Azure Key Vault, etc.)
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Globalization;
|
||||
using System.Net.Http;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Doctor.Models;
|
||||
using StellaOps.Doctor.Plugins;
|
||||
|
||||
namespace StellaOps.Doctor.Plugins.Integration.Checks;
|
||||
|
||||
/// <summary>
|
||||
/// Verifies connectivity to configured secrets managers.
|
||||
/// Checks authentication, seal status (Vault), and API accessibility.
|
||||
/// </summary>
|
||||
public sealed class SecretsManagerConnectivityCheck : IDoctorCheck
|
||||
{
|
||||
private const string PluginId = "stellaops.doctor.integration";
|
||||
private const string CategoryName = "Integration";
|
||||
|
||||
/// <inheritdoc />
|
||||
public string CheckId => "check.integration.secrets.manager";
|
||||
|
||||
/// <inheritdoc />
|
||||
public string Name => "Secrets Manager Connectivity";
|
||||
|
||||
/// <inheritdoc />
|
||||
public string Description => "Verify connectivity to secrets managers (Vault, AWS Secrets Manager, Azure Key Vault)";
|
||||
|
||||
/// <inheritdoc />
|
||||
public DoctorSeverity DefaultSeverity => DoctorSeverity.Fail;
|
||||
|
||||
/// <inheritdoc />
|
||||
public IReadOnlyList<string> Tags => ["integration", "secrets", "vault", "security", "keyvault"];
|
||||
|
||||
/// <inheritdoc />
|
||||
public TimeSpan EstimatedDuration => TimeSpan.FromSeconds(10);
|
||||
|
||||
/// <inheritdoc />
|
||||
public bool CanRun(DoctorPluginContext context)
|
||||
{
|
||||
var secretsConfig = context.Configuration.GetSection("Secrets");
|
||||
return secretsConfig.Exists() && secretsConfig.GetChildren().Any();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<DoctorCheckResult> RunAsync(DoctorPluginContext context, CancellationToken ct)
|
||||
{
|
||||
var builder = context.CreateResult(CheckId, PluginId, CategoryName);
|
||||
|
||||
var managers = GetConfiguredSecretsManagers(context.Configuration);
|
||||
|
||||
if (managers.Count == 0)
|
||||
{
|
||||
return builder
|
||||
.Skip("No secrets managers configured")
|
||||
.WithEvidence("Secrets Managers", eb => eb.Add("configured_managers", "0"))
|
||||
.Build();
|
||||
}
|
||||
|
||||
var httpClientFactory = context.Services.GetService<IHttpClientFactory>();
|
||||
if (httpClientFactory == null)
|
||||
{
|
||||
return builder
|
||||
.Skip("IHttpClientFactory not available")
|
||||
.Build();
|
||||
}
|
||||
|
||||
var httpClient = httpClientFactory.CreateClient("DoctorHealthCheck");
|
||||
httpClient.Timeout = TimeSpan.FromSeconds(10);
|
||||
|
||||
var results = new List<SecretsManagerResult>();
|
||||
var unhealthy = new List<string>();
|
||||
var sealed_ = new List<string>(); // 'sealed' is a keyword
|
||||
|
||||
foreach (var mgr in managers)
|
||||
{
|
||||
var result = await CheckSecretsManagerAsync(httpClient, mgr, ct);
|
||||
results.Add(result);
|
||||
|
||||
if (!result.Reachable || !result.AuthSuccess)
|
||||
{
|
||||
unhealthy.Add(mgr.Name);
|
||||
}
|
||||
else if (result.IsSealed)
|
||||
{
|
||||
sealed_.Add(mgr.Name);
|
||||
}
|
||||
}
|
||||
|
||||
// Secrets manager issues are critical - blocks deployments
|
||||
if (unhealthy.Count > 0)
|
||||
{
|
||||
return builder
|
||||
.Fail($"{unhealthy.Count} secrets manager(s) unreachable")
|
||||
.WithEvidence("Secrets Managers", eb =>
|
||||
{
|
||||
eb.Add("total_managers", managers.Count.ToString(CultureInfo.InvariantCulture));
|
||||
eb.Add("healthy_managers", (managers.Count - unhealthy.Count).ToString(CultureInfo.InvariantCulture));
|
||||
eb.Add("unhealthy_managers", string.Join(", ", unhealthy));
|
||||
AddSecretsEvidence(eb, results);
|
||||
})
|
||||
.WithCauses(
|
||||
"Secrets manager is down",
|
||||
"Network connectivity issue",
|
||||
"Authentication token expired",
|
||||
"TLS certificate issue")
|
||||
.WithRemediation(rb =>
|
||||
{
|
||||
rb.AddStep(1, "Test secrets manager connectivity",
|
||||
$"stella secrets ping {unhealthy[0]}",
|
||||
CommandType.Shell);
|
||||
rb.AddStep(2, "Refresh authentication",
|
||||
$"stella secrets auth refresh {unhealthy[0]}",
|
||||
CommandType.Manual);
|
||||
})
|
||||
.WithVerification($"stella doctor --check {CheckId}")
|
||||
.Build();
|
||||
}
|
||||
|
||||
if (sealed_.Count > 0)
|
||||
{
|
||||
return builder
|
||||
.Fail($"{sealed_.Count} Vault instance(s) are sealed")
|
||||
.WithEvidence("Secrets Managers", eb =>
|
||||
{
|
||||
eb.Add("total_managers", managers.Count.ToString(CultureInfo.InvariantCulture));
|
||||
eb.Add("healthy_managers", (managers.Count - sealed_.Count).ToString(CultureInfo.InvariantCulture));
|
||||
eb.Add("sealed_vaults", string.Join(", ", sealed_));
|
||||
AddSecretsEvidence(eb, results);
|
||||
})
|
||||
.WithCauses(
|
||||
"Vault was restarted and needs unseal",
|
||||
"Vault auto-seal triggered",
|
||||
"HSM connectivity lost")
|
||||
.WithRemediation(rb =>
|
||||
{
|
||||
rb.AddStep(1, "Unseal Vault",
|
||||
$"vault operator unseal",
|
||||
CommandType.Manual);
|
||||
rb.AddStep(2, "Check seal status",
|
||||
$"stella secrets status {sealed_[0]}",
|
||||
CommandType.Shell);
|
||||
})
|
||||
.WithVerification($"stella doctor --check {CheckId}")
|
||||
.Build();
|
||||
}
|
||||
|
||||
return builder
|
||||
.Pass($"{managers.Count} secrets manager(s) healthy")
|
||||
.WithEvidence("Secrets Managers", eb =>
|
||||
{
|
||||
eb.Add("total_managers", managers.Count.ToString(CultureInfo.InvariantCulture));
|
||||
eb.Add("healthy_managers", managers.Count.ToString(CultureInfo.InvariantCulture));
|
||||
AddSecretsEvidence(eb, results);
|
||||
})
|
||||
.Build();
|
||||
}
|
||||
|
||||
private static async Task<SecretsManagerResult> CheckSecretsManagerAsync(
|
||||
HttpClient client, SecretsManagerConfig mgr, CancellationToken ct)
|
||||
{
|
||||
var result = new SecretsManagerResult { Name = mgr.Name, Type = mgr.Type };
|
||||
|
||||
try
|
||||
{
|
||||
var healthEndpoint = GetHealthEndpoint(mgr);
|
||||
var request = new HttpRequestMessage(HttpMethod.Get, healthEndpoint);
|
||||
|
||||
// Add auth headers based on type
|
||||
if (!string.IsNullOrEmpty(mgr.Token))
|
||||
{
|
||||
if (mgr.Type.Equals("vault", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
request.Headers.Add("X-Vault-Token", mgr.Token);
|
||||
}
|
||||
else
|
||||
{
|
||||
request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", mgr.Token);
|
||||
}
|
||||
}
|
||||
|
||||
var sw = System.Diagnostics.Stopwatch.StartNew();
|
||||
var response = await client.SendAsync(request, ct);
|
||||
sw.Stop();
|
||||
|
||||
result.LatencyMs = (int)sw.ElapsedMilliseconds;
|
||||
result.Reachable = true;
|
||||
result.AuthSuccess = response.StatusCode != System.Net.HttpStatusCode.Unauthorized &&
|
||||
response.StatusCode != System.Net.HttpStatusCode.Forbidden;
|
||||
|
||||
// Parse response for type-specific info
|
||||
if (response.IsSuccessStatusCode)
|
||||
{
|
||||
var json = await response.Content.ReadAsStringAsync(ct);
|
||||
ParseSecretsManagerResponse(json, mgr.Type, result);
|
||||
}
|
||||
}
|
||||
catch (HttpRequestException ex)
|
||||
{
|
||||
result.Reachable = false;
|
||||
result.ErrorMessage = ex.Message;
|
||||
}
|
||||
catch (TaskCanceledException)
|
||||
{
|
||||
result.Reachable = false;
|
||||
result.ErrorMessage = "Timeout";
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static void ParseSecretsManagerResponse(string json, string type, SecretsManagerResult result)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var doc = JsonDocument.Parse(json);
|
||||
|
||||
if (type.Equals("vault", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
// Vault health endpoint returns sealed status
|
||||
if (doc.RootElement.TryGetProperty("sealed", out var sealedEl))
|
||||
{
|
||||
result.IsSealed = sealedEl.GetBoolean();
|
||||
}
|
||||
if (doc.RootElement.TryGetProperty("initialized", out var initEl))
|
||||
{
|
||||
result.IsInitialized = initEl.GetBoolean();
|
||||
}
|
||||
if (doc.RootElement.TryGetProperty("version", out var verEl))
|
||||
{
|
||||
result.Version = verEl.GetString();
|
||||
}
|
||||
}
|
||||
}
|
||||
catch { }
|
||||
}
|
||||
|
||||
private static string GetHealthEndpoint(SecretsManagerConfig mgr)
|
||||
{
|
||||
return mgr.Type.ToLowerInvariant() switch
|
||||
{
|
||||
"vault" => $"{mgr.Url.TrimEnd('/')}/v1/sys/health?standbyok=true&sealedcode=200&uninitcode=200",
|
||||
"aws" => mgr.Url, // AWS uses SDK, URL is just for config reference
|
||||
"azure" => $"{mgr.Url.TrimEnd('/')}/healthstatus",
|
||||
"gcp" => mgr.Url, // GCP uses SDK
|
||||
_ => $"{mgr.Url.TrimEnd('/')}/health"
|
||||
};
|
||||
}
|
||||
|
||||
private static void AddSecretsEvidence(EvidenceBuilder eb, List<SecretsManagerResult> results)
|
||||
{
|
||||
foreach (var r in results)
|
||||
{
|
||||
var prefix = $"secrets_{r.Name.ToLowerInvariant().Replace(" ", "_").Replace("-", "_")}";
|
||||
eb.Add($"{prefix}_type", r.Type);
|
||||
eb.Add($"{prefix}_reachable", r.Reachable.ToString().ToLowerInvariant());
|
||||
eb.Add($"{prefix}_auth_success", r.AuthSuccess.ToString().ToLowerInvariant());
|
||||
eb.Add($"{prefix}_latency_ms", r.LatencyMs.ToString(CultureInfo.InvariantCulture));
|
||||
if (r.Type.Equals("vault", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
eb.Add($"{prefix}_sealed", r.IsSealed.ToString().ToLowerInvariant());
|
||||
eb.Add($"{prefix}_initialized", r.IsInitialized.ToString().ToLowerInvariant());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static List<SecretsManagerConfig> GetConfiguredSecretsManagers(IConfiguration config)
|
||||
{
|
||||
var managers = new List<SecretsManagerConfig>();
|
||||
|
||||
var secretsSection = config.GetSection("Secrets:Managers");
|
||||
if (secretsSection.Exists())
|
||||
{
|
||||
foreach (var child in secretsSection.GetChildren())
|
||||
{
|
||||
var name = child.GetValue<string>("Name") ?? child.Key;
|
||||
var url = child.GetValue<string>("Url");
|
||||
var type = child.GetValue<string>("Type") ?? "vault";
|
||||
var token = child.GetValue<string>("Token");
|
||||
|
||||
if (!string.IsNullOrEmpty(url))
|
||||
{
|
||||
managers.Add(new SecretsManagerConfig
|
||||
{
|
||||
Name = name,
|
||||
Url = url,
|
||||
Type = type,
|
||||
Token = token
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check legacy single-manager config
|
||||
var legacyUrl = config.GetValue<string>("Secrets:Vault:Url")
|
||||
?? config.GetValue<string>("Vault:Url");
|
||||
if (!string.IsNullOrEmpty(legacyUrl) && managers.Count == 0)
|
||||
{
|
||||
managers.Add(new SecretsManagerConfig
|
||||
{
|
||||
Name = "vault",
|
||||
Url = legacyUrl,
|
||||
Type = "vault",
|
||||
Token = config.GetValue<string>("Secrets:Vault:Token") ?? config.GetValue<string>("Vault:Token")
|
||||
});
|
||||
}
|
||||
|
||||
return managers;
|
||||
}
|
||||
|
||||
private sealed class SecretsManagerConfig
|
||||
{
|
||||
public required string Name { get; init; }
|
||||
public required string Url { get; init; }
|
||||
public required string Type { get; init; }
|
||||
public string? Token { get; init; }
|
||||
}
|
||||
|
||||
private sealed class SecretsManagerResult
|
||||
{
|
||||
public required string Name { get; init; }
|
||||
public required string Type { get; init; }
|
||||
public bool Reachable { get; set; }
|
||||
public bool AuthSuccess { get; set; }
|
||||
public int LatencyMs { get; set; }
|
||||
public bool IsSealed { get; set; }
|
||||
public bool IsInitialized { get; set; } = true;
|
||||
public string? Version { get; set; }
|
||||
public string? ErrorMessage { get; set; }
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user