75f6942769
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
- Implemented MigrationCategoryTests to validate migration categorization for startup, release, seed, and data migrations. - Added tests for edge cases, including null, empty, and whitespace migration names. - Created StartupMigrationHostTests to verify the behavior of the migration host with real PostgreSQL instances using Testcontainers. - Included tests for migration execution, schema creation, and handling of pending release migrations. - Added SQL migration files for testing: creating a test table, adding a column, a release migration, and seeding data.
559 lines
20 KiB
C#
559 lines
20 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Net.Http;
|
|
using System.Net.Http.Headers;
|
|
using System.Text.Json;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using Microsoft.Extensions.Logging;
|
|
using StellaOps.Auth.Abstractions;
|
|
using StellaOps.Auth.Client;
|
|
using StellaOps.Cli.Configuration;
|
|
using StellaOps.Cli.Extensions;
|
|
using StellaOps.Cli.Services.Models;
|
|
|
|
namespace StellaOps.Cli.Services;
|
|
|
|
/// <summary>
|
|
/// Client for observability API operations.
|
|
/// Per CLI-OBS-51-001.
|
|
/// </summary>
|
|
internal sealed class ObservabilityClient : IObservabilityClient
|
|
{
|
|
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
|
|
private static readonly TimeSpan TokenRefreshSkew = TimeSpan.FromSeconds(30);
|
|
|
|
private readonly HttpClient httpClient;
|
|
private readonly StellaOpsCliOptions options;
|
|
private readonly ILogger<ObservabilityClient> logger;
|
|
private readonly IStellaOpsTokenClient? tokenClient;
|
|
private readonly object tokenSync = new();
|
|
|
|
private string? cachedAccessToken;
|
|
private DateTimeOffset cachedAccessTokenExpiresAt = DateTimeOffset.MinValue;
|
|
|
|
public ObservabilityClient(
|
|
HttpClient httpClient,
|
|
StellaOpsCliOptions options,
|
|
ILogger<ObservabilityClient> logger,
|
|
IStellaOpsTokenClient? tokenClient = null)
|
|
{
|
|
this.httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
|
|
this.options = options ?? throw new ArgumentNullException(nameof(options));
|
|
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
|
this.tokenClient = tokenClient;
|
|
|
|
if (!string.IsNullOrWhiteSpace(options.BackendUrl) && httpClient.BaseAddress is null)
|
|
{
|
|
if (Uri.TryCreate(options.BackendUrl, UriKind.Absolute, out var baseUri))
|
|
{
|
|
httpClient.BaseAddress = baseUri;
|
|
}
|
|
}
|
|
}
|
|
|
|
public async Task<ObsTopResult> GetHealthSummaryAsync(
|
|
ObsTopRequest request,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(request);
|
|
|
|
try
|
|
{
|
|
EnsureConfigured();
|
|
|
|
var requestUri = BuildHealthSummaryUri(request);
|
|
using var httpRequest = new HttpRequestMessage(HttpMethod.Get, requestUri);
|
|
await AuthorizeRequestAsync(httpRequest, cancellationToken).ConfigureAwait(false);
|
|
|
|
using var response = await httpClient.SendAsync(httpRequest, cancellationToken).ConfigureAwait(false);
|
|
|
|
if (!response.IsSuccessStatusCode)
|
|
{
|
|
var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
|
|
logger.LogError(
|
|
"Failed to get health summary (status {StatusCode}). Response: {Payload}",
|
|
(int)response.StatusCode,
|
|
string.IsNullOrWhiteSpace(payload) ? "<empty>" : payload);
|
|
|
|
return new ObsTopResult
|
|
{
|
|
Success = false,
|
|
Errors = [$"API returned {(int)response.StatusCode}: {response.ReasonPhrase}"]
|
|
};
|
|
}
|
|
|
|
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
|
|
var summary = await JsonSerializer
|
|
.DeserializeAsync<PlatformHealthSummary>(stream, SerializerOptions, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
|
|
return new ObsTopResult
|
|
{
|
|
Success = true,
|
|
Summary = summary ?? new PlatformHealthSummary()
|
|
};
|
|
}
|
|
catch (HttpRequestException ex)
|
|
{
|
|
logger.LogError(ex, "HTTP error while fetching health summary");
|
|
return new ObsTopResult
|
|
{
|
|
Success = false,
|
|
Errors = [$"Connection error: {ex.Message}"]
|
|
};
|
|
}
|
|
catch (TaskCanceledException ex) when (!cancellationToken.IsCancellationRequested)
|
|
{
|
|
logger.LogError(ex, "Request timed out while fetching health summary");
|
|
return new ObsTopResult
|
|
{
|
|
Success = false,
|
|
Errors = ["Request timed out"]
|
|
};
|
|
}
|
|
}
|
|
|
|
private static string BuildHealthSummaryUri(ObsTopRequest request)
|
|
{
|
|
var queryParams = new List<string>();
|
|
|
|
if (request.Services.Count > 0)
|
|
{
|
|
foreach (var service in request.Services)
|
|
{
|
|
queryParams.Add($"service={Uri.EscapeDataString(service)}");
|
|
}
|
|
}
|
|
|
|
if (!string.IsNullOrWhiteSpace(request.Tenant))
|
|
{
|
|
queryParams.Add($"tenant={Uri.EscapeDataString(request.Tenant)}");
|
|
}
|
|
|
|
queryParams.Add($"includeQueues={request.IncludeQueues.ToString().ToLowerInvariant()}");
|
|
queryParams.Add($"maxAlerts={request.MaxAlerts}");
|
|
|
|
var query = queryParams.Count > 0 ? "?" + string.Join("&", queryParams) : string.Empty;
|
|
return $"/api/v1/observability/health{query}";
|
|
}
|
|
|
|
private void EnsureConfigured()
|
|
{
|
|
if (string.IsNullOrWhiteSpace(options.BackendUrl) && httpClient.BaseAddress is null)
|
|
{
|
|
throw new InvalidOperationException(
|
|
"Backend URL not configured. Set STELLAOPS_BACKEND_URL or use --backend-url.");
|
|
}
|
|
}
|
|
|
|
private async Task AuthorizeRequestAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
|
{
|
|
var token = await GetAccessTokenAsync(cancellationToken).ConfigureAwait(false);
|
|
if (!string.IsNullOrWhiteSpace(token))
|
|
{
|
|
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token);
|
|
}
|
|
}
|
|
|
|
private async Task<string?> GetAccessTokenAsync(CancellationToken cancellationToken)
|
|
{
|
|
if (tokenClient is null)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
lock (tokenSync)
|
|
{
|
|
if (cachedAccessToken is not null && DateTimeOffset.UtcNow < cachedAccessTokenExpiresAt - TokenRefreshSkew)
|
|
{
|
|
return cachedAccessToken;
|
|
}
|
|
}
|
|
|
|
try
|
|
{
|
|
var result = await tokenClient.GetAccessTokenAsync(StellaOpsScopes.ObservabilityRead, cancellationToken).ConfigureAwait(false);
|
|
|
|
lock (tokenSync)
|
|
{
|
|
cachedAccessToken = result.AccessToken;
|
|
cachedAccessTokenExpiresAt = result.ExpiresAt;
|
|
}
|
|
return result.AccessToken;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
logger.LogWarning(ex, "Token acquisition failed");
|
|
return null;
|
|
}
|
|
}
|
|
|
|
// CLI-OBS-52-001: Trace retrieval
|
|
public async Task<ObsTraceResult> GetTraceAsync(
|
|
ObsTraceRequest request,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(request);
|
|
|
|
try
|
|
{
|
|
EnsureConfigured();
|
|
|
|
var requestUri = BuildTraceUri(request);
|
|
using var httpRequest = new HttpRequestMessage(HttpMethod.Get, requestUri);
|
|
await AuthorizeRequestAsync(httpRequest, cancellationToken).ConfigureAwait(false);
|
|
|
|
using var response = await httpClient.SendAsync(httpRequest, cancellationToken).ConfigureAwait(false);
|
|
|
|
if (response.StatusCode == System.Net.HttpStatusCode.NotFound)
|
|
{
|
|
return new ObsTraceResult
|
|
{
|
|
Success = false,
|
|
Errors = [$"Trace not found: {request.TraceId}"]
|
|
};
|
|
}
|
|
|
|
if (!response.IsSuccessStatusCode)
|
|
{
|
|
var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
|
|
logger.LogError(
|
|
"Failed to get trace (status {StatusCode}). Response: {Payload}",
|
|
(int)response.StatusCode,
|
|
string.IsNullOrWhiteSpace(payload) ? "<empty>" : payload);
|
|
|
|
return new ObsTraceResult
|
|
{
|
|
Success = false,
|
|
Errors = [$"API returned {(int)response.StatusCode}: {response.ReasonPhrase}"]
|
|
};
|
|
}
|
|
|
|
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
|
|
var trace = await JsonSerializer
|
|
.DeserializeAsync<DistributedTrace>(stream, SerializerOptions, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
|
|
return new ObsTraceResult
|
|
{
|
|
Success = true,
|
|
Trace = trace
|
|
};
|
|
}
|
|
catch (HttpRequestException ex)
|
|
{
|
|
logger.LogError(ex, "HTTP error while fetching trace");
|
|
return new ObsTraceResult
|
|
{
|
|
Success = false,
|
|
Errors = [$"Connection error: {ex.Message}"]
|
|
};
|
|
}
|
|
catch (TaskCanceledException ex) when (!cancellationToken.IsCancellationRequested)
|
|
{
|
|
logger.LogError(ex, "Request timed out while fetching trace");
|
|
return new ObsTraceResult
|
|
{
|
|
Success = false,
|
|
Errors = ["Request timed out"]
|
|
};
|
|
}
|
|
}
|
|
|
|
private static string BuildTraceUri(ObsTraceRequest request)
|
|
{
|
|
var queryParams = new List<string>();
|
|
|
|
if (!string.IsNullOrWhiteSpace(request.Tenant))
|
|
{
|
|
queryParams.Add($"tenant={Uri.EscapeDataString(request.Tenant)}");
|
|
}
|
|
|
|
queryParams.Add($"includeEvidence={request.IncludeEvidence.ToString().ToLowerInvariant()}");
|
|
|
|
var query = queryParams.Count > 0 ? "?" + string.Join("&", queryParams) : string.Empty;
|
|
return $"/api/v1/observability/traces/{Uri.EscapeDataString(request.TraceId)}{query}";
|
|
}
|
|
|
|
// CLI-OBS-52-001: Logs retrieval
|
|
public async Task<ObsLogsResult> GetLogsAsync(
|
|
ObsLogsRequest request,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(request);
|
|
|
|
try
|
|
{
|
|
EnsureConfigured();
|
|
|
|
var requestUri = BuildLogsUri(request);
|
|
using var httpRequest = new HttpRequestMessage(HttpMethod.Get, requestUri);
|
|
await AuthorizeRequestAsync(httpRequest, cancellationToken).ConfigureAwait(false);
|
|
|
|
using var response = await httpClient.SendAsync(httpRequest, cancellationToken).ConfigureAwait(false);
|
|
|
|
if (!response.IsSuccessStatusCode)
|
|
{
|
|
var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
|
|
logger.LogError(
|
|
"Failed to get logs (status {StatusCode}). Response: {Payload}",
|
|
(int)response.StatusCode,
|
|
string.IsNullOrWhiteSpace(payload) ? "<empty>" : payload);
|
|
|
|
return new ObsLogsResult
|
|
{
|
|
Success = false,
|
|
Errors = [$"API returned {(int)response.StatusCode}: {response.ReasonPhrase}"]
|
|
};
|
|
}
|
|
|
|
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
|
|
var result = await JsonSerializer
|
|
.DeserializeAsync<ObsLogsResult>(stream, SerializerOptions, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
|
|
return result ?? new ObsLogsResult { Success = true };
|
|
}
|
|
catch (HttpRequestException ex)
|
|
{
|
|
logger.LogError(ex, "HTTP error while fetching logs");
|
|
return new ObsLogsResult
|
|
{
|
|
Success = false,
|
|
Errors = [$"Connection error: {ex.Message}"]
|
|
};
|
|
}
|
|
catch (TaskCanceledException ex) when (!cancellationToken.IsCancellationRequested)
|
|
{
|
|
logger.LogError(ex, "Request timed out while fetching logs");
|
|
return new ObsLogsResult
|
|
{
|
|
Success = false,
|
|
Errors = ["Request timed out"]
|
|
};
|
|
}
|
|
}
|
|
|
|
private static string BuildLogsUri(ObsLogsRequest request)
|
|
{
|
|
var queryParams = new List<string>
|
|
{
|
|
$"from={Uri.EscapeDataString(request.From.ToString("o"))}",
|
|
$"to={Uri.EscapeDataString(request.To.ToString("o"))}"
|
|
};
|
|
|
|
if (!string.IsNullOrWhiteSpace(request.Tenant))
|
|
{
|
|
queryParams.Add($"tenant={Uri.EscapeDataString(request.Tenant)}");
|
|
}
|
|
|
|
foreach (var service in request.Services)
|
|
{
|
|
queryParams.Add($"service={Uri.EscapeDataString(service)}");
|
|
}
|
|
|
|
foreach (var level in request.Levels)
|
|
{
|
|
queryParams.Add($"level={Uri.EscapeDataString(level)}");
|
|
}
|
|
|
|
if (!string.IsNullOrWhiteSpace(request.Query))
|
|
{
|
|
queryParams.Add($"q={Uri.EscapeDataString(request.Query)}");
|
|
}
|
|
|
|
queryParams.Add($"pageSize={request.PageSize}");
|
|
|
|
if (!string.IsNullOrWhiteSpace(request.PageToken))
|
|
{
|
|
queryParams.Add($"pageToken={Uri.EscapeDataString(request.PageToken)}");
|
|
}
|
|
|
|
var query = "?" + string.Join("&", queryParams);
|
|
return $"/api/v1/observability/logs{query}";
|
|
}
|
|
|
|
// CLI-OBS-55-001: Incident mode operations
|
|
public async Task<IncidentModeResult> GetIncidentModeStatusAsync(
|
|
string? tenant,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
try
|
|
{
|
|
EnsureConfigured();
|
|
|
|
var query = !string.IsNullOrWhiteSpace(tenant)
|
|
? $"?tenant={Uri.EscapeDataString(tenant)}"
|
|
: string.Empty;
|
|
|
|
using var httpRequest = new HttpRequestMessage(HttpMethod.Get, $"/api/v1/observability/incident-mode{query}");
|
|
await AuthorizeRequestAsync(httpRequest, cancellationToken).ConfigureAwait(false);
|
|
|
|
using var response = await httpClient.SendAsync(httpRequest, cancellationToken).ConfigureAwait(false);
|
|
|
|
if (!response.IsSuccessStatusCode)
|
|
{
|
|
var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
|
|
logger.LogError(
|
|
"Failed to get incident mode status (status {StatusCode}). Response: {Payload}",
|
|
(int)response.StatusCode,
|
|
string.IsNullOrWhiteSpace(payload) ? "<empty>" : payload);
|
|
|
|
return new IncidentModeResult
|
|
{
|
|
Success = false,
|
|
Errors = [$"API returned {(int)response.StatusCode}: {response.ReasonPhrase}"]
|
|
};
|
|
}
|
|
|
|
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
|
|
var state = await JsonSerializer
|
|
.DeserializeAsync<IncidentModeState>(stream, SerializerOptions, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
|
|
return new IncidentModeResult
|
|
{
|
|
Success = true,
|
|
State = state
|
|
};
|
|
}
|
|
catch (HttpRequestException ex)
|
|
{
|
|
logger.LogError(ex, "HTTP error while fetching incident mode status");
|
|
return new IncidentModeResult
|
|
{
|
|
Success = false,
|
|
Errors = [$"Connection error: {ex.Message}"]
|
|
};
|
|
}
|
|
catch (TaskCanceledException ex) when (!cancellationToken.IsCancellationRequested)
|
|
{
|
|
logger.LogError(ex, "Request timed out while fetching incident mode status");
|
|
return new IncidentModeResult
|
|
{
|
|
Success = false,
|
|
Errors = ["Request timed out"]
|
|
};
|
|
}
|
|
}
|
|
|
|
public async Task<IncidentModeResult> EnableIncidentModeAsync(
|
|
IncidentModeEnableRequest request,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(request);
|
|
|
|
try
|
|
{
|
|
EnsureConfigured();
|
|
|
|
using var httpRequest = new HttpRequestMessage(HttpMethod.Post, "/api/v1/observability/incident-mode/enable");
|
|
var json = JsonSerializer.Serialize(request, SerializerOptions);
|
|
httpRequest.Content = new StringContent(json, System.Text.Encoding.UTF8, "application/json");
|
|
await AuthorizeRequestAsync(httpRequest, cancellationToken).ConfigureAwait(false);
|
|
|
|
using var response = await httpClient.SendAsync(httpRequest, cancellationToken).ConfigureAwait(false);
|
|
|
|
if (!response.IsSuccessStatusCode)
|
|
{
|
|
var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
|
|
logger.LogError(
|
|
"Failed to enable incident mode (status {StatusCode}). Response: {Payload}",
|
|
(int)response.StatusCode,
|
|
string.IsNullOrWhiteSpace(payload) ? "<empty>" : payload);
|
|
|
|
return new IncidentModeResult
|
|
{
|
|
Success = false,
|
|
Errors = [$"API returned {(int)response.StatusCode}: {response.ReasonPhrase}"]
|
|
};
|
|
}
|
|
|
|
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
|
|
var result = await JsonSerializer
|
|
.DeserializeAsync<IncidentModeResult>(stream, SerializerOptions, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
|
|
return result ?? new IncidentModeResult { Success = true };
|
|
}
|
|
catch (HttpRequestException ex)
|
|
{
|
|
logger.LogError(ex, "HTTP error while enabling incident mode");
|
|
return new IncidentModeResult
|
|
{
|
|
Success = false,
|
|
Errors = [$"Connection error: {ex.Message}"]
|
|
};
|
|
}
|
|
catch (TaskCanceledException ex) when (!cancellationToken.IsCancellationRequested)
|
|
{
|
|
logger.LogError(ex, "Request timed out while enabling incident mode");
|
|
return new IncidentModeResult
|
|
{
|
|
Success = false,
|
|
Errors = ["Request timed out"]
|
|
};
|
|
}
|
|
}
|
|
|
|
public async Task<IncidentModeResult> DisableIncidentModeAsync(
|
|
IncidentModeDisableRequest request,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(request);
|
|
|
|
try
|
|
{
|
|
EnsureConfigured();
|
|
|
|
using var httpRequest = new HttpRequestMessage(HttpMethod.Post, "/api/v1/observability/incident-mode/disable");
|
|
var json = JsonSerializer.Serialize(request, SerializerOptions);
|
|
httpRequest.Content = new StringContent(json, System.Text.Encoding.UTF8, "application/json");
|
|
await AuthorizeRequestAsync(httpRequest, cancellationToken).ConfigureAwait(false);
|
|
|
|
using var response = await httpClient.SendAsync(httpRequest, cancellationToken).ConfigureAwait(false);
|
|
|
|
if (!response.IsSuccessStatusCode)
|
|
{
|
|
var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
|
|
logger.LogError(
|
|
"Failed to disable incident mode (status {StatusCode}). Response: {Payload}",
|
|
(int)response.StatusCode,
|
|
string.IsNullOrWhiteSpace(payload) ? "<empty>" : payload);
|
|
|
|
return new IncidentModeResult
|
|
{
|
|
Success = false,
|
|
Errors = [$"API returned {(int)response.StatusCode}: {response.ReasonPhrase}"]
|
|
};
|
|
}
|
|
|
|
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
|
|
var result = await JsonSerializer
|
|
.DeserializeAsync<IncidentModeResult>(stream, SerializerOptions, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
|
|
return result ?? new IncidentModeResult { Success = true };
|
|
}
|
|
catch (HttpRequestException ex)
|
|
{
|
|
logger.LogError(ex, "HTTP error while disabling incident mode");
|
|
return new IncidentModeResult
|
|
{
|
|
Success = false,
|
|
Errors = [$"Connection error: {ex.Message}"]
|
|
};
|
|
}
|
|
catch (TaskCanceledException ex) when (!cancellationToken.IsCancellationRequested)
|
|
{
|
|
logger.LogError(ex, "Request timed out while disabling incident mode");
|
|
return new IncidentModeResult
|
|
{
|
|
Success = false,
|
|
Errors = ["Request timed out"]
|
|
};
|
|
}
|
|
}
|
|
}
|