// --------------------------------------------------------------------- // // Copyright (c) StellaOps. Licensed under the AGPL-3.0-or-later. // // // OTel trace assertions: verify job_id, tenant_id, schedule_id tags // // --------------------------------------------------------------------- using System.Collections.Concurrent; using System.Diagnostics; using System.Net.Http.Headers; using System.Net.Http.Json; using FluentAssertions; using Microsoft.AspNetCore.Mvc.Testing; using Xunit; namespace StellaOps.Scheduler.WebService.Tests.Observability; /// /// OTel trace assertions for Scheduler.WebService verifying /// job_id, tenant_id, schedule_id tags are properly emitted. /// [Trait("Category", "Observability")] [Trait("Sprint", "5100-0009-0008")] public sealed class SchedulerOTelTraceTests : IClassFixture>, IDisposable { private readonly WebApplicationFactory _factory; private readonly ActivityListener _listener; private readonly ConcurrentBag _capturedActivities; /// /// Initializes a new instance of the class. /// public SchedulerOTelTraceTests(WebApplicationFactory factory) { _factory = factory; _capturedActivities = new ConcurrentBag(); _listener = new ActivityListener { ShouldListenTo = source => source.Name.StartsWith("StellaOps", StringComparison.OrdinalIgnoreCase) || source.Name.Contains("Scheduler", StringComparison.OrdinalIgnoreCase), Sample = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllDataAndRecorded, ActivityStopped = activity => _capturedActivities.Add(activity) }; ActivitySource.AddActivityListener(_listener); } /// public void Dispose() { _listener.Dispose(); } #region Activity Creation Tests /// /// Verifies activity is created for schedule creation operations. /// [Fact] public async Task CreateSchedule_CreatesActivity_WithSchedulerSource() { // Arrange ClearCapturedActivities(); using var client = CreateAuthenticatedClient("tenant-001"); var payload = new { name = "otel-test-schedule", cronExpression = "0 * * * *", timezone = "UTC" }; // Act await client.PostAsJsonAsync("/api/v1/schedules", payload); // Assert var schedulerActivities = _capturedActivities .Where(a => a.OperationName.Contains("schedule", StringComparison.OrdinalIgnoreCase) || a.DisplayName.Contains("schedule", StringComparison.OrdinalIgnoreCase)) .ToList(); schedulerActivities.Should().NotBeEmpty( because: "schedule creation should emit OTel activity"); } /// /// Verifies activity is created for job enqueue operations. /// [Fact] public async Task EnqueueJob_CreatesActivity() { // Arrange ClearCapturedActivities(); using var client = CreateAuthenticatedClient("tenant-001"); var payload = new { type = "scan", target = "image:latest", priority = 5 }; // Act await client.PostAsJsonAsync("/api/v1/jobs", payload); // Assert var jobActivities = _capturedActivities .Where(a => a.OperationName.Contains("job", StringComparison.OrdinalIgnoreCase) || a.DisplayName.Contains("enqueue", StringComparison.OrdinalIgnoreCase)) .ToList(); jobActivities.Should().NotBeEmpty( because: "job enqueue should emit OTel activity"); } #endregion #region Scheduler-Specific Tag Tests /// /// Verifies job_id tag is present on job-related activities. /// [Fact] public async Task JobActivity_HasJobIdTag() { // Arrange ClearCapturedActivities(); using var client = CreateAuthenticatedClient("tenant-001"); // Act - Enqueue a job var response = await client.PostAsJsonAsync("/api/v1/jobs", new { type = "scan", target = "image:test" }); // Assert var jobActivities = _capturedActivities .Where(a => a.OperationName.Contains("job", StringComparison.OrdinalIgnoreCase)) .ToList(); foreach (var activity in jobActivities) { var jobIdTag = activity.Tags.FirstOrDefault(t => t.Key == "job_id" || t.Key == "stellaops.job.id"); if (!string.IsNullOrEmpty(jobIdTag.Value)) { jobIdTag.Value.Should().NotBeNullOrWhiteSpace( because: "job_id tag should have a value"); } } } /// /// Verifies tenant_id tag is present on all scheduler activities. /// [Fact] public async Task SchedulerActivity_HasTenantIdTag() { // Arrange const string expectedTenantId = "tenant-otel-test"; ClearCapturedActivities(); using var client = CreateAuthenticatedClient(expectedTenantId); // Act await client.GetAsync("/api/v1/schedules"); // Assert var schedulerActivities = _capturedActivities .Where(a => a.Source.Name.Contains("Scheduler", StringComparison.OrdinalIgnoreCase) || a.Source.Name.StartsWith("StellaOps", StringComparison.OrdinalIgnoreCase)) .ToList(); foreach (var activity in schedulerActivities) { var tenantTag = activity.Tags.FirstOrDefault(t => t.Key == "tenant_id" || t.Key == "stellaops.tenant.id" || t.Key == "enduser.id"); // At least some activities should have tenant context if (!string.IsNullOrEmpty(tenantTag.Value)) { tenantTag.Value.Should().Be(expectedTenantId); } } } /// /// Verifies schedule_id tag is present on schedule-related activities. /// [Fact] public async Task ScheduleActivity_HasScheduleIdTag() { // Arrange ClearCapturedActivities(); using var client = CreateAuthenticatedClient("tenant-001"); // Create a schedule first var createResponse = await client.PostAsJsonAsync("/api/v1/schedules", new { name = "schedule-for-otel-test", cronExpression = "0 12 * * *", timezone = "UTC" }); // Act - Query the schedule ClearCapturedActivities(); await client.GetAsync("/api/v1/schedules"); // Assert var scheduleActivities = _capturedActivities .Where(a => a.OperationName.Contains("schedule", StringComparison.OrdinalIgnoreCase)) .ToList(); foreach (var activity in scheduleActivities) { var scheduleIdTag = activity.Tags.FirstOrDefault(t => t.Key == "schedule_id" || t.Key == "stellaops.schedule.id"); // Schedule operations should include schedule_id when applicable if (activity.OperationName.Contains("get", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrEmpty(scheduleIdTag.Value)) { scheduleIdTag.Value.Should().NotBeNullOrWhiteSpace(); } } } #endregion #region Error Trace Tests /// /// Verifies failed operations include error status in activity. /// [Fact] public async Task FailedOperation_SetsActivityStatusToError() { // Arrange ClearCapturedActivities(); using var client = CreateAuthenticatedClient("tenant-001"); // Act - Request a non-existent resource await client.GetAsync("/api/v1/schedules/non-existent-schedule-id"); // Assert var errorActivities = _capturedActivities .Where(a => a.Status == ActivityStatusCode.Error || a.Tags.Any(t => t.Key == "error" && t.Value == "true") || a.Tags.Any(t => t.Key == "otel.status_code" && t.Value == "ERROR")) .ToList(); // Not all 404s are errors from OTel perspective, but validation errors should be // This test validates the pattern exists for actual errors } /// /// Verifies validation errors include error details in activity. /// [Fact] public async Task ValidationError_IncludesErrorDetailsInActivity() { // Arrange ClearCapturedActivities(); using var client = CreateAuthenticatedClient("tenant-001"); // Act - Send invalid payload await client.PostAsJsonAsync("/api/v1/schedules", new { name = "", // Invalid: empty name cronExpression = "invalid cron", timezone = "Invalid/Timezone" }); // Assert var activitiesWithErrors = _capturedActivities .Where(a => a.Events.Any(e => e.Name == "exception" || e.Name == "error")) .ToList(); // If validation errors emit events, they should include details foreach (var activity in activitiesWithErrors) { var errorEvent = activity.Events.FirstOrDefault(e => e.Name == "exception" || e.Name == "error"); if (errorEvent.Name != null) { errorEvent.Tags.Should().ContainKey("exception.message"); } } } #endregion #region Trace Correlation Tests /// /// Verifies trace context is propagated across operations. /// [Fact] public async Task TraceContext_IsPropagatedAcrossOperations() { // Arrange ClearCapturedActivities(); using var client = CreateAuthenticatedClient("tenant-001"); // Set explicit trace context var traceId = ActivityTraceId.CreateRandom(); var spanId = ActivitySpanId.CreateRandom(); var traceparent = $"00-{traceId}-{spanId}-01"; client.DefaultRequestHeaders.Add("traceparent", traceparent); // Act await client.GetAsync("/api/v1/schedules"); // Assert var activitiesWithTraceId = _capturedActivities .Where(a => a.TraceId == traceId) .ToList(); // Activities should inherit the trace context activitiesWithTraceId.Should().NotBeEmpty( because: "activities should propagate incoming trace context"); } /// /// Verifies parent-child relationships are established correctly. /// [Fact] public async Task Activities_HaveProperParentChildRelationships() { // Arrange ClearCapturedActivities(); using var client = CreateAuthenticatedClient("tenant-001"); // Act await client.PostAsJsonAsync("/api/v1/schedules", new { name = "parent-child-test", cronExpression = "0 * * * *", timezone = "UTC" }); // Assert var activitiesWithParent = _capturedActivities .Where(a => a.ParentId != null) .ToList(); foreach (var activity in activitiesWithParent) { // Parent should exist and be from the same trace var parent = _capturedActivities.FirstOrDefault(p => p.Id == activity.ParentId); if (parent != null) { parent.TraceId.Should().Be(activity.TraceId); } } } /// /// Verifies correlation ID header is included in trace baggage. /// [Fact] public async Task CorrelationId_IsIncludedInTraceBaggage() { // Arrange ClearCapturedActivities(); using var client = CreateAuthenticatedClient("tenant-001"); const string correlationId = "test-correlation-12345"; client.DefaultRequestHeaders.Add("X-Correlation-Id", correlationId); // Act await client.GetAsync("/api/v1/schedules"); // Assert var activitiesWithCorrelation = _capturedActivities .Where(a => a.Baggage.Any(b => b.Key == "correlation_id" && b.Value == correlationId) || a.Tags.Any(t => t.Key == "correlation_id" && t.Value == correlationId)) .ToList(); // Correlation ID should be propagated // Note: Implementation may use either baggage or tags } #endregion #region Span Attributes Tests /// /// Verifies HTTP-related attributes are present on activities. /// [Fact] public async Task HttpActivity_HasStandardHttpAttributes() { // Arrange ClearCapturedActivities(); using var client = CreateAuthenticatedClient("tenant-001"); // Act await client.GetAsync("/api/v1/schedules"); // Assert var httpActivities = _capturedActivities .Where(a => a.Kind == ActivityKind.Server || a.Tags.Any(t => t.Key.StartsWith("http."))) .ToList(); foreach (var activity in httpActivities) { var tags = activity.Tags.ToDictionary(t => t.Key, t => t.Value); // Standard OTel HTTP semantic conventions if (tags.ContainsKey("http.method") || tags.ContainsKey("http.request.method")) { var method = tags.GetValueOrDefault("http.method") ?? tags.GetValueOrDefault("http.request.method"); method.Should().Be("GET"); } if (tags.ContainsKey("http.status_code") || tags.ContainsKey("http.response.status_code")) { var statusCode = tags.GetValueOrDefault("http.status_code") ?? tags.GetValueOrDefault("http.response.status_code"); statusCode.Should().NotBeNullOrWhiteSpace(); } } } /// /// Verifies service name is set correctly on activities. /// [Fact] public async Task Activity_HasCorrectServiceName() { // Arrange ClearCapturedActivities(); using var client = CreateAuthenticatedClient("tenant-001"); // Act await client.GetAsync("/api/v1/schedules"); // Assert var serviceActivities = _capturedActivities .Where(a => a.Tags.Any(t => t.Key == "service.name")) .ToList(); foreach (var activity in serviceActivities) { var serviceName = activity.Tags.First(t => t.Key == "service.name").Value; serviceName.Should().ContainAny("Scheduler", "scheduler", "stellaops"); } } #endregion #region Metric Tag Consistency Tests /// /// Verifies tag naming follows OpenTelemetry semantic conventions. /// [Fact] public async Task Tags_FollowSemanticConventions() { // Arrange ClearCapturedActivities(); using var client = CreateAuthenticatedClient("tenant-001"); // Act await client.GetAsync("/api/v1/schedules"); // Assert foreach (var activity in _capturedActivities) { foreach (var tag in activity.Tags) { // Tags should use lowercase and underscores per OTel convention tag.Key.Should().MatchRegex(@"^[a-z][a-z0-9_.]*$", because: $"tag '{tag.Key}' should follow semantic convention naming"); // No null values tag.Value.Should().NotBeNull( because: $"tag '{tag.Key}' should not have null value"); } } } /// /// Verifies custom StellaOps tags use consistent prefix. /// [Fact] public async Task CustomTags_UseConsistentPrefix() { // Arrange ClearCapturedActivities(); using var client = CreateAuthenticatedClient("tenant-001"); // Act await client.PostAsJsonAsync("/api/v1/jobs", new { type = "scan", target = "image:v1" }); // Assert var stellaOpsTags = _capturedActivities .SelectMany(a => a.Tags) .Where(t => t.Key.Contains("stellaops") || t.Key.Contains("job") || t.Key.Contains("schedule")) .ToList(); foreach (var tag in stellaOpsTags) { // Custom tags should use stellaops. prefix or be standard OTel attributes tag.Key.Should().MatchRegex(@"^(stellaops\.|http\.|net\.|rpc\.|db\.|messaging\.|[a-z_]+)"); } } #endregion #region Test Helpers private HttpClient CreateAuthenticatedClient(string tenantId) { var client = _factory.CreateClient(); var token = CreateTestToken(tenantId); client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token); return client; } private static string CreateTestToken(string tenantId) { // Simplified test token - real implementation would use proper JWT var header = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes("""{"alg":"RS256","typ":"JWT"}""")); var payload = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes( $$"""{"sub":"user@{{tenantId}}","tenant_id":"{{tenantId}}","permissions":["scheduler:read","scheduler:write"],"exp":{{DateTimeOffset.UtcNow.AddHours(1).ToUnixTimeSeconds()}}}""")); return $"{header}.{payload}.test-signature"; } private void ClearCapturedActivities() { while (_capturedActivities.TryTake(out _)) { } } #endregion }