// ---------------------------------------------------------------------
//
// Copyright (c) StellaOps. Licensed under the AGPL-3.0-or-later.
//
//
// OTel trace assertions: verify job_id, tenant_id, schedule_id tags
//
// ---------------------------------------------------------------------
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Net.Http.Headers;
using System.Net.Http.Json;
using FluentAssertions;
using Microsoft.AspNetCore.Mvc.Testing;
using Xunit;
namespace StellaOps.Scheduler.WebService.Tests.Observability;
///
/// OTel trace assertions for Scheduler.WebService verifying
/// job_id, tenant_id, schedule_id tags are properly emitted.
///
[Trait("Category", "Observability")]
[Trait("Sprint", "5100-0009-0008")]
public sealed class SchedulerOTelTraceTests : IClassFixture>, IDisposable
{
private readonly WebApplicationFactory _factory;
private readonly ActivityListener _listener;
private readonly ConcurrentBag _capturedActivities;
///
/// Initializes a new instance of the class.
///
public SchedulerOTelTraceTests(WebApplicationFactory factory)
{
_factory = factory;
_capturedActivities = new ConcurrentBag();
_listener = new ActivityListener
{
ShouldListenTo = source => source.Name.StartsWith("StellaOps", StringComparison.OrdinalIgnoreCase)
|| source.Name.Contains("Scheduler", StringComparison.OrdinalIgnoreCase),
Sample = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllDataAndRecorded,
ActivityStopped = activity => _capturedActivities.Add(activity)
};
ActivitySource.AddActivityListener(_listener);
}
///
public void Dispose()
{
_listener.Dispose();
}
#region Activity Creation Tests
///
/// Verifies activity is created for schedule creation operations.
///
[Fact]
public async Task CreateSchedule_CreatesActivity_WithSchedulerSource()
{
// Arrange
ClearCapturedActivities();
using var client = CreateAuthenticatedClient("tenant-001");
var payload = new
{
name = "otel-test-schedule",
cronExpression = "0 * * * *",
timezone = "UTC"
};
// Act
await client.PostAsJsonAsync("/api/v1/schedules", payload);
// Assert
var schedulerActivities = _capturedActivities
.Where(a => a.OperationName.Contains("schedule", StringComparison.OrdinalIgnoreCase)
|| a.DisplayName.Contains("schedule", StringComparison.OrdinalIgnoreCase))
.ToList();
schedulerActivities.Should().NotBeEmpty(
because: "schedule creation should emit OTel activity");
}
///
/// Verifies activity is created for job enqueue operations.
///
[Fact]
public async Task EnqueueJob_CreatesActivity()
{
// Arrange
ClearCapturedActivities();
using var client = CreateAuthenticatedClient("tenant-001");
var payload = new
{
type = "scan",
target = "image:latest",
priority = 5
};
// Act
await client.PostAsJsonAsync("/api/v1/jobs", payload);
// Assert
var jobActivities = _capturedActivities
.Where(a => a.OperationName.Contains("job", StringComparison.OrdinalIgnoreCase)
|| a.DisplayName.Contains("enqueue", StringComparison.OrdinalIgnoreCase))
.ToList();
jobActivities.Should().NotBeEmpty(
because: "job enqueue should emit OTel activity");
}
#endregion
#region Scheduler-Specific Tag Tests
///
/// Verifies job_id tag is present on job-related activities.
///
[Fact]
public async Task JobActivity_HasJobIdTag()
{
// Arrange
ClearCapturedActivities();
using var client = CreateAuthenticatedClient("tenant-001");
// Act - Enqueue a job
var response = await client.PostAsJsonAsync("/api/v1/jobs", new
{
type = "scan",
target = "image:test"
});
// Assert
var jobActivities = _capturedActivities
.Where(a => a.OperationName.Contains("job", StringComparison.OrdinalIgnoreCase))
.ToList();
foreach (var activity in jobActivities)
{
var jobIdTag = activity.Tags.FirstOrDefault(t => t.Key == "job_id" || t.Key == "stellaops.job.id");
if (!string.IsNullOrEmpty(jobIdTag.Value))
{
jobIdTag.Value.Should().NotBeNullOrWhiteSpace(
because: "job_id tag should have a value");
}
}
}
///
/// Verifies tenant_id tag is present on all scheduler activities.
///
[Fact]
public async Task SchedulerActivity_HasTenantIdTag()
{
// Arrange
const string expectedTenantId = "tenant-otel-test";
ClearCapturedActivities();
using var client = CreateAuthenticatedClient(expectedTenantId);
// Act
await client.GetAsync("/api/v1/schedules");
// Assert
var schedulerActivities = _capturedActivities
.Where(a => a.Source.Name.Contains("Scheduler", StringComparison.OrdinalIgnoreCase)
|| a.Source.Name.StartsWith("StellaOps", StringComparison.OrdinalIgnoreCase))
.ToList();
foreach (var activity in schedulerActivities)
{
var tenantTag = activity.Tags.FirstOrDefault(t =>
t.Key == "tenant_id" ||
t.Key == "stellaops.tenant.id" ||
t.Key == "enduser.id");
// At least some activities should have tenant context
if (!string.IsNullOrEmpty(tenantTag.Value))
{
tenantTag.Value.Should().Be(expectedTenantId);
}
}
}
///
/// Verifies schedule_id tag is present on schedule-related activities.
///
[Fact]
public async Task ScheduleActivity_HasScheduleIdTag()
{
// Arrange
ClearCapturedActivities();
using var client = CreateAuthenticatedClient("tenant-001");
// Create a schedule first
var createResponse = await client.PostAsJsonAsync("/api/v1/schedules", new
{
name = "schedule-for-otel-test",
cronExpression = "0 12 * * *",
timezone = "UTC"
});
// Act - Query the schedule
ClearCapturedActivities();
await client.GetAsync("/api/v1/schedules");
// Assert
var scheduleActivities = _capturedActivities
.Where(a => a.OperationName.Contains("schedule", StringComparison.OrdinalIgnoreCase))
.ToList();
foreach (var activity in scheduleActivities)
{
var scheduleIdTag = activity.Tags.FirstOrDefault(t =>
t.Key == "schedule_id" ||
t.Key == "stellaops.schedule.id");
// Schedule operations should include schedule_id when applicable
if (activity.OperationName.Contains("get", StringComparison.OrdinalIgnoreCase) &&
!string.IsNullOrEmpty(scheduleIdTag.Value))
{
scheduleIdTag.Value.Should().NotBeNullOrWhiteSpace();
}
}
}
#endregion
#region Error Trace Tests
///
/// Verifies failed operations include error status in activity.
///
[Fact]
public async Task FailedOperation_SetsActivityStatusToError()
{
// Arrange
ClearCapturedActivities();
using var client = CreateAuthenticatedClient("tenant-001");
// Act - Request a non-existent resource
await client.GetAsync("/api/v1/schedules/non-existent-schedule-id");
// Assert
var errorActivities = _capturedActivities
.Where(a => a.Status == ActivityStatusCode.Error ||
a.Tags.Any(t => t.Key == "error" && t.Value == "true") ||
a.Tags.Any(t => t.Key == "otel.status_code" && t.Value == "ERROR"))
.ToList();
// Not all 404s are errors from OTel perspective, but validation errors should be
// This test validates the pattern exists for actual errors
}
///
/// Verifies validation errors include error details in activity.
///
[Fact]
public async Task ValidationError_IncludesErrorDetailsInActivity()
{
// Arrange
ClearCapturedActivities();
using var client = CreateAuthenticatedClient("tenant-001");
// Act - Send invalid payload
await client.PostAsJsonAsync("/api/v1/schedules", new
{
name = "", // Invalid: empty name
cronExpression = "invalid cron",
timezone = "Invalid/Timezone"
});
// Assert
var activitiesWithErrors = _capturedActivities
.Where(a => a.Events.Any(e => e.Name == "exception" || e.Name == "error"))
.ToList();
// If validation errors emit events, they should include details
foreach (var activity in activitiesWithErrors)
{
var errorEvent = activity.Events.FirstOrDefault(e =>
e.Name == "exception" || e.Name == "error");
if (errorEvent.Name != null)
{
errorEvent.Tags.Should().ContainKey("exception.message");
}
}
}
#endregion
#region Trace Correlation Tests
///
/// Verifies trace context is propagated across operations.
///
[Fact]
public async Task TraceContext_IsPropagatedAcrossOperations()
{
// Arrange
ClearCapturedActivities();
using var client = CreateAuthenticatedClient("tenant-001");
// Set explicit trace context
var traceId = ActivityTraceId.CreateRandom();
var spanId = ActivitySpanId.CreateRandom();
var traceparent = $"00-{traceId}-{spanId}-01";
client.DefaultRequestHeaders.Add("traceparent", traceparent);
// Act
await client.GetAsync("/api/v1/schedules");
// Assert
var activitiesWithTraceId = _capturedActivities
.Where(a => a.TraceId == traceId)
.ToList();
// Activities should inherit the trace context
activitiesWithTraceId.Should().NotBeEmpty(
because: "activities should propagate incoming trace context");
}
///
/// Verifies parent-child relationships are established correctly.
///
[Fact]
public async Task Activities_HaveProperParentChildRelationships()
{
// Arrange
ClearCapturedActivities();
using var client = CreateAuthenticatedClient("tenant-001");
// Act
await client.PostAsJsonAsync("/api/v1/schedules", new
{
name = "parent-child-test",
cronExpression = "0 * * * *",
timezone = "UTC"
});
// Assert
var activitiesWithParent = _capturedActivities
.Where(a => a.ParentId != null)
.ToList();
foreach (var activity in activitiesWithParent)
{
// Parent should exist and be from the same trace
var parent = _capturedActivities.FirstOrDefault(p => p.Id == activity.ParentId);
if (parent != null)
{
parent.TraceId.Should().Be(activity.TraceId);
}
}
}
///
/// Verifies correlation ID header is included in trace baggage.
///
[Fact]
public async Task CorrelationId_IsIncludedInTraceBaggage()
{
// Arrange
ClearCapturedActivities();
using var client = CreateAuthenticatedClient("tenant-001");
const string correlationId = "test-correlation-12345";
client.DefaultRequestHeaders.Add("X-Correlation-Id", correlationId);
// Act
await client.GetAsync("/api/v1/schedules");
// Assert
var activitiesWithCorrelation = _capturedActivities
.Where(a => a.Baggage.Any(b => b.Key == "correlation_id" && b.Value == correlationId) ||
a.Tags.Any(t => t.Key == "correlation_id" && t.Value == correlationId))
.ToList();
// Correlation ID should be propagated
// Note: Implementation may use either baggage or tags
}
#endregion
#region Span Attributes Tests
///
/// Verifies HTTP-related attributes are present on activities.
///
[Fact]
public async Task HttpActivity_HasStandardHttpAttributes()
{
// Arrange
ClearCapturedActivities();
using var client = CreateAuthenticatedClient("tenant-001");
// Act
await client.GetAsync("/api/v1/schedules");
// Assert
var httpActivities = _capturedActivities
.Where(a => a.Kind == ActivityKind.Server ||
a.Tags.Any(t => t.Key.StartsWith("http.")))
.ToList();
foreach (var activity in httpActivities)
{
var tags = activity.Tags.ToDictionary(t => t.Key, t => t.Value);
// Standard OTel HTTP semantic conventions
if (tags.ContainsKey("http.method") || tags.ContainsKey("http.request.method"))
{
var method = tags.GetValueOrDefault("http.method") ?? tags.GetValueOrDefault("http.request.method");
method.Should().Be("GET");
}
if (tags.ContainsKey("http.status_code") || tags.ContainsKey("http.response.status_code"))
{
var statusCode = tags.GetValueOrDefault("http.status_code") ?? tags.GetValueOrDefault("http.response.status_code");
statusCode.Should().NotBeNullOrWhiteSpace();
}
}
}
///
/// Verifies service name is set correctly on activities.
///
[Fact]
public async Task Activity_HasCorrectServiceName()
{
// Arrange
ClearCapturedActivities();
using var client = CreateAuthenticatedClient("tenant-001");
// Act
await client.GetAsync("/api/v1/schedules");
// Assert
var serviceActivities = _capturedActivities
.Where(a => a.Tags.Any(t => t.Key == "service.name"))
.ToList();
foreach (var activity in serviceActivities)
{
var serviceName = activity.Tags.First(t => t.Key == "service.name").Value;
serviceName.Should().ContainAny("Scheduler", "scheduler", "stellaops");
}
}
#endregion
#region Metric Tag Consistency Tests
///
/// Verifies tag naming follows OpenTelemetry semantic conventions.
///
[Fact]
public async Task Tags_FollowSemanticConventions()
{
// Arrange
ClearCapturedActivities();
using var client = CreateAuthenticatedClient("tenant-001");
// Act
await client.GetAsync("/api/v1/schedules");
// Assert
foreach (var activity in _capturedActivities)
{
foreach (var tag in activity.Tags)
{
// Tags should use lowercase and underscores per OTel convention
tag.Key.Should().MatchRegex(@"^[a-z][a-z0-9_.]*$",
because: $"tag '{tag.Key}' should follow semantic convention naming");
// No null values
tag.Value.Should().NotBeNull(
because: $"tag '{tag.Key}' should not have null value");
}
}
}
///
/// Verifies custom StellaOps tags use consistent prefix.
///
[Fact]
public async Task CustomTags_UseConsistentPrefix()
{
// Arrange
ClearCapturedActivities();
using var client = CreateAuthenticatedClient("tenant-001");
// Act
await client.PostAsJsonAsync("/api/v1/jobs", new { type = "scan", target = "image:v1" });
// Assert
var stellaOpsTags = _capturedActivities
.SelectMany(a => a.Tags)
.Where(t => t.Key.Contains("stellaops") || t.Key.Contains("job") || t.Key.Contains("schedule"))
.ToList();
foreach (var tag in stellaOpsTags)
{
// Custom tags should use stellaops. prefix or be standard OTel attributes
tag.Key.Should().MatchRegex(@"^(stellaops\.|http\.|net\.|rpc\.|db\.|messaging\.|[a-z_]+)");
}
}
#endregion
#region Test Helpers
private HttpClient CreateAuthenticatedClient(string tenantId)
{
var client = _factory.CreateClient();
var token = CreateTestToken(tenantId);
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token);
return client;
}
private static string CreateTestToken(string tenantId)
{
// Simplified test token - real implementation would use proper JWT
var header = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes("""{"alg":"RS256","typ":"JWT"}"""));
var payload = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(
$$"""{"sub":"user@{{tenantId}}","tenant_id":"{{tenantId}}","permissions":["scheduler:read","scheduler:write"],"exp":{{DateTimeOffset.UtcNow.AddHours(1).ToUnixTimeSeconds()}}}"""));
return $"{header}.{payload}.test-signature";
}
private void ClearCapturedActivities()
{
while (_capturedActivities.TryTake(out _)) { }
}
#endregion
}