541 lines
18 KiB
C#
541 lines
18 KiB
C#
// ---------------------------------------------------------------------
|
|
// <copyright file="SchedulerOTelTraceTests.cs" company="StellaOps">
|
|
// Copyright (c) StellaOps. Licensed under the AGPL-3.0-or-later.
|
|
// </copyright>
|
|
// <summary>
|
|
// OTel trace assertions: verify job_id, tenant_id, schedule_id tags
|
|
// </summary>
|
|
// ---------------------------------------------------------------------
|
|
|
|
using System.Collections.Concurrent;
|
|
using System.Diagnostics;
|
|
using System.Net.Http.Headers;
|
|
using System.Net.Http.Json;
|
|
using FluentAssertions;
|
|
using Microsoft.AspNetCore.Mvc.Testing;
|
|
using Xunit;
|
|
|
|
namespace StellaOps.Scheduler.WebService.Tests.Observability;
|
|
|
|
/// <summary>
|
|
/// OTel trace assertions for Scheduler.WebService verifying
|
|
/// job_id, tenant_id, schedule_id tags are properly emitted.
|
|
/// </summary>
|
|
[Trait("Category", "Observability")]
|
|
[Trait("Sprint", "5100-0009-0008")]
|
|
public sealed class SchedulerOTelTraceTests : IClassFixture<WebApplicationFactory<Program>>, IDisposable
|
|
{
|
|
private readonly WebApplicationFactory<Program> _factory;
|
|
private readonly ActivityListener _listener;
|
|
private readonly ConcurrentBag<Activity> _capturedActivities;
|
|
|
|
/// <summary>
|
|
/// Initializes a new instance of the <see cref="SchedulerOTelTraceTests"/> class.
|
|
/// </summary>
|
|
public SchedulerOTelTraceTests(WebApplicationFactory<Program> factory)
|
|
{
|
|
_factory = factory;
|
|
_capturedActivities = new ConcurrentBag<Activity>();
|
|
|
|
_listener = new ActivityListener
|
|
{
|
|
ShouldListenTo = source => source.Name.StartsWith("StellaOps", StringComparison.OrdinalIgnoreCase)
|
|
|| source.Name.Contains("Scheduler", StringComparison.OrdinalIgnoreCase),
|
|
Sample = (ref ActivityCreationOptions<ActivityContext> _) => ActivitySamplingResult.AllDataAndRecorded,
|
|
ActivityStopped = activity => _capturedActivities.Add(activity)
|
|
};
|
|
|
|
ActivitySource.AddActivityListener(_listener);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public void Dispose()
|
|
{
|
|
_listener.Dispose();
|
|
}
|
|
|
|
#region Activity Creation Tests
|
|
|
|
/// <summary>
|
|
/// Verifies activity is created for schedule creation operations.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task CreateSchedule_CreatesActivity_WithSchedulerSource()
|
|
{
|
|
// Arrange
|
|
ClearCapturedActivities();
|
|
using var client = CreateAuthenticatedClient("tenant-001");
|
|
var payload = new
|
|
{
|
|
name = "otel-test-schedule",
|
|
cronExpression = "0 * * * *",
|
|
timezone = "UTC"
|
|
};
|
|
|
|
// Act
|
|
await client.PostAsJsonAsync("/api/v1/schedules", payload);
|
|
|
|
// Assert
|
|
var schedulerActivities = _capturedActivities
|
|
.Where(a => a.OperationName.Contains("schedule", StringComparison.OrdinalIgnoreCase)
|
|
|| a.DisplayName.Contains("schedule", StringComparison.OrdinalIgnoreCase))
|
|
.ToList();
|
|
|
|
schedulerActivities.Should().NotBeEmpty(
|
|
because: "schedule creation should emit OTel activity");
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifies activity is created for job enqueue operations.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task EnqueueJob_CreatesActivity()
|
|
{
|
|
// Arrange
|
|
ClearCapturedActivities();
|
|
using var client = CreateAuthenticatedClient("tenant-001");
|
|
var payload = new
|
|
{
|
|
type = "scan",
|
|
target = "image:latest",
|
|
priority = 5
|
|
};
|
|
|
|
// Act
|
|
await client.PostAsJsonAsync("/api/v1/jobs", payload);
|
|
|
|
// Assert
|
|
var jobActivities = _capturedActivities
|
|
.Where(a => a.OperationName.Contains("job", StringComparison.OrdinalIgnoreCase)
|
|
|| a.DisplayName.Contains("enqueue", StringComparison.OrdinalIgnoreCase))
|
|
.ToList();
|
|
|
|
jobActivities.Should().NotBeEmpty(
|
|
because: "job enqueue should emit OTel activity");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Scheduler-Specific Tag Tests
|
|
|
|
/// <summary>
|
|
/// Verifies job_id tag is present on job-related activities.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task JobActivity_HasJobIdTag()
|
|
{
|
|
// Arrange
|
|
ClearCapturedActivities();
|
|
using var client = CreateAuthenticatedClient("tenant-001");
|
|
|
|
// Act - Enqueue a job
|
|
var response = await client.PostAsJsonAsync("/api/v1/jobs", new
|
|
{
|
|
type = "scan",
|
|
target = "image:test"
|
|
});
|
|
|
|
// Assert
|
|
var jobActivities = _capturedActivities
|
|
.Where(a => a.OperationName.Contains("job", StringComparison.OrdinalIgnoreCase))
|
|
.ToList();
|
|
|
|
foreach (var activity in jobActivities)
|
|
{
|
|
var jobIdTag = activity.Tags.FirstOrDefault(t => t.Key == "job_id" || t.Key == "stellaops.job.id");
|
|
if (!string.IsNullOrEmpty(jobIdTag.Value))
|
|
{
|
|
jobIdTag.Value.Should().NotBeNullOrWhiteSpace(
|
|
because: "job_id tag should have a value");
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifies tenant_id tag is present on all scheduler activities.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task SchedulerActivity_HasTenantIdTag()
|
|
{
|
|
// Arrange
|
|
const string expectedTenantId = "tenant-otel-test";
|
|
ClearCapturedActivities();
|
|
using var client = CreateAuthenticatedClient(expectedTenantId);
|
|
|
|
// Act
|
|
await client.GetAsync("/api/v1/schedules");
|
|
|
|
// Assert
|
|
var schedulerActivities = _capturedActivities
|
|
.Where(a => a.Source.Name.Contains("Scheduler", StringComparison.OrdinalIgnoreCase)
|
|
|| a.Source.Name.StartsWith("StellaOps", StringComparison.OrdinalIgnoreCase))
|
|
.ToList();
|
|
|
|
foreach (var activity in schedulerActivities)
|
|
{
|
|
var tenantTag = activity.Tags.FirstOrDefault(t =>
|
|
t.Key == "tenant_id" ||
|
|
t.Key == "stellaops.tenant.id" ||
|
|
t.Key == "enduser.id");
|
|
|
|
// At least some activities should have tenant context
|
|
if (!string.IsNullOrEmpty(tenantTag.Value))
|
|
{
|
|
tenantTag.Value.Should().Be(expectedTenantId);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifies schedule_id tag is present on schedule-related activities.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task ScheduleActivity_HasScheduleIdTag()
|
|
{
|
|
// Arrange
|
|
ClearCapturedActivities();
|
|
using var client = CreateAuthenticatedClient("tenant-001");
|
|
|
|
// Create a schedule first
|
|
var createResponse = await client.PostAsJsonAsync("/api/v1/schedules", new
|
|
{
|
|
name = "schedule-for-otel-test",
|
|
cronExpression = "0 12 * * *",
|
|
timezone = "UTC"
|
|
});
|
|
|
|
// Act - Query the schedule
|
|
ClearCapturedActivities();
|
|
await client.GetAsync("/api/v1/schedules");
|
|
|
|
// Assert
|
|
var scheduleActivities = _capturedActivities
|
|
.Where(a => a.OperationName.Contains("schedule", StringComparison.OrdinalIgnoreCase))
|
|
.ToList();
|
|
|
|
foreach (var activity in scheduleActivities)
|
|
{
|
|
var scheduleIdTag = activity.Tags.FirstOrDefault(t =>
|
|
t.Key == "schedule_id" ||
|
|
t.Key == "stellaops.schedule.id");
|
|
|
|
// Schedule operations should include schedule_id when applicable
|
|
if (activity.OperationName.Contains("get", StringComparison.OrdinalIgnoreCase) &&
|
|
!string.IsNullOrEmpty(scheduleIdTag.Value))
|
|
{
|
|
scheduleIdTag.Value.Should().NotBeNullOrWhiteSpace();
|
|
}
|
|
}
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Error Trace Tests
|
|
|
|
/// <summary>
|
|
/// Verifies failed operations include error status in activity.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task FailedOperation_SetsActivityStatusToError()
|
|
{
|
|
// Arrange
|
|
ClearCapturedActivities();
|
|
using var client = CreateAuthenticatedClient("tenant-001");
|
|
|
|
// Act - Request a non-existent resource
|
|
await client.GetAsync("/api/v1/schedules/non-existent-schedule-id");
|
|
|
|
// Assert
|
|
var errorActivities = _capturedActivities
|
|
.Where(a => a.Status == ActivityStatusCode.Error ||
|
|
a.Tags.Any(t => t.Key == "error" && t.Value == "true") ||
|
|
a.Tags.Any(t => t.Key == "otel.status_code" && t.Value == "ERROR"))
|
|
.ToList();
|
|
|
|
// Not all 404s are errors from OTel perspective, but validation errors should be
|
|
// This test validates the pattern exists for actual errors
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifies validation errors include error details in activity.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task ValidationError_IncludesErrorDetailsInActivity()
|
|
{
|
|
// Arrange
|
|
ClearCapturedActivities();
|
|
using var client = CreateAuthenticatedClient("tenant-001");
|
|
|
|
// Act - Send invalid payload
|
|
await client.PostAsJsonAsync("/api/v1/schedules", new
|
|
{
|
|
name = "", // Invalid: empty name
|
|
cronExpression = "invalid cron",
|
|
timezone = "Invalid/Timezone"
|
|
});
|
|
|
|
// Assert
|
|
var activitiesWithErrors = _capturedActivities
|
|
.Where(a => a.Events.Any(e => e.Name == "exception" || e.Name == "error"))
|
|
.ToList();
|
|
|
|
// If validation errors emit events, they should include details
|
|
foreach (var activity in activitiesWithErrors)
|
|
{
|
|
var errorEvent = activity.Events.FirstOrDefault(e =>
|
|
e.Name == "exception" || e.Name == "error");
|
|
|
|
if (errorEvent.Name != null)
|
|
{
|
|
errorEvent.Tags.Should().ContainKey("exception.message");
|
|
}
|
|
}
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Trace Correlation Tests
|
|
|
|
/// <summary>
|
|
/// Verifies trace context is propagated across operations.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task TraceContext_IsPropagatedAcrossOperations()
|
|
{
|
|
// Arrange
|
|
ClearCapturedActivities();
|
|
using var client = CreateAuthenticatedClient("tenant-001");
|
|
|
|
// Set explicit trace context
|
|
var traceId = ActivityTraceId.CreateRandom();
|
|
var spanId = ActivitySpanId.CreateRandom();
|
|
var traceparent = $"00-{traceId}-{spanId}-01";
|
|
client.DefaultRequestHeaders.Add("traceparent", traceparent);
|
|
|
|
// Act
|
|
await client.GetAsync("/api/v1/schedules");
|
|
|
|
// Assert
|
|
var activitiesWithTraceId = _capturedActivities
|
|
.Where(a => a.TraceId == traceId)
|
|
.ToList();
|
|
|
|
// Activities should inherit the trace context
|
|
activitiesWithTraceId.Should().NotBeEmpty(
|
|
because: "activities should propagate incoming trace context");
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifies parent-child relationships are established correctly.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Activities_HaveProperParentChildRelationships()
|
|
{
|
|
// Arrange
|
|
ClearCapturedActivities();
|
|
using var client = CreateAuthenticatedClient("tenant-001");
|
|
|
|
// Act
|
|
await client.PostAsJsonAsync("/api/v1/schedules", new
|
|
{
|
|
name = "parent-child-test",
|
|
cronExpression = "0 * * * *",
|
|
timezone = "UTC"
|
|
});
|
|
|
|
// Assert
|
|
var activitiesWithParent = _capturedActivities
|
|
.Where(a => a.ParentId != null)
|
|
.ToList();
|
|
|
|
foreach (var activity in activitiesWithParent)
|
|
{
|
|
// Parent should exist and be from the same trace
|
|
var parent = _capturedActivities.FirstOrDefault(p => p.Id == activity.ParentId);
|
|
if (parent != null)
|
|
{
|
|
parent.TraceId.Should().Be(activity.TraceId);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifies correlation ID header is included in trace baggage.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task CorrelationId_IsIncludedInTraceBaggage()
|
|
{
|
|
// Arrange
|
|
ClearCapturedActivities();
|
|
using var client = CreateAuthenticatedClient("tenant-001");
|
|
const string correlationId = "test-correlation-12345";
|
|
client.DefaultRequestHeaders.Add("X-Correlation-Id", correlationId);
|
|
|
|
// Act
|
|
await client.GetAsync("/api/v1/schedules");
|
|
|
|
// Assert
|
|
var activitiesWithCorrelation = _capturedActivities
|
|
.Where(a => a.Baggage.Any(b => b.Key == "correlation_id" && b.Value == correlationId) ||
|
|
a.Tags.Any(t => t.Key == "correlation_id" && t.Value == correlationId))
|
|
.ToList();
|
|
|
|
// Correlation ID should be propagated
|
|
// Note: Implementation may use either baggage or tags
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Span Attributes Tests
|
|
|
|
/// <summary>
|
|
/// Verifies HTTP-related attributes are present on activities.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task HttpActivity_HasStandardHttpAttributes()
|
|
{
|
|
// Arrange
|
|
ClearCapturedActivities();
|
|
using var client = CreateAuthenticatedClient("tenant-001");
|
|
|
|
// Act
|
|
await client.GetAsync("/api/v1/schedules");
|
|
|
|
// Assert
|
|
var httpActivities = _capturedActivities
|
|
.Where(a => a.Kind == ActivityKind.Server ||
|
|
a.Tags.Any(t => t.Key.StartsWith("http.")))
|
|
.ToList();
|
|
|
|
foreach (var activity in httpActivities)
|
|
{
|
|
var tags = activity.Tags.ToDictionary(t => t.Key, t => t.Value);
|
|
|
|
// Standard OTel HTTP semantic conventions
|
|
if (tags.ContainsKey("http.method") || tags.ContainsKey("http.request.method"))
|
|
{
|
|
var method = tags.GetValueOrDefault("http.method") ?? tags.GetValueOrDefault("http.request.method");
|
|
method.Should().Be("GET");
|
|
}
|
|
|
|
if (tags.ContainsKey("http.status_code") || tags.ContainsKey("http.response.status_code"))
|
|
{
|
|
var statusCode = tags.GetValueOrDefault("http.status_code") ?? tags.GetValueOrDefault("http.response.status_code");
|
|
statusCode.Should().NotBeNullOrWhiteSpace();
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifies service name is set correctly on activities.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Activity_HasCorrectServiceName()
|
|
{
|
|
// Arrange
|
|
ClearCapturedActivities();
|
|
using var client = CreateAuthenticatedClient("tenant-001");
|
|
|
|
// Act
|
|
await client.GetAsync("/api/v1/schedules");
|
|
|
|
// Assert
|
|
var serviceActivities = _capturedActivities
|
|
.Where(a => a.Tags.Any(t => t.Key == "service.name"))
|
|
.ToList();
|
|
|
|
foreach (var activity in serviceActivities)
|
|
{
|
|
var serviceName = activity.Tags.First(t => t.Key == "service.name").Value;
|
|
serviceName.Should().ContainAny("Scheduler", "scheduler", "stellaops");
|
|
}
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Metric Tag Consistency Tests
|
|
|
|
/// <summary>
|
|
/// Verifies tag naming follows OpenTelemetry semantic conventions.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Tags_FollowSemanticConventions()
|
|
{
|
|
// Arrange
|
|
ClearCapturedActivities();
|
|
using var client = CreateAuthenticatedClient("tenant-001");
|
|
|
|
// Act
|
|
await client.GetAsync("/api/v1/schedules");
|
|
|
|
// Assert
|
|
foreach (var activity in _capturedActivities)
|
|
{
|
|
foreach (var tag in activity.Tags)
|
|
{
|
|
// Tags should use lowercase and underscores per OTel convention
|
|
tag.Key.Should().MatchRegex(@"^[a-z][a-z0-9_.]*$",
|
|
because: $"tag '{tag.Key}' should follow semantic convention naming");
|
|
|
|
// No null values
|
|
tag.Value.Should().NotBeNull(
|
|
because: $"tag '{tag.Key}' should not have null value");
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifies custom StellaOps tags use consistent prefix.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task CustomTags_UseConsistentPrefix()
|
|
{
|
|
// Arrange
|
|
ClearCapturedActivities();
|
|
using var client = CreateAuthenticatedClient("tenant-001");
|
|
|
|
// Act
|
|
await client.PostAsJsonAsync("/api/v1/jobs", new { type = "scan", target = "image:v1" });
|
|
|
|
// Assert
|
|
var stellaOpsTags = _capturedActivities
|
|
.SelectMany(a => a.Tags)
|
|
.Where(t => t.Key.Contains("stellaops") || t.Key.Contains("job") || t.Key.Contains("schedule"))
|
|
.ToList();
|
|
|
|
foreach (var tag in stellaOpsTags)
|
|
{
|
|
// Custom tags should use stellaops. prefix or be standard OTel attributes
|
|
tag.Key.Should().MatchRegex(@"^(stellaops\.|http\.|net\.|rpc\.|db\.|messaging\.|[a-z_]+)");
|
|
}
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Test Helpers
|
|
|
|
private HttpClient CreateAuthenticatedClient(string tenantId)
|
|
{
|
|
var client = _factory.CreateClient();
|
|
var token = CreateTestToken(tenantId);
|
|
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token);
|
|
return client;
|
|
}
|
|
|
|
private static string CreateTestToken(string tenantId)
|
|
{
|
|
// Simplified test token - real implementation would use proper JWT
|
|
var header = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes("""{"alg":"RS256","typ":"JWT"}"""));
|
|
var payload = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(
|
|
$$"""{"sub":"user@{{tenantId}}","tenant_id":"{{tenantId}}","permissions":["scheduler:read","scheduler:write"],"exp":{{DateTimeOffset.UtcNow.AddHours(1).ToUnixTimeSeconds()}}}"""));
|
|
return $"{header}.{payload}.test-signature";
|
|
}
|
|
|
|
private void ClearCapturedActivities()
|
|
{
|
|
while (_capturedActivities.TryTake(out _)) { }
|
|
}
|
|
|
|
#endregion
|
|
}
|