stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 3 Releases Wiki Activity

Restructure solution layout by module

Browse Source
This commit is contained in:
master
2025-10-28 15:10:40 +02:00
parent 95daa159c4
commit d870da18ce
4103 changed files with 192899 additions and 187024 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
src/Scheduler/__Tests/StellaOps.Scheduler.Queue.Tests/PlannerAndRunnerMessageTests.cs Normal file
View File

@@ -0,0 +1,110 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using FluentAssertions;
using StellaOps.Scheduler.Models;
using Xunit;
namespace StellaOps.Scheduler.Queue.Tests;
public sealed class PlannerAndRunnerMessageTests
{
[Fact]
public void PlannerMessage_CanonicalSerialization_RoundTrips()
{
var schedule = new Schedule(
id: "sch-tenant-nightly",
tenantId: "tenant-alpha",
name: "Nightly Deltas",
enabled: true,
cronExpression: "0 2 * * *",
timezone: "UTC",
mode: ScheduleMode.AnalysisOnly,
selection: new Selector(SelectorScope.AllImages, tenantId: "tenant-alpha"),
onlyIf: new ScheduleOnlyIf(lastReportOlderThanDays: 3),
notify: new ScheduleNotify(onNewFindings: true, SeverityRank.High, includeKev: true),
limits: new ScheduleLimits(maxJobs: 10, ratePerSecond: 5, parallelism: 3),
createdAt: DateTimeOffset.Parse("2025-10-01T02:00:00Z"),
createdBy: "system",
updatedAt: DateTimeOffset.Parse("2025-10-02T02:00:00Z"),
updatedBy: "system",
subscribers: ImmutableArray<string>.Empty,
schemaVersion: "1.0.0");
var run = new Run(
id: "run-123",
tenantId: "tenant-alpha",
trigger: RunTrigger.Cron,
state: RunState.Planning,
stats: new RunStats(candidates: 5, deduped: 4, queued: 0, completed: 0, deltas: 0),
createdAt: DateTimeOffset.Parse("2025-10-02T02:05:00Z"),
reason: new RunReason(manualReason: null, feedserExportId: null, vexerExportId: null, cursor: null)
with { ImpactWindowFrom = "2025-10-01T00:00:00Z", ImpactWindowTo = "2025-10-02T00:00:00Z" },
scheduleId: "sch-tenant-nightly");
var impactSet = new ImpactSet(
selector: new Selector(SelectorScope.AllImages, tenantId: "tenant-alpha"),
images: new[]
{
new ImpactImage(
imageDigest: "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
registry: "registry",
repository: "repo",
namespaces: new[] { "prod" },
tags: new[] { "latest" },
usedByEntrypoint: true,
labels: new[] { KeyValuePair.Create("team", "appsec") })
},
usageOnly: true,
generatedAt: DateTimeOffset.Parse("2025-10-02T02:06:00Z"),
total: 1,
snapshotId: "snap-001");
var message = new PlannerQueueMessage(run, impactSet, schedule, correlationId: "corr-1");
var json = CanonicalJsonSerializer.Serialize(message);
var roundTrip = CanonicalJsonSerializer.Deserialize<PlannerQueueMessage>(json);
roundTrip.Should().BeEquivalentTo(message, options => options.WithStrictOrdering());
}
[Fact]
public void RunnerSegmentMessage_RequiresAtLeastOneDigest()
{
var act = () => new RunnerSegmentQueueMessage(
segmentId: "segment-empty",
runId: "run-123",
tenantId: "tenant-alpha",
imageDigests: Array.Empty<string>());
act.Should().Throw<ArgumentException>();
}
[Fact]
public void RunnerSegmentMessage_CanonicalSerialization_RoundTrips()
{
var message = new RunnerSegmentQueueMessage(
segmentId: "segment-01",
runId: "run-123",
tenantId: "tenant-alpha",
imageDigests: new[]
{
"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
},
scheduleId: "sch-tenant-nightly",
ratePerSecond: 25,
usageOnly: true,
attributes: new Dictionary<string, string>
{
["plannerShard"] = "0",
["priority"] = "kev"
},
correlationId: "corr-2");
var json = CanonicalJsonSerializer.Serialize(message);
var roundTrip = CanonicalJsonSerializer.Deserialize<RunnerSegmentQueueMessage>(json);
roundTrip.Should().BeEquivalentTo(message, options => options.WithStrictOrdering());
}
}

346
src/Scheduler/__Tests/StellaOps.Scheduler.Queue.Tests/RedisSchedulerQueueTests.cs Normal file
View File

@@ -0,0 +1,346 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using DotNet.Testcontainers.Builders;
using DotNet.Testcontainers.Containers;
using DotNet.Testcontainers.Configurations;
using FluentAssertions;
using Microsoft.Extensions.Logging.Abstractions;
using StackExchange.Redis;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Queue.Redis;
using Xunit;
namespace StellaOps.Scheduler.Queue.Tests;
public sealed class RedisSchedulerQueueTests : IAsyncLifetime
{
private readonly RedisTestcontainer _redis;
private string? _skipReason;
public RedisSchedulerQueueTests()
{
var configuration = new RedisTestcontainerConfiguration();
_redis = new TestcontainersBuilder<RedisTestcontainer>()
.WithDatabase(configuration)
.Build();
}
public async Task InitializeAsync()
{
try
{
await _redis.StartAsync();
}
catch (Exception ex) when (IsDockerUnavailable(ex))
{
_skipReason = $"Docker engine is not available for Redis-backed tests: {ex.Message}";
}
}
public async Task DisposeAsync()
{
if (_skipReason is not null)
{
return;
}
await _redis.DisposeAsync().AsTask();
}
[Fact]
public async Task PlannerQueue_EnqueueLeaseAck_RemovesMessage()
{
if (SkipIfUnavailable())
{
return;
}
var options = CreateOptions();
await using var queue = new RedisSchedulerPlannerQueue(
options,
options.Redis,
NullLogger<RedisSchedulerPlannerQueue>.Instance,
TimeProvider.System,
async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
var message = TestData.CreatePlannerMessage();
var enqueue = await queue.EnqueueAsync(message);
enqueue.Deduplicated.Should().BeFalse();
var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("planner-1", batchSize: 5, options.DefaultLeaseDuration));
leases.Should().HaveCount(1);
var lease = leases[0];
lease.Message.Run.Id.Should().Be(message.Run.Id);
lease.TenantId.Should().Be(message.TenantId);
lease.ScheduleId.Should().Be(message.ScheduleId);
await lease.AcknowledgeAsync();
var afterAck = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("planner-1", 5, options.DefaultLeaseDuration));
afterAck.Should().BeEmpty();
}
[Fact]
public async Task RunnerQueue_Retry_IncrementsDeliveryAttempt()
{
if (SkipIfUnavailable())
{
return;
}
var options = CreateOptions();
options.RetryInitialBackoff = TimeSpan.Zero;
options.RetryMaxBackoff = TimeSpan.Zero;
await using var queue = new RedisSchedulerRunnerQueue(
options,
options.Redis,
NullLogger<RedisSchedulerRunnerQueue>.Instance,
TimeProvider.System,
async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
var message = TestData.CreateRunnerMessage();
await queue.EnqueueAsync(message);
var firstLease = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("runner-1", batchSize: 1, options.DefaultLeaseDuration));
firstLease.Should().ContainSingle();
var lease = firstLease[0];
lease.Attempt.Should().Be(1);
await lease.ReleaseAsync(SchedulerQueueReleaseDisposition.Retry);
var secondLease = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("runner-1", batchSize: 1, options.DefaultLeaseDuration));
secondLease.Should().ContainSingle();
secondLease[0].Attempt.Should().Be(2);
}
[Fact]
public async Task PlannerQueue_ClaimExpired_ReassignsLease()
{
if (SkipIfUnavailable())
{
return;
}
var options = CreateOptions();
await using var queue = new RedisSchedulerPlannerQueue(
options,
options.Redis,
NullLogger<RedisSchedulerPlannerQueue>.Instance,
TimeProvider.System,
async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
var message = TestData.CreatePlannerMessage();
await queue.EnqueueAsync(message);
var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("planner-a", 1, options.DefaultLeaseDuration));
leases.Should().ContainSingle();
await Task.Delay(50);
var reclaimed = await queue.ClaimExpiredAsync(new SchedulerQueueClaimOptions("planner-b", batchSize: 1, minIdleTime: TimeSpan.Zero));
reclaimed.Should().ContainSingle();
reclaimed[0].Consumer.Should().Be("planner-b");
reclaimed[0].RunId.Should().Be(message.Run.Id);
await reclaimed[0].AcknowledgeAsync();
}
[Fact]
public async Task PlannerQueue_RecordsDepthMetrics()
{
if (SkipIfUnavailable())
{
return;
}
var options = CreateOptions();
await using var queue = new RedisSchedulerPlannerQueue(
options,
options.Redis,
NullLogger<RedisSchedulerPlannerQueue>.Instance,
TimeProvider.System,
async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
var message = TestData.CreatePlannerMessage();
await queue.EnqueueAsync(message);
var depths = SchedulerQueueMetrics.SnapshotDepths();
depths.TryGetValue(("redis", "planner"), out var plannerDepth)
.Should().BeTrue();
plannerDepth.Should().Be(1);
var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("planner-depth", 1, options.DefaultLeaseDuration));
leases.Should().ContainSingle();
await leases[0].AcknowledgeAsync();
depths = SchedulerQueueMetrics.SnapshotDepths();
depths.TryGetValue(("redis", "planner"), out plannerDepth).Should().BeTrue();
plannerDepth.Should().Be(0);
}
[Fact]
public async Task RunnerQueue_DropWhenDeadLetterDisabled()
{
if (SkipIfUnavailable())
{
return;
}
var options = CreateOptions();
options.MaxDeliveryAttempts = 1;
options.DeadLetterEnabled = false;
await using var queue = new RedisSchedulerRunnerQueue(
options,
options.Redis,
NullLogger<RedisSchedulerRunnerQueue>.Instance,
TimeProvider.System,
async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
var message = TestData.CreateRunnerMessage();
await queue.EnqueueAsync(message);
var lease = (await queue.LeaseAsync(new SchedulerQueueLeaseRequest("runner-drop", 1, options.DefaultLeaseDuration)))[0];
await lease.ReleaseAsync(SchedulerQueueReleaseDisposition.Retry);
var depths = SchedulerQueueMetrics.SnapshotDepths();
depths.TryGetValue(("redis", "runner"), out var runnerDepth).Should().BeTrue();
runnerDepth.Should().Be(0);
}
private SchedulerQueueOptions CreateOptions()
{
var unique = Guid.NewGuid().ToString("N");
return new SchedulerQueueOptions
{
Kind = SchedulerQueueTransportKind.Redis,
DefaultLeaseDuration = TimeSpan.FromSeconds(2),
MaxDeliveryAttempts = 5,
RetryInitialBackoff = TimeSpan.FromMilliseconds(10),
RetryMaxBackoff = TimeSpan.FromMilliseconds(50),
Redis = new SchedulerRedisQueueOptions
{
ConnectionString = _redis.ConnectionString,
Database = 0,
InitializationTimeout = TimeSpan.FromSeconds(10),
Planner = new RedisSchedulerStreamOptions
{
Stream = $"scheduler:test:planner:{unique}",
ConsumerGroup = $"planner-consumers-{unique}",
DeadLetterStream = $"scheduler:test:planner:{unique}:dead",
IdempotencyKeyPrefix = $"scheduler:test:planner:{unique}:idemp:",
IdempotencyWindow = TimeSpan.FromMinutes(5)
},
Runner = new RedisSchedulerStreamOptions
{
Stream = $"scheduler:test:runner:{unique}",
ConsumerGroup = $"runner-consumers-{unique}",
DeadLetterStream = $"scheduler:test:runner:{unique}:dead",
IdempotencyKeyPrefix = $"scheduler:test:runner:{unique}:idemp:",
IdempotencyWindow = TimeSpan.FromMinutes(5)
}
}
};
}
private bool SkipIfUnavailable()
{
if (_skipReason is not null)
{
return true;
}
return false;
}
private static bool IsDockerUnavailable(Exception exception)
{
while (exception is AggregateException aggregate && aggregate.InnerException is not null)
{
exception = aggregate.InnerException;
}
return exception is TimeoutException
|| exception.GetType().Name.Contains("Docker", StringComparison.OrdinalIgnoreCase);
}
private static class TestData
{
public static PlannerQueueMessage CreatePlannerMessage()
{
var schedule = new Schedule(
id: "sch-test",
tenantId: "tenant-alpha",
name: "Test",
enabled: true,
cronExpression: "0 0 * * *",
timezone: "UTC",
mode: ScheduleMode.AnalysisOnly,
selection: new Selector(SelectorScope.AllImages, tenantId: "tenant-alpha"),
onlyIf: ScheduleOnlyIf.Default,
notify: ScheduleNotify.Default,
limits: ScheduleLimits.Default,
createdAt: DateTimeOffset.UtcNow,
createdBy: "tests",
updatedAt: DateTimeOffset.UtcNow,
updatedBy: "tests");
var run = new Run(
id: "run-test",
tenantId: "tenant-alpha",
trigger: RunTrigger.Manual,
state: RunState.Planning,
stats: RunStats.Empty,
createdAt: DateTimeOffset.UtcNow,
reason: RunReason.Empty,
scheduleId: schedule.Id);
var impactSet = new ImpactSet(
selector: new Selector(SelectorScope.AllImages, tenantId: "tenant-alpha"),
images: new[]
{
new ImpactImage(
imageDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
registry: "registry",
repository: "repo",
namespaces: new[] { "prod" },
tags: new[] { "latest" })
},
usageOnly: true,
generatedAt: DateTimeOffset.UtcNow,
total: 1);
return new PlannerQueueMessage(run, impactSet, schedule, correlationId: "corr-test");
}
public static RunnerSegmentQueueMessage CreateRunnerMessage()
{
return new RunnerSegmentQueueMessage(
segmentId: "segment-test",
runId: "run-test",
tenantId: "tenant-alpha",
imageDigests: new[]
{
"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
},
scheduleId: "sch-test",
ratePerSecond: 10,
usageOnly: true,
attributes: new Dictionary<string, string> { ["priority"] = "kev" },
correlationId: "corr-runner");
}
}
}

115
src/Scheduler/__Tests/StellaOps.Scheduler.Queue.Tests/SchedulerQueueServiceCollectionExtensionsTests.cs Normal file
View File

@@ -0,0 +1,115 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using FluentAssertions;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Queue;
using StellaOps.Scheduler.Queue.Nats;
using Xunit;
namespace StellaOps.Scheduler.Queue.Tests;
public sealed class SchedulerQueueServiceCollectionExtensionsTests
{
[Fact]
public async Task AddSchedulerQueues_RegistersNatsTransport()
{
var services = new ServiceCollection();
services.AddSingleton<ILoggerFactory>(_ => NullLoggerFactory.Instance);
services.AddSchedulerQueues(new ConfigurationBuilder().Build());
var optionsDescriptor = services.First(descriptor => descriptor.ServiceType == typeof(SchedulerQueueOptions));
var options = (SchedulerQueueOptions)optionsDescriptor.ImplementationInstance!;
options.Kind = SchedulerQueueTransportKind.Nats;
options.Nats.Url = "nats://localhost:4222";
await using var provider = services.BuildServiceProvider();
var plannerQueue = provider.GetRequiredService<ISchedulerPlannerQueue>();
var runnerQueue = provider.GetRequiredService<ISchedulerRunnerQueue>();
plannerQueue.Should().BeOfType<NatsSchedulerPlannerQueue>();
runnerQueue.Should().BeOfType<NatsSchedulerRunnerQueue>();
}
[Fact]
public async Task SchedulerQueueHealthCheck_ReturnsHealthy_WhenTransportsReachable()
{
var healthCheck = new SchedulerQueueHealthCheck(
new FakePlannerQueue(failPing: false),
new FakeRunnerQueue(failPing: false),
NullLogger<SchedulerQueueHealthCheck>.Instance);
var context = new HealthCheckContext
{
Registration = new HealthCheckRegistration("scheduler-queue", healthCheck, HealthStatus.Unhealthy, Array.Empty<string>())
};
var result = await healthCheck.CheckHealthAsync(context);
result.Status.Should().Be(HealthStatus.Healthy);
}
[Fact]
public async Task SchedulerQueueHealthCheck_ReturnsUnhealthy_WhenRunnerPingFails()
{
var healthCheck = new SchedulerQueueHealthCheck(
new FakePlannerQueue(failPing: false),
new FakeRunnerQueue(failPing: true),
NullLogger<SchedulerQueueHealthCheck>.Instance);
var context = new HealthCheckContext
{
Registration = new HealthCheckRegistration("scheduler-queue", healthCheck, HealthStatus.Unhealthy, Array.Empty<string>())
};
var result = await healthCheck.CheckHealthAsync(context);
result.Status.Should().Be(HealthStatus.Unhealthy);
result.Description.Should().Contain("runner transport unreachable");
}
private abstract class FakeQueue<TMessage> : ISchedulerQueue<TMessage>, ISchedulerQueueTransportDiagnostics
{
private readonly bool _failPing;
protected FakeQueue(bool failPing)
{
_failPing = failPing;
}
public ValueTask<SchedulerQueueEnqueueResult> EnqueueAsync(TMessage message, CancellationToken cancellationToken = default)
=> ValueTask.FromResult(new SchedulerQueueEnqueueResult("stub", false));
public ValueTask<IReadOnlyList<ISchedulerQueueLease<TMessage>>> LeaseAsync(SchedulerQueueLeaseRequest request, CancellationToken cancellationToken = default)
=> ValueTask.FromResult<IReadOnlyList<ISchedulerQueueLease<TMessage>>>(Array.Empty<ISchedulerQueueLease<TMessage>>());
public ValueTask<IReadOnlyList<ISchedulerQueueLease<TMessage>>> ClaimExpiredAsync(SchedulerQueueClaimOptions options, CancellationToken cancellationToken = default)
=> ValueTask.FromResult<IReadOnlyList<ISchedulerQueueLease<TMessage>>>(Array.Empty<ISchedulerQueueLease<TMessage>>());
public ValueTask PingAsync(CancellationToken cancellationToken)
=> _failPing
? ValueTask.FromException(new InvalidOperationException("ping failed"))
: ValueTask.CompletedTask;
}
private sealed class FakePlannerQueue : FakeQueue<PlannerQueueMessage>, ISchedulerPlannerQueue
{
public FakePlannerQueue(bool failPing) : base(failPing)
{
}
}
private sealed class FakeRunnerQueue : FakeQueue<RunnerSegmentQueueMessage>, ISchedulerRunnerQueue
{
public FakeRunnerQueue(bool failPing) : base(failPing)
{
}
}
}

28
src/Scheduler/__Tests/StellaOps.Scheduler.Queue.Tests/StellaOps.Scheduler.Queue.Tests.csproj Normal file
View File

@@ -0,0 +1,28 @@
<?xml version='1.0' encoding='utf-8'?>
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>
<UseConcelierTestInfra>false</UseConcelierTestInfra>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="FluentAssertions" Version="6.12.0" />
<PackageReference Include="DotNet.Testcontainers" Version="1.7.0-beta.2269" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.0" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="xunit" Version="2.9.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="coverlet.collector" Version="6.0.4">
<PrivateAssets>all</PrivateAssets>
</PackageReference>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../../__Libraries/StellaOps.Scheduler.Models/StellaOps.Scheduler.Models.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Scheduler.Queue/StellaOps.Scheduler.Queue.csproj" />
</ItemGroup>
</Project>