feat: Add Go module and workspace test fixtures

- Created expected JSON files for Go modules and workspaces.
- Added go.mod and go.sum files for example projects.
- Implemented private module structure with expected JSON output.
- Introduced vendored dependencies with corresponding expected JSON.
- Developed PostgresGraphJobStore for managing graph jobs.
- Established SQL migration scripts for graph jobs schema.
- Implemented GraphJobRepository for CRUD operations on graph jobs.
- Created IGraphJobRepository interface for repository abstraction.
- Added unit tests for GraphJobRepository to ensure functionality.
This commit is contained in:
StellaOps Bot
2025-12-06 20:04:03 +02:00
parent a6f1406509
commit 05597616d6
178 changed files with 12022 additions and 4545 deletions

View File

@@ -0,0 +1,82 @@
using System.Collections.Generic;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Postgres.Repositories;
namespace StellaOps.Scheduler.WebService.GraphJobs;
internal sealed class PostgresGraphJobStore : IGraphJobStore
{
private readonly IGraphJobRepository _repository;
public PostgresGraphJobStore(IGraphJobRepository repository)
{
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
}
public async ValueTask<GraphBuildJob> AddAsync(GraphBuildJob job, CancellationToken cancellationToken)
{
await _repository.InsertAsync(job, cancellationToken);
return job;
}
public async ValueTask<GraphOverlayJob> AddAsync(GraphOverlayJob job, CancellationToken cancellationToken)
{
await _repository.InsertAsync(job, cancellationToken);
return job;
}
public async ValueTask<GraphJobCollection> GetJobsAsync(string tenantId, GraphJobQuery query, CancellationToken cancellationToken)
{
var normalized = query.Normalize();
var builds = normalized.Type is null or GraphJobQueryType.Build
? await _repository.ListBuildJobsAsync(tenantId, normalized.Status, normalized.Limit ?? 50, cancellationToken)
: Array.Empty<GraphBuildJob>();
var overlays = normalized.Type is null or GraphJobQueryType.Overlay
? await _repository.ListOverlayJobsAsync(tenantId, normalized.Status, normalized.Limit ?? 50, cancellationToken)
: Array.Empty<GraphOverlayJob>();
return GraphJobCollection.From(builds, overlays);
}
public async ValueTask<GraphBuildJob?> GetBuildJobAsync(string tenantId, string jobId, CancellationToken cancellationToken)
=> await _repository.GetBuildJobAsync(tenantId, jobId, cancellationToken);
public async ValueTask<GraphOverlayJob?> GetOverlayJobAsync(string tenantId, string jobId, CancellationToken cancellationToken)
=> await _repository.GetOverlayJobAsync(tenantId, jobId, cancellationToken);
public async ValueTask<GraphJobUpdateResult<GraphBuildJob>> UpdateAsync(GraphBuildJob job, GraphJobStatus expectedStatus, CancellationToken cancellationToken)
{
if (await _repository.TryReplaceAsync(job, expectedStatus, cancellationToken).ConfigureAwait(false))
{
return GraphJobUpdateResult<GraphBuildJob>.UpdatedResult(job);
}
var existing = await _repository.GetBuildJobAsync(job.TenantId, job.Id, cancellationToken).ConfigureAwait(false);
if (existing is null)
{
throw new KeyNotFoundException($"Graph build job '{job.Id}' not found.");
}
return GraphJobUpdateResult<GraphBuildJob>.NotUpdated(existing);
}
public async ValueTask<GraphJobUpdateResult<GraphOverlayJob>> UpdateAsync(GraphOverlayJob job, GraphJobStatus expectedStatus, CancellationToken cancellationToken)
{
if (await _repository.TryReplaceOverlayAsync(job, expectedStatus, cancellationToken).ConfigureAwait(false))
{
return GraphJobUpdateResult<GraphOverlayJob>.UpdatedResult(job);
}
var existing = await _repository.GetOverlayJobAsync(job.TenantId, job.Id, cancellationToken).ConfigureAwait(false);
if (existing is null)
{
throw new KeyNotFoundException($"Graph overlay job '{job.Id}' not found.");
}
return GraphJobUpdateResult<GraphOverlayJob>.NotUpdated(existing);
}
public async ValueTask<IReadOnlyCollection<GraphOverlayJob>> GetOverlayJobsAsync(string tenantId, CancellationToken cancellationToken)
=> await _repository.ListOverlayJobsAsync(tenantId, cancellationToken);
}

View File

@@ -8,9 +8,8 @@ using StellaOps.Plugin.DependencyInjection;
using StellaOps.Plugin.Hosting;
using StellaOps.Scheduler.WebService.Hosting;
using StellaOps.Scheduler.ImpactIndex;
using StellaOps.Scheduler.Storage.Mongo;
using StellaOps.Scheduler.Storage.Mongo.Repositories;
using StellaOps.Scheduler.Storage.Mongo.Services;
using StellaOps.Scheduler.Storage.Postgres;
using StellaOps.Scheduler.Storage.Postgres.Repositories;
using StellaOps.Scheduler.WebService;
using StellaOps.Scheduler.WebService.Auth;
using StellaOps.Scheduler.WebService.EventWebhooks;
@@ -83,8 +82,9 @@ builder.Services.AddOptions<SchedulerCartographerOptions>()
var storageSection = builder.Configuration.GetSection("Scheduler:Storage");
if (storageSection.Exists())
{
builder.Services.AddSchedulerMongoStorage(storageSection);
builder.Services.AddSingleton<IGraphJobStore, MongoGraphJobStore>();
builder.Services.AddSchedulerPostgresStorage(storageSection);
builder.Services.AddScoped<IGraphJobRepository, GraphJobRepository>();
builder.Services.AddSingleton<IGraphJobStore, PostgresGraphJobStore>();
builder.Services.AddSingleton<IPolicyRunService, PolicyRunService>();
builder.Services.AddSingleton<IPolicySimulationMetricsProvider, PolicySimulationMetricsProvider>();
builder.Services.AddSingleton<IPolicySimulationMetricsRecorder>(static sp => (IPolicySimulationMetricsRecorder)sp.GetRequiredService<IPolicySimulationMetricsProvider>());

View File

@@ -9,6 +9,7 @@
<ProjectReference Include="../__Libraries/StellaOps.Scheduler.Models/StellaOps.Scheduler.Models.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Scheduler.ImpactIndex/StellaOps.Scheduler.ImpactIndex.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Scheduler.Queue/StellaOps.Scheduler.Queue.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Scheduler.Storage.Postgres/StellaOps.Scheduler.Storage.Postgres.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Plugin/StellaOps.Plugin.csproj" />
<ProjectReference Include="../../Authority/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOps.Auth.Abstractions.csproj" />
<ProjectReference Include="../../Authority/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOps.Auth.ServerIntegration.csproj" />

View File

@@ -4,7 +4,7 @@ using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using StellaOps.Scheduler.Queue;
using StellaOps.Scheduler.Storage.Mongo;
using StellaOps.Scheduler.Storage.Postgres;
using StellaOps.Scheduler.Worker.DependencyInjection;
var builder = Host.CreateApplicationBuilder(args);
@@ -21,7 +21,7 @@ builder.Services.AddSchedulerQueues(builder.Configuration);
var storageSection = builder.Configuration.GetSection("Scheduler:Storage");
if (storageSection.Exists())
{
builder.Services.AddSchedulerMongoStorage(storageSection);
builder.Services.AddSchedulerPostgresStorage(storageSection);
}
builder.Services.AddSchedulerWorker(builder.Configuration.GetSection("Scheduler:Worker"));

View File

@@ -10,7 +10,7 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Scheduler.Queue\StellaOps.Scheduler.Queue.csproj" />
<ProjectReference Include="..\StellaOps.Scheduler.Storage.Mongo\StellaOps.Scheduler.Storage.Mongo.csproj" />
<ProjectReference Include="..\StellaOps.Scheduler.Storage.Postgres\StellaOps.Scheduler.Storage.Postgres.csproj" />
<ProjectReference Include="..\StellaOps.Scheduler.Worker\StellaOps.Scheduler.Worker.csproj" />
</ItemGroup>
</Project>

View File

@@ -9,8 +9,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "__Libraries", "__Libraries"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scheduler.Models", "__Libraries\StellaOps.Scheduler.Models\StellaOps.Scheduler.Models.csproj", "{382FA1C0-5F5F-424A-8485-7FED0ADE9F6B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scheduler.Storage.Mongo", "__Libraries\StellaOps.Scheduler.Storage.Mongo\StellaOps.Scheduler.Storage.Mongo.csproj", "{33770BC5-6802-45AD-A866-10027DD360E2}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scheduler.Storage.Postgres", "__Libraries\StellaOps.Scheduler.Storage.Postgres\StellaOps.Scheduler.Storage.Postgres.csproj", "{167198F1-43CF-42F4-BEF2-5ABC87116A37}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scheduler.ImpactIndex", "__Libraries\StellaOps.Scheduler.ImpactIndex\StellaOps.Scheduler.ImpactIndex.csproj", "{56209C24-3CE7-4F8E-8B8C-F052CB919DE2}"
@@ -61,8 +59,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scheduler.Models.
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scheduler.Queue.Tests", "__Tests\StellaOps.Scheduler.Queue.Tests\StellaOps.Scheduler.Queue.Tests.csproj", "{7C22F6B7-095E-459B-BCCF-87098EA9F192}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scheduler.Storage.Mongo.Tests", "__Tests\StellaOps.Scheduler.Storage.Mongo.Tests\StellaOps.Scheduler.Storage.Mongo.Tests.csproj", "{972CEB4D-510B-4701-B4A2-F14A85F11CC7}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scheduler.WebService.Tests", "__Tests\StellaOps.Scheduler.WebService.Tests\StellaOps.Scheduler.WebService.Tests.csproj", "{7B4C9EAC-316E-4890-A715-7BB9C1577F96}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scheduler.Backfill.Tests", "__Tests\StellaOps.Scheduler.Backfill.Tests\StellaOps.Scheduler.Backfill.Tests.csproj", "{B13D1DF0-1B9E-4557-919C-0A4E0FC9A8C7}"

View File

@@ -0,0 +1,34 @@
-- Scheduler graph jobs schema (Postgres)
DO $$ BEGIN
CREATE TYPE scheduler.graph_job_type AS ENUM ('build', 'overlay');
EXCEPTION WHEN duplicate_object THEN NULL; END $$;
DO $$ BEGIN
CREATE TYPE scheduler.graph_job_status AS ENUM ('pending', 'running', 'completed', 'failed', 'canceled');
EXCEPTION WHEN duplicate_object THEN NULL; END $$;
CREATE TABLE IF NOT EXISTS scheduler.graph_jobs (
id UUID PRIMARY KEY,
tenant_id TEXT NOT NULL,
type scheduler.graph_job_type NOT NULL,
status scheduler.graph_job_status NOT NULL,
payload JSONB NOT NULL,
correlation_id TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_graph_jobs_tenant_status ON scheduler.graph_jobs(tenant_id, status, created_at DESC);
CREATE INDEX IF NOT EXISTS idx_graph_jobs_tenant_type_status ON scheduler.graph_jobs(tenant_id, type, status, created_at DESC);
CREATE TABLE IF NOT EXISTS scheduler.graph_job_events (
id BIGSERIAL PRIMARY KEY,
job_id UUID NOT NULL REFERENCES scheduler.graph_jobs(id) ON DELETE CASCADE,
tenant_id TEXT NOT NULL,
status scheduler.graph_job_status NOT NULL,
payload JSONB NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_graph_job_events_job ON scheduler.graph_job_events(job_id, created_at DESC);

View File

@@ -0,0 +1,157 @@
using System.Collections.Generic;
using System.Text.Json;
using Dapper;
using Npgsql;
using StellaOps.Infrastructure.Postgres;
using StellaOps.Scheduler.Models;
namespace StellaOps.Scheduler.Storage.Postgres.Repositories;
public sealed class GraphJobRepository : IGraphJobRepository
{
private readonly SchedulerDataSource _dataSource;
private readonly JsonSerializerOptions _json;
public GraphJobRepository(SchedulerDataSource dataSource)
{
_dataSource = dataSource;
_json = CanonicalJsonSerializer.Options;
}
public async ValueTask InsertAsync(GraphBuildJob job, CancellationToken cancellationToken)
{
const string sql = @"INSERT INTO scheduler.graph_jobs
(id, tenant_id, type, status, payload, created_at, updated_at, correlation_id)
VALUES (@Id, @TenantId, @Type, @Status, @Payload, @CreatedAt, @UpdatedAt, @CorrelationId);";
await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
await conn.ExecuteAsync(sql, new
{
job.Id,
job.TenantId,
Type = (short)GraphJobQueryType.Build,
Status = (short)job.Status,
Payload = JsonSerializer.Serialize(job, _json),
job.CreatedAt,
UpdatedAt = job.UpdatedAt ?? job.CreatedAt,
job.CorrelationId
});
}
public async ValueTask InsertAsync(GraphOverlayJob job, CancellationToken cancellationToken)
{
const string sql = @"INSERT INTO scheduler.graph_jobs
(id, tenant_id, type, status, payload, created_at, updated_at, correlation_id)
VALUES (@Id, @TenantId, @Type, @Status, @Payload, @CreatedAt, @UpdatedAt, @CorrelationId);";
await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
await conn.ExecuteAsync(sql, new
{
job.Id,
job.TenantId,
Type = (short)GraphJobQueryType.Overlay,
Status = (short)job.Status,
Payload = JsonSerializer.Serialize(job, _json),
job.CreatedAt,
UpdatedAt = job.UpdatedAt ?? job.CreatedAt,
job.CorrelationId
});
}
public async ValueTask<GraphBuildJob?> GetBuildJobAsync(string tenantId, string jobId, CancellationToken cancellationToken)
{
const string sql = "SELECT payload FROM scheduler.graph_jobs WHERE tenant_id=@TenantId AND id=@Id AND type=@Type LIMIT 1";
await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
var payload = await conn.ExecuteScalarAsync<string?>(sql, new { TenantId = tenantId, Id = jobId, Type = (short)GraphJobQueryType.Build });
return payload is null ? null : JsonSerializer.Deserialize<GraphBuildJob>(payload, _json);
}
public async ValueTask<GraphOverlayJob?> GetOverlayJobAsync(string tenantId, string jobId, CancellationToken cancellationToken)
{
const string sql = "SELECT payload FROM scheduler.graph_jobs WHERE tenant_id=@TenantId AND id=@Id AND type=@Type LIMIT 1";
await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
var payload = await conn.ExecuteScalarAsync<string?>(sql, new { TenantId = tenantId, Id = jobId, Type = (short)GraphJobQueryType.Overlay });
return payload is null ? null : JsonSerializer.Deserialize<GraphOverlayJob>(payload, _json);
}
public async ValueTask<IReadOnlyCollection<GraphBuildJob>> ListBuildJobsAsync(string tenantId, GraphJobStatus? status, int limit, CancellationToken cancellationToken)
{
var sql = "SELECT payload FROM scheduler.graph_jobs WHERE tenant_id=@TenantId AND type=@Type";
if (status is not null)
{
sql += " AND status=@Status";
}
sql += " ORDER BY created_at DESC LIMIT @Limit";
await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
var rows = await conn.QueryAsync<string>(sql, new
{
TenantId = tenantId,
Type = (short)GraphJobQueryType.Build,
Status = status is null ? null : (short)status,
Limit = limit
});
return rows.Select(r => JsonSerializer.Deserialize<GraphBuildJob>(r, _json)!).ToArray();
}
public async ValueTask<IReadOnlyCollection<GraphOverlayJob>> ListOverlayJobsAsync(string tenantId, GraphJobStatus? status, int limit, CancellationToken cancellationToken)
{
var sql = "SELECT payload FROM scheduler.graph_jobs WHERE tenant_id=@TenantId AND type=@Type";
if (status is not null)
{
sql += " AND status=@Status";
}
sql += " ORDER BY created_at DESC LIMIT @Limit";
await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
var rows = await conn.QueryAsync<string>(sql, new
{
TenantId = tenantId,
Type = (short)GraphJobQueryType.Overlay,
Status = status is null ? null : (short)status,
Limit = limit
});
return rows.Select(r => JsonSerializer.Deserialize<GraphOverlayJob>(r, _json)!).ToArray();
}
public ValueTask<IReadOnlyCollection<GraphOverlayJob>> ListOverlayJobsAsync(string tenantId, CancellationToken cancellationToken)
=> ListOverlayJobsAsync(tenantId, status: null, limit: 50, cancellationToken);
public async ValueTask<bool> TryReplaceAsync(GraphBuildJob job, GraphJobStatus expectedStatus, CancellationToken cancellationToken)
{
const string sql = @"UPDATE scheduler.graph_jobs
SET status=@NewStatus, payload=@Payload, updated_at=NOW()
WHERE tenant_id=@TenantId AND id=@Id AND status=@ExpectedStatus AND type=@Type";
await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
var rows = await conn.ExecuteAsync(sql, new
{
job.TenantId,
job.Id,
ExpectedStatus = (short)expectedStatus,
NewStatus = (short)job.Status,
Type = (short)GraphJobQueryType.Build,
Payload = JsonSerializer.Serialize(job, _json)
});
return rows == 1;
}
public async ValueTask<bool> TryReplaceOverlayAsync(GraphOverlayJob job, GraphJobStatus expectedStatus, CancellationToken cancellationToken)
{
const string sql = @"UPDATE scheduler.graph_jobs
SET status=@NewStatus, payload=@Payload, updated_at=NOW()
WHERE tenant_id=@TenantId AND id=@Id AND status=@ExpectedStatus AND type=@Type";
await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
var rows = await conn.ExecuteAsync(sql, new
{
job.TenantId,
job.Id,
ExpectedStatus = (short)expectedStatus,
NewStatus = (short)job.Status,
Type = (short)GraphJobQueryType.Overlay,
Payload = JsonSerializer.Serialize(job, _json)
});
return rows == 1;
}
}

View File

@@ -0,0 +1,22 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Scheduler.Models;
namespace StellaOps.Scheduler.Storage.Postgres.Repositories;
public interface IGraphJobRepository
{
ValueTask InsertAsync(GraphBuildJob job, CancellationToken cancellationToken);
ValueTask InsertAsync(GraphOverlayJob job, CancellationToken cancellationToken);
ValueTask<bool> TryReplaceAsync(GraphBuildJob job, GraphJobStatus expectedStatus, CancellationToken cancellationToken);
ValueTask<bool> TryReplaceOverlayAsync(GraphOverlayJob job, GraphJobStatus expectedStatus, CancellationToken cancellationToken);
ValueTask<GraphBuildJob?> GetBuildJobAsync(string tenantId, string jobId, CancellationToken cancellationToken);
ValueTask<GraphOverlayJob?> GetOverlayJobAsync(string tenantId, string jobId, CancellationToken cancellationToken);
ValueTask<IReadOnlyCollection<GraphBuildJob>> ListBuildJobsAsync(string tenantId, GraphJobStatus? status, int limit, CancellationToken cancellationToken);
ValueTask<IReadOnlyCollection<GraphOverlayJob>> ListOverlayJobsAsync(string tenantId, GraphJobStatus? status, int limit, CancellationToken cancellationToken);
ValueTask<IReadOnlyCollection<GraphOverlayJob>> ListOverlayJobsAsync(string tenantId, CancellationToken cancellationToken);
}

View File

@@ -33,6 +33,7 @@ public static class ServiceCollectionExtensions
services.AddScoped<IDistributedLockRepository, DistributedLockRepository>();
services.AddScoped<IJobHistoryRepository, JobHistoryRepository>();
services.AddScoped<IMetricsRepository, MetricsRepository>();
services.AddScoped<IGraphJobRepository, GraphJobRepository>();
return services;
}
@@ -57,6 +58,7 @@ public static class ServiceCollectionExtensions
services.AddScoped<IDistributedLockRepository, DistributedLockRepository>();
services.AddScoped<IJobHistoryRepository, JobHistoryRepository>();
services.AddScoped<IMetricsRepository, MetricsRepository>();
services.AddScoped<IGraphJobRepository, GraphJobRepository>();
return services;
}

View File

@@ -16,6 +16,11 @@
<ItemGroup>
<ProjectReference Include="..\..\..\__Libraries\StellaOps.Infrastructure.Postgres\StellaOps.Infrastructure.Postgres.csproj" />
<ProjectReference Include="..\StellaOps.Scheduler.Models\StellaOps.Scheduler.Models.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Dapper" Version="2.1.24" />
</ItemGroup>
</Project>

View File

@@ -6,8 +6,8 @@ using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Queue;
using StellaOps.Scheduler.Storage.Mongo.Repositories;
using StellaOps.Scheduler.Storage.Mongo.Services;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Repositories;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Services;
using StellaOps.Scheduler.Worker.Events;
using StellaOps.Scheduler.Worker.Observability;

View File

@@ -1,129 +1,129 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Mongo.Repositories;
using StellaOps.Scheduler.Worker.Options;
namespace StellaOps.Scheduler.Worker.Graph;
internal sealed class GraphBuildBackgroundService : BackgroundService
{
private readonly IGraphJobRepository _repository;
private readonly GraphBuildExecutionService _executionService;
private readonly IOptions<SchedulerWorkerOptions> _options;
private readonly ILogger<GraphBuildBackgroundService> _logger;
public GraphBuildBackgroundService(
IGraphJobRepository repository,
GraphBuildExecutionService executionService,
IOptions<SchedulerWorkerOptions> options,
ILogger<GraphBuildBackgroundService> logger)
{
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
_executionService = executionService ?? throw new ArgumentNullException(nameof(executionService));
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Graph build worker started.");
while (!stoppingToken.IsCancellationRequested)
{
try
{
var graphOptions = _options.Value.Graph;
if (!graphOptions.Enabled)
{
await DelayAsync(graphOptions.IdleDelay, stoppingToken).ConfigureAwait(false);
continue;
}
var jobs = await _repository.ListBuildJobsAsync(GraphJobStatus.Pending, graphOptions.BatchSize, stoppingToken).ConfigureAwait(false);
if (jobs.Count == 0)
{
await DelayAsync(graphOptions.IdleDelay, stoppingToken).ConfigureAwait(false);
continue;
}
foreach (var job in jobs)
{
try
{
var result = await _executionService.ExecuteAsync(job, stoppingToken).ConfigureAwait(false);
LogResult(result);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
catch (Exception ex)
{
_logger.LogError(ex, "Unhandled exception while processing graph build job {JobId}.", job.Id);
}
}
await DelayAsync(graphOptions.PollInterval, stoppingToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
catch (Exception ex)
{
_logger.LogError(ex, "Graph build worker encountered an error; backing off.");
await DelayAsync(TimeSpan.FromSeconds(5), stoppingToken).ConfigureAwait(false);
}
}
_logger.LogInformation("Graph build worker stopping.");
}
private async Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken)
{
if (delay <= TimeSpan.Zero)
{
return;
}
try
{
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
}
catch (TaskCanceledException)
{
}
}
private void LogResult(GraphBuildExecutionResult result)
{
switch (result.Type)
{
case GraphBuildExecutionResultType.Completed:
_logger.LogInformation(
"Graph build job {JobId} completed (tenant={TenantId}).",
result.Job.Id,
result.Job.TenantId);
break;
case GraphBuildExecutionResultType.Failed:
_logger.LogWarning(
"Graph build job {JobId} failed (tenant={TenantId}): {Reason}.",
result.Job.Id,
result.Job.TenantId,
result.Reason ?? "unknown error");
break;
case GraphBuildExecutionResultType.Skipped:
_logger.LogDebug(
"Graph build job {JobId} skipped: {Reason}.",
result.Job.Id,
result.Reason ?? "no reason");
break;
}
}
}
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Repositories;
using StellaOps.Scheduler.Worker.Options;
namespace StellaOps.Scheduler.Worker.Graph;
internal sealed class GraphBuildBackgroundService : BackgroundService
{
private readonly IGraphJobRepository _repository;
private readonly GraphBuildExecutionService _executionService;
private readonly IOptions<SchedulerWorkerOptions> _options;
private readonly ILogger<GraphBuildBackgroundService> _logger;
public GraphBuildBackgroundService(
IGraphJobRepository repository,
GraphBuildExecutionService executionService,
IOptions<SchedulerWorkerOptions> options,
ILogger<GraphBuildBackgroundService> logger)
{
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
_executionService = executionService ?? throw new ArgumentNullException(nameof(executionService));
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Graph build worker started.");
while (!stoppingToken.IsCancellationRequested)
{
try
{
var graphOptions = _options.Value.Graph;
if (!graphOptions.Enabled)
{
await DelayAsync(graphOptions.IdleDelay, stoppingToken).ConfigureAwait(false);
continue;
}
var jobs = await _repository.ListBuildJobsAsync(GraphJobStatus.Pending, graphOptions.BatchSize, stoppingToken).ConfigureAwait(false);
if (jobs.Count == 0)
{
await DelayAsync(graphOptions.IdleDelay, stoppingToken).ConfigureAwait(false);
continue;
}
foreach (var job in jobs)
{
try
{
var result = await _executionService.ExecuteAsync(job, stoppingToken).ConfigureAwait(false);
LogResult(result);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
catch (Exception ex)
{
_logger.LogError(ex, "Unhandled exception while processing graph build job {JobId}.", job.Id);
}
}
await DelayAsync(graphOptions.PollInterval, stoppingToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
catch (Exception ex)
{
_logger.LogError(ex, "Graph build worker encountered an error; backing off.");
await DelayAsync(TimeSpan.FromSeconds(5), stoppingToken).ConfigureAwait(false);
}
}
_logger.LogInformation("Graph build worker stopping.");
}
private async Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken)
{
if (delay <= TimeSpan.Zero)
{
return;
}
try
{
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
}
catch (TaskCanceledException)
{
}
}
private void LogResult(GraphBuildExecutionResult result)
{
switch (result.Type)
{
case GraphBuildExecutionResultType.Completed:
_logger.LogInformation(
"Graph build job {JobId} completed (tenant={TenantId}).",
result.Job.Id,
result.Job.TenantId);
break;
case GraphBuildExecutionResultType.Failed:
_logger.LogWarning(
"Graph build job {JobId} failed (tenant={TenantId}): {Reason}.",
result.Job.Id,
result.Job.TenantId,
result.Reason ?? "unknown error");
break;
case GraphBuildExecutionResultType.Skipped:
_logger.LogDebug(
"Graph build job {JobId} skipped: {Reason}.",
result.Job.Id,
result.Reason ?? "no reason");
break;
}
}
}

View File

@@ -1,76 +1,76 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Mongo.Repositories;
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Repositories;
using StellaOps.Scheduler.Worker.Graph.Cartographer;
using StellaOps.Scheduler.Worker.Graph.Scheduler;
using StellaOps.Scheduler.Worker.Options;
using StellaOps.Scheduler.Worker.Observability;
namespace StellaOps.Scheduler.Worker.Graph;
internal sealed class GraphBuildExecutionService
{
private readonly IGraphJobRepository _repository;
private readonly ICartographerBuildClient _cartographerClient;
private readonly IGraphJobCompletionClient _completionClient;
private readonly IOptions<SchedulerWorkerOptions> _options;
private readonly SchedulerWorkerMetrics _metrics;
private readonly TimeProvider _timeProvider;
private readonly ILogger<GraphBuildExecutionService> _logger;
public GraphBuildExecutionService(
IGraphJobRepository repository,
ICartographerBuildClient cartographerClient,
IGraphJobCompletionClient completionClient,
IOptions<SchedulerWorkerOptions> options,
SchedulerWorkerMetrics metrics,
TimeProvider? timeProvider,
ILogger<GraphBuildExecutionService> logger)
{
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
_cartographerClient = cartographerClient ?? throw new ArgumentNullException(nameof(cartographerClient));
_completionClient = completionClient ?? throw new ArgumentNullException(nameof(completionClient));
_options = options ?? throw new ArgumentNullException(nameof(options));
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<GraphBuildExecutionResult> ExecuteAsync(GraphBuildJob job, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(job);
var graphOptions = _options.Value.Graph;
if (!graphOptions.Enabled)
{
_metrics.RecordGraphJobResult("build", "skipped");
return GraphBuildExecutionResult.Skipped(job, "graph_processing_disabled");
}
if (job.Status != GraphJobStatus.Pending)
{
_metrics.RecordGraphJobResult("build", "skipped");
return GraphBuildExecutionResult.Skipped(job, "status_not_pending");
}
var now = _timeProvider.GetUtcNow();
GraphBuildJob running;
try
{
running = GraphJobStateMachine.EnsureTransition(job, GraphJobStatus.Running, now, attempts: job.Attempts + 1);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to transition graph job {JobId} to running state.", job.Id);
_metrics.RecordGraphJobResult("build", "skipped");
return GraphBuildExecutionResult.Skipped(job, "transition_invalid");
}
namespace StellaOps.Scheduler.Worker.Graph;
internal sealed class GraphBuildExecutionService
{
private readonly IGraphJobRepository _repository;
private readonly ICartographerBuildClient _cartographerClient;
private readonly IGraphJobCompletionClient _completionClient;
private readonly IOptions<SchedulerWorkerOptions> _options;
private readonly SchedulerWorkerMetrics _metrics;
private readonly TimeProvider _timeProvider;
private readonly ILogger<GraphBuildExecutionService> _logger;
public GraphBuildExecutionService(
IGraphJobRepository repository,
ICartographerBuildClient cartographerClient,
IGraphJobCompletionClient completionClient,
IOptions<SchedulerWorkerOptions> options,
SchedulerWorkerMetrics metrics,
TimeProvider? timeProvider,
ILogger<GraphBuildExecutionService> logger)
{
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
_cartographerClient = cartographerClient ?? throw new ArgumentNullException(nameof(cartographerClient));
_completionClient = completionClient ?? throw new ArgumentNullException(nameof(completionClient));
_options = options ?? throw new ArgumentNullException(nameof(options));
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<GraphBuildExecutionResult> ExecuteAsync(GraphBuildJob job, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(job);
var graphOptions = _options.Value.Graph;
if (!graphOptions.Enabled)
{
_metrics.RecordGraphJobResult("build", "skipped");
return GraphBuildExecutionResult.Skipped(job, "graph_processing_disabled");
}
if (job.Status != GraphJobStatus.Pending)
{
_metrics.RecordGraphJobResult("build", "skipped");
return GraphBuildExecutionResult.Skipped(job, "status_not_pending");
}
var now = _timeProvider.GetUtcNow();
GraphBuildJob running;
try
{
running = GraphJobStateMachine.EnsureTransition(job, GraphJobStatus.Running, now, attempts: job.Attempts + 1);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to transition graph job {JobId} to running state.", job.Id);
_metrics.RecordGraphJobResult("build", "skipped");
return GraphBuildExecutionResult.Skipped(job, "transition_invalid");
}
if (!await _repository.TryReplaceAsync(running, job.Status, cancellationToken).ConfigureAwait(false))
{
_metrics.RecordGraphJobResult("build", "skipped");
@@ -78,161 +78,161 @@ internal sealed class GraphBuildExecutionService
}
_metrics.RecordGraphJobStart("build", running.TenantId, running.GraphSnapshotId ?? running.SbomId);
var attempt = 0;
CartographerBuildResult? lastResult = null;
Exception? lastException = null;
var backoff = graphOptions.RetryBackoff;
while (attempt < graphOptions.MaxAttempts)
{
cancellationToken.ThrowIfCancellationRequested();
attempt++;
try
{
var response = await _cartographerClient.StartBuildAsync(running, cancellationToken).ConfigureAwait(false);
lastResult = response;
if (!string.IsNullOrWhiteSpace(response.CartographerJobId) && response.CartographerJobId != running.CartographerJobId)
{
var updated = running with { CartographerJobId = response.CartographerJobId };
if (await _repository.TryReplaceAsync(updated, GraphJobStatus.Running, cancellationToken).ConfigureAwait(false))
{
running = updated;
}
}
if (!string.IsNullOrWhiteSpace(response.GraphSnapshotId) && response.GraphSnapshotId != running.GraphSnapshotId)
{
var updated = running with { GraphSnapshotId = response.GraphSnapshotId };
if (await _repository.TryReplaceAsync(updated, GraphJobStatus.Running, cancellationToken).ConfigureAwait(false))
{
running = updated;
}
}
if (response.Status == GraphJobStatus.Completed || response.Status == GraphJobStatus.Cancelled || response.Status == GraphJobStatus.Running)
{
var completionTime = _timeProvider.GetUtcNow();
await NotifyCompletionAsync(running, GraphJobStatus.Completed, completionTime, response.GraphSnapshotId, response.ResultUri, response.Error, cancellationToken).ConfigureAwait(false);
var attempt = 0;
CartographerBuildResult? lastResult = null;
Exception? lastException = null;
var backoff = graphOptions.RetryBackoff;
while (attempt < graphOptions.MaxAttempts)
{
cancellationToken.ThrowIfCancellationRequested();
attempt++;
try
{
var response = await _cartographerClient.StartBuildAsync(running, cancellationToken).ConfigureAwait(false);
lastResult = response;
if (!string.IsNullOrWhiteSpace(response.CartographerJobId) && response.CartographerJobId != running.CartographerJobId)
{
var updated = running with { CartographerJobId = response.CartographerJobId };
if (await _repository.TryReplaceAsync(updated, GraphJobStatus.Running, cancellationToken).ConfigureAwait(false))
{
running = updated;
}
}
if (!string.IsNullOrWhiteSpace(response.GraphSnapshotId) && response.GraphSnapshotId != running.GraphSnapshotId)
{
var updated = running with { GraphSnapshotId = response.GraphSnapshotId };
if (await _repository.TryReplaceAsync(updated, GraphJobStatus.Running, cancellationToken).ConfigureAwait(false))
{
running = updated;
}
}
if (response.Status == GraphJobStatus.Completed || response.Status == GraphJobStatus.Cancelled || response.Status == GraphJobStatus.Running)
{
var completionTime = _timeProvider.GetUtcNow();
await NotifyCompletionAsync(running, GraphJobStatus.Completed, completionTime, response.GraphSnapshotId, response.ResultUri, response.Error, cancellationToken).ConfigureAwait(false);
var duration = completionTime - running.CreatedAt;
_metrics.RecordGraphJobResult("build", "completed", duration);
_metrics.RecordGraphJobCompletion("build", running.TenantId, running.GraphSnapshotId ?? running.SbomId, "completed", duration);
return GraphBuildExecutionResult.Completed(running, response.ResultUri);
}
if (response.Status == GraphJobStatus.Failed)
{
if (attempt >= graphOptions.MaxAttempts)
{
var completionTime = _timeProvider.GetUtcNow();
await NotifyCompletionAsync(running, GraphJobStatus.Failed, completionTime, response.GraphSnapshotId, response.ResultUri, response.Error, cancellationToken).ConfigureAwait(false);
if (response.Status == GraphJobStatus.Failed)
{
if (attempt >= graphOptions.MaxAttempts)
{
var completionTime = _timeProvider.GetUtcNow();
await NotifyCompletionAsync(running, GraphJobStatus.Failed, completionTime, response.GraphSnapshotId, response.ResultUri, response.Error, cancellationToken).ConfigureAwait(false);
var duration = completionTime - running.CreatedAt;
_metrics.RecordGraphJobResult("build", "failed", duration);
_metrics.RecordGraphJobCompletion("build", running.TenantId, running.GraphSnapshotId ?? running.SbomId, "failed", duration);
return GraphBuildExecutionResult.Failed(running, response.Error);
}
_logger.LogWarning(
"Cartographer build attempt {Attempt} failed for job {JobId}; retrying in {Delay} (reason: {Reason}).",
attempt,
job.Id,
backoff,
response.Error ?? "unknown");
await Task.Delay(backoff, cancellationToken).ConfigureAwait(false);
continue;
}
// If Cartographer reports pending/queued we wait and retry.
if (attempt >= graphOptions.MaxAttempts)
{
var completionTime = _timeProvider.GetUtcNow();
await NotifyCompletionAsync(running, GraphJobStatus.Failed, completionTime, response.GraphSnapshotId, response.ResultUri, response.Error ?? "Cartographer did not complete the build.", cancellationToken).ConfigureAwait(false);
_logger.LogWarning(
"Cartographer build attempt {Attempt} failed for job {JobId}; retrying in {Delay} (reason: {Reason}).",
attempt,
job.Id,
backoff,
response.Error ?? "unknown");
await Task.Delay(backoff, cancellationToken).ConfigureAwait(false);
continue;
}
// If Cartographer reports pending/queued we wait and retry.
if (attempt >= graphOptions.MaxAttempts)
{
var completionTime = _timeProvider.GetUtcNow();
await NotifyCompletionAsync(running, GraphJobStatus.Failed, completionTime, response.GraphSnapshotId, response.ResultUri, response.Error ?? "Cartographer did not complete the build.", cancellationToken).ConfigureAwait(false);
var duration = completionTime - running.CreatedAt;
_metrics.RecordGraphJobResult("build", "failed", duration);
_metrics.RecordGraphJobCompletion("build", running.TenantId, running.GraphSnapshotId ?? running.SbomId, "failed", duration);
return GraphBuildExecutionResult.Failed(running, response.Error);
}
await Task.Delay(backoff, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
{
lastException = ex;
if (attempt >= graphOptions.MaxAttempts)
{
var completionTime = _timeProvider.GetUtcNow();
await NotifyCompletionAsync(running, GraphJobStatus.Failed, completionTime, running.GraphSnapshotId, null, ex.Message, cancellationToken).ConfigureAwait(false);
_metrics.RecordGraphJobResult("build", "failed", completionTime - running.CreatedAt);
return GraphBuildExecutionResult.Failed(running, ex.Message);
}
_logger.LogWarning(ex, "Cartographer build attempt {Attempt} failed for job {JobId}; retrying in {Delay}.", attempt, job.Id, backoff);
await Task.Delay(backoff, cancellationToken).ConfigureAwait(false);
}
}
var error = lastResult?.Error ?? lastException?.Message ?? "Cartographer build failed";
var finalTime = _timeProvider.GetUtcNow();
await NotifyCompletionAsync(running, GraphJobStatus.Failed, finalTime, lastResult?.GraphSnapshotId ?? running.GraphSnapshotId, lastResult?.ResultUri, error, cancellationToken).ConfigureAwait(false);
await Task.Delay(backoff, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
{
lastException = ex;
if (attempt >= graphOptions.MaxAttempts)
{
var completionTime = _timeProvider.GetUtcNow();
await NotifyCompletionAsync(running, GraphJobStatus.Failed, completionTime, running.GraphSnapshotId, null, ex.Message, cancellationToken).ConfigureAwait(false);
_metrics.RecordGraphJobResult("build", "failed", completionTime - running.CreatedAt);
return GraphBuildExecutionResult.Failed(running, ex.Message);
}
_logger.LogWarning(ex, "Cartographer build attempt {Attempt} failed for job {JobId}; retrying in {Delay}.", attempt, job.Id, backoff);
await Task.Delay(backoff, cancellationToken).ConfigureAwait(false);
}
}
var error = lastResult?.Error ?? lastException?.Message ?? "Cartographer build failed";
var finalTime = _timeProvider.GetUtcNow();
await NotifyCompletionAsync(running, GraphJobStatus.Failed, finalTime, lastResult?.GraphSnapshotId ?? running.GraphSnapshotId, lastResult?.ResultUri, error, cancellationToken).ConfigureAwait(false);
var finalDuration = finalTime - running.CreatedAt;
_metrics.RecordGraphJobResult("build", "failed", finalDuration);
_metrics.RecordGraphJobCompletion("build", running.TenantId, running.GraphSnapshotId ?? running.SbomId, "failed", finalDuration);
return GraphBuildExecutionResult.Failed(running, error);
}
private async Task NotifyCompletionAsync(
GraphBuildJob job,
GraphJobStatus status,
DateTimeOffset occurredAt,
string? graphSnapshotId,
string? resultUri,
string? error,
CancellationToken cancellationToken)
{
var dto = new GraphJobCompletionRequestDto(
job.Id,
"Build",
status,
occurredAt,
graphSnapshotId ?? job.GraphSnapshotId,
resultUri,
job.CorrelationId,
status == GraphJobStatus.Failed ? (error ?? "Cartographer build failed.") : null);
try
{
await _completionClient.NotifyAsync(dto, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
{
_logger.LogError(ex, "Failed notifying Scheduler completion for graph job {JobId}.", job.Id);
}
}
}
internal enum GraphBuildExecutionResultType
{
Completed,
Failed,
Skipped
}
internal readonly record struct GraphBuildExecutionResult(
GraphBuildExecutionResultType Type,
GraphBuildJob Job,
string? Reason = null,
string? ResultUri = null)
{
public static GraphBuildExecutionResult Completed(GraphBuildJob job, string? resultUri)
=> new(GraphBuildExecutionResultType.Completed, job, ResultUri: resultUri);
public static GraphBuildExecutionResult Failed(GraphBuildJob job, string? error)
=> new(GraphBuildExecutionResultType.Failed, job, error);
public static GraphBuildExecutionResult Skipped(GraphBuildJob job, string reason)
=> new(GraphBuildExecutionResultType.Skipped, job, reason);
}
private async Task NotifyCompletionAsync(
GraphBuildJob job,
GraphJobStatus status,
DateTimeOffset occurredAt,
string? graphSnapshotId,
string? resultUri,
string? error,
CancellationToken cancellationToken)
{
var dto = new GraphJobCompletionRequestDto(
job.Id,
"Build",
status,
occurredAt,
graphSnapshotId ?? job.GraphSnapshotId,
resultUri,
job.CorrelationId,
status == GraphJobStatus.Failed ? (error ?? "Cartographer build failed.") : null);
try
{
await _completionClient.NotifyAsync(dto, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
{
_logger.LogError(ex, "Failed notifying Scheduler completion for graph job {JobId}.", job.Id);
}
}
}
internal enum GraphBuildExecutionResultType
{
Completed,
Failed,
Skipped
}
internal readonly record struct GraphBuildExecutionResult(
GraphBuildExecutionResultType Type,
GraphBuildJob Job,
string? Reason = null,
string? ResultUri = null)
{
public static GraphBuildExecutionResult Completed(GraphBuildJob job, string? resultUri)
=> new(GraphBuildExecutionResultType.Completed, job, ResultUri: resultUri);
public static GraphBuildExecutionResult Failed(GraphBuildJob job, string? error)
=> new(GraphBuildExecutionResultType.Failed, job, error);
public static GraphBuildExecutionResult Skipped(GraphBuildJob job, string reason)
=> new(GraphBuildExecutionResultType.Skipped, job, reason);
}

View File

@@ -1,128 +1,128 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Mongo.Repositories;
using StellaOps.Scheduler.Worker.Options;
namespace StellaOps.Scheduler.Worker.Graph;
internal sealed class GraphOverlayBackgroundService : BackgroundService
{
private readonly IGraphJobRepository _repository;
private readonly GraphOverlayExecutionService _executionService;
private readonly IOptions<SchedulerWorkerOptions> _options;
private readonly ILogger<GraphOverlayBackgroundService> _logger;
public GraphOverlayBackgroundService(
IGraphJobRepository repository,
GraphOverlayExecutionService executionService,
IOptions<SchedulerWorkerOptions> options,
ILogger<GraphOverlayBackgroundService> logger)
{
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
_executionService = executionService ?? throw new ArgumentNullException(nameof(executionService));
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Graph overlay worker started.");
while (!stoppingToken.IsCancellationRequested)
{
try
{
var graphOptions = _options.Value.Graph;
if (!graphOptions.Enabled)
{
await DelayAsync(graphOptions.IdleDelay, stoppingToken).ConfigureAwait(false);
continue;
}
var jobs = await _repository.ListOverlayJobsAsync(GraphJobStatus.Pending, graphOptions.BatchSize, stoppingToken).ConfigureAwait(false);
if (jobs.Count == 0)
{
await DelayAsync(graphOptions.IdleDelay, stoppingToken).ConfigureAwait(false);
continue;
}
foreach (var job in jobs)
{
try
{
var result = await _executionService.ExecuteAsync(job, stoppingToken).ConfigureAwait(false);
LogResult(result);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
catch (Exception ex)
{
_logger.LogError(ex, "Unhandled exception while processing graph overlay job {JobId}.", job.Id);
}
}
await DelayAsync(graphOptions.PollInterval, stoppingToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
catch (Exception ex)
{
_logger.LogError(ex, "Graph overlay worker encountered an error; backing off.");
await DelayAsync(TimeSpan.FromSeconds(5), stoppingToken).ConfigureAwait(false);
}
}
_logger.LogInformation("Graph overlay worker stopping.");
}
private async Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken)
{
if (delay <= TimeSpan.Zero)
{
return;
}
try
{
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
}
catch (TaskCanceledException)
{
}
}
private void LogResult(GraphOverlayExecutionResult result)
{
switch (result.Type)
{
case GraphOverlayExecutionResultType.Completed:
_logger.LogInformation(
"Graph overlay job {JobId} completed (tenant={TenantId}).",
result.Job.Id,
result.Job.TenantId);
break;
case GraphOverlayExecutionResultType.Failed:
_logger.LogWarning(
"Graph overlay job {JobId} failed (tenant={TenantId}): {Reason}.",
result.Job.Id,
result.Job.TenantId,
result.Reason ?? "unknown error");
break;
case GraphOverlayExecutionResultType.Skipped:
_logger.LogDebug(
"Graph overlay job {JobId} skipped: {Reason}.",
result.Job.Id,
result.Reason ?? "no reason");
break;
}
}
}
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Repositories;
using StellaOps.Scheduler.Worker.Options;
namespace StellaOps.Scheduler.Worker.Graph;
internal sealed class GraphOverlayBackgroundService : BackgroundService
{
private readonly IGraphJobRepository _repository;
private readonly GraphOverlayExecutionService _executionService;
private readonly IOptions<SchedulerWorkerOptions> _options;
private readonly ILogger<GraphOverlayBackgroundService> _logger;
public GraphOverlayBackgroundService(
IGraphJobRepository repository,
GraphOverlayExecutionService executionService,
IOptions<SchedulerWorkerOptions> options,
ILogger<GraphOverlayBackgroundService> logger)
{
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
_executionService = executionService ?? throw new ArgumentNullException(nameof(executionService));
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Graph overlay worker started.");
while (!stoppingToken.IsCancellationRequested)
{
try
{
var graphOptions = _options.Value.Graph;
if (!graphOptions.Enabled)
{
await DelayAsync(graphOptions.IdleDelay, stoppingToken).ConfigureAwait(false);
continue;
}
var jobs = await _repository.ListOverlayJobsAsync(GraphJobStatus.Pending, graphOptions.BatchSize, stoppingToken).ConfigureAwait(false);
if (jobs.Count == 0)
{
await DelayAsync(graphOptions.IdleDelay, stoppingToken).ConfigureAwait(false);
continue;
}
foreach (var job in jobs)
{
try
{
var result = await _executionService.ExecuteAsync(job, stoppingToken).ConfigureAwait(false);
LogResult(result);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
catch (Exception ex)
{
_logger.LogError(ex, "Unhandled exception while processing graph overlay job {JobId}.", job.Id);
}
}
await DelayAsync(graphOptions.PollInterval, stoppingToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
catch (Exception ex)
{
_logger.LogError(ex, "Graph overlay worker encountered an error; backing off.");
await DelayAsync(TimeSpan.FromSeconds(5), stoppingToken).ConfigureAwait(false);
}
}
_logger.LogInformation("Graph overlay worker stopping.");
}
private async Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken)
{
if (delay <= TimeSpan.Zero)
{
return;
}
try
{
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
}
catch (TaskCanceledException)
{
}
}
private void LogResult(GraphOverlayExecutionResult result)
{
switch (result.Type)
{
case GraphOverlayExecutionResultType.Completed:
_logger.LogInformation(
"Graph overlay job {JobId} completed (tenant={TenantId}).",
result.Job.Id,
result.Job.TenantId);
break;
case GraphOverlayExecutionResultType.Failed:
_logger.LogWarning(
"Graph overlay job {JobId} failed (tenant={TenantId}): {Reason}.",
result.Job.Id,
result.Job.TenantId,
result.Reason ?? "unknown error");
break;
case GraphOverlayExecutionResultType.Skipped:
_logger.LogDebug(
"Graph overlay job {JobId} skipped: {Reason}.",
result.Job.Id,
result.Reason ?? "no reason");
break;
}
}
}

View File

@@ -1,76 +1,76 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Mongo.Repositories;
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Repositories;
using StellaOps.Scheduler.Worker.Graph.Cartographer;
using StellaOps.Scheduler.Worker.Graph.Scheduler;
using StellaOps.Scheduler.Worker.Options;
using StellaOps.Scheduler.Worker.Observability;
namespace StellaOps.Scheduler.Worker.Graph;
internal sealed class GraphOverlayExecutionService
{
private readonly IGraphJobRepository _repository;
private readonly ICartographerOverlayClient _overlayClient;
private readonly IGraphJobCompletionClient _completionClient;
private readonly IOptions<SchedulerWorkerOptions> _options;
private readonly SchedulerWorkerMetrics _metrics;
private readonly TimeProvider _timeProvider;
private readonly ILogger<GraphOverlayExecutionService> _logger;
public GraphOverlayExecutionService(
IGraphJobRepository repository,
ICartographerOverlayClient overlayClient,
IGraphJobCompletionClient completionClient,
IOptions<SchedulerWorkerOptions> options,
SchedulerWorkerMetrics metrics,
TimeProvider? timeProvider,
ILogger<GraphOverlayExecutionService> logger)
{
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
_overlayClient = overlayClient ?? throw new ArgumentNullException(nameof(overlayClient));
_completionClient = completionClient ?? throw new ArgumentNullException(nameof(completionClient));
_options = options ?? throw new ArgumentNullException(nameof(options));
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<GraphOverlayExecutionResult> ExecuteAsync(GraphOverlayJob job, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(job);
var graphOptions = _options.Value.Graph;
if (!graphOptions.Enabled)
{
_metrics.RecordGraphJobResult("overlay", "skipped");
return GraphOverlayExecutionResult.Skipped(job, "graph_processing_disabled");
}
if (job.Status != GraphJobStatus.Pending)
{
_metrics.RecordGraphJobResult("overlay", "skipped");
return GraphOverlayExecutionResult.Skipped(job, "status_not_pending");
}
var now = _timeProvider.GetUtcNow();
GraphOverlayJob running;
try
{
running = GraphJobStateMachine.EnsureTransition(job, GraphJobStatus.Running, now, attempts: job.Attempts + 1);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to transition graph overlay job {JobId} to running state.", job.Id);
_metrics.RecordGraphJobResult("overlay", "skipped");
return GraphOverlayExecutionResult.Skipped(job, "transition_invalid");
}
namespace StellaOps.Scheduler.Worker.Graph;
internal sealed class GraphOverlayExecutionService
{
private readonly IGraphJobRepository _repository;
private readonly ICartographerOverlayClient _overlayClient;
private readonly IGraphJobCompletionClient _completionClient;
private readonly IOptions<SchedulerWorkerOptions> _options;
private readonly SchedulerWorkerMetrics _metrics;
private readonly TimeProvider _timeProvider;
private readonly ILogger<GraphOverlayExecutionService> _logger;
public GraphOverlayExecutionService(
IGraphJobRepository repository,
ICartographerOverlayClient overlayClient,
IGraphJobCompletionClient completionClient,
IOptions<SchedulerWorkerOptions> options,
SchedulerWorkerMetrics metrics,
TimeProvider? timeProvider,
ILogger<GraphOverlayExecutionService> logger)
{
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
_overlayClient = overlayClient ?? throw new ArgumentNullException(nameof(overlayClient));
_completionClient = completionClient ?? throw new ArgumentNullException(nameof(completionClient));
_options = options ?? throw new ArgumentNullException(nameof(options));
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<GraphOverlayExecutionResult> ExecuteAsync(GraphOverlayJob job, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(job);
var graphOptions = _options.Value.Graph;
if (!graphOptions.Enabled)
{
_metrics.RecordGraphJobResult("overlay", "skipped");
return GraphOverlayExecutionResult.Skipped(job, "graph_processing_disabled");
}
if (job.Status != GraphJobStatus.Pending)
{
_metrics.RecordGraphJobResult("overlay", "skipped");
return GraphOverlayExecutionResult.Skipped(job, "status_not_pending");
}
var now = _timeProvider.GetUtcNow();
GraphOverlayJob running;
try
{
running = GraphJobStateMachine.EnsureTransition(job, GraphJobStatus.Running, now, attempts: job.Attempts + 1);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to transition graph overlay job {JobId} to running state.", job.Id);
_metrics.RecordGraphJobResult("overlay", "skipped");
return GraphOverlayExecutionResult.Skipped(job, "transition_invalid");
}
if (!await _repository.TryReplaceOverlayAsync(running, job.Status, cancellationToken).ConfigureAwait(false))
{
_metrics.RecordGraphJobResult("overlay", "skipped");
@@ -78,142 +78,142 @@ internal sealed class GraphOverlayExecutionService
}
_metrics.RecordGraphJobStart("overlay", running.TenantId, running.GraphSnapshotId);
var attempt = 0;
CartographerOverlayResult? lastResult = null;
Exception? lastException = null;
var backoff = graphOptions.RetryBackoff;
while (attempt < graphOptions.MaxAttempts)
{
cancellationToken.ThrowIfCancellationRequested();
attempt++;
try
{
var response = await _overlayClient.StartOverlayAsync(running, cancellationToken).ConfigureAwait(false);
lastResult = response;
if (response.Status == GraphJobStatus.Completed || response.Status == GraphJobStatus.Cancelled || response.Status == GraphJobStatus.Running)
{
var completionTime = _timeProvider.GetUtcNow();
await NotifyCompletionAsync(running, GraphJobStatus.Completed, completionTime, response.GraphSnapshotId ?? running.GraphSnapshotId, response.ResultUri, response.Error, cancellationToken).ConfigureAwait(false);
var attempt = 0;
CartographerOverlayResult? lastResult = null;
Exception? lastException = null;
var backoff = graphOptions.RetryBackoff;
while (attempt < graphOptions.MaxAttempts)
{
cancellationToken.ThrowIfCancellationRequested();
attempt++;
try
{
var response = await _overlayClient.StartOverlayAsync(running, cancellationToken).ConfigureAwait(false);
lastResult = response;
if (response.Status == GraphJobStatus.Completed || response.Status == GraphJobStatus.Cancelled || response.Status == GraphJobStatus.Running)
{
var completionTime = _timeProvider.GetUtcNow();
await NotifyCompletionAsync(running, GraphJobStatus.Completed, completionTime, response.GraphSnapshotId ?? running.GraphSnapshotId, response.ResultUri, response.Error, cancellationToken).ConfigureAwait(false);
var duration = completionTime - running.CreatedAt;
_metrics.RecordGraphJobResult("overlay", "completed", duration);
_metrics.RecordGraphJobCompletion("overlay", running.TenantId, running.GraphSnapshotId, "completed", duration);
return GraphOverlayExecutionResult.Completed(running, response.ResultUri);
}
if (response.Status == GraphJobStatus.Failed)
{
if (attempt >= graphOptions.MaxAttempts)
{
var completionTime = _timeProvider.GetUtcNow();
await NotifyCompletionAsync(running, GraphJobStatus.Failed, completionTime, response.GraphSnapshotId ?? running.GraphSnapshotId, response.ResultUri, response.Error, cancellationToken).ConfigureAwait(false);
if (response.Status == GraphJobStatus.Failed)
{
if (attempt >= graphOptions.MaxAttempts)
{
var completionTime = _timeProvider.GetUtcNow();
await NotifyCompletionAsync(running, GraphJobStatus.Failed, completionTime, response.GraphSnapshotId ?? running.GraphSnapshotId, response.ResultUri, response.Error, cancellationToken).ConfigureAwait(false);
var duration = completionTime - running.CreatedAt;
_metrics.RecordGraphJobResult("overlay", "failed", duration);
_metrics.RecordGraphJobCompletion("overlay", running.TenantId, running.GraphSnapshotId, "failed", duration);
return GraphOverlayExecutionResult.Failed(running, response.Error);
}
_logger.LogWarning(
"Cartographer overlay attempt {Attempt} failed for job {JobId}; retrying in {Delay} (reason: {Reason}).",
attempt,
job.Id,
backoff,
response.Error ?? "unknown");
await Task.Delay(backoff, cancellationToken).ConfigureAwait(false);
continue;
}
if (attempt >= graphOptions.MaxAttempts)
{
var completionTime = _timeProvider.GetUtcNow();
await NotifyCompletionAsync(running, GraphJobStatus.Failed, completionTime, response.GraphSnapshotId ?? running.GraphSnapshotId, response.ResultUri, response.Error ?? "Cartographer did not complete the overlay.", cancellationToken).ConfigureAwait(false);
_logger.LogWarning(
"Cartographer overlay attempt {Attempt} failed for job {JobId}; retrying in {Delay} (reason: {Reason}).",
attempt,
job.Id,
backoff,
response.Error ?? "unknown");
await Task.Delay(backoff, cancellationToken).ConfigureAwait(false);
continue;
}
if (attempt >= graphOptions.MaxAttempts)
{
var completionTime = _timeProvider.GetUtcNow();
await NotifyCompletionAsync(running, GraphJobStatus.Failed, completionTime, response.GraphSnapshotId ?? running.GraphSnapshotId, response.ResultUri, response.Error ?? "Cartographer did not complete the overlay.", cancellationToken).ConfigureAwait(false);
var duration = completionTime - running.CreatedAt;
_metrics.RecordGraphJobResult("overlay", "failed", duration);
_metrics.RecordGraphJobCompletion("overlay", running.TenantId, running.GraphSnapshotId, "failed", duration);
return GraphOverlayExecutionResult.Failed(running, response.Error);
}
await Task.Delay(backoff, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
{
lastException = ex;
if (attempt >= graphOptions.MaxAttempts)
{
var completionTime = _timeProvider.GetUtcNow();
await NotifyCompletionAsync(running, GraphJobStatus.Failed, completionTime, running.GraphSnapshotId, null, ex.Message, cancellationToken).ConfigureAwait(false);
_metrics.RecordGraphJobResult("overlay", "failed", completionTime - running.CreatedAt);
return GraphOverlayExecutionResult.Failed(running, ex.Message);
}
_logger.LogWarning(ex, "Cartographer overlay attempt {Attempt} failed for job {JobId}; retrying in {Delay}.", attempt, job.Id, backoff);
await Task.Delay(backoff, cancellationToken).ConfigureAwait(false);
}
}
var error = lastResult?.Error ?? lastException?.Message ?? "Cartographer overlay failed";
var finalTime = _timeProvider.GetUtcNow();
await NotifyCompletionAsync(running, GraphJobStatus.Failed, finalTime, lastResult?.GraphSnapshotId ?? running.GraphSnapshotId, lastResult?.ResultUri, error, cancellationToken).ConfigureAwait(false);
await Task.Delay(backoff, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
{
lastException = ex;
if (attempt >= graphOptions.MaxAttempts)
{
var completionTime = _timeProvider.GetUtcNow();
await NotifyCompletionAsync(running, GraphJobStatus.Failed, completionTime, running.GraphSnapshotId, null, ex.Message, cancellationToken).ConfigureAwait(false);
_metrics.RecordGraphJobResult("overlay", "failed", completionTime - running.CreatedAt);
return GraphOverlayExecutionResult.Failed(running, ex.Message);
}
_logger.LogWarning(ex, "Cartographer overlay attempt {Attempt} failed for job {JobId}; retrying in {Delay}.", attempt, job.Id, backoff);
await Task.Delay(backoff, cancellationToken).ConfigureAwait(false);
}
}
var error = lastResult?.Error ?? lastException?.Message ?? "Cartographer overlay failed";
var finalTime = _timeProvider.GetUtcNow();
await NotifyCompletionAsync(running, GraphJobStatus.Failed, finalTime, lastResult?.GraphSnapshotId ?? running.GraphSnapshotId, lastResult?.ResultUri, error, cancellationToken).ConfigureAwait(false);
var finalDuration = finalTime - running.CreatedAt;
_metrics.RecordGraphJobResult("overlay", "failed", finalDuration);
_metrics.RecordGraphJobCompletion("overlay", running.TenantId, running.GraphSnapshotId, "failed", finalDuration);
return GraphOverlayExecutionResult.Failed(running, error);
}
private async Task NotifyCompletionAsync(
GraphOverlayJob job,
GraphJobStatus status,
DateTimeOffset occurredAt,
string? graphSnapshotId,
string? resultUri,
string? error,
CancellationToken cancellationToken)
{
var dto = new GraphJobCompletionRequestDto(
job.Id,
"Overlay",
status,
occurredAt,
graphSnapshotId ?? job.GraphSnapshotId,
resultUri,
job.CorrelationId,
status == GraphJobStatus.Failed ? (error ?? "Cartographer overlay failed.") : null);
try
{
await _completionClient.NotifyAsync(dto, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
{
_logger.LogError(ex, "Failed notifying Scheduler completion for graph overlay job {JobId}.", job.Id);
}
}
}
internal enum GraphOverlayExecutionResultType
{
Completed,
Failed,
Skipped
}
internal readonly record struct GraphOverlayExecutionResult(
GraphOverlayExecutionResultType Type,
GraphOverlayJob Job,
string? Reason = null,
string? ResultUri = null)
{
public static GraphOverlayExecutionResult Completed(GraphOverlayJob job, string? resultUri)
=> new(GraphOverlayExecutionResultType.Completed, job, ResultUri: resultUri);
public static GraphOverlayExecutionResult Failed(GraphOverlayJob job, string? error)
=> new(GraphOverlayExecutionResultType.Failed, job, error);
public static GraphOverlayExecutionResult Skipped(GraphOverlayJob job, string reason)
=> new(GraphOverlayExecutionResultType.Skipped, job, reason);
}
private async Task NotifyCompletionAsync(
GraphOverlayJob job,
GraphJobStatus status,
DateTimeOffset occurredAt,
string? graphSnapshotId,
string? resultUri,
string? error,
CancellationToken cancellationToken)
{
var dto = new GraphJobCompletionRequestDto(
job.Id,
"Overlay",
status,
occurredAt,
graphSnapshotId ?? job.GraphSnapshotId,
resultUri,
job.CorrelationId,
status == GraphJobStatus.Failed ? (error ?? "Cartographer overlay failed.") : null);
try
{
await _completionClient.NotifyAsync(dto, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
{
_logger.LogError(ex, "Failed notifying Scheduler completion for graph overlay job {JobId}.", job.Id);
}
}
}
internal enum GraphOverlayExecutionResultType
{
Completed,
Failed,
Skipped
}
internal readonly record struct GraphOverlayExecutionResult(
GraphOverlayExecutionResultType Type,
GraphOverlayJob Job,
string? Reason = null,
string? ResultUri = null)
{
public static GraphOverlayExecutionResult Completed(GraphOverlayJob job, string? resultUri)
=> new(GraphOverlayExecutionResultType.Completed, job, ResultUri: resultUri);
public static GraphOverlayExecutionResult Failed(GraphOverlayJob job, string? error)
=> new(GraphOverlayExecutionResultType.Failed, job, error);
public static GraphOverlayExecutionResult Skipped(GraphOverlayJob job, string reason)
=> new(GraphOverlayExecutionResultType.Skipped, job, reason);
}

View File

@@ -1,7 +1,7 @@
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Mongo.Repositories;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Repositories;
using StellaOps.Scheduler.Worker.Options;
namespace StellaOps.Scheduler.Worker.Planning;

View File

@@ -2,8 +2,8 @@ using System.Collections.Immutable;
using Microsoft.Extensions.Logging;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Queue;
using StellaOps.Scheduler.Storage.Mongo.Repositories;
using StellaOps.Scheduler.Storage.Mongo.Services;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Repositories;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Services;
using StellaOps.Scheduler.Worker.Options;
using StellaOps.Scheduler.Worker.Observability;

View File

@@ -1,188 +1,188 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Mongo.Repositories;
using StellaOps.Scheduler.Worker.Options;
namespace StellaOps.Scheduler.Worker.Policy;
internal sealed class PolicyRunDispatchBackgroundService : BackgroundService
{
private readonly IPolicyRunJobRepository _repository;
private readonly PolicyRunExecutionService _executionService;
private readonly IOptions<SchedulerWorkerOptions> _options;
private readonly TimeProvider _timeProvider;
private readonly ILogger<PolicyRunDispatchBackgroundService> _logger;
private readonly string _leaseOwner;
public PolicyRunDispatchBackgroundService(
IPolicyRunJobRepository repository,
PolicyRunExecutionService executionService,
IOptions<SchedulerWorkerOptions> options,
TimeProvider? timeProvider,
ILogger<PolicyRunDispatchBackgroundService> logger)
{
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
_executionService = executionService ?? throw new ArgumentNullException(nameof(executionService));
_options = options ?? throw new ArgumentNullException(nameof(options));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_leaseOwner = options.Value.Policy.Dispatch.LeaseOwner;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Policy run dispatcher loop started with lease owner {LeaseOwner}.", _leaseOwner);
while (!stoppingToken.IsCancellationRequested)
{
try
{
var policyOptions = _options.Value.Policy;
if (!policyOptions.Enabled)
{
await DelayAsync(policyOptions.Dispatch.IdleDelay, stoppingToken).ConfigureAwait(false);
continue;
}
var batch = await LeaseBatchAsync(policyOptions.Dispatch, stoppingToken).ConfigureAwait(false);
if (batch.Count == 0)
{
await DelayAsync(policyOptions.Dispatch.IdleDelay, stoppingToken).ConfigureAwait(false);
continue;
}
foreach (var job in batch)
{
try
{
var result = await _executionService.ExecuteAsync(job, stoppingToken).ConfigureAwait(false);
LogResult(result);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex)
{
_logger.LogError(ex, "Unhandled exception while processing policy run job {JobId}.", job.Id);
}
}
}
catch (OperationCanceledException)
{
break;
}
catch (Exception ex)
{
_logger.LogError(ex, "Policy run dispatcher encountered an error; backing off.");
await DelayAsync(TimeSpan.FromSeconds(5), stoppingToken).ConfigureAwait(false);
}
}
_logger.LogInformation("Policy run dispatcher loop stopping.");
}
private async Task<IReadOnlyList<PolicyRunJob>> LeaseBatchAsync(
SchedulerWorkerOptions.PolicyOptions.DispatchOptions dispatchOptions,
CancellationToken cancellationToken)
{
var jobs = new List<PolicyRunJob>(dispatchOptions.BatchSize);
for (var i = 0; i < dispatchOptions.BatchSize; i++)
{
var now = _timeProvider.GetUtcNow();
PolicyRunJob? leased;
try
{
leased = await _repository
.LeaseAsync(_leaseOwner, now, dispatchOptions.LeaseDuration, dispatchOptions.MaxAttempts, cancellationToken: cancellationToken)
.ConfigureAwait(false);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to lease policy run job on attempt {Attempt}.", i + 1);
break;
}
if (leased is null)
{
break;
}
jobs.Add(leased);
}
return jobs;
}
private void LogResult(PolicyRunExecutionResult result)
{
switch (result.Type)
{
case PolicyRunExecutionResultType.Submitted:
_logger.LogInformation(
"Policy run job {JobId} submitted for tenant {TenantId} policy {PolicyId} (runId={RunId}).",
result.UpdatedJob.Id,
result.UpdatedJob.TenantId,
result.UpdatedJob.PolicyId,
result.UpdatedJob.RunId);
break;
case PolicyRunExecutionResultType.Retrying:
_logger.LogWarning(
"Policy run job {JobId} will retry for tenant {TenantId} policy {PolicyId}: {Error}.",
result.UpdatedJob.Id,
result.UpdatedJob.TenantId,
result.UpdatedJob.PolicyId,
result.Error);
break;
case PolicyRunExecutionResultType.Failed:
_logger.LogError(
"Policy run job {JobId} failed permanently for tenant {TenantId} policy {PolicyId}: {Error}.",
result.UpdatedJob.Id,
result.UpdatedJob.TenantId,
result.UpdatedJob.PolicyId,
result.Error);
break;
case PolicyRunExecutionResultType.Cancelled:
_logger.LogInformation(
"Policy run job {JobId} cancelled for tenant {TenantId} policy {PolicyId}.",
result.UpdatedJob.Id,
result.UpdatedJob.TenantId,
result.UpdatedJob.PolicyId);
break;
case PolicyRunExecutionResultType.NoOp:
_logger.LogInformation(
"Policy run job {JobId} completed without submission for tenant {TenantId} policy {PolicyId} (reason={Reason}).",
result.UpdatedJob.Id,
result.UpdatedJob.TenantId,
result.UpdatedJob.PolicyId,
result.Error ?? "none");
break;
}
}
private async Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken)
{
if (delay <= TimeSpan.Zero)
{
return;
}
try
{
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
}
catch (TaskCanceledException)
{
}
}
}
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Repositories;
using StellaOps.Scheduler.Worker.Options;
namespace StellaOps.Scheduler.Worker.Policy;
internal sealed class PolicyRunDispatchBackgroundService : BackgroundService
{
private readonly IPolicyRunJobRepository _repository;
private readonly PolicyRunExecutionService _executionService;
private readonly IOptions<SchedulerWorkerOptions> _options;
private readonly TimeProvider _timeProvider;
private readonly ILogger<PolicyRunDispatchBackgroundService> _logger;
private readonly string _leaseOwner;
public PolicyRunDispatchBackgroundService(
IPolicyRunJobRepository repository,
PolicyRunExecutionService executionService,
IOptions<SchedulerWorkerOptions> options,
TimeProvider? timeProvider,
ILogger<PolicyRunDispatchBackgroundService> logger)
{
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
_executionService = executionService ?? throw new ArgumentNullException(nameof(executionService));
_options = options ?? throw new ArgumentNullException(nameof(options));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_leaseOwner = options.Value.Policy.Dispatch.LeaseOwner;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Policy run dispatcher loop started with lease owner {LeaseOwner}.", _leaseOwner);
while (!stoppingToken.IsCancellationRequested)
{
try
{
var policyOptions = _options.Value.Policy;
if (!policyOptions.Enabled)
{
await DelayAsync(policyOptions.Dispatch.IdleDelay, stoppingToken).ConfigureAwait(false);
continue;
}
var batch = await LeaseBatchAsync(policyOptions.Dispatch, stoppingToken).ConfigureAwait(false);
if (batch.Count == 0)
{
await DelayAsync(policyOptions.Dispatch.IdleDelay, stoppingToken).ConfigureAwait(false);
continue;
}
foreach (var job in batch)
{
try
{
var result = await _executionService.ExecuteAsync(job, stoppingToken).ConfigureAwait(false);
LogResult(result);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex)
{
_logger.LogError(ex, "Unhandled exception while processing policy run job {JobId}.", job.Id);
}
}
}
catch (OperationCanceledException)
{
break;
}
catch (Exception ex)
{
_logger.LogError(ex, "Policy run dispatcher encountered an error; backing off.");
await DelayAsync(TimeSpan.FromSeconds(5), stoppingToken).ConfigureAwait(false);
}
}
_logger.LogInformation("Policy run dispatcher loop stopping.");
}
private async Task<IReadOnlyList<PolicyRunJob>> LeaseBatchAsync(
SchedulerWorkerOptions.PolicyOptions.DispatchOptions dispatchOptions,
CancellationToken cancellationToken)
{
var jobs = new List<PolicyRunJob>(dispatchOptions.BatchSize);
for (var i = 0; i < dispatchOptions.BatchSize; i++)
{
var now = _timeProvider.GetUtcNow();
PolicyRunJob? leased;
try
{
leased = await _repository
.LeaseAsync(_leaseOwner, now, dispatchOptions.LeaseDuration, dispatchOptions.MaxAttempts, cancellationToken: cancellationToken)
.ConfigureAwait(false);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to lease policy run job on attempt {Attempt}.", i + 1);
break;
}
if (leased is null)
{
break;
}
jobs.Add(leased);
}
return jobs;
}
private void LogResult(PolicyRunExecutionResult result)
{
switch (result.Type)
{
case PolicyRunExecutionResultType.Submitted:
_logger.LogInformation(
"Policy run job {JobId} submitted for tenant {TenantId} policy {PolicyId} (runId={RunId}).",
result.UpdatedJob.Id,
result.UpdatedJob.TenantId,
result.UpdatedJob.PolicyId,
result.UpdatedJob.RunId);
break;
case PolicyRunExecutionResultType.Retrying:
_logger.LogWarning(
"Policy run job {JobId} will retry for tenant {TenantId} policy {PolicyId}: {Error}.",
result.UpdatedJob.Id,
result.UpdatedJob.TenantId,
result.UpdatedJob.PolicyId,
result.Error);
break;
case PolicyRunExecutionResultType.Failed:
_logger.LogError(
"Policy run job {JobId} failed permanently for tenant {TenantId} policy {PolicyId}: {Error}.",
result.UpdatedJob.Id,
result.UpdatedJob.TenantId,
result.UpdatedJob.PolicyId,
result.Error);
break;
case PolicyRunExecutionResultType.Cancelled:
_logger.LogInformation(
"Policy run job {JobId} cancelled for tenant {TenantId} policy {PolicyId}.",
result.UpdatedJob.Id,
result.UpdatedJob.TenantId,
result.UpdatedJob.PolicyId);
break;
case PolicyRunExecutionResultType.NoOp:
_logger.LogInformation(
"Policy run job {JobId} completed without submission for tenant {TenantId} policy {PolicyId} (reason={Reason}).",
result.UpdatedJob.Id,
result.UpdatedJob.TenantId,
result.UpdatedJob.PolicyId,
result.Error ?? "none");
break;
}
}
private async Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken)
{
if (delay <= TimeSpan.Zero)
{
return;
}
try
{
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
}
catch (TaskCanceledException)
{
}
}
}

View File

@@ -1,18 +1,18 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Mongo.Repositories;
using StellaOps.Scheduler.Worker.Observability;
using StellaOps.Scheduler.Worker.Options;
namespace StellaOps.Scheduler.Worker.Policy;
internal sealed class PolicyRunExecutionService
{
private readonly IPolicyRunJobRepository _repository;
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Repositories;
using StellaOps.Scheduler.Worker.Observability;
using StellaOps.Scheduler.Worker.Options;
namespace StellaOps.Scheduler.Worker.Policy;
internal sealed class PolicyRunExecutionService
{
private readonly IPolicyRunJobRepository _repository;
private readonly IPolicyRunClient _client;
private readonly IOptions<SchedulerWorkerOptions> _options;
private readonly TimeProvider _timeProvider;
@@ -40,31 +40,31 @@ internal sealed class PolicyRunExecutionService
_webhookClient = webhookClient ?? throw new ArgumentNullException(nameof(webhookClient));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<PolicyRunExecutionResult> ExecuteAsync(PolicyRunJob job, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(job);
cancellationToken.ThrowIfCancellationRequested();
if (job.CancellationRequested)
{
var cancelledAt = _timeProvider.GetUtcNow();
var cancelled = job with
{
Status = PolicyRunJobStatus.Cancelled,
CancelledAt = cancelledAt,
UpdatedAt = cancelledAt,
LeaseOwner = null,
LeaseExpiresAt = null,
AvailableAt = cancelledAt
};
var replaced = await _repository.ReplaceAsync(cancelled, job.LeaseOwner, cancellationToken: cancellationToken).ConfigureAwait(false);
if (!replaced)
{
_logger.LogWarning("Failed to update cancelled policy run job {JobId}.", job.Id);
}
public async Task<PolicyRunExecutionResult> ExecuteAsync(PolicyRunJob job, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(job);
cancellationToken.ThrowIfCancellationRequested();
if (job.CancellationRequested)
{
var cancelledAt = _timeProvider.GetUtcNow();
var cancelled = job with
{
Status = PolicyRunJobStatus.Cancelled,
CancelledAt = cancelledAt,
UpdatedAt = cancelledAt,
LeaseOwner = null,
LeaseExpiresAt = null,
AvailableAt = cancelledAt
};
var replaced = await _repository.ReplaceAsync(cancelled, job.LeaseOwner, cancellationToken: cancellationToken).ConfigureAwait(false);
if (!replaced)
{
_logger.LogWarning("Failed to update cancelled policy run job {JobId}.", job.Id);
}
_metrics.RecordPolicyRunEvent(
cancelled.TenantId,
cancelled.PolicyId,
@@ -83,38 +83,38 @@ internal sealed class PolicyRunExecutionService
await _webhookClient.NotifyAsync(cancelledPayload, cancellationToken).ConfigureAwait(false);
return PolicyRunExecutionResult.Cancelled(cancelled);
}
var targeting = await _targetingService
.EnsureTargetsAsync(job, cancellationToken)
.ConfigureAwait(false);
if (targeting.Status == PolicyRunTargetingStatus.NoWork)
{
var completionTime = _timeProvider.GetUtcNow();
var completed = targeting.Job with
{
Status = PolicyRunJobStatus.Completed,
CompletedAt = completionTime,
UpdatedAt = completionTime,
LeaseOwner = null,
LeaseExpiresAt = null,
AvailableAt = completionTime,
LastError = null
};
var replaced = await _repository.ReplaceAsync(
completed,
job.LeaseOwner,
cancellationToken: cancellationToken)
.ConfigureAwait(false);
if (!replaced)
{
_logger.LogWarning("Failed to persist no-work completion for policy run job {JobId}.", job.Id);
}
var latency = CalculateLatency(job, completionTime);
}
var targeting = await _targetingService
.EnsureTargetsAsync(job, cancellationToken)
.ConfigureAwait(false);
if (targeting.Status == PolicyRunTargetingStatus.NoWork)
{
var completionTime = _timeProvider.GetUtcNow();
var completed = targeting.Job with
{
Status = PolicyRunJobStatus.Completed,
CompletedAt = completionTime,
UpdatedAt = completionTime,
LeaseOwner = null,
LeaseExpiresAt = null,
AvailableAt = completionTime,
LastError = null
};
var replaced = await _repository.ReplaceAsync(
completed,
job.LeaseOwner,
cancellationToken: cancellationToken)
.ConfigureAwait(false);
if (!replaced)
{
_logger.LogWarning("Failed to persist no-work completion for policy run job {JobId}.", job.Id);
}
var latency = CalculateLatency(job, completionTime);
_metrics.RecordPolicyRunEvent(
completed.TenantId,
completed.PolicyId,
@@ -132,85 +132,85 @@ internal sealed class PolicyRunExecutionService
await _webhookClient.NotifyAsync(completedPayload, cancellationToken).ConfigureAwait(false);
return PolicyRunExecutionResult.NoOp(completed, targeting.Reason);
}
job = targeting.Job;
var now = _timeProvider.GetUtcNow();
var request = job.ToPolicyRunRequest(now);
var submission = await _client.SubmitAsync(job, request, cancellationToken).ConfigureAwait(false);
var dispatchOptions = _options.Value.Policy.Dispatch;
var attemptCount = job.AttemptCount + 1;
if (submission.Success)
{
var updated = job with
{
Status = PolicyRunJobStatus.Submitted,
RunId = submission.RunId ?? job.RunId,
SubmittedAt = submission.QueuedAt ?? now,
UpdatedAt = now,
AttemptCount = attemptCount,
LastAttemptAt = now,
LastError = null,
LeaseOwner = null,
LeaseExpiresAt = null,
AvailableAt = now
};
var replaced = await _repository.ReplaceAsync(updated, job.LeaseOwner, cancellationToken: cancellationToken).ConfigureAwait(false);
if (!replaced)
{
_logger.LogWarning("Failed to persist submitted policy run job {JobId}.", job.Id);
}
var latency = CalculateLatency(job, now);
_metrics.RecordPolicyRunEvent(
updated.TenantId,
updated.PolicyId,
updated.Mode,
"submitted",
latency);
_logger.LogInformation(
"Policy run job {JobId} submitted (tenant={TenantId}, policy={PolicyId}, runId={RunId}, attempts={Attempts}).",
updated.Id,
updated.TenantId,
updated.PolicyId,
updated.RunId ?? "(pending)",
attemptCount);
return PolicyRunExecutionResult.Submitted(updated);
}
var nextStatus = attemptCount >= dispatchOptions.MaxAttempts
? PolicyRunJobStatus.Failed
: PolicyRunJobStatus.Pending;
var nextAvailable = nextStatus == PolicyRunJobStatus.Pending
? now.Add(dispatchOptions.RetryBackoff)
: now;
var failedJob = job with
{
Status = nextStatus,
AttemptCount = attemptCount,
LastAttemptAt = now,
LastError = submission.Error,
LeaseOwner = null,
LeaseExpiresAt = null,
UpdatedAt = now,
AvailableAt = nextAvailable
};
var updateSuccess = await _repository.ReplaceAsync(failedJob, job.LeaseOwner, cancellationToken: cancellationToken).ConfigureAwait(false);
if (!updateSuccess)
{
_logger.LogWarning("Failed to update policy run job {JobId} after submission failure.", job.Id);
}
var latencyForFailure = CalculateLatency(job, now);
var reason = string.IsNullOrWhiteSpace(submission.Error) ? null : submission.Error;
if (nextStatus == PolicyRunJobStatus.Failed)
{
}
job = targeting.Job;
var now = _timeProvider.GetUtcNow();
var request = job.ToPolicyRunRequest(now);
var submission = await _client.SubmitAsync(job, request, cancellationToken).ConfigureAwait(false);
var dispatchOptions = _options.Value.Policy.Dispatch;
var attemptCount = job.AttemptCount + 1;
if (submission.Success)
{
var updated = job with
{
Status = PolicyRunJobStatus.Submitted,
RunId = submission.RunId ?? job.RunId,
SubmittedAt = submission.QueuedAt ?? now,
UpdatedAt = now,
AttemptCount = attemptCount,
LastAttemptAt = now,
LastError = null,
LeaseOwner = null,
LeaseExpiresAt = null,
AvailableAt = now
};
var replaced = await _repository.ReplaceAsync(updated, job.LeaseOwner, cancellationToken: cancellationToken).ConfigureAwait(false);
if (!replaced)
{
_logger.LogWarning("Failed to persist submitted policy run job {JobId}.", job.Id);
}
var latency = CalculateLatency(job, now);
_metrics.RecordPolicyRunEvent(
updated.TenantId,
updated.PolicyId,
updated.Mode,
"submitted",
latency);
_logger.LogInformation(
"Policy run job {JobId} submitted (tenant={TenantId}, policy={PolicyId}, runId={RunId}, attempts={Attempts}).",
updated.Id,
updated.TenantId,
updated.PolicyId,
updated.RunId ?? "(pending)",
attemptCount);
return PolicyRunExecutionResult.Submitted(updated);
}
var nextStatus = attemptCount >= dispatchOptions.MaxAttempts
? PolicyRunJobStatus.Failed
: PolicyRunJobStatus.Pending;
var nextAvailable = nextStatus == PolicyRunJobStatus.Pending
? now.Add(dispatchOptions.RetryBackoff)
: now;
var failedJob = job with
{
Status = nextStatus,
AttemptCount = attemptCount,
LastAttemptAt = now,
LastError = submission.Error,
LeaseOwner = null,
LeaseExpiresAt = null,
UpdatedAt = now,
AvailableAt = nextAvailable
};
var updateSuccess = await _repository.ReplaceAsync(failedJob, job.LeaseOwner, cancellationToken: cancellationToken).ConfigureAwait(false);
if (!updateSuccess)
{
_logger.LogWarning("Failed to update policy run job {JobId} after submission failure.", job.Id);
}
var latencyForFailure = CalculateLatency(job, now);
var reason = string.IsNullOrWhiteSpace(submission.Error) ? null : submission.Error;
if (nextStatus == PolicyRunJobStatus.Failed)
{
_metrics.RecordPolicyRunEvent(
failedJob.TenantId,
failedJob.PolicyId,
@@ -233,31 +233,31 @@ internal sealed class PolicyRunExecutionService
await _webhookClient.NotifyAsync(failedPayload, cancellationToken).ConfigureAwait(false);
return PolicyRunExecutionResult.Failed(failedJob, submission.Error);
}
_metrics.RecordPolicyRunEvent(
failedJob.TenantId,
failedJob.PolicyId,
failedJob.Mode,
"retry",
latencyForFailure,
reason);
_logger.LogWarning(
"Policy run job {JobId} retry scheduled (tenant={TenantId}, policy={PolicyId}, runId={RunId}, attempt={Attempt}). Error: {Error}",
failedJob.Id,
failedJob.TenantId,
failedJob.PolicyId,
failedJob.RunId ?? "(pending)",
attemptCount,
submission.Error ?? "unknown");
return PolicyRunExecutionResult.Retrying(failedJob, submission.Error);
}
private static TimeSpan CalculateLatency(PolicyRunJob job, DateTimeOffset now)
{
var origin = job.QueuedAt ?? job.CreatedAt;
var latency = now - origin;
return latency < TimeSpan.Zero ? TimeSpan.Zero : latency;
}
}
}
_metrics.RecordPolicyRunEvent(
failedJob.TenantId,
failedJob.PolicyId,
failedJob.Mode,
"retry",
latencyForFailure,
reason);
_logger.LogWarning(
"Policy run job {JobId} retry scheduled (tenant={TenantId}, policy={PolicyId}, runId={RunId}, attempt={Attempt}). Error: {Error}",
failedJob.Id,
failedJob.TenantId,
failedJob.PolicyId,
failedJob.RunId ?? "(pending)",
attemptCount,
submission.Error ?? "unknown");
return PolicyRunExecutionResult.Retrying(failedJob, submission.Error);
}
private static TimeSpan CalculateLatency(PolicyRunJob job, DateTimeOffset now)
{
var origin = job.QueuedAt ?? job.CreatedAt;
var latency = now - origin;
return latency < TimeSpan.Zero ? TimeSpan.Zero : latency;
}
}

View File

@@ -8,7 +8,7 @@
<ItemGroup>
<ProjectReference Include="../StellaOps.Scheduler.ImpactIndex/StellaOps.Scheduler.ImpactIndex.csproj" />
<ProjectReference Include="../StellaOps.Scheduler.Models/StellaOps.Scheduler.Models.csproj" />
<ProjectReference Include="../StellaOps.Scheduler.Storage.Mongo/StellaOps.Scheduler.Storage.Mongo.csproj" />
<ProjectReference Include="../StellaOps.Scheduler.Storage.Postgres/StellaOps.Scheduler.Storage.Postgres.csproj" />
<ProjectReference Include="../StellaOps.Scheduler.Queue/StellaOps.Scheduler.Queue.csproj" />
<ProjectReference Include="../../../Notify/__Libraries/StellaOps.Notify.Models/StellaOps.Notify.Models.csproj" />
<ProjectReference Include="../../../Notify/__Libraries/StellaOps.Notify.Queue/StellaOps.Notify.Queue.csproj" />

View File

@@ -1,12 +1,12 @@
global using System.Text.Json;
global using System.Text.Json.Nodes;
global using Microsoft.Extensions.Logging.Abstractions;
global using Microsoft.Extensions.Options;
global using Mongo2Go;
global using MongoDB.Bson;
global using MongoDB.Driver;
global using StellaOps.Scheduler.Models;
global using StellaOps.Scheduler.Storage.Mongo.Internal;
global using StellaOps.Scheduler.Storage.Mongo.Migrations;
global using StellaOps.Scheduler.Storage.Mongo.Options;
global using Xunit;
global using System.Text.Json;
global using System.Text.Json.Nodes;
global using Microsoft.Extensions.Logging.Abstractions;
global using Microsoft.Extensions.Options;
global using Mongo2Go;
global using MongoDB.Bson;
global using MongoDB.Driver;
global using StellaOps.Scheduler.Models;
global using StellaOps.Scheduler.Storage.Postgres.Repositories.Internal;
global using StellaOps.Scheduler.Storage.Postgres.Repositories.Migrations;
global using StellaOps.Scheduler.Storage.Postgres.Repositories.Options;
global using Xunit;

View File

@@ -1,70 +1,70 @@
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Mongo.Repositories;
using StellaOps.Scheduler.WebService.GraphJobs;
using Xunit;
namespace StellaOps.Scheduler.Storage.Mongo.Tests.Integration;
public sealed class GraphJobStoreTests
{
private static readonly DateTimeOffset OccurredAt = new(2025, 11, 4, 10, 30, 0, TimeSpan.Zero);
[Fact]
public async Task UpdateAsync_SucceedsWhenExpectedStatusMatches()
{
using var harness = new SchedulerMongoTestHarness();
var repository = new GraphJobRepository(harness.Context);
var store = new MongoGraphJobStore(repository);
var initial = CreateBuildJob();
await store.AddAsync(initial, CancellationToken.None);
var running = GraphJobStateMachine.EnsureTransition(initial, GraphJobStatus.Running, OccurredAt, attempts: initial.Attempts);
var completed = GraphJobStateMachine.EnsureTransition(running, GraphJobStatus.Completed, OccurredAt, attempts: running.Attempts + 1);
var updateResult = await store.UpdateAsync(completed, GraphJobStatus.Pending, CancellationToken.None);
Assert.True(updateResult.Updated);
var persisted = await store.GetBuildJobAsync(initial.TenantId, initial.Id, CancellationToken.None);
Assert.NotNull(persisted);
Assert.Equal(GraphJobStatus.Completed, persisted!.Status);
}
[Fact]
public async Task UpdateAsync_ReturnsExistingWhenExpectedStatusMismatch()
{
using var harness = new SchedulerMongoTestHarness();
var repository = new GraphJobRepository(harness.Context);
var store = new MongoGraphJobStore(repository);
var initial = CreateBuildJob();
await store.AddAsync(initial, CancellationToken.None);
var running = GraphJobStateMachine.EnsureTransition(initial, GraphJobStatus.Running, OccurredAt, attempts: initial.Attempts);
var completed = GraphJobStateMachine.EnsureTransition(running, GraphJobStatus.Completed, OccurredAt, attempts: running.Attempts + 1);
await store.UpdateAsync(completed, GraphJobStatus.Pending, CancellationToken.None);
var result = await store.UpdateAsync(completed, GraphJobStatus.Pending, CancellationToken.None);
Assert.False(result.Updated);
Assert.Equal(GraphJobStatus.Completed, result.Job.Status);
}
private static GraphBuildJob CreateBuildJob()
{
var digest = "sha256:" + new string('b', 64);
return new GraphBuildJob(
id: "gbj_store_test",
tenantId: "tenant-store",
sbomId: "sbom-alpha",
sbomVersionId: "sbom-alpha-v1",
sbomDigest: digest,
status: GraphJobStatus.Pending,
trigger: GraphBuildJobTrigger.SbomVersion,
createdAt: OccurredAt,
metadata: null);
}
}
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Repositories;
using StellaOps.Scheduler.WebService.GraphJobs;
using Xunit;
namespace StellaOps.Scheduler.Storage.Postgres.Repositories.Tests.Integration;
public sealed class GraphJobStoreTests
{
private static readonly DateTimeOffset OccurredAt = new(2025, 11, 4, 10, 30, 0, TimeSpan.Zero);
[Fact]
public async Task UpdateAsync_SucceedsWhenExpectedStatusMatches()
{
using var harness = new SchedulerMongoTestHarness();
var repository = new GraphJobRepository(harness.Context);
var store = new MongoGraphJobStore(repository);
var initial = CreateBuildJob();
await store.AddAsync(initial, CancellationToken.None);
var running = GraphJobStateMachine.EnsureTransition(initial, GraphJobStatus.Running, OccurredAt, attempts: initial.Attempts);
var completed = GraphJobStateMachine.EnsureTransition(running, GraphJobStatus.Completed, OccurredAt, attempts: running.Attempts + 1);
var updateResult = await store.UpdateAsync(completed, GraphJobStatus.Pending, CancellationToken.None);
Assert.True(updateResult.Updated);
var persisted = await store.GetBuildJobAsync(initial.TenantId, initial.Id, CancellationToken.None);
Assert.NotNull(persisted);
Assert.Equal(GraphJobStatus.Completed, persisted!.Status);
}
[Fact]
public async Task UpdateAsync_ReturnsExistingWhenExpectedStatusMismatch()
{
using var harness = new SchedulerMongoTestHarness();
var repository = new GraphJobRepository(harness.Context);
var store = new MongoGraphJobStore(repository);
var initial = CreateBuildJob();
await store.AddAsync(initial, CancellationToken.None);
var running = GraphJobStateMachine.EnsureTransition(initial, GraphJobStatus.Running, OccurredAt, attempts: initial.Attempts);
var completed = GraphJobStateMachine.EnsureTransition(running, GraphJobStatus.Completed, OccurredAt, attempts: running.Attempts + 1);
await store.UpdateAsync(completed, GraphJobStatus.Pending, CancellationToken.None);
var result = await store.UpdateAsync(completed, GraphJobStatus.Pending, CancellationToken.None);
Assert.False(result.Updated);
Assert.Equal(GraphJobStatus.Completed, result.Job.Status);
}
private static GraphBuildJob CreateBuildJob()
{
var digest = "sha256:" + new string('b', 64);
return new GraphBuildJob(
id: "gbj_store_test",
tenantId: "tenant-store",
sbomId: "sbom-alpha",
sbomVersionId: "sbom-alpha-v1",
sbomDigest: digest,
status: GraphJobStatus.Pending,
trigger: GraphBuildJobTrigger.SbomVersion,
createdAt: OccurredAt,
metadata: null);
}
}

View File

@@ -1,126 +1,126 @@
using System.Text.Json.Nodes;
namespace StellaOps.Scheduler.Storage.Mongo.Tests.Integration;
public sealed class SchedulerMongoRoundTripTests : IDisposable
{
private readonly MongoDbRunner _runner;
private readonly SchedulerMongoContext _context;
public SchedulerMongoRoundTripTests()
{
_runner = MongoDbRunner.Start(additionalMongodArguments: "--quiet");
var options = new SchedulerMongoOptions
{
ConnectionString = _runner.ConnectionString,
Database = $"scheduler_roundtrip_{Guid.NewGuid():N}"
};
_context = new SchedulerMongoContext(Microsoft.Extensions.Options.Options.Create(options), NullLogger<SchedulerMongoContext>.Instance);
var migrations = new ISchedulerMongoMigration[]
{
new EnsureSchedulerCollectionsMigration(NullLogger<EnsureSchedulerCollectionsMigration>.Instance),
new EnsureSchedulerIndexesMigration()
};
var runner = new SchedulerMongoMigrationRunner(_context, migrations, NullLogger<SchedulerMongoMigrationRunner>.Instance);
runner.RunAsync(CancellationToken.None).GetAwaiter().GetResult();
}
[Fact]
public async Task SamplesRoundTripThroughMongoWithoutLosingCanonicalShape()
{
var samplesRoot = LocateSamplesRoot();
var scheduleJson = await File.ReadAllTextAsync(Path.Combine(samplesRoot, "schedule.json"), CancellationToken.None);
await AssertRoundTripAsync(
scheduleJson,
_context.Options.SchedulesCollection,
CanonicalJsonSerializer.Deserialize<Schedule>,
schedule => schedule.Id);
var runJson = await File.ReadAllTextAsync(Path.Combine(samplesRoot, "run.json"), CancellationToken.None);
await AssertRoundTripAsync(
runJson,
_context.Options.RunsCollection,
CanonicalJsonSerializer.Deserialize<Run>,
run => run.Id);
var impactJson = await File.ReadAllTextAsync(Path.Combine(samplesRoot, "impact-set.json"), CancellationToken.None);
await AssertRoundTripAsync(
impactJson,
_context.Options.ImpactSnapshotsCollection,
CanonicalJsonSerializer.Deserialize<ImpactSet>,
_ => null);
var auditJson = await File.ReadAllTextAsync(Path.Combine(samplesRoot, "audit.json"), CancellationToken.None);
await AssertRoundTripAsync(
auditJson,
_context.Options.AuditCollection,
CanonicalJsonSerializer.Deserialize<AuditRecord>,
audit => audit.Id);
}
private async Task AssertRoundTripAsync<TModel>(
string json,
string collectionName,
Func<string, TModel> deserialize,
Func<TModel, string?> resolveId)
{
ArgumentNullException.ThrowIfNull(deserialize);
ArgumentNullException.ThrowIfNull(resolveId);
var model = deserialize(json);
var canonical = CanonicalJsonSerializer.Serialize(model);
var document = BsonDocument.Parse(canonical);
var identifier = resolveId(model);
if (!string.IsNullOrEmpty(identifier))
{
document["_id"] = identifier;
}
var collection = _context.Database.GetCollection<BsonDocument>(collectionName);
await collection.InsertOneAsync(document, cancellationToken: CancellationToken.None);
var filter = identifier is null ? Builders<BsonDocument>.Filter.Empty : Builders<BsonDocument>.Filter.Eq("_id", identifier);
var stored = await collection.Find(filter).FirstOrDefaultAsync();
Assert.NotNull(stored);
var sanitized = stored!.DeepClone().AsBsonDocument;
sanitized.Remove("_id");
var storedJson = sanitized.ToJson();
var parsedExpected = JsonNode.Parse(canonical) ?? throw new InvalidOperationException("Canonical node null.");
var parsedActual = JsonNode.Parse(storedJson) ?? throw new InvalidOperationException("Stored node null.");
Assert.True(JsonNode.DeepEquals(parsedExpected, parsedActual), "Document changed shape after Mongo round-trip.");
}
private static string LocateSamplesRoot()
{
var current = AppContext.BaseDirectory;
while (!string.IsNullOrEmpty(current))
{
var candidate = Path.Combine(current, "samples", "api", "scheduler");
if (Directory.Exists(candidate))
{
return candidate;
}
var parent = Path.GetDirectoryName(current.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar));
if (string.Equals(parent, current, StringComparison.Ordinal))
{
break;
}
current = parent;
}
throw new DirectoryNotFoundException("Unable to locate samples/api/scheduler in repository tree.");
}
public void Dispose()
{
_runner.Dispose();
}
}
using System.Text.Json.Nodes;
namespace StellaOps.Scheduler.Storage.Postgres.Repositories.Tests.Integration;
public sealed class SchedulerMongoRoundTripTests : IDisposable
{
private readonly MongoDbRunner _runner;
private readonly SchedulerMongoContext _context;
public SchedulerMongoRoundTripTests()
{
_runner = MongoDbRunner.Start(additionalMongodArguments: "--quiet");
var options = new SchedulerMongoOptions
{
ConnectionString = _runner.ConnectionString,
Database = $"scheduler_roundtrip_{Guid.NewGuid():N}"
};
_context = new SchedulerMongoContext(Microsoft.Extensions.Options.Options.Create(options), NullLogger<SchedulerMongoContext>.Instance);
var migrations = new ISchedulerMongoMigration[]
{
new EnsureSchedulerCollectionsMigration(NullLogger<EnsureSchedulerCollectionsMigration>.Instance),
new EnsureSchedulerIndexesMigration()
};
var runner = new SchedulerMongoMigrationRunner(_context, migrations, NullLogger<SchedulerMongoMigrationRunner>.Instance);
runner.RunAsync(CancellationToken.None).GetAwaiter().GetResult();
}
[Fact]
public async Task SamplesRoundTripThroughMongoWithoutLosingCanonicalShape()
{
var samplesRoot = LocateSamplesRoot();
var scheduleJson = await File.ReadAllTextAsync(Path.Combine(samplesRoot, "schedule.json"), CancellationToken.None);
await AssertRoundTripAsync(
scheduleJson,
_context.Options.SchedulesCollection,
CanonicalJsonSerializer.Deserialize<Schedule>,
schedule => schedule.Id);
var runJson = await File.ReadAllTextAsync(Path.Combine(samplesRoot, "run.json"), CancellationToken.None);
await AssertRoundTripAsync(
runJson,
_context.Options.RunsCollection,
CanonicalJsonSerializer.Deserialize<Run>,
run => run.Id);
var impactJson = await File.ReadAllTextAsync(Path.Combine(samplesRoot, "impact-set.json"), CancellationToken.None);
await AssertRoundTripAsync(
impactJson,
_context.Options.ImpactSnapshotsCollection,
CanonicalJsonSerializer.Deserialize<ImpactSet>,
_ => null);
var auditJson = await File.ReadAllTextAsync(Path.Combine(samplesRoot, "audit.json"), CancellationToken.None);
await AssertRoundTripAsync(
auditJson,
_context.Options.AuditCollection,
CanonicalJsonSerializer.Deserialize<AuditRecord>,
audit => audit.Id);
}
private async Task AssertRoundTripAsync<TModel>(
string json,
string collectionName,
Func<string, TModel> deserialize,
Func<TModel, string?> resolveId)
{
ArgumentNullException.ThrowIfNull(deserialize);
ArgumentNullException.ThrowIfNull(resolveId);
var model = deserialize(json);
var canonical = CanonicalJsonSerializer.Serialize(model);
var document = BsonDocument.Parse(canonical);
var identifier = resolveId(model);
if (!string.IsNullOrEmpty(identifier))
{
document["_id"] = identifier;
}
var collection = _context.Database.GetCollection<BsonDocument>(collectionName);
await collection.InsertOneAsync(document, cancellationToken: CancellationToken.None);
var filter = identifier is null ? Builders<BsonDocument>.Filter.Empty : Builders<BsonDocument>.Filter.Eq("_id", identifier);
var stored = await collection.Find(filter).FirstOrDefaultAsync();
Assert.NotNull(stored);
var sanitized = stored!.DeepClone().AsBsonDocument;
sanitized.Remove("_id");
var storedJson = sanitized.ToJson();
var parsedExpected = JsonNode.Parse(canonical) ?? throw new InvalidOperationException("Canonical node null.");
var parsedActual = JsonNode.Parse(storedJson) ?? throw new InvalidOperationException("Stored node null.");
Assert.True(JsonNode.DeepEquals(parsedExpected, parsedActual), "Document changed shape after Mongo round-trip.");
}
private static string LocateSamplesRoot()
{
var current = AppContext.BaseDirectory;
while (!string.IsNullOrEmpty(current))
{
var candidate = Path.Combine(current, "samples", "api", "scheduler");
if (Directory.Exists(candidate))
{
return candidate;
}
var parent = Path.GetDirectoryName(current.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar));
if (string.Equals(parent, current, StringComparison.Ordinal))
{
break;
}
current = parent;
}
throw new DirectoryNotFoundException("Unable to locate samples/api/scheduler in repository tree.");
}
public void Dispose()
{
_runner.Dispose();
}
}

View File

@@ -1,106 +1,106 @@
namespace StellaOps.Scheduler.Storage.Mongo.Tests.Migrations;
public sealed class SchedulerMongoMigrationTests : IDisposable
{
private readonly MongoDbRunner _runner;
public SchedulerMongoMigrationTests()
{
_runner = MongoDbRunner.Start(additionalMongodArguments: "--quiet");
}
[Fact]
public async Task RunAsync_CreatesCollectionsAndIndexes()
{
var options = new SchedulerMongoOptions
{
ConnectionString = _runner.ConnectionString,
Database = $"scheduler_tests_{Guid.NewGuid():N}"
};
var context = new SchedulerMongoContext(Microsoft.Extensions.Options.Options.Create(options), NullLogger<SchedulerMongoContext>.Instance);
var migrations = new ISchedulerMongoMigration[]
{
new EnsureSchedulerCollectionsMigration(NullLogger<EnsureSchedulerCollectionsMigration>.Instance),
new EnsureSchedulerIndexesMigration()
};
var runner = new SchedulerMongoMigrationRunner(context, migrations, NullLogger<SchedulerMongoMigrationRunner>.Instance);
await runner.RunAsync(CancellationToken.None);
var cursor = await context.Database.ListCollectionNamesAsync(cancellationToken: CancellationToken.None);
var collections = await cursor.ToListAsync();
Assert.Contains(options.SchedulesCollection, collections);
Assert.Contains(options.RunsCollection, collections);
Assert.Contains(options.ImpactSnapshotsCollection, collections);
Assert.Contains(options.AuditCollection, collections);
Assert.Contains(options.LocksCollection, collections);
Assert.Contains(options.MigrationsCollection, collections);
await AssertScheduleIndexesAsync(context, options);
await AssertRunIndexesAsync(context, options);
await AssertImpactSnapshotIndexesAsync(context, options);
await AssertAuditIndexesAsync(context, options);
await AssertLockIndexesAsync(context, options);
}
private static async Task AssertScheduleIndexesAsync(SchedulerMongoContext context, SchedulerMongoOptions options)
{
var names = await ListIndexNamesAsync(context.Database.GetCollection<BsonDocument>(options.SchedulesCollection));
Assert.Contains("tenant_enabled", names);
Assert.Contains("cron_timezone", names);
}
private static async Task AssertRunIndexesAsync(SchedulerMongoContext context, SchedulerMongoOptions options)
{
var collection = context.Database.GetCollection<BsonDocument>(options.RunsCollection);
var indexes = await ListIndexesAsync(collection);
Assert.Contains(indexes, doc => string.Equals(doc["name"].AsString, "tenant_createdAt_desc", StringComparison.Ordinal));
Assert.Contains(indexes, doc => string.Equals(doc["name"].AsString, "state_lookup", StringComparison.Ordinal));
Assert.Contains(indexes, doc => string.Equals(doc["name"].AsString, "schedule_createdAt_desc", StringComparison.Ordinal));
var ttl = indexes.FirstOrDefault(doc => doc.TryGetValue("name", out var name) && name == "finishedAt_ttl");
Assert.NotNull(ttl);
Assert.Equal(options.CompletedRunRetention.TotalSeconds, ttl!["expireAfterSeconds"].ToDouble());
}
private static async Task AssertImpactSnapshotIndexesAsync(SchedulerMongoContext context, SchedulerMongoOptions options)
{
var names = await ListIndexNamesAsync(context.Database.GetCollection<BsonDocument>(options.ImpactSnapshotsCollection));
Assert.Contains("selector_tenant_scope", names);
Assert.Contains("snapshotId_unique", names);
}
private static async Task AssertAuditIndexesAsync(SchedulerMongoContext context, SchedulerMongoOptions options)
{
var names = await ListIndexNamesAsync(context.Database.GetCollection<BsonDocument>(options.AuditCollection));
Assert.Contains("tenant_occurredAt_desc", names);
Assert.Contains("correlation_lookup", names);
}
private static async Task AssertLockIndexesAsync(SchedulerMongoContext context, SchedulerMongoOptions options)
{
var names = await ListIndexNamesAsync(context.Database.GetCollection<BsonDocument>(options.LocksCollection));
Assert.Contains("tenant_resource_unique", names);
Assert.Contains("expiresAt_ttl", names);
}
private static async Task<IReadOnlyList<string>> ListIndexNamesAsync(IMongoCollection<BsonDocument> collection)
{
var documents = await ListIndexesAsync(collection);
return documents.Select(doc => doc["name"].AsString).ToArray();
}
private static async Task<List<BsonDocument>> ListIndexesAsync(IMongoCollection<BsonDocument> collection)
{
using var cursor = await collection.Indexes.ListAsync();
return await cursor.ToListAsync();
}
public void Dispose()
{
_runner.Dispose();
}
}
namespace StellaOps.Scheduler.Storage.Postgres.Repositories.Tests.Migrations;
public sealed class SchedulerMongoMigrationTests : IDisposable
{
private readonly MongoDbRunner _runner;
public SchedulerMongoMigrationTests()
{
_runner = MongoDbRunner.Start(additionalMongodArguments: "--quiet");
}
[Fact]
public async Task RunAsync_CreatesCollectionsAndIndexes()
{
var options = new SchedulerMongoOptions
{
ConnectionString = _runner.ConnectionString,
Database = $"scheduler_tests_{Guid.NewGuid():N}"
};
var context = new SchedulerMongoContext(Microsoft.Extensions.Options.Options.Create(options), NullLogger<SchedulerMongoContext>.Instance);
var migrations = new ISchedulerMongoMigration[]
{
new EnsureSchedulerCollectionsMigration(NullLogger<EnsureSchedulerCollectionsMigration>.Instance),
new EnsureSchedulerIndexesMigration()
};
var runner = new SchedulerMongoMigrationRunner(context, migrations, NullLogger<SchedulerMongoMigrationRunner>.Instance);
await runner.RunAsync(CancellationToken.None);
var cursor = await context.Database.ListCollectionNamesAsync(cancellationToken: CancellationToken.None);
var collections = await cursor.ToListAsync();
Assert.Contains(options.SchedulesCollection, collections);
Assert.Contains(options.RunsCollection, collections);
Assert.Contains(options.ImpactSnapshotsCollection, collections);
Assert.Contains(options.AuditCollection, collections);
Assert.Contains(options.LocksCollection, collections);
Assert.Contains(options.MigrationsCollection, collections);
await AssertScheduleIndexesAsync(context, options);
await AssertRunIndexesAsync(context, options);
await AssertImpactSnapshotIndexesAsync(context, options);
await AssertAuditIndexesAsync(context, options);
await AssertLockIndexesAsync(context, options);
}
private static async Task AssertScheduleIndexesAsync(SchedulerMongoContext context, SchedulerMongoOptions options)
{
var names = await ListIndexNamesAsync(context.Database.GetCollection<BsonDocument>(options.SchedulesCollection));
Assert.Contains("tenant_enabled", names);
Assert.Contains("cron_timezone", names);
}
private static async Task AssertRunIndexesAsync(SchedulerMongoContext context, SchedulerMongoOptions options)
{
var collection = context.Database.GetCollection<BsonDocument>(options.RunsCollection);
var indexes = await ListIndexesAsync(collection);
Assert.Contains(indexes, doc => string.Equals(doc["name"].AsString, "tenant_createdAt_desc", StringComparison.Ordinal));
Assert.Contains(indexes, doc => string.Equals(doc["name"].AsString, "state_lookup", StringComparison.Ordinal));
Assert.Contains(indexes, doc => string.Equals(doc["name"].AsString, "schedule_createdAt_desc", StringComparison.Ordinal));
var ttl = indexes.FirstOrDefault(doc => doc.TryGetValue("name", out var name) && name == "finishedAt_ttl");
Assert.NotNull(ttl);
Assert.Equal(options.CompletedRunRetention.TotalSeconds, ttl!["expireAfterSeconds"].ToDouble());
}
private static async Task AssertImpactSnapshotIndexesAsync(SchedulerMongoContext context, SchedulerMongoOptions options)
{
var names = await ListIndexNamesAsync(context.Database.GetCollection<BsonDocument>(options.ImpactSnapshotsCollection));
Assert.Contains("selector_tenant_scope", names);
Assert.Contains("snapshotId_unique", names);
}
private static async Task AssertAuditIndexesAsync(SchedulerMongoContext context, SchedulerMongoOptions options)
{
var names = await ListIndexNamesAsync(context.Database.GetCollection<BsonDocument>(options.AuditCollection));
Assert.Contains("tenant_occurredAt_desc", names);
Assert.Contains("correlation_lookup", names);
}
private static async Task AssertLockIndexesAsync(SchedulerMongoContext context, SchedulerMongoOptions options)
{
var names = await ListIndexNamesAsync(context.Database.GetCollection<BsonDocument>(options.LocksCollection));
Assert.Contains("tenant_resource_unique", names);
Assert.Contains("expiresAt_ttl", names);
}
private static async Task<IReadOnlyList<string>> ListIndexNamesAsync(IMongoCollection<BsonDocument> collection)
{
var documents = await ListIndexesAsync(collection);
return documents.Select(doc => doc["name"].AsString).ToArray();
}
private static async Task<List<BsonDocument>> ListIndexesAsync(IMongoCollection<BsonDocument> collection)
{
using var cursor = await collection.Indexes.ListAsync();
return await cursor.ToListAsync();
}
public void Dispose()
{
_runner.Dispose();
}
}

View File

@@ -1,60 +1,60 @@
using System;
using System.Linq;
using System.Threading;
using StellaOps.Scheduler.Storage.Mongo.Repositories;
namespace StellaOps.Scheduler.Storage.Mongo.Tests.Repositories;
public sealed class AuditRepositoryTests
{
[Fact]
public async Task InsertAndListAsync_ReturnsTenantScopedEntries()
{
using var harness = new SchedulerMongoTestHarness();
var repository = new AuditRepository(harness.Context);
var record1 = TestDataFactory.CreateAuditRecord("tenant-alpha", "1");
var record2 = TestDataFactory.CreateAuditRecord("tenant-alpha", "2");
var otherTenant = TestDataFactory.CreateAuditRecord("tenant-beta", "3");
await repository.InsertAsync(record1);
await repository.InsertAsync(record2);
await repository.InsertAsync(otherTenant);
var results = await repository.ListAsync("tenant-alpha");
Assert.Equal(2, results.Count);
Assert.DoesNotContain(results, record => record.TenantId == "tenant-beta");
}
[Fact]
public async Task ListAsync_AppliesFilters()
{
using var harness = new SchedulerMongoTestHarness();
var repository = new AuditRepository(harness.Context);
var older = TestDataFactory.CreateAuditRecord(
"tenant-alpha",
"old",
occurredAt: DateTimeOffset.UtcNow.AddMinutes(-30),
scheduleId: "sch-a");
var newer = TestDataFactory.CreateAuditRecord(
"tenant-alpha",
"new",
occurredAt: DateTimeOffset.UtcNow,
scheduleId: "sch-a");
await repository.InsertAsync(older);
await repository.InsertAsync(newer);
var options = new AuditQueryOptions
{
Since = DateTimeOffset.UtcNow.AddMinutes(-5),
ScheduleId = "sch-a",
Limit = 5
};
var results = await repository.ListAsync("tenant-alpha", options);
Assert.Single(results);
Assert.Equal("audit_new", results.Single().Id);
}
}
using System;
using System.Linq;
using System.Threading;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Repositories;
namespace StellaOps.Scheduler.Storage.Postgres.Repositories.Tests.Repositories;
public sealed class AuditRepositoryTests
{
[Fact]
public async Task InsertAndListAsync_ReturnsTenantScopedEntries()
{
using var harness = new SchedulerMongoTestHarness();
var repository = new AuditRepository(harness.Context);
var record1 = TestDataFactory.CreateAuditRecord("tenant-alpha", "1");
var record2 = TestDataFactory.CreateAuditRecord("tenant-alpha", "2");
var otherTenant = TestDataFactory.CreateAuditRecord("tenant-beta", "3");
await repository.InsertAsync(record1);
await repository.InsertAsync(record2);
await repository.InsertAsync(otherTenant);
var results = await repository.ListAsync("tenant-alpha");
Assert.Equal(2, results.Count);
Assert.DoesNotContain(results, record => record.TenantId == "tenant-beta");
}
[Fact]
public async Task ListAsync_AppliesFilters()
{
using var harness = new SchedulerMongoTestHarness();
var repository = new AuditRepository(harness.Context);
var older = TestDataFactory.CreateAuditRecord(
"tenant-alpha",
"old",
occurredAt: DateTimeOffset.UtcNow.AddMinutes(-30),
scheduleId: "sch-a");
var newer = TestDataFactory.CreateAuditRecord(
"tenant-alpha",
"new",
occurredAt: DateTimeOffset.UtcNow,
scheduleId: "sch-a");
await repository.InsertAsync(older);
await repository.InsertAsync(newer);
var options = new AuditQueryOptions
{
Since = DateTimeOffset.UtcNow.AddMinutes(-5),
ScheduleId = "sch-a",
Limit = 5
};
var results = await repository.ListAsync("tenant-alpha", options);
Assert.Single(results);
Assert.Equal("audit_new", results.Single().Id);
}
}

View File

@@ -1,41 +1,41 @@
using System;
using System.Threading;
using StellaOps.Scheduler.Storage.Mongo.Repositories;
namespace StellaOps.Scheduler.Storage.Mongo.Tests.Repositories;
public sealed class ImpactSnapshotRepositoryTests
{
[Fact]
public async Task UpsertAndGetAsync_RoundTripsSnapshot()
{
using var harness = new SchedulerMongoTestHarness();
var repository = new ImpactSnapshotRepository(harness.Context);
var snapshot = TestDataFactory.CreateImpactSet("tenant-alpha", "impact-1", DateTimeOffset.UtcNow.AddMinutes(-5));
await repository.UpsertAsync(snapshot, cancellationToken: CancellationToken.None);
var stored = await repository.GetBySnapshotIdAsync("impact-1", cancellationToken: CancellationToken.None);
Assert.NotNull(stored);
Assert.Equal(snapshot.SnapshotId, stored!.SnapshotId);
Assert.Equal(snapshot.Images[0].ImageDigest, stored.Images[0].ImageDigest);
}
[Fact]
public async Task GetLatestBySelectorAsync_ReturnsMostRecent()
{
using var harness = new SchedulerMongoTestHarness();
var repository = new ImpactSnapshotRepository(harness.Context);
var selectorTenant = "tenant-alpha";
var first = TestDataFactory.CreateImpactSet(selectorTenant, "impact-old", DateTimeOffset.UtcNow.AddMinutes(-10));
var latest = TestDataFactory.CreateImpactSet(selectorTenant, "impact-new", DateTimeOffset.UtcNow);
await repository.UpsertAsync(first);
await repository.UpsertAsync(latest);
var resolved = await repository.GetLatestBySelectorAsync(latest.Selector);
Assert.NotNull(resolved);
Assert.Equal("impact-new", resolved!.SnapshotId);
}
}
using System;
using System.Threading;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Repositories;
namespace StellaOps.Scheduler.Storage.Postgres.Repositories.Tests.Repositories;
public sealed class ImpactSnapshotRepositoryTests
{
[Fact]
public async Task UpsertAndGetAsync_RoundTripsSnapshot()
{
using var harness = new SchedulerMongoTestHarness();
var repository = new ImpactSnapshotRepository(harness.Context);
var snapshot = TestDataFactory.CreateImpactSet("tenant-alpha", "impact-1", DateTimeOffset.UtcNow.AddMinutes(-5));
await repository.UpsertAsync(snapshot, cancellationToken: CancellationToken.None);
var stored = await repository.GetBySnapshotIdAsync("impact-1", cancellationToken: CancellationToken.None);
Assert.NotNull(stored);
Assert.Equal(snapshot.SnapshotId, stored!.SnapshotId);
Assert.Equal(snapshot.Images[0].ImageDigest, stored.Images[0].ImageDigest);
}
[Fact]
public async Task GetLatestBySelectorAsync_ReturnsMostRecent()
{
using var harness = new SchedulerMongoTestHarness();
var repository = new ImpactSnapshotRepository(harness.Context);
var selectorTenant = "tenant-alpha";
var first = TestDataFactory.CreateImpactSet(selectorTenant, "impact-old", DateTimeOffset.UtcNow.AddMinutes(-10));
var latest = TestDataFactory.CreateImpactSet(selectorTenant, "impact-new", DateTimeOffset.UtcNow);
await repository.UpsertAsync(first);
await repository.UpsertAsync(latest);
var resolved = await repository.GetLatestBySelectorAsync(latest.Selector);
Assert.NotNull(resolved);
Assert.Equal("impact-new", resolved!.SnapshotId);
}
}

View File

@@ -1,76 +1,76 @@
using System;
using System.Collections.Immutable;
using System.Linq;
using System.Threading;
using StellaOps.Scheduler.Storage.Mongo.Repositories;
namespace StellaOps.Scheduler.Storage.Mongo.Tests.Repositories;
public sealed class RunRepositoryTests
{
[Fact]
public async Task InsertAndGetAsync_RoundTripsRun()
{
using var harness = new SchedulerMongoTestHarness();
var repository = new RunRepository(harness.Context);
var run = TestDataFactory.CreateRun("run_1", "tenant-alpha", RunState.Planning);
await repository.InsertAsync(run, cancellationToken: CancellationToken.None);
var stored = await repository.GetAsync(run.TenantId, run.Id, cancellationToken: CancellationToken.None);
Assert.NotNull(stored);
Assert.Equal(run.State, stored!.State);
Assert.Equal(run.Trigger, stored.Trigger);
}
[Fact]
public async Task UpdateAsync_ChangesStateAndStats()
{
using var harness = new SchedulerMongoTestHarness();
var repository = new RunRepository(harness.Context);
var run = TestDataFactory.CreateRun("run_update", "tenant-alpha", RunState.Planning);
await repository.InsertAsync(run);
var updated = run with
{
State = RunState.Completed,
FinishedAt = DateTimeOffset.UtcNow,
Stats = new RunStats(candidates: 10, deduped: 10, queued: 10, completed: 10, deltas: 2)
};
var result = await repository.UpdateAsync(updated);
Assert.True(result);
var stored = await repository.GetAsync(updated.TenantId, updated.Id);
Assert.NotNull(stored);
Assert.Equal(RunState.Completed, stored!.State);
Assert.Equal(10, stored.Stats.Completed);
}
[Fact]
public async Task ListAsync_FiltersByStateAndSchedule()
{
using var harness = new SchedulerMongoTestHarness();
var repository = new RunRepository(harness.Context);
var run1 = TestDataFactory.CreateRun("run_state_1", "tenant-alpha", RunState.Planning, scheduleId: "sch_a");
var run2 = TestDataFactory.CreateRun("run_state_2", "tenant-alpha", RunState.Running, scheduleId: "sch_a");
var run3 = TestDataFactory.CreateRun("run_state_3", "tenant-alpha", RunState.Completed, scheduleId: "sch_b");
await repository.InsertAsync(run1);
await repository.InsertAsync(run2);
await repository.InsertAsync(run3);
var options = new RunQueryOptions
{
ScheduleId = "sch_a",
States = new[] { RunState.Running }.ToImmutableArray(),
Limit = 10
};
var results = await repository.ListAsync("tenant-alpha", options);
Assert.Single(results);
Assert.Equal("run_state_2", results.Single().Id);
}
}
using System;
using System.Collections.Immutable;
using System.Linq;
using System.Threading;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Repositories;
namespace StellaOps.Scheduler.Storage.Postgres.Repositories.Tests.Repositories;
public sealed class RunRepositoryTests
{
[Fact]
public async Task InsertAndGetAsync_RoundTripsRun()
{
using var harness = new SchedulerMongoTestHarness();
var repository = new RunRepository(harness.Context);
var run = TestDataFactory.CreateRun("run_1", "tenant-alpha", RunState.Planning);
await repository.InsertAsync(run, cancellationToken: CancellationToken.None);
var stored = await repository.GetAsync(run.TenantId, run.Id, cancellationToken: CancellationToken.None);
Assert.NotNull(stored);
Assert.Equal(run.State, stored!.State);
Assert.Equal(run.Trigger, stored.Trigger);
}
[Fact]
public async Task UpdateAsync_ChangesStateAndStats()
{
using var harness = new SchedulerMongoTestHarness();
var repository = new RunRepository(harness.Context);
var run = TestDataFactory.CreateRun("run_update", "tenant-alpha", RunState.Planning);
await repository.InsertAsync(run);
var updated = run with
{
State = RunState.Completed,
FinishedAt = DateTimeOffset.UtcNow,
Stats = new RunStats(candidates: 10, deduped: 10, queued: 10, completed: 10, deltas: 2)
};
var result = await repository.UpdateAsync(updated);
Assert.True(result);
var stored = await repository.GetAsync(updated.TenantId, updated.Id);
Assert.NotNull(stored);
Assert.Equal(RunState.Completed, stored!.State);
Assert.Equal(10, stored.Stats.Completed);
}
[Fact]
public async Task ListAsync_FiltersByStateAndSchedule()
{
using var harness = new SchedulerMongoTestHarness();
var repository = new RunRepository(harness.Context);
var run1 = TestDataFactory.CreateRun("run_state_1", "tenant-alpha", RunState.Planning, scheduleId: "sch_a");
var run2 = TestDataFactory.CreateRun("run_state_2", "tenant-alpha", RunState.Running, scheduleId: "sch_a");
var run3 = TestDataFactory.CreateRun("run_state_3", "tenant-alpha", RunState.Completed, scheduleId: "sch_b");
await repository.InsertAsync(run1);
await repository.InsertAsync(run2);
await repository.InsertAsync(run3);
var options = new RunQueryOptions
{
ScheduleId = "sch_a",
States = new[] { RunState.Running }.ToImmutableArray(),
Limit = 10
};
var results = await repository.ListAsync("tenant-alpha", options);
Assert.Single(results);
Assert.Equal("run_state_2", results.Single().Id);
}
}

View File

@@ -1,74 +1,74 @@
using System;
using System.Threading;
using StellaOps.Scheduler.Storage.Mongo.Repositories;
namespace StellaOps.Scheduler.Storage.Mongo.Tests.Repositories;
public sealed class ScheduleRepositoryTests
{
[Fact]
public async Task UpsertAsync_PersistsScheduleWithCanonicalShape()
{
using var harness = new SchedulerMongoTestHarness();
var repository = new ScheduleRepository(harness.Context);
var schedule = TestDataFactory.CreateSchedule("sch_unit_1", "tenant-alpha");
await repository.UpsertAsync(schedule, cancellationToken: CancellationToken.None);
var stored = await repository.GetAsync(schedule.TenantId, schedule.Id, cancellationToken: CancellationToken.None);
Assert.NotNull(stored);
Assert.Equal(schedule.Id, stored!.Id);
Assert.Equal(schedule.Name, stored.Name);
Assert.Equal(schedule.Selection.Scope, stored.Selection.Scope);
}
[Fact]
public async Task ListAsync_ExcludesDisabledAndDeletedByDefault()
{
using var harness = new SchedulerMongoTestHarness();
var repository = new ScheduleRepository(harness.Context);
var tenantId = "tenant-alpha";
var enabled = TestDataFactory.CreateSchedule("sch_enabled", tenantId, enabled: true, name: "Enabled");
var disabled = TestDataFactory.CreateSchedule("sch_disabled", tenantId, enabled: false, name: "Disabled");
await repository.UpsertAsync(enabled);
await repository.UpsertAsync(disabled);
await repository.SoftDeleteAsync(tenantId, enabled.Id, "svc_scheduler", DateTimeOffset.UtcNow);
var results = await repository.ListAsync(tenantId);
Assert.Empty(results);
var includeDisabled = await repository.ListAsync(
tenantId,
new ScheduleQueryOptions { IncludeDisabled = true, IncludeDeleted = true });
Assert.Equal(2, includeDisabled.Count);
Assert.Contains(includeDisabled, schedule => schedule.Id == enabled.Id);
Assert.Contains(includeDisabled, schedule => schedule.Id == disabled.Id);
}
[Fact]
public async Task SoftDeleteAsync_SetsMetadataAndExcludesFromQueries()
{
using var harness = new SchedulerMongoTestHarness();
var repository = new ScheduleRepository(harness.Context);
var schedule = TestDataFactory.CreateSchedule("sch_delete", "tenant-beta");
await repository.UpsertAsync(schedule);
var deletedAt = DateTimeOffset.UtcNow;
var deleted = await repository.SoftDeleteAsync(schedule.TenantId, schedule.Id, "svc_delete", deletedAt);
Assert.True(deleted);
var retrieved = await repository.GetAsync(schedule.TenantId, schedule.Id);
Assert.Null(retrieved);
var includeDeleted = await repository.ListAsync(
schedule.TenantId,
new ScheduleQueryOptions { IncludeDeleted = true, IncludeDisabled = true });
Assert.Single(includeDeleted);
Assert.Equal("sch_delete", includeDeleted[0].Id);
}
}
using System;
using System.Threading;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Repositories;
namespace StellaOps.Scheduler.Storage.Postgres.Repositories.Tests.Repositories;
public sealed class ScheduleRepositoryTests
{
[Fact]
public async Task UpsertAsync_PersistsScheduleWithCanonicalShape()
{
using var harness = new SchedulerMongoTestHarness();
var repository = new ScheduleRepository(harness.Context);
var schedule = TestDataFactory.CreateSchedule("sch_unit_1", "tenant-alpha");
await repository.UpsertAsync(schedule, cancellationToken: CancellationToken.None);
var stored = await repository.GetAsync(schedule.TenantId, schedule.Id, cancellationToken: CancellationToken.None);
Assert.NotNull(stored);
Assert.Equal(schedule.Id, stored!.Id);
Assert.Equal(schedule.Name, stored.Name);
Assert.Equal(schedule.Selection.Scope, stored.Selection.Scope);
}
[Fact]
public async Task ListAsync_ExcludesDisabledAndDeletedByDefault()
{
using var harness = new SchedulerMongoTestHarness();
var repository = new ScheduleRepository(harness.Context);
var tenantId = "tenant-alpha";
var enabled = TestDataFactory.CreateSchedule("sch_enabled", tenantId, enabled: true, name: "Enabled");
var disabled = TestDataFactory.CreateSchedule("sch_disabled", tenantId, enabled: false, name: "Disabled");
await repository.UpsertAsync(enabled);
await repository.UpsertAsync(disabled);
await repository.SoftDeleteAsync(tenantId, enabled.Id, "svc_scheduler", DateTimeOffset.UtcNow);
var results = await repository.ListAsync(tenantId);
Assert.Empty(results);
var includeDisabled = await repository.ListAsync(
tenantId,
new ScheduleQueryOptions { IncludeDisabled = true, IncludeDeleted = true });
Assert.Equal(2, includeDisabled.Count);
Assert.Contains(includeDisabled, schedule => schedule.Id == enabled.Id);
Assert.Contains(includeDisabled, schedule => schedule.Id == disabled.Id);
}
[Fact]
public async Task SoftDeleteAsync_SetsMetadataAndExcludesFromQueries()
{
using var harness = new SchedulerMongoTestHarness();
var repository = new ScheduleRepository(harness.Context);
var schedule = TestDataFactory.CreateSchedule("sch_delete", "tenant-beta");
await repository.UpsertAsync(schedule);
var deletedAt = DateTimeOffset.UtcNow;
var deleted = await repository.SoftDeleteAsync(schedule.TenantId, schedule.Id, "svc_delete", deletedAt);
Assert.True(deleted);
var retrieved = await repository.GetAsync(schedule.TenantId, schedule.Id);
Assert.Null(retrieved);
var includeDeleted = await repository.ListAsync(
schedule.TenantId,
new ScheduleQueryOptions { IncludeDeleted = true, IncludeDisabled = true });
Assert.Single(includeDeleted);
Assert.Equal("sch_delete", includeDeleted[0].Id);
}
}

View File

@@ -1,36 +1,36 @@
using System;
using System.Threading;
using Microsoft.Extensions.Logging.Abstractions;
namespace StellaOps.Scheduler.Storage.Mongo.Tests;
internal sealed class SchedulerMongoTestHarness : IDisposable
{
private readonly MongoDbRunner _runner;
public SchedulerMongoTestHarness()
{
_runner = MongoDbRunner.Start(additionalMongodArguments: "--quiet");
var options = new SchedulerMongoOptions
{
ConnectionString = _runner.ConnectionString,
Database = $"scheduler_tests_{Guid.NewGuid():N}"
};
Context = new SchedulerMongoContext(Microsoft.Extensions.Options.Options.Create(options), NullLogger<SchedulerMongoContext>.Instance);
var migrations = new ISchedulerMongoMigration[]
{
new EnsureSchedulerCollectionsMigration(NullLogger<EnsureSchedulerCollectionsMigration>.Instance),
new EnsureSchedulerIndexesMigration()
};
var runner = new SchedulerMongoMigrationRunner(Context, migrations, NullLogger<SchedulerMongoMigrationRunner>.Instance);
runner.RunAsync(CancellationToken.None).GetAwaiter().GetResult();
}
public SchedulerMongoContext Context { get; }
public void Dispose()
{
_runner.Dispose();
}
}
using System;
using System.Threading;
using Microsoft.Extensions.Logging.Abstractions;
namespace StellaOps.Scheduler.Storage.Postgres.Repositories.Tests;
internal sealed class SchedulerMongoTestHarness : IDisposable
{
private readonly MongoDbRunner _runner;
public SchedulerMongoTestHarness()
{
_runner = MongoDbRunner.Start(additionalMongodArguments: "--quiet");
var options = new SchedulerMongoOptions
{
ConnectionString = _runner.ConnectionString,
Database = $"scheduler_tests_{Guid.NewGuid():N}"
};
Context = new SchedulerMongoContext(Microsoft.Extensions.Options.Options.Create(options), NullLogger<SchedulerMongoContext>.Instance);
var migrations = new ISchedulerMongoMigration[]
{
new EnsureSchedulerCollectionsMigration(NullLogger<EnsureSchedulerCollectionsMigration>.Instance),
new EnsureSchedulerIndexesMigration()
};
var runner = new SchedulerMongoMigrationRunner(Context, migrations, NullLogger<SchedulerMongoMigrationRunner>.Instance);
runner.RunAsync(CancellationToken.None).GetAwaiter().GetResult();
}
public SchedulerMongoContext Context { get; }
public void Dispose()
{
_runner.Dispose();
}
}

View File

@@ -1,116 +1,116 @@
using Microsoft.Extensions.Logging.Abstractions;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Mongo.Repositories;
using StellaOps.Scheduler.Storage.Mongo.Services;
namespace StellaOps.Scheduler.Storage.Mongo.Tests.Services;
public sealed class RunSummaryServiceTests : IDisposable
{
private readonly SchedulerMongoTestHarness _harness;
private readonly RunSummaryRepository _repository;
private readonly StubTimeProvider _timeProvider;
private readonly RunSummaryService _service;
public RunSummaryServiceTests()
{
_harness = new SchedulerMongoTestHarness();
_repository = new RunSummaryRepository(_harness.Context);
_timeProvider = new StubTimeProvider(DateTimeOffset.Parse("2025-10-26T10:00:00Z"));
_service = new RunSummaryService(_repository, _timeProvider, NullLogger<RunSummaryService>.Instance);
}
[Fact]
public async Task ProjectAsync_FirstRunCreatesProjection()
{
var run = TestDataFactory.CreateRun("run-1", "tenant-alpha", RunState.Planning, "sch-alpha");
var projection = await _service.ProjectAsync(run, CancellationToken.None);
Assert.Equal("tenant-alpha", projection.TenantId);
Assert.Equal("sch-alpha", projection.ScheduleId);
Assert.NotNull(projection.LastRun);
Assert.Equal(RunState.Planning, projection.LastRun!.State);
Assert.Equal(1, projection.Counters.Total);
Assert.Equal(1, projection.Counters.Planning);
Assert.Equal(0, projection.Counters.Completed);
Assert.Single(projection.Recent);
Assert.Equal(run.Id, projection.Recent[0].RunId);
}
[Fact]
public async Task ProjectAsync_UpdateRunReplacesExistingEntry()
{
var createdAt = DateTimeOffset.Parse("2025-10-26T09:55:00Z");
var run = TestDataFactory.CreateRun(
"run-update",
"tenant-alpha",
RunState.Planning,
"sch-alpha",
createdAt: createdAt,
startedAt: createdAt.AddMinutes(1));
await _service.ProjectAsync(run, CancellationToken.None);
var updated = run with
{
State = RunState.Completed,
StartedAt = run.StartedAt,
FinishedAt = run.CreatedAt.AddMinutes(5),
Stats = new RunStats(candidates: 10, deduped: 8, queued: 5, completed: 10, deltas: 2, newCriticals: 1)
};
_timeProvider.Advance(TimeSpan.FromMinutes(10));
var projection = await _service.ProjectAsync(updated, CancellationToken.None);
Assert.NotNull(projection.LastRun);
Assert.Equal(RunState.Completed, projection.LastRun!.State);
Assert.Equal(1, projection.Counters.Completed);
Assert.Equal(0, projection.Counters.Planning);
Assert.Single(projection.Recent);
Assert.Equal(updated.Stats.Completed, projection.LastRun!.Stats.Completed);
Assert.True(projection.UpdatedAt > run.CreatedAt);
}
[Fact]
public async Task ProjectAsync_TrimsRecentEntriesBeyondLimit()
{
var baseTime = DateTimeOffset.Parse("2025-10-26T00:00:00Z");
for (var i = 0; i < 25; i++)
{
var run = TestDataFactory.CreateRun(
$"run-{i}",
"tenant-alpha",
RunState.Completed,
"sch-alpha",
stats: new RunStats(candidates: 5, deduped: 4, queued: 3, completed: 5, deltas: 1),
createdAt: baseTime.AddMinutes(i));
await _service.ProjectAsync(run, CancellationToken.None);
}
var projections = await _service.ListAsync("tenant-alpha", CancellationToken.None);
Assert.Single(projections);
var projection = projections[0];
Assert.Equal(20, projection.Recent.Length);
Assert.Equal(20, projection.Counters.Total);
Assert.Equal("run-24", projection.Recent[0].RunId);
}
public void Dispose()
{
_harness.Dispose();
}
private sealed class StubTimeProvider : TimeProvider
{
private DateTimeOffset _utcNow;
public StubTimeProvider(DateTimeOffset initial)
=> _utcNow = initial;
public override DateTimeOffset GetUtcNow() => _utcNow;
public void Advance(TimeSpan delta) => _utcNow = _utcNow.Add(delta);
}
}
using Microsoft.Extensions.Logging.Abstractions;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Repositories;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Services;
namespace StellaOps.Scheduler.Storage.Postgres.Repositories.Tests.Services;
public sealed class RunSummaryServiceTests : IDisposable
{
private readonly SchedulerMongoTestHarness _harness;
private readonly RunSummaryRepository _repository;
private readonly StubTimeProvider _timeProvider;
private readonly RunSummaryService _service;
public RunSummaryServiceTests()
{
_harness = new SchedulerMongoTestHarness();
_repository = new RunSummaryRepository(_harness.Context);
_timeProvider = new StubTimeProvider(DateTimeOffset.Parse("2025-10-26T10:00:00Z"));
_service = new RunSummaryService(_repository, _timeProvider, NullLogger<RunSummaryService>.Instance);
}
[Fact]
public async Task ProjectAsync_FirstRunCreatesProjection()
{
var run = TestDataFactory.CreateRun("run-1", "tenant-alpha", RunState.Planning, "sch-alpha");
var projection = await _service.ProjectAsync(run, CancellationToken.None);
Assert.Equal("tenant-alpha", projection.TenantId);
Assert.Equal("sch-alpha", projection.ScheduleId);
Assert.NotNull(projection.LastRun);
Assert.Equal(RunState.Planning, projection.LastRun!.State);
Assert.Equal(1, projection.Counters.Total);
Assert.Equal(1, projection.Counters.Planning);
Assert.Equal(0, projection.Counters.Completed);
Assert.Single(projection.Recent);
Assert.Equal(run.Id, projection.Recent[0].RunId);
}
[Fact]
public async Task ProjectAsync_UpdateRunReplacesExistingEntry()
{
var createdAt = DateTimeOffset.Parse("2025-10-26T09:55:00Z");
var run = TestDataFactory.CreateRun(
"run-update",
"tenant-alpha",
RunState.Planning,
"sch-alpha",
createdAt: createdAt,
startedAt: createdAt.AddMinutes(1));
await _service.ProjectAsync(run, CancellationToken.None);
var updated = run with
{
State = RunState.Completed,
StartedAt = run.StartedAt,
FinishedAt = run.CreatedAt.AddMinutes(5),
Stats = new RunStats(candidates: 10, deduped: 8, queued: 5, completed: 10, deltas: 2, newCriticals: 1)
};
_timeProvider.Advance(TimeSpan.FromMinutes(10));
var projection = await _service.ProjectAsync(updated, CancellationToken.None);
Assert.NotNull(projection.LastRun);
Assert.Equal(RunState.Completed, projection.LastRun!.State);
Assert.Equal(1, projection.Counters.Completed);
Assert.Equal(0, projection.Counters.Planning);
Assert.Single(projection.Recent);
Assert.Equal(updated.Stats.Completed, projection.LastRun!.Stats.Completed);
Assert.True(projection.UpdatedAt > run.CreatedAt);
}
[Fact]
public async Task ProjectAsync_TrimsRecentEntriesBeyondLimit()
{
var baseTime = DateTimeOffset.Parse("2025-10-26T00:00:00Z");
for (var i = 0; i < 25; i++)
{
var run = TestDataFactory.CreateRun(
$"run-{i}",
"tenant-alpha",
RunState.Completed,
"sch-alpha",
stats: new RunStats(candidates: 5, deduped: 4, queued: 3, completed: 5, deltas: 1),
createdAt: baseTime.AddMinutes(i));
await _service.ProjectAsync(run, CancellationToken.None);
}
var projections = await _service.ListAsync("tenant-alpha", CancellationToken.None);
Assert.Single(projections);
var projection = projections[0];
Assert.Equal(20, projection.Recent.Length);
Assert.Equal(20, projection.Counters.Total);
Assert.Equal("run-24", projection.Recent[0].RunId);
}
public void Dispose()
{
_harness.Dispose();
}
private sealed class StubTimeProvider : TimeProvider
{
private DateTimeOffset _utcNow;
public StubTimeProvider(DateTimeOffset initial)
=> _utcNow = initial;
public override DateTimeOffset GetUtcNow() => _utcNow;
public void Advance(TimeSpan delta) => _utcNow = _utcNow.Add(delta);
}
}

View File

@@ -1,82 +1,82 @@
using Microsoft.Extensions.Logging.Abstractions;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Mongo.Repositories;
using StellaOps.Scheduler.Storage.Mongo.Services;
namespace StellaOps.Scheduler.Storage.Mongo.Tests.Services;
public sealed class SchedulerAuditServiceTests : IDisposable
{
private readonly SchedulerMongoTestHarness _harness;
private readonly AuditRepository _repository;
private readonly StubTimeProvider _timeProvider;
private readonly SchedulerAuditService _service;
public SchedulerAuditServiceTests()
{
_harness = new SchedulerMongoTestHarness();
_repository = new AuditRepository(_harness.Context);
_timeProvider = new StubTimeProvider(DateTimeOffset.Parse("2025-10-26T11:30:00Z"));
_service = new SchedulerAuditService(_repository, _timeProvider, NullLogger<SchedulerAuditService>.Instance);
}
[Fact]
public async Task WriteAsync_PersistsRecordWithGeneratedId()
{
var auditEvent = new SchedulerAuditEvent(
TenantId: "tenant-alpha",
Category: "scheduler",
Action: "create",
Actor: new AuditActor("user_admin", "Admin", "user"),
ScheduleId: "sch-alpha",
CorrelationId: "corr-1",
Metadata: new Dictionary<string, string>
{
["Reason"] = "initial",
},
Message: "created schedule");
var record = await _service.WriteAsync(auditEvent, CancellationToken.None);
Assert.StartsWith("audit_", record.Id, StringComparison.Ordinal);
Assert.Equal(_timeProvider.GetUtcNow(), record.OccurredAt);
var stored = await _repository.ListAsync("tenant-alpha", new AuditQueryOptions { ScheduleId = "sch-alpha" }, session: null, CancellationToken.None);
Assert.Single(stored);
Assert.Equal(record.Id, stored[0].Id);
Assert.Equal("created schedule", stored[0].Message);
Assert.Contains(stored[0].Metadata, pair => pair.Key == "reason" && pair.Value == "initial");
}
[Fact]
public async Task WriteAsync_HonoursProvidedAuditId()
{
var auditEvent = new SchedulerAuditEvent(
TenantId: "tenant-alpha",
Category: "scheduler",
Action: "update",
Actor: new AuditActor("user_admin", "Admin", "user"),
ScheduleId: "sch-alpha",
AuditId: "audit_custom_1",
OccurredAt: DateTimeOffset.Parse("2025-10-26T12:00:00Z"));
var record = await _service.WriteAsync(auditEvent, CancellationToken.None);
Assert.Equal("audit_custom_1", record.Id);
Assert.Equal(DateTimeOffset.Parse("2025-10-26T12:00:00Z"), record.OccurredAt);
}
public void Dispose()
{
_harness.Dispose();
}
private sealed class StubTimeProvider : TimeProvider
{
private DateTimeOffset _utcNow;
public StubTimeProvider(DateTimeOffset initial)
=> _utcNow = initial;
public override DateTimeOffset GetUtcNow() => _utcNow;
}
}
using Microsoft.Extensions.Logging.Abstractions;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Repositories;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Services;
namespace StellaOps.Scheduler.Storage.Postgres.Repositories.Tests.Services;
public sealed class SchedulerAuditServiceTests : IDisposable
{
private readonly SchedulerMongoTestHarness _harness;
private readonly AuditRepository _repository;
private readonly StubTimeProvider _timeProvider;
private readonly SchedulerAuditService _service;
public SchedulerAuditServiceTests()
{
_harness = new SchedulerMongoTestHarness();
_repository = new AuditRepository(_harness.Context);
_timeProvider = new StubTimeProvider(DateTimeOffset.Parse("2025-10-26T11:30:00Z"));
_service = new SchedulerAuditService(_repository, _timeProvider, NullLogger<SchedulerAuditService>.Instance);
}
[Fact]
public async Task WriteAsync_PersistsRecordWithGeneratedId()
{
var auditEvent = new SchedulerAuditEvent(
TenantId: "tenant-alpha",
Category: "scheduler",
Action: "create",
Actor: new AuditActor("user_admin", "Admin", "user"),
ScheduleId: "sch-alpha",
CorrelationId: "corr-1",
Metadata: new Dictionary<string, string>
{
["Reason"] = "initial",
},
Message: "created schedule");
var record = await _service.WriteAsync(auditEvent, CancellationToken.None);
Assert.StartsWith("audit_", record.Id, StringComparison.Ordinal);
Assert.Equal(_timeProvider.GetUtcNow(), record.OccurredAt);
var stored = await _repository.ListAsync("tenant-alpha", new AuditQueryOptions { ScheduleId = "sch-alpha" }, session: null, CancellationToken.None);
Assert.Single(stored);
Assert.Equal(record.Id, stored[0].Id);
Assert.Equal("created schedule", stored[0].Message);
Assert.Contains(stored[0].Metadata, pair => pair.Key == "reason" && pair.Value == "initial");
}
[Fact]
public async Task WriteAsync_HonoursProvidedAuditId()
{
var auditEvent = new SchedulerAuditEvent(
TenantId: "tenant-alpha",
Category: "scheduler",
Action: "update",
Actor: new AuditActor("user_admin", "Admin", "user"),
ScheduleId: "sch-alpha",
AuditId: "audit_custom_1",
OccurredAt: DateTimeOffset.Parse("2025-10-26T12:00:00Z"));
var record = await _service.WriteAsync(auditEvent, CancellationToken.None);
Assert.Equal("audit_custom_1", record.Id);
Assert.Equal(DateTimeOffset.Parse("2025-10-26T12:00:00Z"), record.OccurredAt);
}
public void Dispose()
{
_harness.Dispose();
}
private sealed class StubTimeProvider : TimeProvider
{
private DateTimeOffset _utcNow;
public StubTimeProvider(DateTimeOffset initial)
=> _utcNow = initial;
public override DateTimeOffset GetUtcNow() => _utcNow;
}
}

View File

@@ -1,35 +1,35 @@
using System.Threading;
using MongoDB.Driver;
using StellaOps.Scheduler.Storage.Mongo.Sessions;
namespace StellaOps.Scheduler.Storage.Mongo.Tests.Sessions;
public sealed class SchedulerMongoSessionFactoryTests
{
[Fact]
public async Task StartSessionAsync_UsesCausalConsistencyByDefault()
{
using var harness = new SchedulerMongoTestHarness();
var factory = new SchedulerMongoSessionFactory(harness.Context);
using var session = await factory.StartSessionAsync(cancellationToken: CancellationToken.None);
Assert.True(session.Options.CausalConsistency.GetValueOrDefault());
}
[Fact]
public async Task StartSessionAsync_AllowsOverridingOptions()
{
using var harness = new SchedulerMongoTestHarness();
var factory = new SchedulerMongoSessionFactory(harness.Context);
var options = new SchedulerMongoSessionOptions
{
CausalConsistency = false,
ReadPreference = ReadPreference.PrimaryPreferred
};
using var session = await factory.StartSessionAsync(options);
Assert.False(session.Options.CausalConsistency.GetValueOrDefault(true));
Assert.Equal(ReadPreference.PrimaryPreferred, session.Options.DefaultTransactionOptions?.ReadPreference);
}
}
using System.Threading;
using MongoDB.Driver;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Sessions;
namespace StellaOps.Scheduler.Storage.Postgres.Repositories.Tests.Sessions;
public sealed class SchedulerMongoSessionFactoryTests
{
[Fact]
public async Task StartSessionAsync_UsesCausalConsistencyByDefault()
{
using var harness = new SchedulerMongoTestHarness();
var factory = new SchedulerMongoSessionFactory(harness.Context);
using var session = await factory.StartSessionAsync(cancellationToken: CancellationToken.None);
Assert.True(session.Options.CausalConsistency.GetValueOrDefault());
}
[Fact]
public async Task StartSessionAsync_AllowsOverridingOptions()
{
using var harness = new SchedulerMongoTestHarness();
var factory = new SchedulerMongoSessionFactory(harness.Context);
var options = new SchedulerMongoSessionOptions
{
CausalConsistency = false,
ReadPreference = ReadPreference.PrimaryPreferred
};
using var session = await factory.StartSessionAsync(options);
Assert.False(session.Options.CausalConsistency.GetValueOrDefault(true));
Assert.Equal(ReadPreference.PrimaryPreferred, session.Options.DefaultTransactionOptions?.ReadPreference);
}
}

View File

@@ -7,7 +7,7 @@
<UseConcelierTestInfra>false</UseConcelierTestInfra>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="../../__Libraries/StellaOps.Scheduler.Storage.Mongo/StellaOps.Scheduler.Storage.Mongo.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Scheduler.Storage.Postgres/StellaOps.Scheduler.Storage.Postgres.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Scheduler.Models/StellaOps.Scheduler.Models.csproj" />
</ItemGroup>
<ItemGroup>

View File

@@ -1,98 +1,98 @@
using System;
using System.Collections.Immutable;
namespace StellaOps.Scheduler.Storage.Mongo.Tests;
internal static class TestDataFactory
{
public static Schedule CreateSchedule(
string id,
string tenantId,
bool enabled = true,
string name = "Nightly Prod")
{
var now = DateTimeOffset.UtcNow;
return new Schedule(
id,
tenantId,
name,
enabled,
"0 2 * * *",
"UTC",
ScheduleMode.AnalysisOnly,
new Selector(SelectorScope.AllImages, tenantId),
ScheduleOnlyIf.Default,
ScheduleNotify.Default,
ScheduleLimits.Default,
now,
"svc_scheduler",
now,
"svc_scheduler",
ImmutableArray<string>.Empty,
SchedulerSchemaVersions.Schedule);
}
public static Run CreateRun(
string id,
string tenantId,
RunState state,
string? scheduleId = null,
RunTrigger trigger = RunTrigger.Manual,
RunStats? stats = null,
DateTimeOffset? createdAt = null,
DateTimeOffset? startedAt = null)
{
var resolvedStats = stats ?? new RunStats(candidates: 10, deduped: 8, queued: 5, completed: 0, deltas: 2);
var created = createdAt ?? DateTimeOffset.UtcNow;
return new Run(
id,
tenantId,
trigger,
state,
resolvedStats,
created,
scheduleId: scheduleId,
reason: new RunReason(manualReason: "test"),
startedAt: startedAt ?? created);
}
public static ImpactSet CreateImpactSet(string tenantId, string snapshotId, DateTimeOffset? generatedAt = null, bool usageOnly = true)
{
var selector = new Selector(SelectorScope.AllImages, tenantId);
var image = new ImpactImage(
"sha256:" + Guid.NewGuid().ToString("N"),
"registry",
"repo/app",
namespaces: new[] { "team-a" },
tags: new[] { "prod" },
usedByEntrypoint: true);
return new ImpactSet(
selector,
new[] { image },
usageOnly: usageOnly,
generatedAt ?? DateTimeOffset.UtcNow,
total: 1,
snapshotId: snapshotId,
schemaVersion: SchedulerSchemaVersions.ImpactSet);
}
public static AuditRecord CreateAuditRecord(
string tenantId,
string idSuffix,
DateTimeOffset? occurredAt = null,
string? scheduleId = null,
string? category = null,
string? action = null)
{
return new AuditRecord(
$"audit_{idSuffix}",
tenantId,
category ?? "scheduler",
action ?? "create",
occurredAt ?? DateTimeOffset.UtcNow,
new AuditActor("user_admin", "Admin", "user"),
scheduleId: scheduleId ?? $"sch_{idSuffix}",
message: "created");
}
}
using System;
using System.Collections.Immutable;
namespace StellaOps.Scheduler.Storage.Postgres.Repositories.Tests;
internal static class TestDataFactory
{
public static Schedule CreateSchedule(
string id,
string tenantId,
bool enabled = true,
string name = "Nightly Prod")
{
var now = DateTimeOffset.UtcNow;
return new Schedule(
id,
tenantId,
name,
enabled,
"0 2 * * *",
"UTC",
ScheduleMode.AnalysisOnly,
new Selector(SelectorScope.AllImages, tenantId),
ScheduleOnlyIf.Default,
ScheduleNotify.Default,
ScheduleLimits.Default,
now,
"svc_scheduler",
now,
"svc_scheduler",
ImmutableArray<string>.Empty,
SchedulerSchemaVersions.Schedule);
}
public static Run CreateRun(
string id,
string tenantId,
RunState state,
string? scheduleId = null,
RunTrigger trigger = RunTrigger.Manual,
RunStats? stats = null,
DateTimeOffset? createdAt = null,
DateTimeOffset? startedAt = null)
{
var resolvedStats = stats ?? new RunStats(candidates: 10, deduped: 8, queued: 5, completed: 0, deltas: 2);
var created = createdAt ?? DateTimeOffset.UtcNow;
return new Run(
id,
tenantId,
trigger,
state,
resolvedStats,
created,
scheduleId: scheduleId,
reason: new RunReason(manualReason: "test"),
startedAt: startedAt ?? created);
}
public static ImpactSet CreateImpactSet(string tenantId, string snapshotId, DateTimeOffset? generatedAt = null, bool usageOnly = true)
{
var selector = new Selector(SelectorScope.AllImages, tenantId);
var image = new ImpactImage(
"sha256:" + Guid.NewGuid().ToString("N"),
"registry",
"repo/app",
namespaces: new[] { "team-a" },
tags: new[] { "prod" },
usedByEntrypoint: true);
return new ImpactSet(
selector,
new[] { image },
usageOnly: usageOnly,
generatedAt ?? DateTimeOffset.UtcNow,
total: 1,
snapshotId: snapshotId,
schemaVersion: SchedulerSchemaVersions.ImpactSet);
}
public static AuditRecord CreateAuditRecord(
string tenantId,
string idSuffix,
DateTimeOffset? occurredAt = null,
string? scheduleId = null,
string? category = null,
string? action = null)
{
return new AuditRecord(
$"audit_{idSuffix}",
tenantId,
category ?? "scheduler",
action ?? "create",
occurredAt ?? DateTimeOffset.UtcNow,
new AuditActor("user_admin", "Admin", "user"),
scheduleId: scheduleId ?? $"sch_{idSuffix}",
message: "created");
}
}

View File

@@ -0,0 +1,123 @@
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using FluentAssertions;
using Microsoft.Extensions.Options;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Postgres.Repositories;
using Xunit;
namespace StellaOps.Scheduler.Storage.Postgres.Tests;
[Collection(SchedulerPostgresCollection.Name)]
public sealed class GraphJobRepositoryTests : IAsyncLifetime
{
private readonly SchedulerPostgresFixture _fixture;
public GraphJobRepositoryTests(SchedulerPostgresFixture fixture)
{
_fixture = fixture;
}
public Task InitializeAsync() => _fixture.TruncateAllTablesAsync();
public Task DisposeAsync() => Task.CompletedTask;
private static GraphBuildJob BuildJob(string tenant, string id, GraphJobStatus status = GraphJobStatus.Pending)
=> new(
id: id,
tenantId: tenant,
sbomId: "sbom-1",
sbomVersionId: "sbom-ver-1",
sbomDigest: "sha256:abc",
status: status,
trigger: GraphBuildJobTrigger.SbomVersion,
createdAt: DateTimeOffset.UtcNow);
private static GraphOverlayJob OverlayJob(string tenant, string id, GraphJobStatus status = GraphJobStatus.Pending)
=> new(
id: id,
tenantId: tenant,
graphSnapshotId: "snap-1",
status: status,
createdAt: DateTimeOffset.UtcNow,
attempts: 0,
targetGraphId: "graph-1",
correlationId: null,
metadata: null);
[Fact]
public async Task InsertAndGetBuildJob()
{
var dataSource = CreateDataSource();
var repo = new GraphJobRepository(dataSource);
var job = BuildJob("t1", Guid.NewGuid().ToString());
await repo.InsertAsync(job, CancellationToken.None);
var fetched = await repo.GetBuildJobAsync("t1", job.Id, CancellationToken.None);
fetched.Should().NotBeNull();
fetched!.Id.Should().Be(job.Id);
fetched.Status.Should().Be(GraphJobStatus.Pending);
}
[Fact]
public async Task TryReplaceSucceedsWithExpectedStatus()
{
var dataSource = CreateDataSource();
var repo = new GraphJobRepository(dataSource);
var job = BuildJob("t1", Guid.NewGuid().ToString());
await repo.InsertAsync(job, CancellationToken.None);
var running = job with { Status = GraphJobStatus.Running };
var updated = await repo.TryReplaceAsync(running, GraphJobStatus.Pending, CancellationToken.None);
updated.Should().BeTrue();
var fetched = await repo.GetBuildJobAsync("t1", job.Id, CancellationToken.None);
fetched!.Status.Should().Be(GraphJobStatus.Running);
}
[Fact]
public async Task TryReplaceFailsOnUnexpectedStatus()
{
var dataSource = CreateDataSource();
var repo = new GraphJobRepository(dataSource);
var job = BuildJob("t1", Guid.NewGuid().ToString(), GraphJobStatus.Completed);
await repo.InsertAsync(job, CancellationToken.None);
var running = job with { Status = GraphJobStatus.Running };
var updated = await repo.TryReplaceAsync(running, GraphJobStatus.Pending, CancellationToken.None);
updated.Should().BeFalse();
}
[Fact]
public async Task ListBuildJobsHonorsStatusAndLimit()
{
var dataSource = CreateDataSource();
var repo = new GraphJobRepository(dataSource);
for (int i = 0; i < 5; i++)
{
await repo.InsertAsync(BuildJob("t1", Guid.NewGuid().ToString(), GraphJobStatus.Pending), CancellationToken.None);
}
var running = BuildJob("t1", Guid.NewGuid().ToString(), GraphJobStatus.Running);
await repo.InsertAsync(running, CancellationToken.None);
var pending = await repo.ListBuildJobsAsync("t1", GraphJobStatus.Pending, 3, CancellationToken.None);
pending.Count.Should().Be(3);
var runningList = await repo.ListBuildJobsAsync("t1", GraphJobStatus.Running, 10, CancellationToken.None);
runningList.Should().ContainSingle(j => j.Id == running.Id);
}
private SchedulerDataSource CreateDataSource()
{
var options = _fixture.Fixture.CreateOptions();
options.SchemaName = _fixture.SchemaName;
return new SchedulerDataSource(Options.Create(options));
}
}

View File

@@ -8,7 +8,7 @@ using System.Threading;
using System.Threading.Tasks;
using MongoDB.Driver;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Mongo.Repositories;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Repositories;
using StellaOps.Scheduler.WebService.PolicySimulations;
using Xunit;

View File

@@ -10,7 +10,7 @@ using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Queue;
using StellaOps.Scheduler.Storage.Mongo.Repositories;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Repositories;
namespace StellaOps.Scheduler.WebService.Tests;
@@ -21,85 +21,85 @@ public sealed class RunEndpointTests : IClassFixture<WebApplicationFactory<Progr
public RunEndpointTests(WebApplicationFactory<Program> factory)
{
_factory = factory;
}
[Fact]
public async Task CreateListCancelRun()
{
using var client = _factory.CreateClient();
client.DefaultRequestHeaders.Add("X-Tenant-Id", "tenant-runs");
}
[Fact]
public async Task CreateListCancelRun()
{
using var client = _factory.CreateClient();
client.DefaultRequestHeaders.Add("X-Tenant-Id", "tenant-runs");
client.DefaultRequestHeaders.Add("X-Scopes", "scheduler.schedules.write scheduler.schedules.read scheduler.runs.write scheduler.runs.read scheduler.runs.preview scheduler.runs.manage");
var scheduleResponse = await client.PostAsJsonAsync("/api/v1/scheduler/schedules", new
{
name = "RunSchedule",
cronExpression = "0 3 * * *",
timezone = "UTC",
mode = "analysis-only",
selection = new
{
scope = "all-images"
}
});
scheduleResponse.EnsureSuccessStatusCode();
var scheduleJson = await scheduleResponse.Content.ReadFromJsonAsync<JsonElement>();
var scheduleId = scheduleJson.GetProperty("schedule").GetProperty("id").GetString();
Assert.False(string.IsNullOrEmpty(scheduleId));
var createRun = await client.PostAsJsonAsync("/api/v1/scheduler/runs", new
{
scheduleId,
trigger = "manual"
});
createRun.EnsureSuccessStatusCode();
Assert.Equal(System.Net.HttpStatusCode.Created, createRun.StatusCode);
var runJson = await createRun.Content.ReadFromJsonAsync<JsonElement>();
var runId = runJson.GetProperty("run").GetProperty("id").GetString();
Assert.False(string.IsNullOrEmpty(runId));
Assert.Equal("planning", runJson.GetProperty("run").GetProperty("state").GetString());
var listResponse = await client.GetAsync("/api/v1/scheduler/runs");
listResponse.EnsureSuccessStatusCode();
var listJson = await listResponse.Content.ReadFromJsonAsync<JsonElement>();
Assert.True(listJson.GetProperty("runs").EnumerateArray().Any());
var cancelResponse = await client.PostAsync($"/api/v1/scheduler/runs/{runId}/cancel", null);
cancelResponse.EnsureSuccessStatusCode();
var cancelled = await cancelResponse.Content.ReadFromJsonAsync<JsonElement>();
Assert.Equal("cancelled", cancelled.GetProperty("run").GetProperty("state").GetString());
var getResponse = await client.GetAsync($"/api/v1/scheduler/runs/{runId}");
getResponse.EnsureSuccessStatusCode();
var runDetail = await getResponse.Content.ReadFromJsonAsync<JsonElement>();
Assert.Equal("cancelled", runDetail.GetProperty("run").GetProperty("state").GetString());
}
[Fact]
var scheduleResponse = await client.PostAsJsonAsync("/api/v1/scheduler/schedules", new
{
name = "RunSchedule",
cronExpression = "0 3 * * *",
timezone = "UTC",
mode = "analysis-only",
selection = new
{
scope = "all-images"
}
});
scheduleResponse.EnsureSuccessStatusCode();
var scheduleJson = await scheduleResponse.Content.ReadFromJsonAsync<JsonElement>();
var scheduleId = scheduleJson.GetProperty("schedule").GetProperty("id").GetString();
Assert.False(string.IsNullOrEmpty(scheduleId));
var createRun = await client.PostAsJsonAsync("/api/v1/scheduler/runs", new
{
scheduleId,
trigger = "manual"
});
createRun.EnsureSuccessStatusCode();
Assert.Equal(System.Net.HttpStatusCode.Created, createRun.StatusCode);
var runJson = await createRun.Content.ReadFromJsonAsync<JsonElement>();
var runId = runJson.GetProperty("run").GetProperty("id").GetString();
Assert.False(string.IsNullOrEmpty(runId));
Assert.Equal("planning", runJson.GetProperty("run").GetProperty("state").GetString());
var listResponse = await client.GetAsync("/api/v1/scheduler/runs");
listResponse.EnsureSuccessStatusCode();
var listJson = await listResponse.Content.ReadFromJsonAsync<JsonElement>();
Assert.True(listJson.GetProperty("runs").EnumerateArray().Any());
var cancelResponse = await client.PostAsync($"/api/v1/scheduler/runs/{runId}/cancel", null);
cancelResponse.EnsureSuccessStatusCode();
var cancelled = await cancelResponse.Content.ReadFromJsonAsync<JsonElement>();
Assert.Equal("cancelled", cancelled.GetProperty("run").GetProperty("state").GetString());
var getResponse = await client.GetAsync($"/api/v1/scheduler/runs/{runId}");
getResponse.EnsureSuccessStatusCode();
var runDetail = await getResponse.Content.ReadFromJsonAsync<JsonElement>();
Assert.Equal("cancelled", runDetail.GetProperty("run").GetProperty("state").GetString());
}
[Fact]
public async Task PreviewImpactForSchedule()
{
using var client = _factory.CreateClient();
client.DefaultRequestHeaders.Add("X-Tenant-Id", "tenant-preview");
client.DefaultRequestHeaders.Add("X-Scopes", "scheduler.schedules.write scheduler.schedules.read scheduler.runs.write scheduler.runs.read scheduler.runs.preview scheduler.runs.manage");
var scheduleResponse = await client.PostAsJsonAsync("/api/v1/scheduler/schedules", new
{
name = "PreviewSchedule",
cronExpression = "0 5 * * *",
timezone = "UTC",
mode = "analysis-only",
selection = new
{
scope = "all-images"
}
});
scheduleResponse.EnsureSuccessStatusCode();
var scheduleJson = await scheduleResponse.Content.ReadFromJsonAsync<JsonElement>();
var scheduleId = scheduleJson.GetProperty("schedule").GetProperty("id").GetString();
Assert.False(string.IsNullOrEmpty(scheduleId));
var scheduleResponse = await client.PostAsJsonAsync("/api/v1/scheduler/schedules", new
{
name = "PreviewSchedule",
cronExpression = "0 5 * * *",
timezone = "UTC",
mode = "analysis-only",
selection = new
{
scope = "all-images"
}
});
scheduleResponse.EnsureSuccessStatusCode();
var scheduleJson = await scheduleResponse.Content.ReadFromJsonAsync<JsonElement>();
var scheduleId = scheduleJson.GetProperty("schedule").GetProperty("id").GetString();
Assert.False(string.IsNullOrEmpty(scheduleId));
var previewResponse = await client.PostAsJsonAsync("/api/v1/scheduler/runs/preview", new
{
scheduleId,

View File

@@ -1,244 +1,244 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Mongo.Repositories;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Repositories;
using StellaOps.Scheduler.Worker.Graph;
using StellaOps.Scheduler.Worker.Graph.Cartographer;
using StellaOps.Scheduler.Worker.Graph.Scheduler;
using StellaOps.Scheduler.Worker.Options;
using StellaOps.Scheduler.Worker.Observability;
using Xunit;
namespace StellaOps.Scheduler.Worker.Tests;
public sealed class GraphBuildExecutionServiceTests
{
[Fact]
public async Task ExecuteAsync_Skips_WhenGraphDisabled()
{
var repository = new RecordingGraphJobRepository();
var cartographer = new StubCartographerBuildClient();
var completion = new RecordingCompletionClient();
using var metrics = new SchedulerWorkerMetrics();
var options = Microsoft.Extensions.Options.Options.Create(new SchedulerWorkerOptions
{
Graph = new SchedulerWorkerOptions.GraphOptions
{
Enabled = false
}
});
var service = new GraphBuildExecutionService(repository, cartographer, completion, options, metrics, TimeProvider.System, NullLogger<GraphBuildExecutionService>.Instance);
var job = CreateGraphJob();
var result = await service.ExecuteAsync(job, CancellationToken.None);
Assert.Equal(GraphBuildExecutionResultType.Skipped, result.Type);
Assert.Equal("graph_processing_disabled", result.Reason);
Assert.Equal(0, repository.ReplaceCalls);
Assert.Equal(0, cartographer.CallCount);
Assert.Empty(completion.Notifications);
}
[Fact]
public async Task ExecuteAsync_CompletesJob_OnSuccess()
{
var repository = new RecordingGraphJobRepository();
var cartographer = new StubCartographerBuildClient
{
Result = new CartographerBuildResult(
GraphJobStatus.Completed,
CartographerJobId: "carto-1",
GraphSnapshotId: "graph_snap",
ResultUri: "oras://graph/result",
Error: null)
};
var completion = new RecordingCompletionClient();
using var metrics = new SchedulerWorkerMetrics();
var options = Microsoft.Extensions.Options.Options.Create(new SchedulerWorkerOptions
{
Graph = new SchedulerWorkerOptions.GraphOptions
{
Enabled = true,
MaxAttempts = 2,
RetryBackoff = TimeSpan.FromMilliseconds(10)
}
});
var service = new GraphBuildExecutionService(repository, cartographer, completion, options, metrics, TimeProvider.System, NullLogger<GraphBuildExecutionService>.Instance);
var job = CreateGraphJob();
var result = await service.ExecuteAsync(job, CancellationToken.None);
Assert.Equal(GraphBuildExecutionResultType.Completed, result.Type);
Assert.Single(completion.Notifications);
var notification = completion.Notifications[0];
Assert.Equal(job.Id, notification.JobId);
Assert.Equal("Build", notification.JobType);
Assert.Equal(GraphJobStatus.Completed, notification.Status);
Assert.Equal("oras://graph/result", notification.ResultUri);
Assert.Equal("graph_snap", notification.GraphSnapshotId);
Assert.Null(notification.Error);
Assert.Equal(1, cartographer.CallCount);
Assert.True(repository.ReplaceCalls >= 1);
}
[Fact]
public async Task ExecuteAsync_Fails_AfterMaxAttempts()
{
var repository = new RecordingGraphJobRepository();
var cartographer = new StubCartographerBuildClient
{
ExceptionToThrow = new InvalidOperationException("network")
};
var completion = new RecordingCompletionClient();
using var metrics = new SchedulerWorkerMetrics();
var options = Microsoft.Extensions.Options.Options.Create(new SchedulerWorkerOptions
{
Graph = new SchedulerWorkerOptions.GraphOptions
{
Enabled = true,
MaxAttempts = 2,
RetryBackoff = TimeSpan.FromMilliseconds(1)
}
});
var service = new GraphBuildExecutionService(repository, cartographer, completion, options, metrics, TimeProvider.System, NullLogger<GraphBuildExecutionService>.Instance);
var job = CreateGraphJob();
var result = await service.ExecuteAsync(job, CancellationToken.None);
Assert.Equal(GraphBuildExecutionResultType.Failed, result.Type);
Assert.Equal(2, cartographer.CallCount);
Assert.Single(completion.Notifications);
Assert.Equal(GraphJobStatus.Failed, completion.Notifications[0].Status);
Assert.Equal("network", completion.Notifications[0].Error);
}
[Fact]
public async Task ExecuteAsync_Skips_WhenConcurrencyConflict()
{
var repository = new RecordingGraphJobRepository
{
ShouldReplaceSucceed = false
};
var cartographer = new StubCartographerBuildClient();
var completion = new RecordingCompletionClient();
using var metrics = new SchedulerWorkerMetrics();
var options = Microsoft.Extensions.Options.Options.Create(new SchedulerWorkerOptions
{
Graph = new SchedulerWorkerOptions.GraphOptions
{
Enabled = true
}
});
var service = new GraphBuildExecutionService(repository, cartographer, completion, options, metrics, TimeProvider.System, NullLogger<GraphBuildExecutionService>.Instance);
var job = CreateGraphJob();
var result = await service.ExecuteAsync(job, CancellationToken.None);
Assert.Equal(GraphBuildExecutionResultType.Skipped, result.Type);
Assert.Equal("concurrency_conflict", result.Reason);
Assert.Equal(0, cartographer.CallCount);
Assert.Empty(completion.Notifications);
}
private static GraphBuildJob CreateGraphJob() => new(
id: "gbj_1",
tenantId: "tenant-alpha",
sbomId: "sbom-1",
sbomVersionId: "sbom-1-v1",
sbomDigest: "sha256:" + new string('a', 64),
status: GraphJobStatus.Pending,
trigger: GraphBuildJobTrigger.SbomVersion,
createdAt: DateTimeOffset.UtcNow,
attempts: 0,
metadata: Array.Empty<KeyValuePair<string, string>>());
private sealed class RecordingGraphJobRepository : IGraphJobRepository
{
public int ReplaceCalls { get; private set; }
public bool ShouldReplaceSucceed { get; set; } = true;
public Task<bool> TryReplaceAsync(GraphBuildJob job, GraphJobStatus expectedStatus, CancellationToken cancellationToken = default)
{
if (!ShouldReplaceSucceed)
{
return Task.FromResult(false);
}
ReplaceCalls++;
return Task.FromResult(true);
}
public Task<GraphBuildJob> ReplaceAsync(GraphBuildJob job, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<GraphOverlayJob> ReplaceAsync(GraphOverlayJob job, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task InsertAsync(GraphBuildJob job, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task InsertAsync(GraphOverlayJob job, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<GraphBuildJob?> GetBuildJobAsync(string tenantId, string jobId, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<GraphOverlayJob?> GetOverlayJobAsync(string tenantId, string jobId, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<IReadOnlyList<GraphBuildJob>> ListBuildJobsAsync(string tenantId, GraphJobStatus? status, int limit, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<IReadOnlyList<GraphBuildJob>> ListBuildJobsAsync(GraphJobStatus? status, int limit, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<IReadOnlyList<GraphOverlayJob>> ListOverlayJobsAsync(string tenantId, GraphJobStatus? status, int limit, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<IReadOnlyList<GraphOverlayJob>> ListOverlayJobsAsync(GraphJobStatus? status, int limit, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<IReadOnlyCollection<GraphOverlayJob>> ListOverlayJobsAsync(string tenantId, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<bool> TryReplaceOverlayAsync(GraphOverlayJob job, GraphJobStatus expectedStatus, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
}
private sealed class StubCartographerBuildClient : ICartographerBuildClient
{
public CartographerBuildResult Result { get; set; } = new(GraphJobStatus.Completed, null, null, null, null);
public Exception? ExceptionToThrow { get; set; }
public int CallCount { get; private set; }
public Task<CartographerBuildResult> StartBuildAsync(GraphBuildJob job, CancellationToken cancellationToken)
{
CallCount++;
if (ExceptionToThrow is not null)
{
throw ExceptionToThrow;
}
return Task.FromResult(Result);
}
}
private sealed class RecordingCompletionClient : IGraphJobCompletionClient
{
public List<GraphJobCompletionRequestDto> Notifications { get; } = new();
public Task NotifyAsync(GraphJobCompletionRequestDto request, CancellationToken cancellationToken)
{
Notifications.Add(request);
return Task.CompletedTask;
}
}
}
using Xunit;
namespace StellaOps.Scheduler.Worker.Tests;
public sealed class GraphBuildExecutionServiceTests
{
[Fact]
public async Task ExecuteAsync_Skips_WhenGraphDisabled()
{
var repository = new RecordingGraphJobRepository();
var cartographer = new StubCartographerBuildClient();
var completion = new RecordingCompletionClient();
using var metrics = new SchedulerWorkerMetrics();
var options = Microsoft.Extensions.Options.Options.Create(new SchedulerWorkerOptions
{
Graph = new SchedulerWorkerOptions.GraphOptions
{
Enabled = false
}
});
var service = new GraphBuildExecutionService(repository, cartographer, completion, options, metrics, TimeProvider.System, NullLogger<GraphBuildExecutionService>.Instance);
var job = CreateGraphJob();
var result = await service.ExecuteAsync(job, CancellationToken.None);
Assert.Equal(GraphBuildExecutionResultType.Skipped, result.Type);
Assert.Equal("graph_processing_disabled", result.Reason);
Assert.Equal(0, repository.ReplaceCalls);
Assert.Equal(0, cartographer.CallCount);
Assert.Empty(completion.Notifications);
}
[Fact]
public async Task ExecuteAsync_CompletesJob_OnSuccess()
{
var repository = new RecordingGraphJobRepository();
var cartographer = new StubCartographerBuildClient
{
Result = new CartographerBuildResult(
GraphJobStatus.Completed,
CartographerJobId: "carto-1",
GraphSnapshotId: "graph_snap",
ResultUri: "oras://graph/result",
Error: null)
};
var completion = new RecordingCompletionClient();
using var metrics = new SchedulerWorkerMetrics();
var options = Microsoft.Extensions.Options.Options.Create(new SchedulerWorkerOptions
{
Graph = new SchedulerWorkerOptions.GraphOptions
{
Enabled = true,
MaxAttempts = 2,
RetryBackoff = TimeSpan.FromMilliseconds(10)
}
});
var service = new GraphBuildExecutionService(repository, cartographer, completion, options, metrics, TimeProvider.System, NullLogger<GraphBuildExecutionService>.Instance);
var job = CreateGraphJob();
var result = await service.ExecuteAsync(job, CancellationToken.None);
Assert.Equal(GraphBuildExecutionResultType.Completed, result.Type);
Assert.Single(completion.Notifications);
var notification = completion.Notifications[0];
Assert.Equal(job.Id, notification.JobId);
Assert.Equal("Build", notification.JobType);
Assert.Equal(GraphJobStatus.Completed, notification.Status);
Assert.Equal("oras://graph/result", notification.ResultUri);
Assert.Equal("graph_snap", notification.GraphSnapshotId);
Assert.Null(notification.Error);
Assert.Equal(1, cartographer.CallCount);
Assert.True(repository.ReplaceCalls >= 1);
}
[Fact]
public async Task ExecuteAsync_Fails_AfterMaxAttempts()
{
var repository = new RecordingGraphJobRepository();
var cartographer = new StubCartographerBuildClient
{
ExceptionToThrow = new InvalidOperationException("network")
};
var completion = new RecordingCompletionClient();
using var metrics = new SchedulerWorkerMetrics();
var options = Microsoft.Extensions.Options.Options.Create(new SchedulerWorkerOptions
{
Graph = new SchedulerWorkerOptions.GraphOptions
{
Enabled = true,
MaxAttempts = 2,
RetryBackoff = TimeSpan.FromMilliseconds(1)
}
});
var service = new GraphBuildExecutionService(repository, cartographer, completion, options, metrics, TimeProvider.System, NullLogger<GraphBuildExecutionService>.Instance);
var job = CreateGraphJob();
var result = await service.ExecuteAsync(job, CancellationToken.None);
Assert.Equal(GraphBuildExecutionResultType.Failed, result.Type);
Assert.Equal(2, cartographer.CallCount);
Assert.Single(completion.Notifications);
Assert.Equal(GraphJobStatus.Failed, completion.Notifications[0].Status);
Assert.Equal("network", completion.Notifications[0].Error);
}
[Fact]
public async Task ExecuteAsync_Skips_WhenConcurrencyConflict()
{
var repository = new RecordingGraphJobRepository
{
ShouldReplaceSucceed = false
};
var cartographer = new StubCartographerBuildClient();
var completion = new RecordingCompletionClient();
using var metrics = new SchedulerWorkerMetrics();
var options = Microsoft.Extensions.Options.Options.Create(new SchedulerWorkerOptions
{
Graph = new SchedulerWorkerOptions.GraphOptions
{
Enabled = true
}
});
var service = new GraphBuildExecutionService(repository, cartographer, completion, options, metrics, TimeProvider.System, NullLogger<GraphBuildExecutionService>.Instance);
var job = CreateGraphJob();
var result = await service.ExecuteAsync(job, CancellationToken.None);
Assert.Equal(GraphBuildExecutionResultType.Skipped, result.Type);
Assert.Equal("concurrency_conflict", result.Reason);
Assert.Equal(0, cartographer.CallCount);
Assert.Empty(completion.Notifications);
}
private static GraphBuildJob CreateGraphJob() => new(
id: "gbj_1",
tenantId: "tenant-alpha",
sbomId: "sbom-1",
sbomVersionId: "sbom-1-v1",
sbomDigest: "sha256:" + new string('a', 64),
status: GraphJobStatus.Pending,
trigger: GraphBuildJobTrigger.SbomVersion,
createdAt: DateTimeOffset.UtcNow,
attempts: 0,
metadata: Array.Empty<KeyValuePair<string, string>>());
private sealed class RecordingGraphJobRepository : IGraphJobRepository
{
public int ReplaceCalls { get; private set; }
public bool ShouldReplaceSucceed { get; set; } = true;
public Task<bool> TryReplaceAsync(GraphBuildJob job, GraphJobStatus expectedStatus, CancellationToken cancellationToken = default)
{
if (!ShouldReplaceSucceed)
{
return Task.FromResult(false);
}
ReplaceCalls++;
return Task.FromResult(true);
}
public Task<GraphBuildJob> ReplaceAsync(GraphBuildJob job, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<GraphOverlayJob> ReplaceAsync(GraphOverlayJob job, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task InsertAsync(GraphBuildJob job, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task InsertAsync(GraphOverlayJob job, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<GraphBuildJob?> GetBuildJobAsync(string tenantId, string jobId, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<GraphOverlayJob?> GetOverlayJobAsync(string tenantId, string jobId, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<IReadOnlyList<GraphBuildJob>> ListBuildJobsAsync(string tenantId, GraphJobStatus? status, int limit, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<IReadOnlyList<GraphBuildJob>> ListBuildJobsAsync(GraphJobStatus? status, int limit, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<IReadOnlyList<GraphOverlayJob>> ListOverlayJobsAsync(string tenantId, GraphJobStatus? status, int limit, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<IReadOnlyList<GraphOverlayJob>> ListOverlayJobsAsync(GraphJobStatus? status, int limit, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<IReadOnlyCollection<GraphOverlayJob>> ListOverlayJobsAsync(string tenantId, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<bool> TryReplaceOverlayAsync(GraphOverlayJob job, GraphJobStatus expectedStatus, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
}
private sealed class StubCartographerBuildClient : ICartographerBuildClient
{
public CartographerBuildResult Result { get; set; } = new(GraphJobStatus.Completed, null, null, null, null);
public Exception? ExceptionToThrow { get; set; }
public int CallCount { get; private set; }
public Task<CartographerBuildResult> StartBuildAsync(GraphBuildJob job, CancellationToken cancellationToken)
{
CallCount++;
if (ExceptionToThrow is not null)
{
throw ExceptionToThrow;
}
return Task.FromResult(Result);
}
}
private sealed class RecordingCompletionClient : IGraphJobCompletionClient
{
public List<GraphJobCompletionRequestDto> Notifications { get; } = new();
public Task NotifyAsync(GraphJobCompletionRequestDto request, CancellationToken cancellationToken)
{
Notifications.Add(request);
return Task.CompletedTask;
}
}
}

View File

@@ -1,238 +1,238 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Mongo.Repositories;
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Repositories;
using StellaOps.Scheduler.Worker.Graph;
using StellaOps.Scheduler.Worker.Graph.Cartographer;
using StellaOps.Scheduler.Worker.Graph.Scheduler;
using StellaOps.Scheduler.Worker.Options;
using StellaOps.Scheduler.Worker.Observability;
using Xunit;
namespace StellaOps.Scheduler.Worker.Tests;
public sealed class GraphOverlayExecutionServiceTests
{
[Fact]
public async Task ExecuteAsync_Skips_WhenGraphDisabled()
{
var repository = new RecordingGraphJobRepository();
var cartographer = new StubCartographerOverlayClient();
var completion = new RecordingCompletionClient();
using var metrics = new SchedulerWorkerMetrics();
var options = Microsoft.Extensions.Options.Options.Create(new SchedulerWorkerOptions
{
Graph = new SchedulerWorkerOptions.GraphOptions
{
Enabled = false
}
});
var service = new GraphOverlayExecutionService(repository, cartographer, completion, options, metrics, TimeProvider.System, NullLogger<GraphOverlayExecutionService>.Instance);
var job = CreateOverlayJob();
var result = await service.ExecuteAsync(job, CancellationToken.None);
Assert.Equal(GraphOverlayExecutionResultType.Skipped, result.Type);
Assert.Equal("graph_processing_disabled", result.Reason);
Assert.Empty(completion.Notifications);
Assert.Equal(0, cartographer.CallCount);
}
[Fact]
public async Task ExecuteAsync_CompletesJob_OnSuccess()
{
var repository = new RecordingGraphJobRepository();
var cartographer = new StubCartographerOverlayClient
{
Result = new CartographerOverlayResult(
GraphJobStatus.Completed,
GraphSnapshotId: "graph_snap_2",
ResultUri: "oras://graph/overlay",
Error: null)
};
var completion = new RecordingCompletionClient();
using var metrics = new SchedulerWorkerMetrics();
var options = Microsoft.Extensions.Options.Options.Create(new SchedulerWorkerOptions
{
Graph = new SchedulerWorkerOptions.GraphOptions
{
Enabled = true,
MaxAttempts = 2,
RetryBackoff = TimeSpan.FromMilliseconds(5)
}
});
var service = new GraphOverlayExecutionService(repository, cartographer, completion, options, metrics, TimeProvider.System, NullLogger<GraphOverlayExecutionService>.Instance);
var job = CreateOverlayJob();
var result = await service.ExecuteAsync(job, CancellationToken.None);
Assert.Equal(GraphOverlayExecutionResultType.Completed, result.Type);
Assert.Single(completion.Notifications);
var notification = completion.Notifications[0];
Assert.Equal("Overlay", notification.JobType);
Assert.Equal(GraphJobStatus.Completed, notification.Status);
Assert.Equal("oras://graph/overlay", notification.ResultUri);
Assert.Equal("graph_snap_2", notification.GraphSnapshotId);
}
[Fact]
public async Task ExecuteAsync_Fails_AfterRetries()
{
var repository = new RecordingGraphJobRepository();
var cartographer = new StubCartographerOverlayClient
{
ExceptionToThrow = new InvalidOperationException("overlay failed")
};
var completion = new RecordingCompletionClient();
using var metrics = new SchedulerWorkerMetrics();
var options = Microsoft.Extensions.Options.Options.Create(new SchedulerWorkerOptions
{
Graph = new SchedulerWorkerOptions.GraphOptions
{
Enabled = true,
MaxAttempts = 2,
RetryBackoff = TimeSpan.FromMilliseconds(1)
}
});
var service = new GraphOverlayExecutionService(repository, cartographer, completion, options, metrics, TimeProvider.System, NullLogger<GraphOverlayExecutionService>.Instance);
var job = CreateOverlayJob();
var result = await service.ExecuteAsync(job, CancellationToken.None);
Assert.Equal(GraphOverlayExecutionResultType.Failed, result.Type);
Assert.Single(completion.Notifications);
Assert.Equal(GraphJobStatus.Failed, completion.Notifications[0].Status);
Assert.Equal("overlay failed", completion.Notifications[0].Error);
}
[Fact]
public async Task ExecuteAsync_Skips_WhenConcurrencyConflict()
{
var repository = new RecordingGraphJobRepository
{
ShouldReplaceSucceed = false
};
var cartographer = new StubCartographerOverlayClient();
var completion = new RecordingCompletionClient();
using var metrics = new SchedulerWorkerMetrics();
var options = Microsoft.Extensions.Options.Options.Create(new SchedulerWorkerOptions
{
Graph = new SchedulerWorkerOptions.GraphOptions
{
Enabled = true
}
});
var service = new GraphOverlayExecutionService(repository, cartographer, completion, options, metrics, TimeProvider.System, NullLogger<GraphOverlayExecutionService>.Instance);
var job = CreateOverlayJob();
var result = await service.ExecuteAsync(job, CancellationToken.None);
Assert.Equal(GraphOverlayExecutionResultType.Skipped, result.Type);
Assert.Equal("concurrency_conflict", result.Reason);
Assert.Empty(completion.Notifications);
Assert.Equal(0, cartographer.CallCount);
}
private static GraphOverlayJob CreateOverlayJob() => new(
id: "goj_1",
tenantId: "tenant-alpha",
graphSnapshotId: "snap-1",
overlayKind: GraphOverlayKind.Policy,
overlayKey: "policy@1",
status: GraphJobStatus.Pending,
trigger: GraphOverlayJobTrigger.Policy,
createdAt: DateTimeOffset.UtcNow,
subjects: Array.Empty<string>(),
attempts: 0,
metadata: Array.Empty<KeyValuePair<string, string>>());
private sealed class RecordingGraphJobRepository : IGraphJobRepository
{
public bool ShouldReplaceSucceed { get; set; } = true;
public int RunningReplacements { get; private set; }
public Task<bool> TryReplaceOverlayAsync(GraphOverlayJob job, GraphJobStatus expectedStatus, CancellationToken cancellationToken = default)
{
if (!ShouldReplaceSucceed)
{
return Task.FromResult(false);
}
RunningReplacements++;
return Task.FromResult(true);
}
public Task<bool> TryReplaceAsync(GraphBuildJob job, GraphJobStatus expectedStatus, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<GraphBuildJob> ReplaceAsync(GraphBuildJob job, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<GraphOverlayJob> ReplaceAsync(GraphOverlayJob job, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task InsertAsync(GraphBuildJob job, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task InsertAsync(GraphOverlayJob job, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<GraphBuildJob?> GetBuildJobAsync(string tenantId, string jobId, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<GraphOverlayJob?> GetOverlayJobAsync(string tenantId, string jobId, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<IReadOnlyList<GraphBuildJob>> ListBuildJobsAsync(string tenantId, GraphJobStatus? status, int limit, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<IReadOnlyList<GraphBuildJob>> ListBuildJobsAsync(GraphJobStatus? status, int limit, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<IReadOnlyList<GraphOverlayJob>> ListOverlayJobsAsync(string tenantId, GraphJobStatus? status, int limit, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<IReadOnlyList<GraphOverlayJob>> ListOverlayJobsAsync(GraphJobStatus? status, int limit, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<IReadOnlyCollection<GraphOverlayJob>> ListOverlayJobsAsync(string tenantId, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
}
private sealed class StubCartographerOverlayClient : ICartographerOverlayClient
{
public CartographerOverlayResult Result { get; set; } = new(GraphJobStatus.Completed, null, null, null);
public Exception? ExceptionToThrow { get; set; }
public int CallCount { get; private set; }
public Task<CartographerOverlayResult> StartOverlayAsync(GraphOverlayJob job, CancellationToken cancellationToken)
{
CallCount++;
if (ExceptionToThrow is not null)
{
throw ExceptionToThrow;
}
return Task.FromResult(Result);
}
}
private sealed class RecordingCompletionClient : IGraphJobCompletionClient
{
public List<GraphJobCompletionRequestDto> Notifications { get; } = new();
public Task NotifyAsync(GraphJobCompletionRequestDto request, CancellationToken cancellationToken)
{
Notifications.Add(request);
return Task.CompletedTask;
}
}
}
using Xunit;
namespace StellaOps.Scheduler.Worker.Tests;
public sealed class GraphOverlayExecutionServiceTests
{
[Fact]
public async Task ExecuteAsync_Skips_WhenGraphDisabled()
{
var repository = new RecordingGraphJobRepository();
var cartographer = new StubCartographerOverlayClient();
var completion = new RecordingCompletionClient();
using var metrics = new SchedulerWorkerMetrics();
var options = Microsoft.Extensions.Options.Options.Create(new SchedulerWorkerOptions
{
Graph = new SchedulerWorkerOptions.GraphOptions
{
Enabled = false
}
});
var service = new GraphOverlayExecutionService(repository, cartographer, completion, options, metrics, TimeProvider.System, NullLogger<GraphOverlayExecutionService>.Instance);
var job = CreateOverlayJob();
var result = await service.ExecuteAsync(job, CancellationToken.None);
Assert.Equal(GraphOverlayExecutionResultType.Skipped, result.Type);
Assert.Equal("graph_processing_disabled", result.Reason);
Assert.Empty(completion.Notifications);
Assert.Equal(0, cartographer.CallCount);
}
[Fact]
public async Task ExecuteAsync_CompletesJob_OnSuccess()
{
var repository = new RecordingGraphJobRepository();
var cartographer = new StubCartographerOverlayClient
{
Result = new CartographerOverlayResult(
GraphJobStatus.Completed,
GraphSnapshotId: "graph_snap_2",
ResultUri: "oras://graph/overlay",
Error: null)
};
var completion = new RecordingCompletionClient();
using var metrics = new SchedulerWorkerMetrics();
var options = Microsoft.Extensions.Options.Options.Create(new SchedulerWorkerOptions
{
Graph = new SchedulerWorkerOptions.GraphOptions
{
Enabled = true,
MaxAttempts = 2,
RetryBackoff = TimeSpan.FromMilliseconds(5)
}
});
var service = new GraphOverlayExecutionService(repository, cartographer, completion, options, metrics, TimeProvider.System, NullLogger<GraphOverlayExecutionService>.Instance);
var job = CreateOverlayJob();
var result = await service.ExecuteAsync(job, CancellationToken.None);
Assert.Equal(GraphOverlayExecutionResultType.Completed, result.Type);
Assert.Single(completion.Notifications);
var notification = completion.Notifications[0];
Assert.Equal("Overlay", notification.JobType);
Assert.Equal(GraphJobStatus.Completed, notification.Status);
Assert.Equal("oras://graph/overlay", notification.ResultUri);
Assert.Equal("graph_snap_2", notification.GraphSnapshotId);
}
[Fact]
public async Task ExecuteAsync_Fails_AfterRetries()
{
var repository = new RecordingGraphJobRepository();
var cartographer = new StubCartographerOverlayClient
{
ExceptionToThrow = new InvalidOperationException("overlay failed")
};
var completion = new RecordingCompletionClient();
using var metrics = new SchedulerWorkerMetrics();
var options = Microsoft.Extensions.Options.Options.Create(new SchedulerWorkerOptions
{
Graph = new SchedulerWorkerOptions.GraphOptions
{
Enabled = true,
MaxAttempts = 2,
RetryBackoff = TimeSpan.FromMilliseconds(1)
}
});
var service = new GraphOverlayExecutionService(repository, cartographer, completion, options, metrics, TimeProvider.System, NullLogger<GraphOverlayExecutionService>.Instance);
var job = CreateOverlayJob();
var result = await service.ExecuteAsync(job, CancellationToken.None);
Assert.Equal(GraphOverlayExecutionResultType.Failed, result.Type);
Assert.Single(completion.Notifications);
Assert.Equal(GraphJobStatus.Failed, completion.Notifications[0].Status);
Assert.Equal("overlay failed", completion.Notifications[0].Error);
}
[Fact]
public async Task ExecuteAsync_Skips_WhenConcurrencyConflict()
{
var repository = new RecordingGraphJobRepository
{
ShouldReplaceSucceed = false
};
var cartographer = new StubCartographerOverlayClient();
var completion = new RecordingCompletionClient();
using var metrics = new SchedulerWorkerMetrics();
var options = Microsoft.Extensions.Options.Options.Create(new SchedulerWorkerOptions
{
Graph = new SchedulerWorkerOptions.GraphOptions
{
Enabled = true
}
});
var service = new GraphOverlayExecutionService(repository, cartographer, completion, options, metrics, TimeProvider.System, NullLogger<GraphOverlayExecutionService>.Instance);
var job = CreateOverlayJob();
var result = await service.ExecuteAsync(job, CancellationToken.None);
Assert.Equal(GraphOverlayExecutionResultType.Skipped, result.Type);
Assert.Equal("concurrency_conflict", result.Reason);
Assert.Empty(completion.Notifications);
Assert.Equal(0, cartographer.CallCount);
}
private static GraphOverlayJob CreateOverlayJob() => new(
id: "goj_1",
tenantId: "tenant-alpha",
graphSnapshotId: "snap-1",
overlayKind: GraphOverlayKind.Policy,
overlayKey: "policy@1",
status: GraphJobStatus.Pending,
trigger: GraphOverlayJobTrigger.Policy,
createdAt: DateTimeOffset.UtcNow,
subjects: Array.Empty<string>(),
attempts: 0,
metadata: Array.Empty<KeyValuePair<string, string>>());
private sealed class RecordingGraphJobRepository : IGraphJobRepository
{
public bool ShouldReplaceSucceed { get; set; } = true;
public int RunningReplacements { get; private set; }
public Task<bool> TryReplaceOverlayAsync(GraphOverlayJob job, GraphJobStatus expectedStatus, CancellationToken cancellationToken = default)
{
if (!ShouldReplaceSucceed)
{
return Task.FromResult(false);
}
RunningReplacements++;
return Task.FromResult(true);
}
public Task<bool> TryReplaceAsync(GraphBuildJob job, GraphJobStatus expectedStatus, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<GraphBuildJob> ReplaceAsync(GraphBuildJob job, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<GraphOverlayJob> ReplaceAsync(GraphOverlayJob job, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task InsertAsync(GraphBuildJob job, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task InsertAsync(GraphOverlayJob job, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<GraphBuildJob?> GetBuildJobAsync(string tenantId, string jobId, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<GraphOverlayJob?> GetOverlayJobAsync(string tenantId, string jobId, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<IReadOnlyList<GraphBuildJob>> ListBuildJobsAsync(string tenantId, GraphJobStatus? status, int limit, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<IReadOnlyList<GraphBuildJob>> ListBuildJobsAsync(GraphJobStatus? status, int limit, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<IReadOnlyList<GraphOverlayJob>> ListOverlayJobsAsync(string tenantId, GraphJobStatus? status, int limit, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<IReadOnlyList<GraphOverlayJob>> ListOverlayJobsAsync(GraphJobStatus? status, int limit, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
public Task<IReadOnlyCollection<GraphOverlayJob>> ListOverlayJobsAsync(string tenantId, CancellationToken cancellationToken = default)
=> throw new NotImplementedException();
}
private sealed class StubCartographerOverlayClient : ICartographerOverlayClient
{
public CartographerOverlayResult Result { get; set; } = new(GraphJobStatus.Completed, null, null, null);
public Exception? ExceptionToThrow { get; set; }
public int CallCount { get; private set; }
public Task<CartographerOverlayResult> StartOverlayAsync(GraphOverlayJob job, CancellationToken cancellationToken)
{
CallCount++;
if (ExceptionToThrow is not null)
{
throw ExceptionToThrow;
}
return Task.FromResult(Result);
}
}
private sealed class RecordingCompletionClient : IGraphJobCompletionClient
{
public List<GraphJobCompletionRequestDto> Notifications { get; } = new();
public Task NotifyAsync(GraphJobCompletionRequestDto request, CancellationToken cancellationToken)
{
Notifications.Add(request);
return Task.CompletedTask;
}
}
}

View File

@@ -6,9 +6,9 @@ using System.Threading.Tasks;
using Microsoft.Extensions.Logging.Abstractions;
using MongoDB.Driver;
using StellaOps.Scheduler.Queue;
using StellaOps.Scheduler.Storage.Mongo.Projections;
using StellaOps.Scheduler.Storage.Mongo.Repositories;
using StellaOps.Scheduler.Storage.Mongo.Services;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Projections;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Repositories;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Services;
using StellaOps.Scheduler.Worker.Options;
using StellaOps.Scheduler.Worker.Observability;
using StellaOps.Scheduler.Worker.Planning;

View File

@@ -5,9 +5,9 @@ using MongoDB.Driver;
using Microsoft.Extensions.Logging.Abstractions;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Queue;
using StellaOps.Scheduler.Storage.Mongo.Projections;
using StellaOps.Scheduler.Storage.Mongo.Repositories;
using StellaOps.Scheduler.Storage.Mongo.Services;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Projections;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Repositories;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Services;
using StellaOps.Scheduler.Worker.Options;
using StellaOps.Scheduler.Worker.Planning;
using StellaOps.Scheduler.Worker.Observability;

View File

@@ -6,7 +6,7 @@ using MongoDB.Driver;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Mongo.Repositories;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Repositories;
using StellaOps.Scheduler.Worker.Options;
using StellaOps.Scheduler.Worker.Policy;
using StellaOps.Scheduler.Worker.Observability;

View File

@@ -1,80 +1,80 @@
using System;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using MongoDB.Driver;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Mongo.Repositories;
using StellaOps.Scheduler.Worker.Options;
using StellaOps.Scheduler.Worker.Observability;
using StellaOps.Scheduler.Worker.Policy;
using Xunit;
namespace StellaOps.Scheduler.Worker.Tests;
public sealed class PolicyRunExecutionServiceTests
{
private static readonly SchedulerWorkerOptions WorkerOptions = new()
{
Policy =
{
Dispatch =
{
LeaseOwner = "test-dispatch",
BatchSize = 1,
LeaseDuration = TimeSpan.FromMinutes(1),
IdleDelay = TimeSpan.FromMilliseconds(10),
MaxAttempts = 2,
RetryBackoff = TimeSpan.FromSeconds(30)
},
Api =
{
BaseAddress = new Uri("https://policy.example.com"),
RunsPath = "/api/policy/policies/{policyId}/runs",
SimulatePath = "/api/policy/policies/{policyId}/simulate"
}
}
};
[Fact]
public async Task ExecuteAsync_CancelsJob_WhenCancellationRequested()
{
var repository = new RecordingPolicyRunJobRepository();
var client = new StubPolicyRunClient();
var options = Microsoft.Extensions.Options.Options.Create(CloneOptions());
var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2025-10-28T10:00:00Z"));
using var metrics = new SchedulerWorkerMetrics();
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using MongoDB.Driver;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Repositories;
using StellaOps.Scheduler.Worker.Options;
using StellaOps.Scheduler.Worker.Observability;
using StellaOps.Scheduler.Worker.Policy;
using Xunit;
namespace StellaOps.Scheduler.Worker.Tests;
public sealed class PolicyRunExecutionServiceTests
{
private static readonly SchedulerWorkerOptions WorkerOptions = new()
{
Policy =
{
Dispatch =
{
LeaseOwner = "test-dispatch",
BatchSize = 1,
LeaseDuration = TimeSpan.FromMinutes(1),
IdleDelay = TimeSpan.FromMilliseconds(10),
MaxAttempts = 2,
RetryBackoff = TimeSpan.FromSeconds(30)
},
Api =
{
BaseAddress = new Uri("https://policy.example.com"),
RunsPath = "/api/policy/policies/{policyId}/runs",
SimulatePath = "/api/policy/policies/{policyId}/simulate"
}
}
};
[Fact]
public async Task ExecuteAsync_CancelsJob_WhenCancellationRequested()
{
var repository = new RecordingPolicyRunJobRepository();
var client = new StubPolicyRunClient();
var options = Microsoft.Extensions.Options.Options.Create(CloneOptions());
var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2025-10-28T10:00:00Z"));
using var metrics = new SchedulerWorkerMetrics();
var targeting = new StubPolicyRunTargetingService
{
OnEnsureTargets = job => PolicyRunTargetingResult.Unchanged(job)
};
var webhook = new RecordingPolicySimulationWebhookClient();
var service = new PolicyRunExecutionService(repository, client, options, timeProvider, metrics, targeting, webhook, NullLogger<PolicyRunExecutionService>.Instance);
var job = CreateJob(status: PolicyRunJobStatus.Dispatching) with
{
CancellationRequested = true,
LeaseOwner = "test-dispatch",
LeaseExpiresAt = timeProvider.GetUtcNow().AddMinutes(1)
};
var result = await service.ExecuteAsync(job, CancellationToken.None);
Assert.Equal(PolicyRunExecutionResultType.Cancelled, result.Type);
Assert.Equal(PolicyRunJobStatus.Cancelled, result.UpdatedJob.Status);
var job = CreateJob(status: PolicyRunJobStatus.Dispatching) with
{
CancellationRequested = true,
LeaseOwner = "test-dispatch",
LeaseExpiresAt = timeProvider.GetUtcNow().AddMinutes(1)
};
var result = await service.ExecuteAsync(job, CancellationToken.None);
Assert.Equal(PolicyRunExecutionResultType.Cancelled, result.Type);
Assert.Equal(PolicyRunJobStatus.Cancelled, result.UpdatedJob.Status);
Assert.True(repository.ReplaceCalled);
Assert.Equal("test-dispatch", repository.ExpectedLeaseOwner);
Assert.Single(webhook.Payloads);
Assert.Equal("cancelled", webhook.Payloads[0].Result);
}
[Fact]
public async Task ExecuteAsync_SubmitsJob_OnSuccess()
{
var repository = new RecordingPolicyRunJobRepository();
}
[Fact]
public async Task ExecuteAsync_SubmitsJob_OnSuccess()
{
var repository = new RecordingPolicyRunJobRepository();
var client = new StubPolicyRunClient
{
Result = PolicyRunSubmissionResult.Succeeded("run:P-7:2025", DateTimeOffset.Parse("2025-10-28T10:01:00Z"))
@@ -88,33 +88,33 @@ public sealed class PolicyRunExecutionServiceTests
};
var webhook = new RecordingPolicySimulationWebhookClient();
var service = new PolicyRunExecutionService(repository, client, options, timeProvider, metrics, targeting, webhook, NullLogger<PolicyRunExecutionService>.Instance);
var job = CreateJob(status: PolicyRunJobStatus.Dispatching) with
{
LeaseOwner = "test-dispatch",
LeaseExpiresAt = timeProvider.GetUtcNow().AddMinutes(1)
};
var result = await service.ExecuteAsync(job, CancellationToken.None);
Assert.Equal(PolicyRunExecutionResultType.Submitted, result.Type);
var job = CreateJob(status: PolicyRunJobStatus.Dispatching) with
{
LeaseOwner = "test-dispatch",
LeaseExpiresAt = timeProvider.GetUtcNow().AddMinutes(1)
};
var result = await service.ExecuteAsync(job, CancellationToken.None);
Assert.Equal(PolicyRunExecutionResultType.Submitted, result.Type);
Assert.Equal(PolicyRunJobStatus.Submitted, result.UpdatedJob.Status);
Assert.Equal("run:P-7:2025", result.UpdatedJob.RunId);
Assert.Equal(job.AttemptCount + 1, result.UpdatedJob.AttemptCount);
Assert.Null(result.UpdatedJob.LastError);
Assert.True(repository.ReplaceCalled);
Assert.Empty(webhook.Payloads);
}
[Fact]
public async Task ExecuteAsync_RetriesJob_OnFailure()
{
var repository = new RecordingPolicyRunJobRepository();
var client = new StubPolicyRunClient
{
Result = PolicyRunSubmissionResult.Failed("timeout")
};
var options = Microsoft.Extensions.Options.Options.Create(CloneOptions());
}
[Fact]
public async Task ExecuteAsync_RetriesJob_OnFailure()
{
var repository = new RecordingPolicyRunJobRepository();
var client = new StubPolicyRunClient
{
Result = PolicyRunSubmissionResult.Failed("timeout")
};
var options = Microsoft.Extensions.Options.Options.Create(CloneOptions());
var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2025-10-28T10:00:00Z"));
using var metrics = new SchedulerWorkerMetrics();
var targeting = new StubPolicyRunTargetingService
@@ -123,35 +123,35 @@ public sealed class PolicyRunExecutionServiceTests
};
var webhook = new RecordingPolicySimulationWebhookClient();
var service = new PolicyRunExecutionService(repository, client, options, timeProvider, metrics, targeting, webhook, NullLogger<PolicyRunExecutionService>.Instance);
var job = CreateJob(status: PolicyRunJobStatus.Dispatching) with
{
LeaseOwner = "test-dispatch",
LeaseExpiresAt = timeProvider.GetUtcNow().AddMinutes(1)
};
var result = await service.ExecuteAsync(job, CancellationToken.None);
Assert.Equal(PolicyRunExecutionResultType.Retrying, result.Type);
Assert.Equal(PolicyRunJobStatus.Pending, result.UpdatedJob.Status);
var job = CreateJob(status: PolicyRunJobStatus.Dispatching) with
{
LeaseOwner = "test-dispatch",
LeaseExpiresAt = timeProvider.GetUtcNow().AddMinutes(1)
};
var result = await service.ExecuteAsync(job, CancellationToken.None);
Assert.Equal(PolicyRunExecutionResultType.Retrying, result.Type);
Assert.Equal(PolicyRunJobStatus.Pending, result.UpdatedJob.Status);
Assert.Equal(job.AttemptCount + 1, result.UpdatedJob.AttemptCount);
Assert.Equal("timeout", result.UpdatedJob.LastError);
Assert.True(result.UpdatedJob.AvailableAt > job.AvailableAt);
Assert.Empty(webhook.Payloads);
}
[Fact]
public async Task ExecuteAsync_MarksJobFailed_WhenAttemptsExceeded()
{
var repository = new RecordingPolicyRunJobRepository();
var client = new StubPolicyRunClient
{
Result = PolicyRunSubmissionResult.Failed("bad request")
};
var optionsValue = CloneOptions();
optionsValue.Policy.Dispatch.MaxAttempts = 1;
var options = Microsoft.Extensions.Options.Options.Create(optionsValue);
var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2025-10-28T10:00:00Z"));
}
[Fact]
public async Task ExecuteAsync_MarksJobFailed_WhenAttemptsExceeded()
{
var repository = new RecordingPolicyRunJobRepository();
var client = new StubPolicyRunClient
{
Result = PolicyRunSubmissionResult.Failed("bad request")
};
var optionsValue = CloneOptions();
optionsValue.Policy.Dispatch.MaxAttempts = 1;
var options = Microsoft.Extensions.Options.Options.Create(optionsValue);
var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2025-10-28T10:00:00Z"));
using var metrics = new SchedulerWorkerMetrics();
var targeting = new StubPolicyRunTargetingService
{
@@ -159,13 +159,13 @@ public sealed class PolicyRunExecutionServiceTests
};
var webhook = new RecordingPolicySimulationWebhookClient();
var service = new PolicyRunExecutionService(repository, client, options, timeProvider, metrics, targeting, webhook, NullLogger<PolicyRunExecutionService>.Instance);
var job = CreateJob(status: PolicyRunJobStatus.Dispatching, attemptCount: 0) with
{
LeaseOwner = "test-dispatch",
LeaseExpiresAt = timeProvider.GetUtcNow().AddMinutes(1)
};
var job = CreateJob(status: PolicyRunJobStatus.Dispatching, attemptCount: 0) with
{
LeaseOwner = "test-dispatch",
LeaseExpiresAt = timeProvider.GetUtcNow().AddMinutes(1)
};
var result = await service.ExecuteAsync(job, CancellationToken.None);
Assert.Equal(PolicyRunExecutionResultType.Failed, result.Type);
@@ -173,100 +173,100 @@ public sealed class PolicyRunExecutionServiceTests
Assert.Equal("bad request", result.UpdatedJob.LastError);
Assert.Single(webhook.Payloads);
Assert.Equal("failed", webhook.Payloads[0].Result);
}
[Fact]
public async Task ExecuteAsync_NoWork_CompletesJob()
{
var repository = new RecordingPolicyRunJobRepository();
var client = new StubPolicyRunClient();
var options = Microsoft.Extensions.Options.Options.Create(CloneOptions());
var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2025-10-28T10:00:00Z"));
using var metrics = new SchedulerWorkerMetrics();
}
[Fact]
public async Task ExecuteAsync_NoWork_CompletesJob()
{
var repository = new RecordingPolicyRunJobRepository();
var client = new StubPolicyRunClient();
var options = Microsoft.Extensions.Options.Options.Create(CloneOptions());
var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2025-10-28T10:00:00Z"));
using var metrics = new SchedulerWorkerMetrics();
var targeting = new StubPolicyRunTargetingService
{
OnEnsureTargets = job => PolicyRunTargetingResult.NoWork(job, "empty")
};
var webhook = new RecordingPolicySimulationWebhookClient();
var service = new PolicyRunExecutionService(repository, client, options, timeProvider, metrics, targeting, webhook, NullLogger<PolicyRunExecutionService>.Instance);
var job = CreateJob(status: PolicyRunJobStatus.Dispatching, inputs: PolicyRunInputs.Empty) with
{
LeaseOwner = "test-dispatch",
LeaseExpiresAt = timeProvider.GetUtcNow().AddMinutes(1)
};
var result = await service.ExecuteAsync(job, CancellationToken.None);
var job = CreateJob(status: PolicyRunJobStatus.Dispatching, inputs: PolicyRunInputs.Empty) with
{
LeaseOwner = "test-dispatch",
LeaseExpiresAt = timeProvider.GetUtcNow().AddMinutes(1)
};
var result = await service.ExecuteAsync(job, CancellationToken.None);
Assert.Equal(PolicyRunExecutionResultType.NoOp, result.Type);
Assert.Equal(PolicyRunJobStatus.Completed, result.UpdatedJob.Status);
Assert.True(repository.ReplaceCalled);
Assert.Equal("test-dispatch", repository.ExpectedLeaseOwner);
Assert.Single(webhook.Payloads);
Assert.Equal("succeeded", webhook.Payloads[0].Result);
}
private static PolicyRunJob CreateJob(PolicyRunJobStatus status, int attemptCount = 0, PolicyRunInputs? inputs = null)
{
var resolvedInputs = inputs ?? new PolicyRunInputs(sbomSet: new[] { "sbom:S-42" }, captureExplain: true);
var metadata = ImmutableSortedDictionary.Create<string, string>(StringComparer.Ordinal);
return new PolicyRunJob(
SchemaVersion: SchedulerSchemaVersions.PolicyRunJob,
Id: "job_1",
TenantId: "tenant-alpha",
PolicyId: "P-7",
PolicyVersion: 4,
Mode: PolicyRunMode.Incremental,
Priority: PolicyRunPriority.Normal,
PriorityRank: -1,
RunId: "run:P-7:2025",
RequestedBy: "user:cli",
CorrelationId: "corr-1",
Metadata: metadata,
Inputs: resolvedInputs,
QueuedAt: DateTimeOffset.Parse("2025-10-28T09:59:00Z"),
Status: status,
AttemptCount: attemptCount,
LastAttemptAt: null,
LastError: null,
CreatedAt: DateTimeOffset.Parse("2025-10-28T09:58:00Z"),
UpdatedAt: DateTimeOffset.Parse("2025-10-28T09:58:00Z"),
AvailableAt: DateTimeOffset.Parse("2025-10-28T09:59:00Z"),
SubmittedAt: null,
CompletedAt: null,
LeaseOwner: null,
LeaseExpiresAt: null,
CancellationRequested: false,
CancellationRequestedAt: null,
CancellationReason: null,
CancelledAt: null);
}
private static SchedulerWorkerOptions CloneOptions()
{
return new SchedulerWorkerOptions
{
Policy = new SchedulerWorkerOptions.PolicyOptions
{
Enabled = WorkerOptions.Policy.Enabled,
Dispatch = new SchedulerWorkerOptions.PolicyOptions.DispatchOptions
{
LeaseOwner = WorkerOptions.Policy.Dispatch.LeaseOwner,
BatchSize = WorkerOptions.Policy.Dispatch.BatchSize,
LeaseDuration = WorkerOptions.Policy.Dispatch.LeaseDuration,
IdleDelay = WorkerOptions.Policy.Dispatch.IdleDelay,
MaxAttempts = WorkerOptions.Policy.Dispatch.MaxAttempts,
RetryBackoff = WorkerOptions.Policy.Dispatch.RetryBackoff
},
Api = new SchedulerWorkerOptions.PolicyOptions.ApiOptions
{
BaseAddress = WorkerOptions.Policy.Api.BaseAddress,
RunsPath = WorkerOptions.Policy.Api.RunsPath,
SimulatePath = WorkerOptions.Policy.Api.SimulatePath,
TenantHeader = WorkerOptions.Policy.Api.TenantHeader,
IdempotencyHeader = WorkerOptions.Policy.Api.IdempotencyHeader,
RequestTimeout = WorkerOptions.Policy.Api.RequestTimeout
},
}
private static PolicyRunJob CreateJob(PolicyRunJobStatus status, int attemptCount = 0, PolicyRunInputs? inputs = null)
{
var resolvedInputs = inputs ?? new PolicyRunInputs(sbomSet: new[] { "sbom:S-42" }, captureExplain: true);
var metadata = ImmutableSortedDictionary.Create<string, string>(StringComparer.Ordinal);
return new PolicyRunJob(
SchemaVersion: SchedulerSchemaVersions.PolicyRunJob,
Id: "job_1",
TenantId: "tenant-alpha",
PolicyId: "P-7",
PolicyVersion: 4,
Mode: PolicyRunMode.Incremental,
Priority: PolicyRunPriority.Normal,
PriorityRank: -1,
RunId: "run:P-7:2025",
RequestedBy: "user:cli",
CorrelationId: "corr-1",
Metadata: metadata,
Inputs: resolvedInputs,
QueuedAt: DateTimeOffset.Parse("2025-10-28T09:59:00Z"),
Status: status,
AttemptCount: attemptCount,
LastAttemptAt: null,
LastError: null,
CreatedAt: DateTimeOffset.Parse("2025-10-28T09:58:00Z"),
UpdatedAt: DateTimeOffset.Parse("2025-10-28T09:58:00Z"),
AvailableAt: DateTimeOffset.Parse("2025-10-28T09:59:00Z"),
SubmittedAt: null,
CompletedAt: null,
LeaseOwner: null,
LeaseExpiresAt: null,
CancellationRequested: false,
CancellationRequestedAt: null,
CancellationReason: null,
CancelledAt: null);
}
private static SchedulerWorkerOptions CloneOptions()
{
return new SchedulerWorkerOptions
{
Policy = new SchedulerWorkerOptions.PolicyOptions
{
Enabled = WorkerOptions.Policy.Enabled,
Dispatch = new SchedulerWorkerOptions.PolicyOptions.DispatchOptions
{
LeaseOwner = WorkerOptions.Policy.Dispatch.LeaseOwner,
BatchSize = WorkerOptions.Policy.Dispatch.BatchSize,
LeaseDuration = WorkerOptions.Policy.Dispatch.LeaseDuration,
IdleDelay = WorkerOptions.Policy.Dispatch.IdleDelay,
MaxAttempts = WorkerOptions.Policy.Dispatch.MaxAttempts,
RetryBackoff = WorkerOptions.Policy.Dispatch.RetryBackoff
},
Api = new SchedulerWorkerOptions.PolicyOptions.ApiOptions
{
BaseAddress = WorkerOptions.Policy.Api.BaseAddress,
RunsPath = WorkerOptions.Policy.Api.RunsPath,
SimulatePath = WorkerOptions.Policy.Api.SimulatePath,
TenantHeader = WorkerOptions.Policy.Api.TenantHeader,
IdempotencyHeader = WorkerOptions.Policy.Api.IdempotencyHeader,
RequestTimeout = WorkerOptions.Policy.Api.RequestTimeout
},
Targeting = new SchedulerWorkerOptions.PolicyOptions.TargetingOptions
{
Enabled = WorkerOptions.Policy.Targeting.Enabled,
@@ -284,15 +284,15 @@ public sealed class PolicyRunExecutionServiceTests
}
};
}
private sealed class StubPolicyRunTargetingService : IPolicyRunTargetingService
{
public Func<PolicyRunJob, PolicyRunTargetingResult>? OnEnsureTargets { get; set; }
public Task<PolicyRunTargetingResult> EnsureTargetsAsync(PolicyRunJob job, CancellationToken cancellationToken)
=> Task.FromResult(OnEnsureTargets?.Invoke(job) ?? PolicyRunTargetingResult.Unchanged(job));
}
private sealed class StubPolicyRunTargetingService : IPolicyRunTargetingService
{
public Func<PolicyRunJob, PolicyRunTargetingResult>? OnEnsureTargets { get; set; }
public Task<PolicyRunTargetingResult> EnsureTargetsAsync(PolicyRunJob job, CancellationToken cancellationToken)
=> Task.FromResult(OnEnsureTargets?.Invoke(job) ?? PolicyRunTargetingResult.Unchanged(job));
}
private sealed class RecordingPolicySimulationWebhookClient : IPolicySimulationWebhookClient
{
public List<PolicySimulationWebhookPayload> Payloads { get; } = new();
@@ -306,13 +306,13 @@ public sealed class PolicyRunExecutionServiceTests
private sealed class RecordingPolicyRunJobRepository : IPolicyRunJobRepository
{
public bool ReplaceCalled { get; private set; }
public string? ExpectedLeaseOwner { get; private set; }
public PolicyRunJob? LastJob { get; private set; }
public Task<PolicyRunJob?> GetAsync(string tenantId, string jobId, IClientSessionHandle? session = null, CancellationToken cancellationToken = default)
=> Task.FromResult<PolicyRunJob?>(null);
public bool ReplaceCalled { get; private set; }
public string? ExpectedLeaseOwner { get; private set; }
public PolicyRunJob? LastJob { get; private set; }
public Task<PolicyRunJob?> GetAsync(string tenantId, string jobId, IClientSessionHandle? session = null, CancellationToken cancellationToken = default)
=> Task.FromResult<PolicyRunJob?>(null);
public Task<PolicyRunJob?> GetByRunIdAsync(string tenantId, string runId, IClientSessionHandle? session = null, CancellationToken cancellationToken = default)
=> Task.FromResult<PolicyRunJob?>(null);
@@ -327,38 +327,38 @@ public sealed class PolicyRunExecutionServiceTests
public Task<PolicyRunJob?> LeaseAsync(string leaseOwner, DateTimeOffset now, TimeSpan leaseDuration, int maxAttempts, IClientSessionHandle? session = null, CancellationToken cancellationToken = default)
=> Task.FromResult<PolicyRunJob?>(null);
public Task<bool> ReplaceAsync(PolicyRunJob job, string? expectedLeaseOwner = null, IClientSessionHandle? session = null, CancellationToken cancellationToken = default)
{
ReplaceCalled = true;
ExpectedLeaseOwner = expectedLeaseOwner;
LastJob = job;
return Task.FromResult(true);
}
public Task<IReadOnlyList<PolicyRunJob>> ListAsync(string tenantId, string? policyId = null, PolicyRunMode? mode = null, IReadOnlyCollection<PolicyRunJobStatus>? statuses = null, DateTimeOffset? queuedAfter = null, int limit = 50, IClientSessionHandle? session = null, CancellationToken cancellationToken = default)
=> Task.FromResult<IReadOnlyList<PolicyRunJob>>(Array.Empty<PolicyRunJob>());
}
private sealed class StubPolicyRunClient : IPolicyRunClient
{
public PolicyRunSubmissionResult Result { get; set; } = PolicyRunSubmissionResult.Succeeded(null, null);
public Task<PolicyRunSubmissionResult> SubmitAsync(PolicyRunJob job, PolicyRunRequest request, CancellationToken cancellationToken)
=> Task.FromResult(Result);
}
private sealed class TestTimeProvider : TimeProvider
{
private DateTimeOffset _now;
public TestTimeProvider(DateTimeOffset now)
{
_now = now;
}
public override DateTimeOffset GetUtcNow() => _now;
public void Advance(TimeSpan delta) => _now = _now.Add(delta);
}
}
public Task<bool> ReplaceAsync(PolicyRunJob job, string? expectedLeaseOwner = null, IClientSessionHandle? session = null, CancellationToken cancellationToken = default)
{
ReplaceCalled = true;
ExpectedLeaseOwner = expectedLeaseOwner;
LastJob = job;
return Task.FromResult(true);
}
public Task<IReadOnlyList<PolicyRunJob>> ListAsync(string tenantId, string? policyId = null, PolicyRunMode? mode = null, IReadOnlyCollection<PolicyRunJobStatus>? statuses = null, DateTimeOffset? queuedAfter = null, int limit = 50, IClientSessionHandle? session = null, CancellationToken cancellationToken = default)
=> Task.FromResult<IReadOnlyList<PolicyRunJob>>(Array.Empty<PolicyRunJob>());
}
private sealed class StubPolicyRunClient : IPolicyRunClient
{
public PolicyRunSubmissionResult Result { get; set; } = PolicyRunSubmissionResult.Succeeded(null, null);
public Task<PolicyRunSubmissionResult> SubmitAsync(PolicyRunJob job, PolicyRunRequest request, CancellationToken cancellationToken)
=> Task.FromResult(Result);
}
private sealed class TestTimeProvider : TimeProvider
{
private DateTimeOffset _now;
public TestTimeProvider(DateTimeOffset now)
{
_now = now;
}
public override DateTimeOffset GetUtcNow() => _now;
public void Advance(TimeSpan delta) => _now = _now.Add(delta);
}
}

View File

@@ -9,9 +9,9 @@ using Microsoft.Extensions.Logging.Abstractions;
using MongoDB.Driver;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Queue;
using StellaOps.Scheduler.Storage.Mongo.Repositories;
using StellaOps.Scheduler.Storage.Mongo.Services;
using StellaOps.Scheduler.Storage.Mongo.Projections;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Repositories;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Services;
using StellaOps.Scheduler.Storage.Postgres.Repositories.Projections;
using StellaOps.Scheduler.Worker.Events;
using StellaOps.Scheduler.Worker.Execution;
using StellaOps.Scheduler.Worker.Observability;