feat: Document completed tasks for KMS, Cryptography, and Plugin Libraries
- Added detailed task completion records for KMS interface implementation and CLI support for file-based keys. - Documented security enhancements including Argon2id password hashing, audit event contracts, and rate limiting configurations. - Included scoped service support and integration updates for the Plugin platform, ensuring proper DI handling and testing coverage.
This commit is contained in:
master
@@ -1,41 +1,185 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scheduler.WebService.Options;
|
||||
|
||||
namespace StellaOps.Scheduler.WebService.GraphJobs.Events;
|
||||
|
||||
internal sealed class GraphJobEventPublisher : IGraphJobCompletionPublisher
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
|
||||
};
|
||||
|
||||
private readonly IOptionsMonitor<SchedulerEventsOptions> _options;
|
||||
private readonly ILogger<GraphJobEventPublisher> _logger;
|
||||
|
||||
public GraphJobEventPublisher(
|
||||
IOptionsMonitor<SchedulerEventsOptions> options,
|
||||
ILogger<GraphJobEventPublisher> logger)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public Task PublishAsync(GraphJobCompletionNotification notification, CancellationToken cancellationToken)
|
||||
{
|
||||
var options = _options.CurrentValue;
|
||||
if (!options.GraphJobs.Enabled)
|
||||
{
|
||||
_logger.LogDebug("Graph job events disabled; skipping emission for {JobId}.", notification.Job.Id);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
var envelope = GraphJobEventFactory.Create(notification);
|
||||
var json = JsonSerializer.Serialize(envelope, SerializerOptions);
|
||||
_logger.LogInformation("{EventJson}", json);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StackExchange.Redis;
|
||||
using StellaOps.Scheduler.WebService.Options;
|
||||
|
||||
namespace StellaOps.Scheduler.WebService.GraphJobs.Events;
|
||||
|
||||
internal sealed class GraphJobEventPublisher : IGraphJobCompletionPublisher, IAsyncDisposable
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
|
||||
};
|
||||
|
||||
private readonly IOptionsMonitor<SchedulerEventsOptions> _options;
|
||||
private readonly IRedisConnectionFactory _connectionFactory;
|
||||
private readonly ILogger<GraphJobEventPublisher> _logger;
|
||||
private readonly SemaphoreSlim _connectionGate = new(1, 1);
|
||||
|
||||
private IConnectionMultiplexer? _connection;
|
||||
private bool _disposed;
|
||||
|
||||
public GraphJobEventPublisher(
|
||||
IOptionsMonitor<SchedulerEventsOptions> options,
|
||||
IRedisConnectionFactory connectionFactory,
|
||||
ILogger<GraphJobEventPublisher> logger)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task PublishAsync(GraphJobCompletionNotification notification, CancellationToken cancellationToken)
|
||||
{
|
||||
if (notification is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(notification));
|
||||
}
|
||||
|
||||
var options = _options.CurrentValue?.GraphJobs ?? new GraphJobEventsOptions();
|
||||
if (!options.Enabled)
|
||||
{
|
||||
_logger.LogDebug("Graph job events disabled; skipping emission for {JobId}.", notification.Job.Id);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!string.Equals(options.Driver, "redis", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Graph job events configured with unsupported driver '{Driver}'. Falling back to logging.",
|
||||
options.Driver);
|
||||
LogEnvelope(notification);
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var database = await GetDatabaseAsync(options, cancellationToken).ConfigureAwait(false);
|
||||
var envelope = GraphJobEventFactory.Create(notification);
|
||||
var payload = JsonSerializer.Serialize(envelope, SerializerOptions);
|
||||
var entries = new[]
|
||||
{
|
||||
new NameValueEntry("event", payload),
|
||||
new NameValueEntry("kind", envelope.Kind),
|
||||
new NameValueEntry("tenant", envelope.Tenant),
|
||||
new NameValueEntry("occurredAt", envelope.Timestamp.ToString("O")),
|
||||
new NameValueEntry("jobId", notification.Job.Id),
|
||||
new NameValueEntry("status", notification.Status.ToString())
|
||||
};
|
||||
|
||||
var streamKey = string.IsNullOrWhiteSpace(options.Stream) ? "stella.events" : options.Stream;
|
||||
var publishTask = CreatePublishTask(database, streamKey, entries, options.MaxStreamLength);
|
||||
|
||||
if (options.PublishTimeoutSeconds > 0)
|
||||
{
|
||||
var timeout = TimeSpan.FromSeconds(options.PublishTimeoutSeconds);
|
||||
await publishTask.WaitAsync(timeout, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await publishTask.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
_logger.LogDebug("Published graph job event {JobId} to stream {Stream}.", notification.Job.Id, streamKey);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to publish graph job completion for {JobId}; logging payload instead.", notification.Job.Id);
|
||||
LogEnvelope(notification);
|
||||
}
|
||||
}
|
||||
|
||||
private Task<RedisValue> CreatePublishTask(IDatabase database, string streamKey, NameValueEntry[] entries, long maxStreamLength)
|
||||
{
|
||||
if (maxStreamLength > 0)
|
||||
{
|
||||
var clamped = (int)Math.Min(maxStreamLength, int.MaxValue);
|
||||
return database.StreamAddAsync(streamKey, entries, maxLength: clamped, useApproximateMaxLength: true);
|
||||
}
|
||||
|
||||
return database.StreamAddAsync(streamKey, entries);
|
||||
}
|
||||
|
||||
private async Task<IDatabase> GetDatabaseAsync(GraphJobEventsOptions options, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (_connection is { IsConnected: true })
|
||||
{
|
||||
return _connection.GetDatabase();
|
||||
}
|
||||
|
||||
await _connectionGate.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_connection is null || !_connection.IsConnected)
|
||||
{
|
||||
var configuration = ConfigurationOptions.Parse(options.Dsn);
|
||||
configuration.AbortOnConnectFail = false;
|
||||
|
||||
if (options.DriverSettings.TryGetValue("clientName", out var clientName) && !string.IsNullOrWhiteSpace(clientName))
|
||||
{
|
||||
configuration.ClientName = clientName;
|
||||
}
|
||||
|
||||
if (options.DriverSettings.TryGetValue("ssl", out var sslValue) && bool.TryParse(sslValue, out var ssl))
|
||||
{
|
||||
configuration.Ssl = ssl;
|
||||
}
|
||||
|
||||
if (options.DriverSettings.TryGetValue("password", out var password) && !string.IsNullOrWhiteSpace(password))
|
||||
{
|
||||
configuration.Password = password;
|
||||
}
|
||||
|
||||
_connection = await _connectionFactory.ConnectAsync(configuration, cancellationToken).ConfigureAwait(false);
|
||||
_logger.LogInformation("Connected graph job publisher to Redis stream {Stream}.", options.Stream);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_connectionGate.Release();
|
||||
}
|
||||
|
||||
return _connection!.GetDatabase();
|
||||
}
|
||||
|
||||
private void LogEnvelope(GraphJobCompletionNotification notification)
|
||||
{
|
||||
var envelope = GraphJobEventFactory.Create(notification);
|
||||
var json = JsonSerializer.Serialize(envelope, SerializerOptions);
|
||||
_logger.LogInformation("{EventJson}", json);
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_disposed = true;
|
||||
|
||||
if (_connection is not null)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _connection.CloseAsync();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "Error while closing graph job Redis connection.");
|
||||
}
|
||||
|
||||
_connection.Dispose();
|
||||
}
|
||||
|
||||
_connectionGate.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
using StackExchange.Redis;
|
||||
|
||||
namespace StellaOps.Scheduler.WebService.GraphJobs.Events;
|
||||
|
||||
internal interface IRedisConnectionFactory
|
||||
{
|
||||
Task<IConnectionMultiplexer> ConnectAsync(ConfigurationOptions options, CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
using StackExchange.Redis;
|
||||
|
||||
namespace StellaOps.Scheduler.WebService.GraphJobs.Events;
|
||||
|
||||
internal sealed class RedisConnectionFactory : IRedisConnectionFactory
|
||||
{
|
||||
public async Task<IConnectionMultiplexer> ConnectAsync(ConfigurationOptions options, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
var completionSource = new TaskCompletionSource<IConnectionMultiplexer>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
cancellationToken.Register(() => completionSource.TrySetCanceled(cancellationToken));
|
||||
|
||||
try
|
||||
{
|
||||
var connection = await ConnectionMultiplexer.ConnectAsync(options).ConfigureAwait(false);
|
||||
completionSource.TrySetResult(connection);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
completionSource.TrySetException(ex);
|
||||
}
|
||||
|
||||
return await completionSource.Task.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,8 @@ namespace StellaOps.Scheduler.WebService.Options;
|
||||
/// <summary>
|
||||
/// Scheduler WebService event options (outbound + inbound).
|
||||
/// </summary>
|
||||
using System;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
public sealed class SchedulerEventsOptions
|
||||
{
|
||||
@@ -12,13 +13,43 @@ public sealed class SchedulerEventsOptions
|
||||
public SchedulerInboundWebhooksOptions Webhooks { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class GraphJobEventsOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Enables emission of legacy <c>scheduler.graph.job.completed@1</c> events.
|
||||
/// </summary>
|
||||
public bool Enabled { get; set; }
|
||||
}
|
||||
public sealed class GraphJobEventsOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Enables emission of legacy <c>scheduler.graph.job.completed@1</c> events.
|
||||
/// </summary>
|
||||
public bool Enabled { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Event transport driver (defaults to <c>redis</c>).
|
||||
/// </summary>
|
||||
public string Driver { get; set; } = "redis";
|
||||
|
||||
/// <summary>
|
||||
/// Connection string for the event transport.
|
||||
/// </summary>
|
||||
public string Dsn { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Stream/topic identifier for published events.
|
||||
/// </summary>
|
||||
public string Stream { get; set; } = "stella.events";
|
||||
|
||||
/// <summary>
|
||||
/// Maximum time in seconds to wait for the transport to accept an event.
|
||||
/// </summary>
|
||||
public double PublishTimeoutSeconds { get; set; } = 5;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of events to retain in the stream.
|
||||
/// </summary>
|
||||
public long MaxStreamLength { get; set; } = 10000;
|
||||
|
||||
/// <summary>
|
||||
/// Additional transport-specific settings (e.g., <c>clientName</c>, <c>ssl</c>).
|
||||
/// </summary>
|
||||
public IDictionary<string, string> DriverSettings { get; set; } = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public sealed class SchedulerInboundWebhooksOptions
|
||||
{
|
||||
|
||||
@@ -48,8 +48,8 @@ if (authorityOptions.Audiences.Count == 0)
|
||||
authorityOptions.Validate();
|
||||
builder.Services.AddSingleton(authorityOptions);
|
||||
|
||||
builder.Services.AddOptions<SchedulerEventsOptions>()
|
||||
.Bind(builder.Configuration.GetSection("Scheduler:Events"))
|
||||
builder.Services.AddOptions<SchedulerEventsOptions>()
|
||||
.Bind(builder.Configuration.GetSection("Scheduler:Events"))
|
||||
.PostConfigure(options =>
|
||||
{
|
||||
options.Webhooks ??= new SchedulerInboundWebhooksOptions();
|
||||
@@ -67,10 +67,11 @@ builder.Services.AddOptions<SchedulerEventsOptions>()
|
||||
options.Webhooks.Vexer.Validate();
|
||||
});
|
||||
|
||||
builder.Services.AddMemoryCache();
|
||||
builder.Services.AddSingleton<IWebhookRateLimiter, InMemoryWebhookRateLimiter>();
|
||||
builder.Services.AddSingleton<IWebhookRequestAuthenticator, WebhookRequestAuthenticator>();
|
||||
builder.Services.AddSingleton<IInboundExportEventSink, LoggingExportEventSink>();
|
||||
builder.Services.AddMemoryCache();
|
||||
builder.Services.AddSingleton<IWebhookRateLimiter, InMemoryWebhookRateLimiter>();
|
||||
builder.Services.AddSingleton<IWebhookRequestAuthenticator, WebhookRequestAuthenticator>();
|
||||
builder.Services.AddSingleton<IInboundExportEventSink, LoggingExportEventSink>();
|
||||
builder.Services.AddSingleton<IRedisConnectionFactory, RedisConnectionFactory>();
|
||||
|
||||
var cartographerOptions = builder.Configuration.GetSection("Scheduler:Cartographer").Get<SchedulerCartographerOptions>() ?? new SchedulerCartographerOptions();
|
||||
builder.Services.AddSingleton(cartographerOptions);
|
||||
|
||||
@@ -13,4 +13,7 @@
|
||||
<ProjectReference Include="../../Authority/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOps.Auth.Abstractions.csproj" />
|
||||
<ProjectReference Include="../../Authority/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOps.Auth.ServerIntegration.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="StackExchange.Redis" Version="2.8.24" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
# Completed Tasks
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| SCHED-WEB-16-101 | DONE (2025-10-27) | Scheduler WebService Guild | SCHED-MODELS-16-101 | Bootstrap Minimal API host with Authority OpTok + DPoP, health endpoints, plug-in discovery per architecture §§1–2. | Service boots with config validation; `/healthz`/`/readyz` pass; restart-only plug-ins enforced. |
|
||||
| SCHED-WEB-16-102 | DONE (2025-10-26) | Scheduler WebService Guild | SCHED-WEB-16-101 | Implement schedules CRUD (tenant-scoped) with cron validation, pause/resume, audit logging. | CRUD operations tested; invalid cron inputs rejected; audit entries persisted. |
|
||||
| SCHED-WEB-16-103 | DONE (2025-10-26) | Scheduler WebService Guild | SCHED-WEB-16-102 | Runs API (list/detail/cancel), ad-hoc run POST, and impact preview endpoints. | Integration tests cover run lifecycle; preview returns counts/sample; cancellation honoured. |
|
||||
| SCHED-WEB-16-104 | DONE (2025-10-27) | Scheduler WebService Guild | SCHED-QUEUE-16-401, SCHED-STORAGE-16-201 | Webhook endpoints for Feeder/Vexer exports with mTLS/HMAC validation and rate limiting. | Webhooks validated via tests; invalid signatures rejected; rate limits documented. |
|
||||
| SCHED-WEB-20-001 | DONE (2025-10-29) | Scheduler WebService Guild, Policy Guild | SCHED-WEB-16-101, POLICY-ENGINE-20-000 | Expose policy run scheduling APIs (`POST /policy/runs`, `GET /policy/runs`) with tenant scoping and RBAC enforcement for `policy:run`. | Endpoints documented; integration tests cover run creation/status; unauthorized access blocked. |
|
||||
| SCHED-WEB-21-001 | DONE (2025-10-26) | Scheduler WebService Guild, Cartographer Guild | SCHED-WEB-16-101, SCHED-MODELS-21-001 | Expose graph build/overlay job APIs (`POST /graphs/build`, `GET /graphs/jobs`) with `graph:write`/`graph:read` enforcement and tenant scoping. | APIs documented in `docs/SCHED-WEB-21-001-GRAPH-APIS.md`; integration tests cover submission/status; unauthorized requests blocked; scope checks now reference `StellaOpsScopes`. |
|
||||
| SCHED-WEB-21-002 | DONE (2025-10-26) | Scheduler WebService Guild | SCHED-WEB-21-001, CARTO-GRAPH-21-007 | Provide overlay lag metrics endpoint and webhook to notify Cartographer of job completions; include correlation IDs. | `POST /graphs/hooks/completed` + `GET /graphs/overlays/lag` documented in `docs/SCHED-WEB-21-001-GRAPH-APIS.md`; integration tests cover completion + metrics. |
|
||||
| SCHED-WEB-21-003 | DONE (2025-10-26) | Scheduler WebService Guild, Authority Core Guild | AUTH-GRAPH-21-001 | Replace temporary `X-Scopes`/`X-Tenant-Id` headers with Authority-issued OpTok verification and scope enforcement for graph endpoints. | Authentication configured via `AddStellaOpsResourceServerAuthentication`; authority scopes enforced end-to-end with `StellaOpsScopes`; header fallback limited to dev mode; tests updated. |
|
||||
@@ -2,16 +2,11 @@
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| SCHED-WEB-16-101 | DONE (2025-10-27) | Scheduler WebService Guild | SCHED-MODELS-16-101 | Bootstrap Minimal API host with Authority OpTok + DPoP, health endpoints, plug-in discovery per architecture §§1–2. | Service boots with config validation; `/healthz`/`/readyz` pass; restart-only plug-ins enforced. |
|
||||
| SCHED-WEB-16-102 | DONE (2025-10-26) | Scheduler WebService Guild | SCHED-WEB-16-101 | Implement schedules CRUD (tenant-scoped) with cron validation, pause/resume, audit logging. | CRUD operations tested; invalid cron inputs rejected; audit entries persisted. |
|
||||
| SCHED-WEB-16-103 | DONE (2025-10-26) | Scheduler WebService Guild | SCHED-WEB-16-102 | Runs API (list/detail/cancel), ad-hoc run POST, and impact preview endpoints. | Integration tests cover run lifecycle; preview returns counts/sample; cancellation honoured. |
|
||||
| SCHED-WEB-16-104 | DONE (2025-10-27) | Scheduler WebService Guild | SCHED-QUEUE-16-401, SCHED-STORAGE-16-201 | Webhook endpoints for Feeder/Vexer exports with mTLS/HMAC validation and rate limiting. | Webhooks validated via tests; invalid signatures rejected; rate limits documented. |
|
||||
|
||||
## Policy Engine v2 (Sprint 20)
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| SCHED-WEB-20-001 | DONE (2025-10-29) | Scheduler WebService Guild, Policy Guild | SCHED-WEB-16-101, POLICY-ENGINE-20-000 | Expose policy run scheduling APIs (`POST /policy/runs`, `GET /policy/runs`) with tenant scoping and RBAC enforcement for `policy:run`. | Endpoints documented; integration tests cover run creation/status; unauthorized access blocked. |
|
||||
> 2025-10-29: Added `/api/v1/scheduler/policy/runs` create/list/get endpoints with in-memory queue, scope/tenant enforcement, and contract docs (`docs/SCHED-WEB-20-001-POLICY-RUNS.md`). Tests cover happy path + auth failures.
|
||||
> 2025-10-26: Use canonical request/response samples from `samples/api/scheduler/policy-*.json`; serializer contract defined in `src/Scheduler/__Libraries/StellaOps.Scheduler.Models/docs/SCHED-MODELS-20-001-POLICY-RUNS.md`.
|
||||
| SCHED-WEB-20-002 | BLOCKED (waiting on SCHED-WORKER-20-301) | Scheduler WebService Guild | SCHED-WEB-20-001, SCHED-WORKER-20-301 | Provide simulation trigger endpoint returning diff preview metadata and job state for UI/CLI consumption. | Simulation endpoint returns deterministic diffs metadata; rate limits enforced; tests cover concurrency. |
|
||||
@@ -21,10 +16,8 @@
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| SCHED-WEB-21-001 | DONE (2025-10-26) | Scheduler WebService Guild, Cartographer Guild | SCHED-WEB-16-101, SCHED-MODELS-21-001 | Expose graph build/overlay job APIs (`POST /graphs/build`, `GET /graphs/jobs`) with `graph:write`/`graph:read` enforcement and tenant scoping. | APIs documented in `docs/SCHED-WEB-21-001-GRAPH-APIS.md`; integration tests cover submission/status; unauthorized requests blocked; scope checks now reference `StellaOpsScopes`. |
|
||||
| SCHED-WEB-21-002 | DONE (2025-10-26) | Scheduler WebService Guild | SCHED-WEB-21-001, CARTO-GRAPH-21-007 | Provide overlay lag metrics endpoint and webhook to notify Cartographer of job completions; include correlation IDs. | `POST /graphs/hooks/completed` + `GET /graphs/overlays/lag` documented in `docs/SCHED-WEB-21-001-GRAPH-APIS.md`; integration tests cover completion + metrics. |
|
||||
| SCHED-WEB-21-003 | DONE (2025-10-26) | Scheduler WebService Guild, Authority Core Guild | AUTH-GRAPH-21-001 | Replace temporary `X-Scopes`/`X-Tenant-Id` headers with Authority-issued OpTok verification and scope enforcement for graph endpoints. | Authentication configured via `AddStellaOpsResourceServerAuthentication`; authority scopes enforced end-to-end with `StellaOpsScopes`; header fallback limited to dev mode; tests updated. |
|
||||
| SCHED-WEB-21-004 | DOING (2025-10-26) | Scheduler WebService Guild, Scheduler Storage Guild | SCHED-WEB-21-001, SCHED-STORAGE-16-201 | Persist graph job lifecycle to Mongo storage and publish `scheduler.graph.job.completed@1` events + outbound webhook to Cartographer. | Storage repositories updated; events emitted; webhook payload documented; integration tests cover storage + event flow. **Note:** Events currently log JSON envelopes while the shared platform bus is provisioned. Cartographer webhook now posts JSON payloads when configured; replace inline logging with bus publisher once the shared event transport is online. |
|
||||
> 2025-10-30: Implemented Redis-backed publisher (`Scheduler:Events:GraphJobs`) emitting `scheduler.graph.job.completed@1` to configured stream with optional logging fallback; docs/configs to be validated with DevOps before closing.
|
||||
|
||||
## StellaOps Console (Sprint 23)
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
# Completed Tasks
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| SCHED-IMPACT-16-300 | DONE (2025-10-20) | Scheduler ImpactIndex Guild | SAMPLES-10-001 | **STUB** ingest/query using fixtures to unblock Scheduler planning (remove by SP16 end). | Stub merges fixture BOM-Index, query API returns deterministic results, removal note tracked. |
|
||||
| SCHED-IMPACT-16-301 | DONE (2025-10-26) | Scheduler ImpactIndex Guild | SCANNER-EMIT-10-605 | Implement ingestion of per-image BOM-Index sidecars into roaring bitmap store (contains/usedBy). | Ingestion tests process sample SBOM index; bitmaps persisted; deterministic IDs assigned. |
|
||||
| SCHED-IMPACT-16-302 | DONE (2025-10-26) | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-301 | Provide query APIs (ResolveByPurls, ResolveByVulns, ResolveAll, selectors) with tenant/namespace filters. | Query functions tested; performance benchmarks documented; selectors enforce filters. |
|
||||
@@ -2,9 +2,6 @@
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| SCHED-IMPACT-16-300 | DONE (2025-10-20) | Scheduler ImpactIndex Guild | SAMPLES-10-001 | **STUB** ingest/query using fixtures to unblock Scheduler planning (remove by SP16 end). | Stub merges fixture BOM-Index, query API returns deterministic results, removal note tracked. |
|
||||
| SCHED-IMPACT-16-301 | DONE (2025-10-26) | Scheduler ImpactIndex Guild | SCANNER-EMIT-10-605 | Implement ingestion of per-image BOM-Index sidecars into roaring bitmap store (contains/usedBy). | Ingestion tests process sample SBOM index; bitmaps persisted; deterministic IDs assigned. |
|
||||
| SCHED-IMPACT-16-302 | DONE (2025-10-26) | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-301 | Provide query APIs (ResolveByPurls, ResolveByVulns, ResolveAll, selectors) with tenant/namespace filters. | Query functions tested; performance benchmarks documented; selectors enforce filters. |
|
||||
| SCHED-IMPACT-16-303 | TODO | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-301 | Snapshot/compaction + invalidation for removed images; persistence to RocksDB/Redis per architecture. | Snapshot routine implemented; invalidation tests pass; docs describe recovery. |
|
||||
|
||||
> Removal tracking note: see `src/Scheduler/__Libraries/StellaOps.Scheduler.ImpactIndex/REMOVAL_NOTE.md` for follow-up actions once the roaring bitmap implementation lands.
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
# Completed Tasks
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| SCHED-MODELS-16-101 | DONE (2025-10-19) | Scheduler Models Guild | — | Define DTOs (Schedule, Run, ImpactSet, Selector, DeltaSummary, AuditRecord) with validation + canonical JSON. | DTOs merged with tests; documentation snippet added; serialization deterministic. |
|
||||
| SCHED-MODELS-16-102 | DONE (2025-10-19) | Scheduler Models Guild | SCHED-MODELS-16-101 | Publish schema docs & sample payloads for UI/Notify integration. | Samples committed; docs referenced; contract tests pass. |
|
||||
| SCHED-MODELS-16-103 | DONE (2025-10-20) | Scheduler Models Guild | SCHED-MODELS-16-101 | Versioning/migration helpers (schedule evolution, run state transitions). | Migration helpers implemented; tests cover upgrade/downgrade; guidelines documented. |
|
||||
| SCHED-MODELS-20-001 | DONE (2025-10-26) | Scheduler Models Guild, Policy Guild | POLICY-ENGINE-20-000 | Define DTOs/schemas for policy runs, diffs, and explain traces (`PolicyRunRequest`, `PolicyRunStatus`, `PolicyDiffSummary`). | DTOs serialize deterministically; schema samples committed; validation helpers added. |
|
||||
| SCHED-MODELS-20-002 | DONE (2025-10-29) | Scheduler Models Guild | SCHED-MODELS-20-001 | Extend scheduler schema docs to include policy run lifecycle, environment metadata, and diff payloads. | Docs updated with compliance checklist; samples validated against JSON schema; consumers notified. |
|
||||
| SCHED-MODELS-21-001 | DONE (2025-10-26) | Scheduler Models Guild, Cartographer Guild | CARTO-GRAPH-21-007 | Define job DTOs for graph builds/overlay refresh (`GraphBuildJob`, `GraphOverlayJob`) with deterministic serialization and status enums. | DTOs serialized deterministically; schema snippets documented in `src/Scheduler/__Libraries/StellaOps.Scheduler.Models/docs/SCHED-MODELS-21-001-GRAPH-JOBS.md`; tests cover transitions. |
|
||||
| SCHED-MODELS-21-002 | DONE (2025-10-26) | Scheduler Models Guild | SCHED-MODELS-21-001 | Publish schema docs/sample payloads for graph jobs and overlay events for downstream workers/UI. | Docs updated with compliance checklist; samples validated; notifications sent to guilds. |
|
||||
@@ -2,21 +2,14 @@
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| SCHED-MODELS-16-101 | DONE (2025-10-19) | Scheduler Models Guild | — | Define DTOs (Schedule, Run, ImpactSet, Selector, DeltaSummary, AuditRecord) with validation + canonical JSON. | DTOs merged with tests; documentation snippet added; serialization deterministic. |
|
||||
| SCHED-MODELS-16-102 | DONE (2025-10-19) | Scheduler Models Guild | SCHED-MODELS-16-101 | Publish schema docs & sample payloads for UI/Notify integration. | Samples committed; docs referenced; contract tests pass. |
|
||||
| SCHED-MODELS-16-103 | DONE (2025-10-20) | Scheduler Models Guild | SCHED-MODELS-16-101 | Versioning/migration helpers (schedule evolution, run state transitions). | Migration helpers implemented; tests cover upgrade/downgrade; guidelines documented. |
|
||||
|
||||
## Policy Engine v2 (Sprint 20)
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| SCHED-MODELS-20-001 | DONE (2025-10-26) | Scheduler Models Guild, Policy Guild | POLICY-ENGINE-20-000 | Define DTOs/schemas for policy runs, diffs, and explain traces (`PolicyRunRequest`, `PolicyRunStatus`, `PolicyDiffSummary`). | DTOs serialize deterministically; schema samples committed; validation helpers added. |
|
||||
| SCHED-MODELS-20-002 | DONE (2025-10-29) | Scheduler Models Guild | SCHED-MODELS-20-001 | Extend scheduler schema docs to include policy run lifecycle, environment metadata, and diff payloads. | Docs updated with compliance checklist; samples validated against JSON schema; consumers notified. |
|
||||
> 2025-10-29: Added lifecycle table, environment metadata section, and diff payload breakdown to `SCHED-MODELS-20-001-POLICY-RUNS.md`; compliance checklist extended to cover new documentation.
|
||||
|
||||
## Graph Explorer v1 (Sprint 21)
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| SCHED-MODELS-21-001 | DONE (2025-10-26) | Scheduler Models Guild, Cartographer Guild | CARTO-GRAPH-21-007 | Define job DTOs for graph builds/overlay refresh (`GraphBuildJob`, `GraphOverlayJob`) with deterministic serialization and status enums. | DTOs serialized deterministically; schema snippets documented in `src/Scheduler/__Libraries/StellaOps.Scheduler.Models/docs/SCHED-MODELS-21-001-GRAPH-JOBS.md`; tests cover transitions. |
|
||||
| SCHED-MODELS-21-002 | DONE (2025-10-26) | Scheduler Models Guild | SCHED-MODELS-21-001 | Publish schema docs/sample payloads for graph jobs and overlay events for downstream workers/UI. | Docs updated with compliance checklist; samples validated; notifications sent to guilds. |
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
# Completed Tasks
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| SCHED-QUEUE-16-401 | DONE (2025-10-20) | Scheduler Queue Guild | SCHED-MODELS-16-101 | Implement queue abstraction + Redis Streams adapter (planner inputs, runner segments) with ack/lease semantics. | Integration tests cover enqueue/dequeue/ack; lease renewal implemented; ordering preserved. |
|
||||
| SCHED-QUEUE-16-402 | DONE (2025-10-20) | Scheduler Queue Guild | SCHED-QUEUE-16-401 | Add NATS JetStream adapter with configuration binding, health probes, failover. | Health endpoints verified; failover documented; adapter tested. |
|
||||
| SCHED-QUEUE-16-403 | DONE (2025-10-20) | Scheduler Queue Guild | SCHED-QUEUE-16-401 | Dead-letter handling + metrics (queue depth, retry counts), configuration toggles. | Dead-letter policy tested; metrics exported; docs updated. |
|
||||
@@ -1,9 +1,6 @@
|
||||
# Scheduler Queue Task Board (Sprint 16)
|
||||
|
||||
> **Status note (2025-10-19):** Scheduler DTOs and sample payloads are now available (SCHED-MODELS-16-102). Queue tasks remain pending on this board.
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| SCHED-QUEUE-16-401 | DONE (2025-10-20) | Scheduler Queue Guild | SCHED-MODELS-16-101 | Implement queue abstraction + Redis Streams adapter (planner inputs, runner segments) with ack/lease semantics. | Integration tests cover enqueue/dequeue/ack; lease renewal implemented; ordering preserved. |
|
||||
| SCHED-QUEUE-16-402 | DONE (2025-10-20) | Scheduler Queue Guild | SCHED-QUEUE-16-401 | Add NATS JetStream adapter with configuration binding, health probes, failover. | Health endpoints verified; failover documented; adapter tested. |
|
||||
| SCHED-QUEUE-16-403 | DONE (2025-10-20) | Scheduler Queue Guild | SCHED-QUEUE-16-401 | Dead-letter handling + metrics (queue depth, retry counts), configuration toggles. | Dead-letter policy tested; metrics exported; docs updated. |
|
||||
# Scheduler Queue Task Board (Sprint 16)
|
||||
|
||||
> **Status note (2025-10-19):** Scheduler DTOs and sample payloads are now available (SCHED-MODELS-16-102). Queue tasks remain pending on this board.
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
# Completed Tasks
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| SCHED-STORAGE-16-201 | DONE (2025-10-19) | Scheduler Storage Guild | SCHED-MODELS-16-101 | Create Mongo collections (schedules, runs, impact_cursors, locks, audit) with indexes/migrations per architecture. | Migration scripts and indexes implemented; integration tests cover CRUD paths. |
|
||||
| SCHED-STORAGE-16-202 | DONE (2025-10-26) | Scheduler Storage Guild | SCHED-STORAGE-16-201 | Implement repositories/services with tenant scoping, soft delete, TTL for completed runs, and causal consistency options. | Unit tests pass; TTL/soft delete validated; documentation updated. |
|
||||
| SCHED-STORAGE-16-203 | DONE (2025-10-26) | Scheduler Storage Guild | SCHED-STORAGE-16-201 | Audit/logging pipeline + run stats materialized views for UI. | Audit entries persisted; stats queries efficient; docs capture usage. |
|
||||
@@ -1,9 +1,6 @@
|
||||
# Scheduler Storage Task Board (Sprint 16)
|
||||
|
||||
> **Status note (2025-10-19):** Scheduler models/samples delivered in SCHED-MODELS-16-102. Tasks below remain pending for the Storage guild.
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| SCHED-STORAGE-16-201 | DONE (2025-10-19) | Scheduler Storage Guild | SCHED-MODELS-16-101 | Create Mongo collections (schedules, runs, impact_cursors, locks, audit) with indexes/migrations per architecture. | Migration scripts and indexes implemented; integration tests cover CRUD paths. |
|
||||
| SCHED-STORAGE-16-202 | DONE (2025-10-26) | Scheduler Storage Guild | SCHED-STORAGE-16-201 | Implement repositories/services with tenant scoping, soft delete, TTL for completed runs, and causal consistency options. | Unit tests pass; TTL/soft delete validated; documentation updated. |
|
||||
| SCHED-STORAGE-16-203 | DONE (2025-10-26) | Scheduler Storage Guild | SCHED-STORAGE-16-201 | Audit/logging pipeline + run stats materialized views for UI. | Audit entries persisted; stats queries efficient; docs capture usage. |
|
||||
# Scheduler Storage Task Board (Sprint 16)
|
||||
|
||||
> **Status note (2025-10-19):** Scheduler models/samples delivered in SCHED-MODELS-16-102. Tasks below remain pending for the Storage guild.
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
# Completed Tasks
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| SCHED-WORKER-16-201 | DONE (2025-10-28) | Scheduler Worker Guild | SCHED-QUEUE-16-401 | Planner loop (cron + event triggers) with lease management, fairness, and rate limiting (§6). | Planner integration tests cover cron/event triggers; rate limits enforced; logs include run IDs. |
|
||||
| SCHED-WORKER-16-202 | DONE (2025-10-27) | Scheduler Worker Guild | SCHED-IMPACT-16-301 | Wire ImpactIndex targeting (ResolveByPurls/vulns), dedupe, shard planning. | Targeting tests confirm correct image selection; dedupe documented; shards evenly distributed. |
|
||||
| SCHED-WORKER-16-203 | DONE (2025-10-27) | Scheduler Worker Guild | SCHED-WORKER-16-202 | Runner execution: call Scanner `/reports` (analysis-only) or `/scans` when configured; collect deltas; handle retries. | Runner tests stub Scanner; retries/backoff validated; deltas aggregated deterministically. |
|
||||
| SCHED-WORKER-16-204 | DONE (2025-10-27) | Scheduler Worker Guild | SCHED-WORKER-16-203 | Emit events (`scheduler.rescan.delta`, `scanner.report.ready`) for Notify/UI with summaries. | Events published to queue; payload schema documented; integration tests verify consumption. |
|
||||
| SCHED-WORKER-16-205 | DONE (2025-10-27) | Scheduler Worker Guild | SCHED-WORKER-16-201 | Metrics/telemetry: run stats, queue depth, planner latency, delta counts. | Metrics exported per spec; dashboards updated; alerts configured. |
|
||||
| SCHED-WORKER-20-301 | DONE (2025-10-28) | Scheduler Worker Guild, Policy Guild | SCHED-WORKER-16-201, POLICY-ENGINE-20-000 | Extend scheduler worker to trigger policy runs (full/incremental/simulate) via Policy Engine API, with idempotent job tracking and tenant scoping. | Worker schedules policy jobs deterministically; job records persisted; integration tests cover modes + cancellation. |
|
||||
| SCHED-WORKER-21-201 | DONE (2025-10-29) | Scheduler Worker Guild, Cartographer Guild | SCHED-MODELS-21-001 | Implement graph build worker that dequeues SBOM graph jobs, invokes Cartographer build APIs, and records status with retries/backoff. | Worker processes fixtures; retries handled; logs include `graph_id`; integration tests pass. |
|
||||
| SCHED-WORKER-20-302 | DONE (2025-10-29) | Scheduler Worker Guild | SCHED-WORKER-20-301, POLICY-ENGINE-20-006 | Implement policy delta targeting to re-evaluate only affected SBOM sets based on change streams and policy metadata. | Targeting reduces workload per design; tests simulate advisory/vex updates; metrics show delta counts. |
|
||||
| SCHED-WORKER-20-303 | DONE (2025-10-29) | Scheduler Worker Guild, Observability Guild | SCHED-WORKER-20-301 | Expose metrics (`policy_runs_scheduled`, `policy_runs_failed`, planner latency) and structured logs with policy/run identifiers. | Metrics registered; dashboards updated; logs validated in integration tests. |
|
||||
| SCHED-WORKER-21-202 | DONE (2025-10-29) | Scheduler Worker Guild | SCHED-WORKER-21-201, CARTO-GRAPH-21-007 | Overlay refresh worker subscribing to policy/SBOM change events, batching affected graph overlays, and enforcing <2 min SLA. | Overlay jobs scheduled deterministically; lag metrics < 2 min in tests; alerts configured. |
|
||||
@@ -2,11 +2,6 @@
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| SCHED-WORKER-16-201 | DONE (2025-10-28) | Scheduler Worker Guild | SCHED-QUEUE-16-401 | Planner loop (cron + event triggers) with lease management, fairness, and rate limiting (§6). | Planner integration tests cover cron/event triggers; rate limits enforced; logs include run IDs. |
|
||||
| SCHED-WORKER-16-202 | DONE (2025-10-27) | Scheduler Worker Guild | SCHED-IMPACT-16-301 | Wire ImpactIndex targeting (ResolveByPurls/vulns), dedupe, shard planning. | Targeting tests confirm correct image selection; dedupe documented; shards evenly distributed. |
|
||||
| SCHED-WORKER-16-203 | DONE (2025-10-27) | Scheduler Worker Guild | SCHED-WORKER-16-202 | Runner execution: call Scanner `/reports` (analysis-only) or `/scans` when configured; collect deltas; handle retries. | Runner tests stub Scanner; retries/backoff validated; deltas aggregated deterministically. |
|
||||
| SCHED-WORKER-16-204 | DONE (2025-10-27) | Scheduler Worker Guild | SCHED-WORKER-16-203 | Emit events (`scheduler.rescan.delta`, `scanner.report.ready`) for Notify/UI with summaries. | Events published to queue; payload schema documented; integration tests verify consumption. |
|
||||
| SCHED-WORKER-16-205 | DONE (2025-10-27) | Scheduler Worker Guild | SCHED-WORKER-16-201 | Metrics/telemetry: run stats, queue depth, planner latency, delta counts. | Metrics exported per spec; dashboards updated; alerts configured. |
|
||||
|
||||
> 2025-10-27: Impact targeting sanitizes selector-constrained results, dedupes digests, and documents shard planning in `docs/SCHED-WORKER-16-202-IMPACT-TARGETING.md`.
|
||||
|
||||
@@ -17,20 +12,15 @@
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| SCHED-WORKER-20-301 | DONE (2025-10-28) | Scheduler Worker Guild, Policy Guild | SCHED-WORKER-16-201, POLICY-ENGINE-20-000 | Extend scheduler worker to trigger policy runs (full/incremental/simulate) via Policy Engine API, with idempotent job tracking and tenant scoping. | Worker schedules policy jobs deterministically; job records persisted; integration tests cover modes + cancellation. |
|
||||
> 2025-10-26: Align worker serializers with `PolicyRunRequest/Status/DiffSummary` contracts from `src/Scheduler/__Libraries/StellaOps.Scheduler.Models`. Reference fixtures in `samples/api/scheduler/` for expected payload ordering.
|
||||
| SCHED-WORKER-20-302 | DONE (2025-10-29) | Scheduler Worker Guild | SCHED-WORKER-20-301, POLICY-ENGINE-20-006 | Implement policy delta targeting to re-evaluate only affected SBOM sets based on change streams and policy metadata. | Targeting reduces workload per design; tests simulate advisory/vex updates; metrics show delta counts. |
|
||||
> 2025-10-29: `PolicyRunTargetingService` translates change-stream metadata into SBOM sets, marks no-work jobs completed, and surfaces targeting options (`Policy.Targeting`). See `docs/SCHED-WORKER-20-302-POLICY-DELTA-TARGETING.md` for supported metadata keys and behaviour.
|
||||
| SCHED-WORKER-20-303 | DONE (2025-10-29) | Scheduler Worker Guild, Observability Guild | SCHED-WORKER-20-301 | Expose metrics (`policy_runs_scheduled`, `policy_runs_failed`, planner latency) and structured logs with policy/run identifiers. | Metrics registered; dashboards updated; logs validated in integration tests. |
|
||||
> 2025-10-29: Added `scheduler_policy_run_events_total` + latency histogram, instrumented policy dispatch success/retry/failure/cancel paths, and upgraded structured logs with tenant/policy/run identifiers. Docs updated in `docs/SCHED-WORKER-20-301-POLICY-RUNS.md` Observability section.
|
||||
|
||||
## Graph Explorer v1 (Sprint 21)
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| SCHED-WORKER-21-201 | DONE (2025-10-29) | Scheduler Worker Guild, Cartographer Guild | SCHED-MODELS-21-001 | Implement graph build worker that dequeues SBOM graph jobs, invokes Cartographer build APIs, and records status with retries/backoff. | Worker processes fixtures; retries handled; logs include `graph_id`; integration tests pass. |
|
||||
> 2025-10-29: Graph build worker background service + execution pipeline added. Cartographer + Scheduler API options documented in `docs/SCHED-WORKER-21-201-GRAPH-BUILD.md`; unit tests cover success/retry/failure paths.
|
||||
| SCHED-WORKER-21-202 | DONE (2025-10-29) | Scheduler Worker Guild | SCHED-WORKER-21-201, CARTO-GRAPH-21-007 | Overlay refresh worker subscribing to policy/SBOM change events, batching affected graph overlays, and enforcing <2 min SLA. | Overlay jobs scheduled deterministically; lag metrics < 2 min in tests; alerts configured. |
|
||||
> 2025-10-29: Overlay worker now polls pending jobs, posts to Cartographer overlay endpoint, and reports completion via Scheduler webhook. Config + behaviour documented in `docs/SCHED-WORKER-21-202-GRAPH-OVERLAY.md`.
|
||||
| SCHED-WORKER-21-203 | TODO | Scheduler Worker Guild, Observability Guild | SCHED-WORKER-21-201 | Export metrics (`graph_build_seconds`, `graph_jobs_inflight`, `overlay_lag_seconds`) and structured logs with tenant/graph identifiers. | Metrics/traces exposed; dashboards updated; integration tests verify metrics emission. |
|
||||
|
||||
|
||||
@@ -1,25 +1,30 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Auth.Abstractions;
|
||||
using StellaOps.Scheduler.Models;
|
||||
using StellaOps.Scheduler.WebService.GraphJobs;
|
||||
using StellaOps.Scheduler.WebService.GraphJobs.Events;
|
||||
using StellaOps.Scheduler.WebService.Options;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Auth.Abstractions;
|
||||
using StellaOps.Scheduler.Models;
|
||||
using StellaOps.Scheduler.WebService.GraphJobs;
|
||||
using StellaOps.Scheduler.WebService.GraphJobs.Events;
|
||||
using StellaOps.Scheduler.WebService.Options;
|
||||
using StackExchange.Redis;
|
||||
|
||||
namespace StellaOps.Scheduler.WebService.Tests;
|
||||
|
||||
public sealed class GraphJobEventPublisherTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task PublishAsync_WritesEventJson_WhenEnabled()
|
||||
{
|
||||
var options = Microsoft.Extensions.Options.Options.Create(new SchedulerEventsOptions
|
||||
{
|
||||
GraphJobs = { Enabled = true }
|
||||
});
|
||||
var loggerProvider = new ListLoggerProvider();
|
||||
using var loggerFactory = LoggerFactory.Create(builder => builder.AddProvider(loggerProvider));
|
||||
var publisher = new GraphJobEventPublisher(new OptionsMonitorStub<SchedulerEventsOptions>(options), loggerFactory.CreateLogger<GraphJobEventPublisher>());
|
||||
public async Task PublishAsync_LogsEvent_WhenDriverUnsupported()
|
||||
{
|
||||
var options = Microsoft.Extensions.Options.Options.Create(new SchedulerEventsOptions
|
||||
{
|
||||
GraphJobs =
|
||||
{
|
||||
Enabled = true,
|
||||
Driver = "memory"
|
||||
}
|
||||
});
|
||||
var loggerProvider = new ListLoggerProvider();
|
||||
using var loggerFactory = LoggerFactory.Create(builder => builder.AddProvider(loggerProvider));
|
||||
var publisher = new GraphJobEventPublisher(new OptionsMonitorStub<SchedulerEventsOptions>(options), new ThrowingRedisConnectionFactory(), loggerFactory.CreateLogger<GraphJobEventPublisher>());
|
||||
|
||||
var buildJob = new GraphBuildJob(
|
||||
id: "gbj_test",
|
||||
@@ -58,12 +63,12 @@ public sealed class GraphJobEventPublisherTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PublishAsync_Suppressed_WhenDisabled()
|
||||
{
|
||||
var options = Microsoft.Extensions.Options.Options.Create(new SchedulerEventsOptions());
|
||||
var loggerProvider = new ListLoggerProvider();
|
||||
using var loggerFactory = LoggerFactory.Create(builder => builder.AddProvider(loggerProvider));
|
||||
var publisher = new GraphJobEventPublisher(new OptionsMonitorStub<SchedulerEventsOptions>(options), loggerFactory.CreateLogger<GraphJobEventPublisher>());
|
||||
public async Task PublishAsync_Suppressed_WhenDisabled()
|
||||
{
|
||||
var options = Microsoft.Extensions.Options.Options.Create(new SchedulerEventsOptions());
|
||||
var loggerProvider = new ListLoggerProvider();
|
||||
using var loggerFactory = LoggerFactory.Create(builder => builder.AddProvider(loggerProvider));
|
||||
var publisher = new GraphJobEventPublisher(new OptionsMonitorStub<SchedulerEventsOptions>(options), new ThrowingRedisConnectionFactory(), loggerFactory.CreateLogger<GraphJobEventPublisher>());
|
||||
|
||||
var overlayJob = new GraphOverlayJob(
|
||||
id: "goj_test",
|
||||
@@ -95,12 +100,12 @@ public sealed class GraphJobEventPublisherTests
|
||||
|
||||
await publisher.PublishAsync(notification, CancellationToken.None);
|
||||
|
||||
Assert.DoesNotContain(loggerProvider.Messages, message => message.Contains(GraphJobEventKinds.GraphJobCompleted, StringComparison.Ordinal));
|
||||
Assert.DoesNotContain(loggerProvider.Messages, message => message.Contains(GraphJobEventKinds.GraphJobCompleted, StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
private sealed class OptionsMonitorStub<T> : IOptionsMonitor<T> where T : class
|
||||
{
|
||||
private readonly IOptions<T> _options;
|
||||
private sealed class OptionsMonitorStub<T> : IOptionsMonitor<T> where T : class
|
||||
{
|
||||
private readonly IOptions<T> _options;
|
||||
|
||||
public OptionsMonitorStub(IOptions<T> options)
|
||||
{
|
||||
@@ -112,7 +117,13 @@ private sealed class OptionsMonitorStub<T> : IOptionsMonitor<T> where T : class
|
||||
public T Get(string? name) => _options.Value;
|
||||
|
||||
public IDisposable? OnChange(Action<T, string?> listener) => null;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class ThrowingRedisConnectionFactory : IRedisConnectionFactory
|
||||
{
|
||||
public Task<IConnectionMultiplexer> ConnectAsync(ConfigurationOptions options, CancellationToken cancellationToken)
|
||||
=> throw new InvalidOperationException("Redis connection should not be established in this test.");
|
||||
}
|
||||
|
||||
private sealed class ListLoggerProvider : ILoggerProvider
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user