up
Some checks failed
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
devportal-offline / build-offline (push) Has been cancelled

This commit is contained in:
StellaOps Bot
2025-12-03 00:10:19 +02:00
parent ea1d58a89b
commit 37cba83708
158 changed files with 147438 additions and 867 deletions

View File

@@ -9,10 +9,12 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "__Libraries", "__Libraries"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scheduler.Models", "__Libraries\StellaOps.Scheduler.Models\StellaOps.Scheduler.Models.csproj", "{382FA1C0-5F5F-424A-8485-7FED0ADE9F6B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scheduler.Storage.Mongo", "__Libraries\StellaOps.Scheduler.Storage.Mongo\StellaOps.Scheduler.Storage.Mongo.csproj", "{33770BC5-6802-45AD-A866-10027DD360E2}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scheduler.ImpactIndex", "__Libraries\StellaOps.Scheduler.ImpactIndex\StellaOps.Scheduler.ImpactIndex.csproj", "{56209C24-3CE7-4F8E-8B8C-F052CB919DE2}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scheduler.Storage.Mongo", "__Libraries\StellaOps.Scheduler.Storage.Mongo\StellaOps.Scheduler.Storage.Mongo.csproj", "{33770BC5-6802-45AD-A866-10027DD360E2}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scheduler.Storage.Postgres", "__Libraries\StellaOps.Scheduler.Storage.Postgres\StellaOps.Scheduler.Storage.Postgres.csproj", "{167198F1-43CF-42F4-BEF2-5ABC87116A37}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scheduler.ImpactIndex", "__Libraries\StellaOps.Scheduler.ImpactIndex\StellaOps.Scheduler.ImpactIndex.csproj", "{56209C24-3CE7-4F8E-8B8C-F052CB919DE2}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Plugin", "..\__Libraries\StellaOps.Plugin\StellaOps.Plugin.csproj", "{2F9CDB3D-7BB5-46B6-A51B-49AB498CC959}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.DependencyInjection", "..\__Libraries\StellaOps.DependencyInjection\StellaOps.DependencyInjection.csproj", "{214ED54A-FA25-4189-9F58-50D11F079ACF}"
@@ -36,9 +38,13 @@ EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Notify.Queue", "..\Notify\__Libraries\StellaOps.Notify.Queue\StellaOps.Notify.Queue.csproj", "{827D179C-A229-439E-A878-4028F30CA670}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scheduler.Worker.Host", "StellaOps.Scheduler.Worker.Host\StellaOps.Scheduler.Worker.Host.csproj", "{37FA8A12-E96E-4F23-AB72-8FA9DD9DA082}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "__Tests", "__Tests", "{56BCE1BF-7CBA-7CE8-203D-A88051F1D642}"
EndProject
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tools", "Tools", "{694D5197-0F28-46B9-BAA2-EFC9825C23D4}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Scheduler.Backfill", "Tools\Scheduler.Backfill\Scheduler.Backfill.csproj", "{9C1AC284-0561-4E78-9EA8-9B55C3180512}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "__Tests", "__Tests", "{56BCE1BF-7CBA-7CE8-203D-A88051F1D642}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scheduler.ImpactIndex.Tests", "__Tests\StellaOps.Scheduler.ImpactIndex.Tests\StellaOps.Scheduler.ImpactIndex.Tests.csproj", "{5ED2BF16-72CE-4DF1-917C-6D832427AE6F}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Emit", "..\Scanner\__Libraries\StellaOps.Scanner.Emit\StellaOps.Scanner.Emit.csproj", "{11D72DD3-3752-4A6A-AA4A-5298D4FD6FA0}"
@@ -57,10 +63,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scheduler.Queue.T
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scheduler.Storage.Mongo.Tests", "__Tests\StellaOps.Scheduler.Storage.Mongo.Tests\StellaOps.Scheduler.Storage.Mongo.Tests.csproj", "{972CEB4D-510B-4701-B4A2-F14A85F11CC7}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scheduler.WebService.Tests", "__Tests\StellaOps.Scheduler.WebService.Tests\StellaOps.Scheduler.WebService.Tests.csproj", "{7B4C9EAC-316E-4890-A715-7BB9C1577F96}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scheduler.Worker.Tests", "__Tests\StellaOps.Scheduler.Worker.Tests\StellaOps.Scheduler.Worker.Tests.csproj", "{D640DBB2-4251-44B3-B949-75FC6BF02B71}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scheduler.WebService.Tests", "__Tests\StellaOps.Scheduler.WebService.Tests\StellaOps.Scheduler.WebService.Tests.csproj", "{7B4C9EAC-316E-4890-A715-7BB9C1577F96}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scheduler.Backfill.Tests", "__Tests\StellaOps.Scheduler.Backfill.Tests\StellaOps.Scheduler.Backfill.Tests.csproj", "{B13D1DF0-1B9E-4557-919C-0A4E0FC9A8C7}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scheduler.Worker.Tests", "__Tests\StellaOps.Scheduler.Worker.Tests\StellaOps.Scheduler.Worker.Tests.csproj", "{D640DBB2-4251-44B3-B949-75FC6BF02B71}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -389,28 +397,67 @@ Global
{D640DBB2-4251-44B3-B949-75FC6BF02B71}.Debug|x64.Build.0 = Debug|Any CPU
{D640DBB2-4251-44B3-B949-75FC6BF02B71}.Debug|x86.ActiveCfg = Debug|Any CPU
{D640DBB2-4251-44B3-B949-75FC6BF02B71}.Debug|x86.Build.0 = Debug|Any CPU
{D640DBB2-4251-44B3-B949-75FC6BF02B71}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D640DBB2-4251-44B3-B949-75FC6BF02B71}.Release|Any CPU.Build.0 = Release|Any CPU
{D640DBB2-4251-44B3-B949-75FC6BF02B71}.Release|x64.ActiveCfg = Release|Any CPU
{D640DBB2-4251-44B3-B949-75FC6BF02B71}.Release|x64.Build.0 = Release|Any CPU
{D640DBB2-4251-44B3-B949-75FC6BF02B71}.Release|x86.ActiveCfg = Release|Any CPU
{D640DBB2-4251-44B3-B949-75FC6BF02B71}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
{D640DBB2-4251-44B3-B949-75FC6BF02B71}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D640DBB2-4251-44B3-B949-75FC6BF02B71}.Release|Any CPU.Build.0 = Release|Any CPU
{D640DBB2-4251-44B3-B949-75FC6BF02B71}.Release|x64.ActiveCfg = Release|Any CPU
{D640DBB2-4251-44B3-B949-75FC6BF02B71}.Release|x64.Build.0 = Release|Any CPU
{D640DBB2-4251-44B3-B949-75FC6BF02B71}.Release|x86.ActiveCfg = Release|Any CPU
{D640DBB2-4251-44B3-B949-75FC6BF02B71}.Release|x86.Build.0 = Release|Any CPU
{167198F1-43CF-42F4-BEF2-5ABC87116A37}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{167198F1-43CF-42F4-BEF2-5ABC87116A37}.Debug|Any CPU.Build.0 = Debug|Any CPU
{167198F1-43CF-42F4-BEF2-5ABC87116A37}.Debug|x64.ActiveCfg = Debug|Any CPU
{167198F1-43CF-42F4-BEF2-5ABC87116A37}.Debug|x64.Build.0 = Debug|Any CPU
{167198F1-43CF-42F4-BEF2-5ABC87116A37}.Debug|x86.ActiveCfg = Debug|Any CPU
{167198F1-43CF-42F4-BEF2-5ABC87116A37}.Debug|x86.Build.0 = Debug|Any CPU
{167198F1-43CF-42F4-BEF2-5ABC87116A37}.Release|Any CPU.ActiveCfg = Release|Any CPU
{167198F1-43CF-42F4-BEF2-5ABC87116A37}.Release|Any CPU.Build.0 = Release|Any CPU
{167198F1-43CF-42F4-BEF2-5ABC87116A37}.Release|x64.ActiveCfg = Release|Any CPU
{167198F1-43CF-42F4-BEF2-5ABC87116A37}.Release|x64.Build.0 = Release|Any CPU
{167198F1-43CF-42F4-BEF2-5ABC87116A37}.Release|x86.ActiveCfg = Release|Any CPU
{167198F1-43CF-42F4-BEF2-5ABC87116A37}.Release|x86.Build.0 = Release|Any CPU
{9C1AC284-0561-4E78-9EA8-9B55C3180512}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{9C1AC284-0561-4E78-9EA8-9B55C3180512}.Debug|Any CPU.Build.0 = Debug|Any CPU
{9C1AC284-0561-4E78-9EA8-9B55C3180512}.Debug|x64.ActiveCfg = Debug|Any CPU
{9C1AC284-0561-4E78-9EA8-9B55C3180512}.Debug|x64.Build.0 = Debug|Any CPU
{9C1AC284-0561-4E78-9EA8-9B55C3180512}.Debug|x86.ActiveCfg = Debug|Any CPU
{9C1AC284-0561-4E78-9EA8-9B55C3180512}.Debug|x86.Build.0 = Debug|Any CPU
{9C1AC284-0561-4E78-9EA8-9B55C3180512}.Release|Any CPU.ActiveCfg = Release|Any CPU
{9C1AC284-0561-4E78-9EA8-9B55C3180512}.Release|Any CPU.Build.0 = Release|Any CPU
{9C1AC284-0561-4E78-9EA8-9B55C3180512}.Release|x64.ActiveCfg = Release|Any CPU
{9C1AC284-0561-4E78-9EA8-9B55C3180512}.Release|x64.Build.0 = Release|Any CPU
{9C1AC284-0561-4E78-9EA8-9B55C3180512}.Release|x86.ActiveCfg = Release|Any CPU
{9C1AC284-0561-4E78-9EA8-9B55C3180512}.Release|x86.Build.0 = Release|Any CPU
{B13D1DF0-1B9E-4557-919C-0A4E0FC9A8C7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{B13D1DF0-1B9E-4557-919C-0A4E0FC9A8C7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B13D1DF0-1B9E-4557-919C-0A4E0FC9A8C7}.Debug|x64.ActiveCfg = Debug|Any CPU
{B13D1DF0-1B9E-4557-919C-0A4E0FC9A8C7}.Debug|x64.Build.0 = Debug|Any CPU
{B13D1DF0-1B9E-4557-919C-0A4E0FC9A8C7}.Debug|x86.ActiveCfg = Debug|Any CPU
{B13D1DF0-1B9E-4557-919C-0A4E0FC9A8C7}.Debug|x86.Build.0 = Debug|Any CPU
{B13D1DF0-1B9E-4557-919C-0A4E0FC9A8C7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B13D1DF0-1B9E-4557-919C-0A4E0FC9A8C7}.Release|Any CPU.Build.0 = Release|Any CPU
{B13D1DF0-1B9E-4557-919C-0A4E0FC9A8C7}.Release|x64.ActiveCfg = Release|Any CPU
{B13D1DF0-1B9E-4557-919C-0A4E0FC9A8C7}.Release|x64.Build.0 = Release|Any CPU
{B13D1DF0-1B9E-4557-919C-0A4E0FC9A8C7}.Release|x86.ActiveCfg = Release|Any CPU
{B13D1DF0-1B9E-4557-919C-0A4E0FC9A8C7}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{382FA1C0-5F5F-424A-8485-7FED0ADE9F6B} = {41F15E67-7190-CF23-3BC4-77E87134CADD}
{33770BC5-6802-45AD-A866-10027DD360E2} = {41F15E67-7190-CF23-3BC4-77E87134CADD}
{56209C24-3CE7-4F8E-8B8C-F052CB919DE2} = {41F15E67-7190-CF23-3BC4-77E87134CADD}
{6A62C12A-8742-4D1E-AEA7-8DDC3C722AC4} = {41F15E67-7190-CF23-3BC4-77E87134CADD}
{C48F2207-8974-43A4-B3D6-6A1761C37605} = {41F15E67-7190-CF23-3BC4-77E87134CADD}
{37FA8A12-E96E-4F23-AB72-8FA9DD9DA082} = {41F15E67-7190-CF23-3BC4-77E87134CADD}
{5ED2BF16-72CE-4DF1-917C-6D832427AE6F} = {56BCE1BF-7CBA-7CE8-203D-A88051F1D642}
{2F097B4B-8F38-45C3-8A42-90250E912F0C} = {56BCE1BF-7CBA-7CE8-203D-A88051F1D642}
{7C22F6B7-095E-459B-BCCF-87098EA9F192} = {56BCE1BF-7CBA-7CE8-203D-A88051F1D642}
{972CEB4D-510B-4701-B4A2-F14A85F11CC7} = {56BCE1BF-7CBA-7CE8-203D-A88051F1D642}
{7B4C9EAC-316E-4890-A715-7BB9C1577F96} = {56BCE1BF-7CBA-7CE8-203D-A88051F1D642}
{D640DBB2-4251-44B3-B949-75FC6BF02B71} = {56BCE1BF-7CBA-7CE8-203D-A88051F1D642}
{56209C24-3CE7-4F8E-8B8C-F052CB919DE2} = {41F15E67-7190-CF23-3BC4-77E87134CADD}
{167198F1-43CF-42F4-BEF2-5ABC87116A37} = {41F15E67-7190-CF23-3BC4-77E87134CADD}
{6A62C12A-8742-4D1E-AEA7-8DDC3C722AC4} = {41F15E67-7190-CF23-3BC4-77E87134CADD}
{C48F2207-8974-43A4-B3D6-6A1761C37605} = {41F15E67-7190-CF23-3BC4-77E87134CADD}
{37FA8A12-E96E-4F23-AB72-8FA9DD9DA082} = {41F15E67-7190-CF23-3BC4-77E87134CADD}
{9C1AC284-0561-4E78-9EA8-9B55C3180512} = {694D5197-0F28-46B9-BAA2-EFC9825C23D4}
{5ED2BF16-72CE-4DF1-917C-6D832427AE6F} = {56BCE1BF-7CBA-7CE8-203D-A88051F1D642}
{2F097B4B-8F38-45C3-8A42-90250E912F0C} = {56BCE1BF-7CBA-7CE8-203D-A88051F1D642}
{7C22F6B7-095E-459B-BCCF-87098EA9F192} = {56BCE1BF-7CBA-7CE8-203D-A88051F1D642}
{972CEB4D-510B-4701-B4A2-F14A85F11CC7} = {56BCE1BF-7CBA-7CE8-203D-A88051F1D642}
{7B4C9EAC-316E-4890-A715-7BB9C1577F96} = {56BCE1BF-7CBA-7CE8-203D-A88051F1D642}
{B13D1DF0-1B9E-4557-919C-0A4E0FC9A8C7} = {56BCE1BF-7CBA-7CE8-203D-A88051F1D642}
{D640DBB2-4251-44B3-B949-75FC6BF02B71} = {56BCE1BF-7CBA-7CE8-203D-A88051F1D642}
EndGlobalSection
EndGlobal

View File

@@ -0,0 +1,20 @@
using StellaOps.Scheduler.Models;
namespace Scheduler.Backfill;
internal static class BackfillMappings
{
public static string ToScheduleMode(ScheduleMode mode)
=> mode switch
{
ScheduleMode.AnalysisOnly => "analysisonly",
ScheduleMode.ContentRefresh => "contentrefresh",
_ => mode.ToString().ToLowerInvariant()
};
public static string ToRunState(RunState state)
=> state.ToString().ToLowerInvariant();
public static string ToRunTrigger(RunTrigger trigger)
=> trigger.ToString().ToLowerInvariant();
}

View File

@@ -0,0 +1,315 @@
using System.Text.Json;
using MongoDB.Bson;
using MongoDB.Bson.Serialization;
using MongoDB.Driver;
using Npgsql;
using Scheduler.Backfill;
using StellaOps.Scheduler.Models;
using StellaOps.Scheduler.Storage.Mongo.Options;
var parsed = ParseArgs(args);
var options = BackfillOptions.From(parsed.MongoConnection, parsed.MongoDatabase, parsed.PostgresConnection, parsed.BatchSize, parsed.DryRun);
var runner = new BackfillRunner(options);
await runner.RunAsync();
return 0;
static BackfillCliOptions ParseArgs(string[] args)
{
string? mongo = null;
string? mongoDb = null;
string? pg = null;
int batch = 500;
bool dryRun = false;
for (var i = 0; i < args.Length; i++)
{
switch (args[i])
{
case "--mongo" or "-m":
mongo = NextValue(args, ref i);
break;
case "--mongo-db":
mongoDb = NextValue(args, ref i);
break;
case "--pg" or "-p":
pg = NextValue(args, ref i);
break;
case "--batch":
batch = int.TryParse(NextValue(args, ref i), out var b) ? b : 500;
break;
case "--dry-run":
dryRun = true;
break;
default:
break;
}
}
return new BackfillCliOptions(mongo, mongoDb, pg, batch, dryRun);
}
static string NextValue(string[] args, ref int index)
{
if (index + 1 >= args.Length)
{
return string.Empty;
}
index++;
return args[index];
}
internal sealed record BackfillCliOptions(
string? MongoConnection,
string? MongoDatabase,
string? PostgresConnection,
int BatchSize,
bool DryRun);
internal sealed record BackfillOptions(
string MongoConnectionString,
string MongoDatabase,
string PostgresConnectionString,
int BatchSize,
bool DryRun)
{
public static BackfillOptions From(string? mongoConn, string? mongoDb, string pgConn, int batchSize, bool dryRun)
{
var mongoOptions = new SchedulerMongoOptions();
var conn = string.IsNullOrWhiteSpace(mongoConn)
? Environment.GetEnvironmentVariable("MONGO_CONNECTION_STRING") ?? mongoOptions.ConnectionString
: mongoConn;
var database = string.IsNullOrWhiteSpace(mongoDb)
? Environment.GetEnvironmentVariable("MONGO_DATABASE") ?? mongoOptions.Database
: mongoDb!;
var pg = string.IsNullOrWhiteSpace(pgConn)
? throw new ArgumentException("PostgreSQL connection string is required (--pg or POSTGRES_CONNECTION_STRING)")
: pgConn;
if (string.IsNullOrWhiteSpace(pg) && Environment.GetEnvironmentVariable("POSTGRES_CONNECTION_STRING") is { } envPg)
{
pg = envPg;
}
if (string.IsNullOrWhiteSpace(pg))
{
throw new ArgumentException("PostgreSQL connection string is required.");
}
return new BackfillOptions(conn, database, pg, Math.Max(50, batchSize), dryRun);
}
}
internal sealed class BackfillRunner
{
private readonly BackfillOptions _options;
private readonly IMongoDatabase _mongo;
private readonly NpgsqlDataSource _pg;
public BackfillRunner(BackfillOptions options)
{
_options = options;
_mongo = new MongoClient(options.MongoConnectionString).GetDatabase(options.MongoDatabase);
_pg = NpgsqlDataSource.Create(options.PostgresConnectionString);
}
public async Task RunAsync()
{
Console.WriteLine($"Mongo -> Postgres backfill starting (dry-run={_options.DryRun})");
await BackfillSchedulesAsync();
await BackfillRunsAsync();
Console.WriteLine("Backfill complete.");
}
private async Task BackfillSchedulesAsync()
{
var collection = _mongo.GetCollection<BsonDocument>(new SchedulerMongoOptions().SchedulesCollection);
using var cursor = await collection.Find(FilterDefinition<BsonDocument>.Empty).ToCursorAsync();
var batch = new List<Schedule>(_options.BatchSize);
long total = 0;
while (await cursor.MoveNextAsync())
{
foreach (var doc in cursor.Current)
{
var schedule = BsonSerializer.Deserialize<Schedule>(doc);
batch.Add(schedule);
if (batch.Count >= _options.BatchSize)
{
total += await PersistSchedulesAsync(batch);
batch.Clear();
}
}
}
if (batch.Count > 0)
{
total += await PersistSchedulesAsync(batch);
}
Console.WriteLine($"Schedules backfilled: {total}");
}
private async Task<long> PersistSchedulesAsync(IEnumerable<Schedule> schedules)
{
if (_options.DryRun)
{
return schedules.LongCount();
}
await using var conn = await _pg.OpenConnectionAsync();
await using var tx = await conn.BeginTransactionAsync();
const string sql = @"
INSERT INTO scheduler.schedules (
id, tenant_id, name, description, enabled, cron_expression, timezone, mode,
selection, only_if, notify, limits, subscribers, created_at, created_by, updated_at, updated_by, deleted_at, deleted_by)
VALUES (
@id, @tenant_id, @name, @description, @enabled, @cron_expression, @timezone, @mode,
@selection, @only_if, @notify, @limits, @subscribers, @created_at, @created_by, @updated_at, @updated_by, @deleted_at, @deleted_by)
ON CONFLICT (id) DO UPDATE SET
tenant_id = EXCLUDED.tenant_id,
name = EXCLUDED.name,
description = EXCLUDED.description,
enabled = EXCLUDED.enabled,
cron_expression = EXCLUDED.cron_expression,
timezone = EXCLUDED.timezone,
mode = EXCLUDED.mode,
selection = EXCLUDED.selection,
only_if = EXCLUDED.only_if,
notify = EXCLUDED.notify,
limits = EXCLUDED.limits,
subscribers = EXCLUDED.subscribers,
created_at = LEAST(scheduler.schedules.created_at, EXCLUDED.created_at),
created_by = EXCLUDED.created_by,
updated_at = EXCLUDED.updated_at,
updated_by = EXCLUDED.updated_by,
deleted_at = EXCLUDED.deleted_at,
deleted_by = EXCLUDED.deleted_by;";
var affected = 0;
foreach (var schedule in schedules)
{
await using var cmd = new NpgsqlCommand(sql, conn, tx);
cmd.Parameters.AddWithValue("id", schedule.Id);
cmd.Parameters.AddWithValue("tenant_id", schedule.TenantId);
cmd.Parameters.AddWithValue("name", schedule.Name);
cmd.Parameters.AddWithValue("description", DBNull.Value);
cmd.Parameters.AddWithValue("enabled", schedule.Enabled);
cmd.Parameters.AddWithValue("cron_expression", schedule.CronExpression);
cmd.Parameters.AddWithValue("timezone", schedule.Timezone);
cmd.Parameters.AddWithValue("mode", BackfillMappings.ToScheduleMode(schedule.Mode));
cmd.Parameters.AddWithValue("selection", CanonicalJsonSerializer.Serialize(schedule.Selection));
cmd.Parameters.AddWithValue("only_if", CanonicalJsonSerializer.Serialize(schedule.OnlyIf));
cmd.Parameters.AddWithValue("notify", CanonicalJsonSerializer.Serialize(schedule.Notify));
cmd.Parameters.AddWithValue("limits", CanonicalJsonSerializer.Serialize(schedule.Limits));
cmd.Parameters.AddWithValue("subscribers", schedule.Subscribers.ToArray());
cmd.Parameters.AddWithValue("created_at", schedule.CreatedAt.UtcDateTime);
cmd.Parameters.AddWithValue("created_by", schedule.CreatedBy);
cmd.Parameters.AddWithValue("updated_at", schedule.UpdatedAt.UtcDateTime);
cmd.Parameters.AddWithValue("updated_by", schedule.UpdatedBy);
cmd.Parameters.AddWithValue("deleted_at", DBNull.Value);
cmd.Parameters.AddWithValue("deleted_by", DBNull.Value);
affected += await cmd.ExecuteNonQueryAsync();
}
await tx.CommitAsync();
return affected;
}
private async Task BackfillRunsAsync()
{
var collection = _mongo.GetCollection<BsonDocument>(new SchedulerMongoOptions().RunsCollection);
using var cursor = await collection.Find(FilterDefinition<BsonDocument>.Empty).ToCursorAsync();
var batch = new List<Run>(_options.BatchSize);
long total = 0;
while (await cursor.MoveNextAsync())
{
foreach (var doc in cursor.Current)
{
var run = BsonSerializer.Deserialize<Run>(doc);
batch.Add(run);
if (batch.Count >= _options.BatchSize)
{
total += await PersistRunsAsync(batch);
batch.Clear();
}
}
}
if (batch.Count > 0)
{
total += await PersistRunsAsync(batch);
}
Console.WriteLine($"Runs backfilled: {total}");
}
private async Task<long> PersistRunsAsync(IEnumerable<Run> runs)
{
if (_options.DryRun)
{
return runs.LongCount();
}
await using var conn = await _pg.OpenConnectionAsync();
await using var tx = await conn.BeginTransactionAsync();
const string sql = @"
INSERT INTO scheduler.runs (
id, tenant_id, schedule_id, state, trigger, stats, deltas, reason, retry_of,
created_at, started_at, finished_at, error, created_by, updated_at, metadata)
VALUES (
@id, @tenant_id, @schedule_id, @state, @trigger, @stats, @deltas, @reason, @retry_of,
@created_at, @started_at, @finished_at, @error, @created_by, @updated_at, @metadata)
ON CONFLICT (id) DO UPDATE SET
tenant_id = EXCLUDED.tenant_id,
schedule_id = EXCLUDED.schedule_id,
state = EXCLUDED.state,
trigger = EXCLUDED.trigger,
stats = EXCLUDED.stats,
deltas = EXCLUDED.deltas,
reason = EXCLUDED.reason,
retry_of = EXCLUDED.retry_of,
created_at = LEAST(scheduler.runs.created_at, EXCLUDED.created_at),
started_at = EXCLUDED.started_at,
finished_at = EXCLUDED.finished_at,
error = EXCLUDED.error,
created_by = COALESCE(EXCLUDED.created_by, scheduler.runs.created_by),
updated_at = EXCLUDED.updated_at,
metadata = EXCLUDED.metadata;";
var affected = 0;
foreach (var run in runs)
{
await using var cmd = new NpgsqlCommand(sql, conn, tx);
cmd.Parameters.AddWithValue("id", run.Id);
cmd.Parameters.AddWithValue("tenant_id", run.TenantId);
cmd.Parameters.AddWithValue("schedule_id", (object?)run.ScheduleId ?? DBNull.Value);
cmd.Parameters.AddWithValue("state", BackfillMappings.ToRunState(run.State));
cmd.Parameters.AddWithValue("trigger", BackfillMappings.ToRunTrigger(run.Trigger));
cmd.Parameters.AddWithValue("stats", CanonicalJsonSerializer.Serialize(run.Stats));
cmd.Parameters.AddWithValue("deltas", CanonicalJsonSerializer.Serialize(run.Deltas));
cmd.Parameters.AddWithValue("reason", CanonicalJsonSerializer.Serialize(run.Reason));
cmd.Parameters.AddWithValue("retry_of", (object?)run.RetryOf ?? DBNull.Value);
cmd.Parameters.AddWithValue("created_at", run.CreatedAt.UtcDateTime);
cmd.Parameters.AddWithValue("started_at", (object?)run.StartedAt?.UtcDateTime ?? DBNull.Value);
cmd.Parameters.AddWithValue("finished_at", (object?)run.FinishedAt?.UtcDateTime ?? DBNull.Value);
cmd.Parameters.AddWithValue("error", (object?)run.Error ?? DBNull.Value);
cmd.Parameters.AddWithValue("created_by", (object?)run.Reason?.ManualReason ?? "system");
cmd.Parameters.AddWithValue("updated_at", DateTime.UtcNow);
cmd.Parameters.AddWithValue("metadata", JsonSerializer.Serialize(new { schema = run.SchemaVersion }));
affected += await cmd.ExecuteNonQueryAsync();
}
await tx.CommitAsync();
return affected;
}
}

View File

@@ -0,0 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Scheduler.Backfill.Tests")]

View File

@@ -0,0 +1,22 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="../../__Libraries/StellaOps.Scheduler.Storage.Mongo/StellaOps.Scheduler.Storage.Mongo.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Scheduler.Storage.Postgres/StellaOps.Scheduler.Storage.Postgres.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Scheduler.Models/StellaOps.Scheduler.Models.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="MongoDB.Driver" Version="3.5.0" />
<PackageReference Include="Npgsql" Version="9.0.2" />
</ItemGroup>
</Project>

View File

@@ -170,3 +170,192 @@ $$ LANGUAGE plpgsql;
CREATE TRIGGER trg_triggers_updated_at
BEFORE UPDATE ON scheduler.triggers
FOR EACH ROW EXECUTE FUNCTION scheduler.update_updated_at();
-- Schedules table (control-plane schedules)
CREATE TABLE IF NOT EXISTS scheduler.schedules (
id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
name TEXT NOT NULL,
description TEXT,
enabled BOOLEAN NOT NULL DEFAULT TRUE,
cron_expression TEXT,
timezone TEXT NOT NULL DEFAULT 'UTC',
mode TEXT NOT NULL CHECK (mode IN ('analysisonly', 'contentrefresh')),
selection JSONB NOT NULL DEFAULT '{}',
only_if JSONB NOT NULL DEFAULT '{}',
notify JSONB NOT NULL DEFAULT '{}',
limits JSONB NOT NULL DEFAULT '{}',
subscribers TEXT[] NOT NULL DEFAULT '{}',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
created_by TEXT NOT NULL,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_by TEXT NOT NULL,
deleted_at TIMESTAMPTZ,
deleted_by TEXT
);
CREATE INDEX IF NOT EXISTS idx_schedules_tenant ON scheduler.schedules(tenant_id) WHERE deleted_at IS NULL;
CREATE INDEX IF NOT EXISTS idx_schedules_enabled ON scheduler.schedules(tenant_id, enabled) WHERE deleted_at IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS uq_schedules_tenant_name_active ON scheduler.schedules(tenant_id, name) WHERE deleted_at IS NULL;
-- Runs table (execution records)
CREATE TABLE IF NOT EXISTS scheduler.runs (
id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
schedule_id TEXT REFERENCES scheduler.schedules(id),
state TEXT NOT NULL CHECK (state IN ('planning','queued','running','completed','error','cancelled')),
trigger TEXT NOT NULL,
stats JSONB NOT NULL DEFAULT '{}',
deltas JSONB NOT NULL DEFAULT '[]',
reason JSONB NOT NULL DEFAULT '{}',
retry_of TEXT REFERENCES scheduler.runs(id),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
started_at TIMESTAMPTZ,
finished_at TIMESTAMPTZ,
error TEXT,
created_by TEXT,
updated_at TIMESTAMPTZ,
metadata JSONB NOT NULL DEFAULT '{}'
);
CREATE INDEX IF NOT EXISTS idx_runs_tenant_state ON scheduler.runs(tenant_id, state);
CREATE INDEX IF NOT EXISTS idx_runs_schedule ON scheduler.runs(schedule_id);
CREATE INDEX IF NOT EXISTS idx_runs_created ON scheduler.runs(created_at DESC);
-- Graph jobs table
CREATE TABLE IF NOT EXISTS scheduler.graph_jobs (
id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
sbom_id TEXT NOT NULL,
sbom_version_id TEXT,
sbom_digest TEXT NOT NULL,
graph_snapshot_id TEXT,
status TEXT NOT NULL CHECK (status IN ('pending','queued','running','completed','failed','cancelled')),
trigger TEXT NOT NULL CHECK (trigger IN ('sbom-version','backfill','manual')),
priority INT NOT NULL DEFAULT 100,
attempts INT NOT NULL DEFAULT 0,
max_attempts INT NOT NULL DEFAULT 3,
cartographer_job_id TEXT,
correlation_id TEXT,
metadata JSONB NOT NULL DEFAULT '{}',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
started_at TIMESTAMPTZ,
completed_at TIMESTAMPTZ,
error TEXT,
error_details JSONB
);
CREATE INDEX IF NOT EXISTS idx_graph_jobs_tenant_status ON scheduler.graph_jobs(tenant_id, status);
CREATE INDEX IF NOT EXISTS idx_graph_jobs_sbom ON scheduler.graph_jobs(sbom_digest);
-- Policy run jobs table
CREATE TABLE IF NOT EXISTS scheduler.policy_jobs (
id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
policy_pack_id TEXT NOT NULL,
policy_version INT,
target_type TEXT NOT NULL,
target_id TEXT NOT NULL,
status TEXT NOT NULL CHECK (status IN ('pending','queued','running','completed','failed','cancelled')),
priority INT NOT NULL DEFAULT 100,
run_id TEXT,
requested_by TEXT,
mode TEXT,
metadata JSONB NOT NULL DEFAULT '{}',
inputs JSONB NOT NULL DEFAULT '{}',
attempt_count INT NOT NULL DEFAULT 0,
max_attempts INT NOT NULL DEFAULT 3,
queued_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
available_at TIMESTAMPTZ,
submitted_at TIMESTAMPTZ,
started_at TIMESTAMPTZ,
completed_at TIMESTAMPTZ,
cancellation_requested BOOLEAN NOT NULL DEFAULT FALSE,
cancellation_reason TEXT,
cancelled_at TIMESTAMPTZ,
last_attempt_at TIMESTAMPTZ,
last_error TEXT,
lease_owner TEXT,
lease_expires_at TIMESTAMPTZ,
correlation_id TEXT
);
CREATE INDEX IF NOT EXISTS idx_policy_jobs_tenant_status ON scheduler.policy_jobs(tenant_id, status);
CREATE INDEX IF NOT EXISTS idx_policy_jobs_run ON scheduler.policy_jobs(run_id);
-- Impact snapshots table
CREATE TABLE IF NOT EXISTS scheduler.impact_snapshots (
id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
run_id TEXT NOT NULL REFERENCES scheduler.runs(id) ON DELETE CASCADE,
image_digest TEXT NOT NULL,
image_reference TEXT,
new_findings INT NOT NULL DEFAULT 0,
new_criticals INT NOT NULL DEFAULT 0,
new_high INT NOT NULL DEFAULT 0,
new_medium INT NOT NULL DEFAULT 0,
new_low INT NOT NULL DEFAULT 0,
total_findings INT NOT NULL DEFAULT 0,
kev_hits TEXT[] NOT NULL DEFAULT '{}',
top_findings JSONB NOT NULL DEFAULT '[]',
report_url TEXT,
attestation JSONB NOT NULL DEFAULT '{}',
detected_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_impact_snapshots_run ON scheduler.impact_snapshots(run_id);
CREATE INDEX IF NOT EXISTS idx_impact_snapshots_tenant ON scheduler.impact_snapshots(tenant_id, detected_at DESC);
-- Execution logs table
CREATE TABLE IF NOT EXISTS scheduler.execution_logs (
id BIGSERIAL PRIMARY KEY,
run_id TEXT NOT NULL REFERENCES scheduler.runs(id) ON DELETE CASCADE,
logged_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
level TEXT NOT NULL,
message TEXT NOT NULL,
logger TEXT,
data JSONB NOT NULL DEFAULT '{}'
);
CREATE INDEX IF NOT EXISTS idx_execution_logs_run ON scheduler.execution_logs(run_id);
-- Run summaries table
CREATE TABLE IF NOT EXISTS scheduler.run_summaries (
id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
schedule_id TEXT REFERENCES scheduler.schedules(id),
period_start TIMESTAMPTZ NOT NULL,
period_end TIMESTAMPTZ NOT NULL,
total_runs INT NOT NULL DEFAULT 0,
successful_runs INT NOT NULL DEFAULT 0,
failed_runs INT NOT NULL DEFAULT 0,
cancelled_runs INT NOT NULL DEFAULT 0,
avg_duration_seconds NUMERIC(10,2),
max_duration_seconds INT,
min_duration_seconds INT,
total_findings_detected INT NOT NULL DEFAULT 0,
new_criticals INT NOT NULL DEFAULT 0,
computed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (tenant_id, schedule_id, period_start)
);
CREATE INDEX IF NOT EXISTS idx_run_summaries_tenant ON scheduler.run_summaries(tenant_id, period_start DESC);
-- Audit table
CREATE TABLE IF NOT EXISTS scheduler.audit (
id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
action TEXT NOT NULL,
entity_type TEXT NOT NULL,
entity_id TEXT NOT NULL,
actor TEXT,
actor_type TEXT,
old_value JSONB,
new_value JSONB,
details JSONB NOT NULL DEFAULT '{}',
ip_address INET,
occurred_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_audit_tenant_time ON scheduler.audit(tenant_id, occurred_at DESC);
CREATE INDEX IF NOT EXISTS idx_audit_entity ON scheduler.audit(entity_type, entity_id);

View File

@@ -0,0 +1,34 @@
using FluentAssertions;
using Scheduler.Backfill;
using StellaOps.Scheduler.Models;
using Xunit;
namespace StellaOps.Scheduler.Backfill.Tests;
public class BackfillMappingsTests
{
[Theory]
[InlineData(ScheduleMode.AnalysisOnly, "analysisonly")]
[InlineData(ScheduleMode.ContentRefresh, "contentrefresh")]
public void ScheduleMode_is_lower_snake(ScheduleMode mode, string expected)
{
BackfillMappings.ToScheduleMode(mode).Should().Be(expected);
}
[Theory]
[InlineData(RunState.Planning, "planning")]
[InlineData(RunState.Completed, "completed")]
[InlineData(RunState.Cancelled, "cancelled")]
public void RunState_is_lower(RunState state, string expected)
{
BackfillMappings.ToRunState(state).Should().Be(expected);
}
[Theory]
[InlineData(RunTrigger.Cron, "cron")]
[InlineData(RunTrigger.Manual, "manual")]
public void RunTrigger_is_lower(RunTrigger trigger, string expected)
{
BackfillMappings.ToRunTrigger(trigger).Should().Be(expected);
}
}

View File

@@ -0,0 +1,20 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<Nullable>enable</Nullable>
<LangVersion>preview</LangVersion>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="FluentAssertions" Version="6.12.0" />
<PackageReference Include="xunit" Version="2.6.3" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.6.3" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../../Tools/Scheduler.Backfill/Scheduler.Backfill.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Scheduler.Models/StellaOps.Scheduler.Models.csproj" />
</ItemGroup>
</Project>