Resolve Concelier/Excititor merge conflicts

This commit is contained in:
root
2025-10-20 14:19:25 +03:00
2687 changed files with 212646 additions and 85913 deletions

View File

@@ -0,0 +1,4 @@
# StellaOps.Scheduler.Storage.Mongo — Agent Charter
## Mission
Implement Mongo persistence (schedules, runs, impact cursors, locks, audit) per `docs/ARCHITECTURE_SCHEDULER.md`.

View File

@@ -0,0 +1,46 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Driver;
using StellaOps.Scheduler.Storage.Mongo.Options;
namespace StellaOps.Scheduler.Storage.Mongo.Internal;
internal sealed class SchedulerMongoContext
{
public SchedulerMongoContext(IOptions<SchedulerMongoOptions> options, ILogger<SchedulerMongoContext> logger)
{
ArgumentNullException.ThrowIfNull(logger);
var value = options?.Value ?? throw new ArgumentNullException(nameof(options));
if (string.IsNullOrWhiteSpace(value.ConnectionString))
{
throw new InvalidOperationException("Scheduler Mongo connection string is not configured.");
}
if (string.IsNullOrWhiteSpace(value.Database))
{
throw new InvalidOperationException("Scheduler Mongo database name is not configured.");
}
Client = new MongoClient(value.ConnectionString);
var settings = new MongoDatabaseSettings();
if (value.UseMajorityReadConcern)
{
settings.ReadConcern = ReadConcern.Majority;
}
if (value.UseMajorityWriteConcern)
{
settings.WriteConcern = WriteConcern.WMajority;
}
Database = Client.GetDatabase(value.Database, settings);
Options = value;
}
public MongoClient Client { get; }
public IMongoDatabase Database { get; }
public SchedulerMongoOptions Options { get; }
}

View File

@@ -0,0 +1,32 @@
using Microsoft.Extensions.Logging;
using StellaOps.Scheduler.Storage.Mongo.Migrations;
namespace StellaOps.Scheduler.Storage.Mongo.Internal;
internal interface ISchedulerMongoInitializer
{
Task EnsureMigrationsAsync(CancellationToken cancellationToken = default);
}
internal sealed class SchedulerMongoInitializer : ISchedulerMongoInitializer
{
private readonly SchedulerMongoContext _context;
private readonly SchedulerMongoMigrationRunner _migrationRunner;
private readonly ILogger<SchedulerMongoInitializer> _logger;
public SchedulerMongoInitializer(
SchedulerMongoContext context,
SchedulerMongoMigrationRunner migrationRunner,
ILogger<SchedulerMongoInitializer> logger)
{
_context = context ?? throw new ArgumentNullException(nameof(context));
_migrationRunner = migrationRunner ?? throw new ArgumentNullException(nameof(migrationRunner));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task EnsureMigrationsAsync(CancellationToken cancellationToken = default)
{
_logger.LogInformation("Ensuring Scheduler Mongo migrations are applied for database {Database}.", _context.Options.Database);
await _migrationRunner.RunAsync(cancellationToken).ConfigureAwait(false);
}
}

View File

@@ -0,0 +1,27 @@
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace StellaOps.Scheduler.Storage.Mongo.Internal;
internal sealed class SchedulerMongoInitializerHostedService : IHostedService
{
private readonly ISchedulerMongoInitializer _initializer;
private readonly ILogger<SchedulerMongoInitializerHostedService> _logger;
public SchedulerMongoInitializerHostedService(
ISchedulerMongoInitializer initializer,
ILogger<SchedulerMongoInitializerHostedService> logger)
{
_initializer = initializer ?? throw new ArgumentNullException(nameof(initializer));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Applying Scheduler Mongo migrations.");
await _initializer.EnsureMigrationsAsync(cancellationToken).ConfigureAwait(false);
}
public Task StopAsync(CancellationToken cancellationToken)
=> Task.CompletedTask;
}

View File

@@ -0,0 +1,47 @@
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using StellaOps.Scheduler.Storage.Mongo.Internal;
namespace StellaOps.Scheduler.Storage.Mongo.Migrations;
internal sealed class EnsureSchedulerCollectionsMigration : ISchedulerMongoMigration
{
private readonly ILogger<EnsureSchedulerCollectionsMigration> _logger;
public EnsureSchedulerCollectionsMigration(ILogger<EnsureSchedulerCollectionsMigration> logger)
=> _logger = logger ?? throw new ArgumentNullException(nameof(logger));
public string Id => "20251019_scheduler_collections_v1";
public async ValueTask ExecuteAsync(SchedulerMongoContext context, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
var requiredCollections = new[]
{
context.Options.SchedulesCollection,
context.Options.RunsCollection,
context.Options.ImpactSnapshotsCollection,
context.Options.AuditCollection,
context.Options.LocksCollection,
context.Options.MigrationsCollection
};
var cursor = await context.Database
.ListCollectionNamesAsync(cancellationToken: cancellationToken)
.ConfigureAwait(false);
var existing = await cursor.ToListAsync(cancellationToken).ConfigureAwait(false);
foreach (var collection in requiredCollections)
{
if (existing.Contains(collection, StringComparer.Ordinal))
{
continue;
}
_logger.LogInformation("Creating Scheduler Mongo collection '{CollectionName}'.", collection);
await context.Database.CreateCollectionAsync(collection, cancellationToken: cancellationToken).ConfigureAwait(false);
}
}
}

View File

@@ -0,0 +1,175 @@
using System;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Scheduler.Storage.Mongo.Internal;
namespace StellaOps.Scheduler.Storage.Mongo.Migrations;
internal sealed class EnsureSchedulerIndexesMigration : ISchedulerMongoMigration
{
public string Id => "20251019_scheduler_indexes_v1";
public async ValueTask ExecuteAsync(SchedulerMongoContext context, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
await EnsureSchedulesIndexesAsync(context, cancellationToken).ConfigureAwait(false);
await EnsureRunsIndexesAsync(context, cancellationToken).ConfigureAwait(false);
await EnsureImpactSnapshotsIndexesAsync(context, cancellationToken).ConfigureAwait(false);
await EnsureAuditIndexesAsync(context, cancellationToken).ConfigureAwait(false);
await EnsureLocksIndexesAsync(context, cancellationToken).ConfigureAwait(false);
}
private static async Task EnsureSchedulesIndexesAsync(SchedulerMongoContext context, CancellationToken cancellationToken)
{
var collection = context.Database.GetCollection<BsonDocument>(context.Options.SchedulesCollection);
var tenantEnabled = new CreateIndexModel<BsonDocument>(
Builders<BsonDocument>.IndexKeys
.Ascending("tenantId")
.Ascending("enabled"),
new CreateIndexOptions<BsonDocument>
{
Name = "tenant_enabled"
});
var cronTimezone = new CreateIndexModel<BsonDocument>(
Builders<BsonDocument>.IndexKeys
.Ascending("cronExpression")
.Ascending("timezone"),
new CreateIndexOptions<BsonDocument>
{
Name = "cron_timezone"
});
await collection.Indexes.CreateManyAsync(new[] { tenantEnabled, cronTimezone }, cancellationToken: cancellationToken)
.ConfigureAwait(false);
}
private static async Task EnsureRunsIndexesAsync(SchedulerMongoContext context, CancellationToken cancellationToken)
{
var collection = context.Database.GetCollection<BsonDocument>(context.Options.RunsCollection);
var tenantCreated = new CreateIndexModel<BsonDocument>(
Builders<BsonDocument>.IndexKeys
.Ascending("tenantId")
.Descending("createdAt"),
new CreateIndexOptions<BsonDocument>
{
Name = "tenant_createdAt_desc"
});
var stateIndex = new CreateIndexModel<BsonDocument>(
Builders<BsonDocument>.IndexKeys
.Ascending("state"),
new CreateIndexOptions<BsonDocument>
{
Name = "state_lookup"
});
var scheduleIndex = new CreateIndexModel<BsonDocument>(
Builders<BsonDocument>.IndexKeys
.Ascending("scheduleId")
.Descending("createdAt"),
new CreateIndexOptions<BsonDocument>
{
Name = "schedule_createdAt_desc"
});
var models = new List<CreateIndexModel<BsonDocument>> { tenantCreated, stateIndex, scheduleIndex };
if (context.Options.CompletedRunRetention > TimeSpan.Zero)
{
var ttlModel = new CreateIndexModel<BsonDocument>(
Builders<BsonDocument>.IndexKeys.Ascending("finishedAt"),
new CreateIndexOptions<BsonDocument>
{
Name = "finishedAt_ttl",
ExpireAfter = context.Options.CompletedRunRetention
});
models.Add(ttlModel);
}
await collection.Indexes.CreateManyAsync(models, cancellationToken: cancellationToken).ConfigureAwait(false);
}
private static async Task EnsureImpactSnapshotsIndexesAsync(SchedulerMongoContext context, CancellationToken cancellationToken)
{
var collection = context.Database.GetCollection<BsonDocument>(context.Options.ImpactSnapshotsCollection);
var tenantScope = new CreateIndexModel<BsonDocument>(
Builders<BsonDocument>.IndexKeys
.Ascending("selector.tenantId")
.Ascending("selector.scope"),
new CreateIndexOptions<BsonDocument>
{
Name = "selector_tenant_scope"
});
var snapshotId = new CreateIndexModel<BsonDocument>(
Builders<BsonDocument>.IndexKeys.Ascending("snapshotId"),
new CreateIndexOptions<BsonDocument>
{
Name = "snapshotId_unique",
Unique = true,
PartialFilterExpression = Builders<BsonDocument>.Filter.Exists("snapshotId", true)
});
await collection.Indexes.CreateManyAsync(new[] { tenantScope, snapshotId }, cancellationToken: cancellationToken)
.ConfigureAwait(false);
}
private static async Task EnsureAuditIndexesAsync(SchedulerMongoContext context, CancellationToken cancellationToken)
{
var collection = context.Database.GetCollection<BsonDocument>(context.Options.AuditCollection);
var tenantOccurred = new CreateIndexModel<BsonDocument>(
Builders<BsonDocument>.IndexKeys
.Ascending("tenantId")
.Descending("occurredAt"),
new CreateIndexOptions<BsonDocument>
{
Name = "tenant_occurredAt_desc"
});
var correlationIndex = new CreateIndexModel<BsonDocument>(
Builders<BsonDocument>.IndexKeys
.Ascending("correlationId"),
new CreateIndexOptions<BsonDocument>
{
Name = "correlation_lookup",
PartialFilterExpression = Builders<BsonDocument>.Filter.Exists("correlationId", true)
});
await collection.Indexes.CreateManyAsync(new[] { tenantOccurred, correlationIndex }, cancellationToken: cancellationToken)
.ConfigureAwait(false);
}
private static async Task EnsureLocksIndexesAsync(SchedulerMongoContext context, CancellationToken cancellationToken)
{
var collection = context.Database.GetCollection<BsonDocument>(context.Options.LocksCollection);
var tenantResource = new CreateIndexModel<BsonDocument>(
Builders<BsonDocument>.IndexKeys
.Ascending("tenantId")
.Ascending("resource"),
new CreateIndexOptions<BsonDocument>
{
Name = "tenant_resource_unique",
Unique = true,
PartialFilterExpression = Builders<BsonDocument>.Filter.Exists("resource", true)
});
var ttlModel = new CreateIndexModel<BsonDocument>(
Builders<BsonDocument>.IndexKeys.Ascending("expiresAt"),
new CreateIndexOptions<BsonDocument>
{
Name = "expiresAt_ttl",
ExpireAfter = TimeSpan.Zero
});
await collection.Indexes.CreateManyAsync(new[] { tenantResource, ttlModel }, cancellationToken: cancellationToken)
.ConfigureAwait(false);
}
}

View File

@@ -0,0 +1,10 @@
using StellaOps.Scheduler.Storage.Mongo.Internal;
namespace StellaOps.Scheduler.Storage.Mongo.Migrations;
internal interface ISchedulerMongoMigration
{
string Id { get; }
ValueTask ExecuteAsync(SchedulerMongoContext context, CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,16 @@
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
namespace StellaOps.Scheduler.Storage.Mongo.Migrations;
internal sealed class SchedulerMongoMigrationRecord
{
[BsonId]
public ObjectId Id { get; set; }
[BsonElement("migrationId")]
public string MigrationId { get; set; } = string.Empty;
[BsonElement("appliedAt")]
public DateTimeOffset AppliedAt { get; set; }
}

View File

@@ -0,0 +1,77 @@
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using StellaOps.Scheduler.Storage.Mongo.Internal;
namespace StellaOps.Scheduler.Storage.Mongo.Migrations;
internal sealed class SchedulerMongoMigrationRunner
{
private readonly SchedulerMongoContext _context;
private readonly IReadOnlyList<ISchedulerMongoMigration> _migrations;
private readonly ILogger<SchedulerMongoMigrationRunner> _logger;
public SchedulerMongoMigrationRunner(
SchedulerMongoContext context,
IEnumerable<ISchedulerMongoMigration> migrations,
ILogger<SchedulerMongoMigrationRunner> logger)
{
_context = context ?? throw new ArgumentNullException(nameof(context));
ArgumentNullException.ThrowIfNull(migrations);
_migrations = migrations.OrderBy(migration => migration.Id, StringComparer.Ordinal).ToArray();
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async ValueTask RunAsync(CancellationToken cancellationToken)
{
if (_migrations.Count == 0)
{
return;
}
var collection = _context.Database.GetCollection<SchedulerMongoMigrationRecord>(_context.Options.MigrationsCollection);
await EnsureMigrationIndexAsync(collection, cancellationToken).ConfigureAwait(false);
var applied = await collection
.Find(FilterDefinition<SchedulerMongoMigrationRecord>.Empty)
.Project(record => record.MigrationId)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
var appliedSet = applied.ToHashSet(StringComparer.Ordinal);
foreach (var migration in _migrations)
{
if (appliedSet.Contains(migration.Id))
{
continue;
}
_logger.LogInformation("Applying Scheduler Mongo migration {MigrationId}.", migration.Id);
await migration.ExecuteAsync(_context, cancellationToken).ConfigureAwait(false);
var record = new SchedulerMongoMigrationRecord
{
Id = MongoDB.Bson.ObjectId.GenerateNewId(),
MigrationId = migration.Id,
AppliedAt = DateTimeOffset.UtcNow
};
await collection.InsertOneAsync(record, cancellationToken: cancellationToken).ConfigureAwait(false);
_logger.LogInformation("Completed Scheduler Mongo migration {MigrationId}.", migration.Id);
}
}
private static async Task EnsureMigrationIndexAsync(
IMongoCollection<SchedulerMongoMigrationRecord> collection,
CancellationToken cancellationToken)
{
var keys = Builders<SchedulerMongoMigrationRecord>.IndexKeys.Ascending(record => record.MigrationId);
var model = new CreateIndexModel<SchedulerMongoMigrationRecord>(keys, new CreateIndexOptions
{
Name = "migrationId_unique",
Unique = true
});
await collection.Indexes.CreateOneAsync(model, cancellationToken: cancellationToken).ConfigureAwait(false);
}
}

View File

@@ -0,0 +1,34 @@
using System;
namespace StellaOps.Scheduler.Storage.Mongo.Options;
/// <summary>
/// Configures MongoDB connectivity and collection names for Scheduler storage.
/// </summary>
public sealed class SchedulerMongoOptions
{
public string ConnectionString { get; set; } = "mongodb://localhost:27017";
public string Database { get; set; } = "stellaops_scheduler";
public string SchedulesCollection { get; set; } = "schedules";
public string RunsCollection { get; set; } = "runs";
public string ImpactSnapshotsCollection { get; set; } = "impact_snapshots";
public string AuditCollection { get; set; } = "audit";
public string LocksCollection { get; set; } = "locks";
public string MigrationsCollection { get; set; } = "_scheduler_migrations";
/// <summary>
/// Optional TTL applied to completed runs. When zero or negative no TTL index is created.
/// </summary>
public TimeSpan CompletedRunRetention { get; set; } = TimeSpan.FromDays(180);
public bool UseMajorityReadConcern { get; set; } = true;
public bool UseMajorityWriteConcern { get; set; } = true;
}

View File

@@ -0,0 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Scheduler.Storage.Mongo.Tests")]

View File

@@ -0,0 +1,25 @@
# Scheduler Storage Mongo — Sprint 16 Handoff
This module now consumes the canonical DTOs defined in `StellaOps.Scheduler.Models`.
Samples covering REST shapes live under `samples/api/scheduler/` and are referenced from `docs/11_DATA_SCHEMAS.md#3.1`.
## Collections & DTO mapping
| Collection | DTO | Notes |
|-------------------|--------------------------|---------------------------------------------------------------------------------------|
| `schedules` | `Schedule` | Persist `Schedule` as-is. `_id``Schedule.Id`. Use compound indexes on `{tenantId, enabled}` and `{whenCron}` per doc. |
| `runs` | `Run` | Store `Run.Stats` inside the document; omit `deltas` array when empty. |
| `impact_snapshots`| `ImpactSet` | Normalise selector filter fields exactly as emitted by the canonical serializer. |
| `audit` | `AuditRecord` | Lower-case metadata keys are already enforced by the model. |
All timestamps are persisted as UTC (`+00:00`). Empty selector filters remain empty arrays (see `impact-set.json` sample).
## Implementation guidance
1. Add a project reference to `StellaOps.Scheduler.Models` and reuse the records directly; avoid duplicate BSON POCOs.
2. When serialising/deserialising to MongoDB, call `CanonicalJsonSerializer` to keep ordering stable for diffable fixtures.
3. Integration tests should load the JSON samples and round-trip through the Mongo persistence layer to guarantee parity.
4. Follow `docs/11_DATA_SCHEMAS.md` for index requirements; update that doc if storage diverges.
5. Register `AddSchedulerMongoStorage` in the host and call `ISchedulerMongoInitializer.EnsureMigrationsAsync` during bootstrap so collections/indexes are created before workers/web APIs start.
With these artefacts in place the dependency on SCHED-MODELS-16-101/102 is cleared—storage work can move to DOING.

View File

@@ -0,0 +1,26 @@
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Scheduler.Storage.Mongo.Internal;
using StellaOps.Scheduler.Storage.Mongo.Migrations;
using StellaOps.Scheduler.Storage.Mongo.Options;
namespace StellaOps.Scheduler.Storage.Mongo;
public static class ServiceCollectionExtensions
{
public static IServiceCollection AddSchedulerMongoStorage(this IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.Configure<SchedulerMongoOptions>(configuration);
services.AddSingleton<SchedulerMongoContext>();
services.AddSingleton<SchedulerMongoMigrationRunner>();
services.AddSingleton<ISchedulerMongoMigration, EnsureSchedulerCollectionsMigration>();
services.AddSingleton<ISchedulerMongoMigration, EnsureSchedulerIndexesMigration>();
services.AddSingleton<ISchedulerMongoInitializer, SchedulerMongoInitializer>();
services.AddHostedService<SchedulerMongoInitializerHostedService>();
return services;
}
}

View File

@@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="8.0.0" />
<PackageReference Include="MongoDB.Bson" Version="3.5.0" />
<PackageReference Include="MongoDB.Driver" Version="3.5.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../StellaOps.Scheduler.Models/StellaOps.Scheduler.Models.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,9 @@
# Scheduler Storage Task Board (Sprint 16)
> **Status note (2025-10-19):** Scheduler models/samples delivered in SCHED-MODELS-16-102. Tasks below remain pending for the Storage guild.
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|----|--------|----------|------------|-------------|---------------|
| SCHED-STORAGE-16-201 | DONE (2025-10-19) | Scheduler Storage Guild | SCHED-MODELS-16-101 | Create Mongo collections (schedules, runs, impact_cursors, locks, audit) with indexes/migrations per architecture. | Migration scripts and indexes implemented; integration tests cover CRUD paths. |
| SCHED-STORAGE-16-202 | TODO | Scheduler Storage Guild | SCHED-STORAGE-16-201 | Implement repositories/services with tenant scoping, soft delete, TTL for completed runs, and causal consistency options. | Unit tests pass; TTL/soft delete validated; documentation updated. |
| SCHED-STORAGE-16-203 | TODO | Scheduler Storage Guild | SCHED-STORAGE-16-201 | Audit/logging pipeline + run stats materialized views for UI. | Audit entries persisted; stats queries efficient; docs capture usage. |