Add determinism tests for verdict artifact generation and update SHA256 sums script
- Implemented comprehensive tests for verdict artifact generation to ensure deterministic outputs across various scenarios, including identical inputs, parallel execution, and change ordering. - Created helper methods for generating sample verdict inputs and computing canonical hashes. - Added tests to validate the stability of canonical hashes, proof spine ordering, and summary statistics. - Introduced a new PowerShell script to update SHA256 sums for files, ensuring accurate hash generation and file integrity checks.
This commit is contained in:
StellaOps Bot
@@ -0,0 +1,269 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// JobIdempotencyTests.cs
|
||||
// Sprint: SPRINT_5100_0009_0008_scheduler_tests
|
||||
// Task: SCHEDULER-5100-006
|
||||
// Description: Model S1 idempotency tests for Scheduler job storage
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scheduler.Storage.Postgres.Models;
|
||||
using StellaOps.Scheduler.Storage.Postgres.Repositories;
|
||||
using StellaOps.TestKit;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scheduler.Storage.Postgres.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Idempotency tests for Scheduler job storage operations.
|
||||
/// Implements Model S1 (Storage/Postgres) test requirements:
|
||||
/// - Same job enqueued twice → single execution
|
||||
/// - Idempotency key uniqueness enforced per tenant
|
||||
/// - Duplicate insertions handled gracefully
|
||||
/// </summary>
|
||||
[Collection(SchedulerPostgresCollection.Name)]
|
||||
[Trait("Category", TestCategories.Integration)]
|
||||
[Trait("Category", "StorageIdempotency")]
|
||||
public sealed class JobIdempotencyTests : IAsyncLifetime
|
||||
{
|
||||
private readonly SchedulerPostgresFixture _fixture;
|
||||
private SchedulerDataSource _dataSource = null!;
|
||||
private JobRepository _jobRepository = null!;
|
||||
private readonly string _tenantId = Guid.NewGuid().ToString();
|
||||
|
||||
public JobIdempotencyTests(SchedulerPostgresFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
}
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
await _fixture.TruncateAllTablesAsync();
|
||||
|
||||
var options = _fixture.Fixture.CreateOptions();
|
||||
options.SchemaName = SchedulerDataSource.DefaultSchemaName;
|
||||
_dataSource = new SchedulerDataSource(Options.Create(options), NullLogger<SchedulerDataSource>.Instance);
|
||||
_jobRepository = new JobRepository(_dataSource, NullLogger<JobRepository>.Instance);
|
||||
}
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
[Fact]
|
||||
public async Task CreateJob_SameIdempotencyKey_SecondInsertFails()
|
||||
{
|
||||
// Arrange
|
||||
var idempotencyKey = $"idem-{Guid.NewGuid():N}";
|
||||
var job1 = CreateJob("job-type-1", idempotencyKey);
|
||||
var job2 = CreateJob("job-type-1", idempotencyKey);
|
||||
|
||||
// Act
|
||||
await _jobRepository.CreateAsync(job1);
|
||||
var createAgain = async () => await _jobRepository.CreateAsync(job2);
|
||||
|
||||
// Assert - Second insert should fail due to unique constraint on idempotency_key
|
||||
await createAgain.Should().ThrowAsync<Exception>(
|
||||
"duplicate idempotency_key should be rejected");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetByIdempotencyKey_ReturnsExistingJob()
|
||||
{
|
||||
// Arrange
|
||||
var idempotencyKey = $"idem-{Guid.NewGuid():N}";
|
||||
var job = CreateJob("job-type-1", idempotencyKey);
|
||||
await _jobRepository.CreateAsync(job);
|
||||
|
||||
// Act
|
||||
var existing = await _jobRepository.GetByIdempotencyKeyAsync(_tenantId, idempotencyKey);
|
||||
|
||||
// Assert
|
||||
existing.Should().NotBeNull();
|
||||
existing!.Id.Should().Be(job.Id);
|
||||
existing.IdempotencyKey.Should().Be(idempotencyKey);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetByIdempotencyKey_DifferentTenant_ReturnsNull()
|
||||
{
|
||||
// Arrange
|
||||
var idempotencyKey = $"idem-{Guid.NewGuid():N}";
|
||||
var job = CreateJob("job-type-1", idempotencyKey);
|
||||
await _jobRepository.CreateAsync(job);
|
||||
|
||||
// Act - Query with different tenant
|
||||
var otherTenant = Guid.NewGuid().ToString();
|
||||
var existing = await _jobRepository.GetByIdempotencyKeyAsync(otherTenant, idempotencyKey);
|
||||
|
||||
// Assert - Should not find job from different tenant
|
||||
existing.Should().BeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SameIdempotencyKey_DifferentTenants_BothSucceed()
|
||||
{
|
||||
// Arrange
|
||||
var idempotencyKey = $"shared-idem-{Guid.NewGuid():N}";
|
||||
var tenant1 = Guid.NewGuid().ToString();
|
||||
var tenant2 = Guid.NewGuid().ToString();
|
||||
|
||||
var job1 = CreateJob("job-type-1", idempotencyKey, tenant1);
|
||||
var job2 = CreateJob("job-type-1", idempotencyKey, tenant2);
|
||||
|
||||
// Act
|
||||
var created1 = await _jobRepository.CreateAsync(job1);
|
||||
var created2 = await _jobRepository.CreateAsync(job2);
|
||||
|
||||
// Assert - Both should succeed (different tenants)
|
||||
created1.Should().NotBeNull();
|
||||
created2.Should().NotBeNull();
|
||||
created1.Id.Should().NotBe(created2.Id);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MultipleJobs_UniqueIdempotencyKeys_AllCreated()
|
||||
{
|
||||
// Arrange
|
||||
var jobs = Enumerable.Range(1, 10)
|
||||
.Select(i => CreateJob($"job-type-{i}", $"idem-{i}-{Guid.NewGuid():N}"))
|
||||
.ToList();
|
||||
|
||||
// Act
|
||||
foreach (var job in jobs)
|
||||
{
|
||||
await _jobRepository.CreateAsync(job);
|
||||
}
|
||||
|
||||
// Assert - All jobs should be created
|
||||
foreach (var job in jobs)
|
||||
{
|
||||
var fetched = await _jobRepository.GetByIdAsync(_tenantId, job.Id);
|
||||
fetched.Should().NotBeNull();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task JobIdempotency_PayloadDigestMatchesExpected()
|
||||
{
|
||||
// Arrange
|
||||
var idempotencyKey = $"idem-{Guid.NewGuid():N}";
|
||||
var payloadDigest = $"sha256:{Guid.NewGuid():N}";
|
||||
var job = CreateJob("job-type-1", idempotencyKey, payloadDigest: payloadDigest);
|
||||
|
||||
// Act
|
||||
await _jobRepository.CreateAsync(job);
|
||||
var fetched = await _jobRepository.GetByIdempotencyKeyAsync(_tenantId, idempotencyKey);
|
||||
|
||||
// Assert
|
||||
fetched.Should().NotBeNull();
|
||||
fetched!.PayloadDigest.Should().Be(payloadDigest);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CompletedJob_SameIdempotencyKey_StillRejectsNewInsert()
|
||||
{
|
||||
// Arrange
|
||||
var idempotencyKey = $"idem-{Guid.NewGuid():N}";
|
||||
var job = CreateJob("job-type-1", idempotencyKey);
|
||||
var created = await _jobRepository.CreateAsync(job);
|
||||
|
||||
// Complete the job
|
||||
var leased = await _jobRepository.TryLeaseJobAsync(
|
||||
_tenantId, created.Id, "worker-1", TimeSpan.FromMinutes(5));
|
||||
if (leased != null)
|
||||
{
|
||||
await _jobRepository.CompleteAsync(_tenantId, created.Id, leased.LeaseId!.Value);
|
||||
}
|
||||
|
||||
// Act - Try to create another job with same idempotency key
|
||||
var newJob = CreateJob("job-type-1", idempotencyKey);
|
||||
var createAgain = async () => await _jobRepository.CreateAsync(newJob);
|
||||
|
||||
// Assert - Should still fail (idempotency key persists after completion)
|
||||
await createAgain.Should().ThrowAsync<Exception>(
|
||||
"completed job's idempotency_key should still block new inserts");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FailedJob_SameIdempotencyKey_StillRejectsNewInsert()
|
||||
{
|
||||
// Arrange
|
||||
var idempotencyKey = $"idem-{Guid.NewGuid():N}";
|
||||
var job = CreateJob("job-type-1", idempotencyKey);
|
||||
var created = await _jobRepository.CreateAsync(job);
|
||||
|
||||
// Fail the job
|
||||
var leased = await _jobRepository.TryLeaseJobAsync(
|
||||
_tenantId, created.Id, "worker-1", TimeSpan.FromMinutes(5));
|
||||
if (leased != null)
|
||||
{
|
||||
await _jobRepository.FailAsync(_tenantId, created.Id, leased.LeaseId!.Value, "test failure", retry: false);
|
||||
}
|
||||
|
||||
// Act - Try to create another job with same idempotency key
|
||||
var newJob = CreateJob("job-type-1", idempotencyKey);
|
||||
var createAgain = async () => await _jobRepository.CreateAsync(newJob);
|
||||
|
||||
// Assert - Should still fail
|
||||
await createAgain.Should().ThrowAsync<Exception>(
|
||||
"failed job's idempotency_key should still block new inserts");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CanceledJob_SameIdempotencyKey_StillRejectsNewInsert()
|
||||
{
|
||||
// Arrange
|
||||
var idempotencyKey = $"idem-{Guid.NewGuid():N}";
|
||||
var job = CreateJob("job-type-1", idempotencyKey);
|
||||
var created = await _jobRepository.CreateAsync(job);
|
||||
|
||||
// Cancel the job
|
||||
await _jobRepository.CancelAsync(_tenantId, created.Id, "test cancellation");
|
||||
|
||||
// Act - Try to create another job with same idempotency key
|
||||
var newJob = CreateJob("job-type-1", idempotencyKey);
|
||||
var createAgain = async () => await _jobRepository.CreateAsync(newJob);
|
||||
|
||||
// Assert - Should still fail
|
||||
await createAgain.Should().ThrowAsync<Exception>(
|
||||
"canceled job's idempotency_key should still block new inserts");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TenantIsolation_JobsOnlyVisibleToOwnTenant()
|
||||
{
|
||||
// Arrange
|
||||
var tenant1 = Guid.NewGuid().ToString();
|
||||
var tenant2 = Guid.NewGuid().ToString();
|
||||
|
||||
var job1 = CreateJob("job-type-1", $"idem-1-{Guid.NewGuid():N}", tenant1);
|
||||
var job2 = CreateJob("job-type-2", $"idem-2-{Guid.NewGuid():N}", tenant2);
|
||||
|
||||
await _jobRepository.CreateAsync(job1);
|
||||
await _jobRepository.CreateAsync(job2);
|
||||
|
||||
// Act
|
||||
var tenant1Jobs = await _jobRepository.GetByStatusAsync(tenant1, JobStatus.Scheduled, limit: 100);
|
||||
var tenant2Jobs = await _jobRepository.GetByStatusAsync(tenant2, JobStatus.Scheduled, limit: 100);
|
||||
|
||||
// Assert
|
||||
tenant1Jobs.Should().NotContain(j => j.TenantId == tenant2);
|
||||
tenant2Jobs.Should().NotContain(j => j.TenantId == tenant1);
|
||||
}
|
||||
|
||||
private JobEntity CreateJob(string jobType, string idempotencyKey, string? tenantId = null, string? payloadDigest = null)
|
||||
{
|
||||
return new JobEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
TenantId = tenantId ?? _tenantId,
|
||||
JobType = jobType,
|
||||
Status = JobStatus.Scheduled,
|
||||
Priority = 0,
|
||||
Payload = """{"test": true}""",
|
||||
PayloadDigest = payloadDigest ?? $"sha256:{Guid.NewGuid():N}",
|
||||
IdempotencyKey = idempotencyKey,
|
||||
MaxAttempts = 3
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,320 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// SchedulerMigrationTests.cs
|
||||
// Sprint: SPRINT_5100_0009_0008_scheduler_tests
|
||||
// Task: SCHEDULER-5100-005
|
||||
// Description: Model S1 migration tests for Scheduler.Storage
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Reflection;
|
||||
using Dapper;
|
||||
using FluentAssertions;
|
||||
using Npgsql;
|
||||
using StellaOps.TestKit;
|
||||
using Testcontainers.PostgreSql;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scheduler.Storage.Postgres.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Migration tests for Scheduler.Storage.
|
||||
/// Implements Model S1 (Storage/Postgres) migration test requirements:
|
||||
/// - Apply all migrations from scratch (fresh database)
|
||||
/// - Apply migrations from N-1 (incremental application)
|
||||
/// - Verify migration idempotency (apply twice → no error)
|
||||
/// </summary>
|
||||
[Trait("Category", TestCategories.Integration)]
|
||||
[Trait("Category", "StorageMigration")]
|
||||
public sealed class SchedulerMigrationTests : IAsyncLifetime
|
||||
{
|
||||
private PostgreSqlContainer _container = null!;
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
_container = new PostgreSqlBuilder()
|
||||
.WithImage("postgres:16-alpine")
|
||||
.WithDatabase("scheduler_migration_test")
|
||||
.WithUsername("postgres")
|
||||
.WithPassword("postgres")
|
||||
.Build();
|
||||
|
||||
await _container.StartAsync();
|
||||
}
|
||||
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
await _container.DisposeAsync();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyMigrations_FromScratch_AllTablesCreated()
|
||||
{
|
||||
// Arrange
|
||||
var connectionString = _container.GetConnectionString();
|
||||
|
||||
// Act - Apply all migrations from scratch
|
||||
await ApplyAllMigrationsAsync(connectionString);
|
||||
|
||||
// Assert - Verify Scheduler tables exist
|
||||
await using var connection = new NpgsqlConnection(connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
var tables = await connection.QueryAsync<string>(
|
||||
@"SELECT table_name FROM information_schema.tables
|
||||
WHERE table_schema = 'scheduler'
|
||||
ORDER BY table_name");
|
||||
|
||||
var tableList = tables.ToList();
|
||||
|
||||
// Verify core Scheduler tables exist
|
||||
tableList.Should().Contain("jobs", "jobs table should exist");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyMigrations_FromScratch_AllMigrationsRecorded()
|
||||
{
|
||||
// Arrange
|
||||
var connectionString = _container.GetConnectionString();
|
||||
await ApplyAllMigrationsAsync(connectionString);
|
||||
|
||||
// Assert - Verify migrations are recorded
|
||||
await using var connection = new NpgsqlConnection(connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
var migrationsApplied = await connection.QueryAsync<string>(
|
||||
"SELECT migration_id FROM __migrations ORDER BY applied_at");
|
||||
|
||||
var migrationList = migrationsApplied.ToList();
|
||||
migrationList.Should().NotBeEmpty("migrations should be tracked");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyMigrations_Twice_IsIdempotent()
|
||||
{
|
||||
// Arrange
|
||||
var connectionString = _container.GetConnectionString();
|
||||
|
||||
// Act - Apply migrations twice
|
||||
await ApplyAllMigrationsAsync(connectionString);
|
||||
var applyAgain = async () => await ApplyAllMigrationsAsync(connectionString);
|
||||
|
||||
// Assert - Second application should not throw
|
||||
await applyAgain.Should().NotThrowAsync(
|
||||
"applying migrations twice should be idempotent");
|
||||
|
||||
// Verify migrations are not duplicated
|
||||
await using var connection = new NpgsqlConnection(connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
var migrationCount = await connection.ExecuteScalarAsync<int>(
|
||||
"SELECT COUNT(*) FROM __migrations");
|
||||
|
||||
// Count unique migrations
|
||||
var uniqueMigrations = await connection.ExecuteScalarAsync<int>(
|
||||
"SELECT COUNT(DISTINCT migration_id) FROM __migrations");
|
||||
|
||||
migrationCount.Should().Be(uniqueMigrations,
|
||||
"each migration should only be recorded once");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyMigrations_VerifySchemaIntegrity()
|
||||
{
|
||||
// Arrange
|
||||
var connectionString = _container.GetConnectionString();
|
||||
await ApplyAllMigrationsAsync(connectionString);
|
||||
|
||||
// Assert - Verify indexes exist
|
||||
await using var connection = new NpgsqlConnection(connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
var indexes = await connection.QueryAsync<string>(
|
||||
@"SELECT indexname FROM pg_indexes
|
||||
WHERE schemaname = 'scheduler'
|
||||
ORDER BY indexname");
|
||||
|
||||
var indexList = indexes.ToList();
|
||||
indexList.Should().NotBeEmpty("scheduler schema should have indexes");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyMigrations_IndividualMigrationsCanRollForward()
|
||||
{
|
||||
// Arrange
|
||||
var connectionString = _container.GetConnectionString();
|
||||
|
||||
// Act - Apply migrations in sequence
|
||||
var migrationFiles = GetMigrationFiles();
|
||||
|
||||
await using var connection = new NpgsqlConnection(connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
// Create migration tracking table first
|
||||
await connection.ExecuteAsync(@"
|
||||
CREATE TABLE IF NOT EXISTS __migrations (
|
||||
id SERIAL PRIMARY KEY,
|
||||
migration_id TEXT NOT NULL UNIQUE,
|
||||
applied_at TIMESTAMPTZ DEFAULT NOW()
|
||||
)");
|
||||
|
||||
// Apply each migration in order
|
||||
int appliedCount = 0;
|
||||
foreach (var migrationFile in migrationFiles.OrderBy(f => f))
|
||||
{
|
||||
var migrationId = Path.GetFileName(migrationFile);
|
||||
|
||||
// Check if already applied
|
||||
var alreadyApplied = await connection.ExecuteScalarAsync<int>(
|
||||
"SELECT COUNT(*) FROM __migrations WHERE migration_id = @Id",
|
||||
new { Id = migrationId });
|
||||
|
||||
if (alreadyApplied > 0)
|
||||
continue;
|
||||
|
||||
// Apply migration
|
||||
var sql = GetMigrationContent(migrationFile);
|
||||
if (!string.IsNullOrWhiteSpace(sql))
|
||||
{
|
||||
await connection.ExecuteAsync(sql);
|
||||
await connection.ExecuteAsync(
|
||||
"INSERT INTO __migrations (migration_id) VALUES (@Id)",
|
||||
new { Id = migrationId });
|
||||
appliedCount++;
|
||||
}
|
||||
}
|
||||
|
||||
// Assert - Migrations should be applied (if any exist)
|
||||
var totalMigrations = await connection.ExecuteScalarAsync<int>(
|
||||
"SELECT COUNT(*) FROM __migrations");
|
||||
|
||||
totalMigrations.Should().BeGreaterThanOrEqualTo(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyMigrations_ForeignKeyConstraintsValid()
|
||||
{
|
||||
// Arrange
|
||||
var connectionString = _container.GetConnectionString();
|
||||
await ApplyAllMigrationsAsync(connectionString);
|
||||
|
||||
// Assert - Verify foreign key constraints exist and are valid
|
||||
await using var connection = new NpgsqlConnection(connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
var foreignKeys = await connection.QueryAsync<string>(
|
||||
@"SELECT tc.constraint_name
|
||||
FROM information_schema.table_constraints tc
|
||||
WHERE tc.constraint_type = 'FOREIGN KEY'
|
||||
AND tc.table_schema = 'scheduler'
|
||||
ORDER BY tc.constraint_name");
|
||||
|
||||
var fkList = foreignKeys.ToList();
|
||||
// Foreign keys may or may not exist depending on schema design
|
||||
fkList.Should().NotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyMigrations_JobsTableHasRequiredColumns()
|
||||
{
|
||||
// Arrange
|
||||
var connectionString = _container.GetConnectionString();
|
||||
await ApplyAllMigrationsAsync(connectionString);
|
||||
|
||||
// Assert - Verify jobs table has required columns
|
||||
await using var connection = new NpgsqlConnection(connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
var columns = await connection.QueryAsync<string>(
|
||||
@"SELECT column_name FROM information_schema.columns
|
||||
WHERE table_name = 'jobs' AND table_schema = 'scheduler'
|
||||
ORDER BY ordinal_position");
|
||||
|
||||
var columnList = columns.ToList();
|
||||
|
||||
if (columnList.Any())
|
||||
{
|
||||
columnList.Should().Contain("id", "jobs table should have id column");
|
||||
columnList.Should().Contain("tenant_id", "jobs table should have tenant_id column");
|
||||
columnList.Should().Contain("status", "jobs table should have status column");
|
||||
columnList.Should().Contain("idempotency_key", "jobs table should have idempotency_key column");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyMigrations_SchedulerSchemaExists()
|
||||
{
|
||||
// Arrange
|
||||
var connectionString = _container.GetConnectionString();
|
||||
await ApplyAllMigrationsAsync(connectionString);
|
||||
|
||||
// Assert - Verify scheduler schema exists
|
||||
await using var connection = new NpgsqlConnection(connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
var schemaExists = await connection.ExecuteScalarAsync<int>(
|
||||
@"SELECT COUNT(*) FROM information_schema.schemata
|
||||
WHERE schema_name = 'scheduler'");
|
||||
|
||||
schemaExists.Should().Be(1, "scheduler schema should exist");
|
||||
}
|
||||
|
||||
private async Task ApplyAllMigrationsAsync(string connectionString)
|
||||
{
|
||||
await using var connection = new NpgsqlConnection(connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
// Create migration tracking table
|
||||
await connection.ExecuteAsync(@"
|
||||
CREATE TABLE IF NOT EXISTS __migrations (
|
||||
id SERIAL PRIMARY KEY,
|
||||
migration_id TEXT NOT NULL UNIQUE,
|
||||
applied_at TIMESTAMPTZ DEFAULT NOW()
|
||||
)");
|
||||
|
||||
// Get and apply all migrations
|
||||
var migrationFiles = GetMigrationFiles();
|
||||
|
||||
foreach (var migrationFile in migrationFiles.OrderBy(f => f))
|
||||
{
|
||||
var migrationId = Path.GetFileName(migrationFile);
|
||||
|
||||
// Skip if already applied
|
||||
var alreadyApplied = await connection.ExecuteScalarAsync<int>(
|
||||
"SELECT COUNT(*) FROM __migrations WHERE migration_id = @Id",
|
||||
new { Id = migrationId });
|
||||
|
||||
if (alreadyApplied > 0)
|
||||
continue;
|
||||
|
||||
// Apply migration
|
||||
var sql = GetMigrationContent(migrationFile);
|
||||
if (!string.IsNullOrWhiteSpace(sql))
|
||||
{
|
||||
await connection.ExecuteAsync(sql);
|
||||
await connection.ExecuteAsync(
|
||||
"INSERT INTO __migrations (migration_id) VALUES (@Id)",
|
||||
new { Id = migrationId });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static IEnumerable<string> GetMigrationFiles()
|
||||
{
|
||||
var assembly = typeof(SchedulerDataSource).Assembly;
|
||||
var resourceNames = assembly.GetManifestResourceNames()
|
||||
.Where(n => n.Contains("Migrations") && n.EndsWith(".sql"))
|
||||
.OrderBy(n => n);
|
||||
|
||||
return resourceNames;
|
||||
}
|
||||
|
||||
private static string GetMigrationContent(string resourceName)
|
||||
{
|
||||
var assembly = typeof(SchedulerDataSource).Assembly;
|
||||
using var stream = assembly.GetManifestResourceStream(resourceName);
|
||||
if (stream == null)
|
||||
return string.Empty;
|
||||
|
||||
using var reader = new StreamReader(stream);
|
||||
return reader.ReadToEnd();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,335 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// SchedulerQueryDeterminismTests.cs
|
||||
// Sprint: SPRINT_5100_0009_0008_scheduler_tests
|
||||
// Task: SCHEDULER-5100-007
|
||||
// Description: Model S1 query determinism tests for Scheduler job queue ordering
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scheduler.Storage.Postgres.Models;
|
||||
using StellaOps.Scheduler.Storage.Postgres.Repositories;
|
||||
using StellaOps.TestKit;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scheduler.Storage.Postgres.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Query determinism tests for Scheduler storage operations.
|
||||
/// Implements Model S1 (Storage/Postgres) test requirements:
|
||||
/// - Explicit ORDER BY checks for job queue queries
|
||||
/// - Same inputs → stable ordering
|
||||
/// - Repeated queries return consistent results
|
||||
/// </summary>
|
||||
[Collection(SchedulerPostgresCollection.Name)]
|
||||
[Trait("Category", TestCategories.Integration)]
|
||||
[Trait("Category", "QueryDeterminism")]
|
||||
public sealed class SchedulerQueryDeterminismTests : IAsyncLifetime
|
||||
{
|
||||
private readonly SchedulerPostgresFixture _fixture;
|
||||
private SchedulerDataSource _dataSource = null!;
|
||||
private JobRepository _jobRepository = null!;
|
||||
private readonly string _tenantId = Guid.NewGuid().ToString();
|
||||
|
||||
public SchedulerQueryDeterminismTests(SchedulerPostgresFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
}
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
await _fixture.TruncateAllTablesAsync();
|
||||
|
||||
var options = _fixture.Fixture.CreateOptions();
|
||||
options.SchemaName = SchedulerDataSource.DefaultSchemaName;
|
||||
_dataSource = new SchedulerDataSource(Options.Create(options), NullLogger<SchedulerDataSource>.Instance);
|
||||
_jobRepository = new JobRepository(_dataSource, NullLogger<JobRepository>.Instance);
|
||||
}
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
[Fact]
|
||||
public async Task GetByStatus_MultipleQueries_ReturnsDeterministicOrder()
|
||||
{
|
||||
// Arrange
|
||||
for (int i = 0; i < 5; i++)
|
||||
{
|
||||
await CreateJobAsync($"job-type-{i}", JobStatus.Scheduled, priority: i % 3);
|
||||
}
|
||||
|
||||
// Act - Query multiple times
|
||||
var results1 = await _jobRepository.GetByStatusAsync(_tenantId, JobStatus.Scheduled);
|
||||
var results2 = await _jobRepository.GetByStatusAsync(_tenantId, JobStatus.Scheduled);
|
||||
var results3 = await _jobRepository.GetByStatusAsync(_tenantId, JobStatus.Scheduled);
|
||||
|
||||
// Assert - All queries should return same order
|
||||
var ids1 = results1.Select(j => j.Id).ToList();
|
||||
var ids2 = results2.Select(j => j.Id).ToList();
|
||||
var ids3 = results3.Select(j => j.Id).ToList();
|
||||
|
||||
ids1.Should().Equal(ids2);
|
||||
ids2.Should().Equal(ids3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetScheduledJobs_ReturnsDeterministicOrder()
|
||||
{
|
||||
// Arrange - Create jobs with different priorities
|
||||
var highPriorityJob = await CreateJobAsync("scan", JobStatus.Scheduled, priority: 10);
|
||||
var mediumPriorityJob = await CreateJobAsync("scan", JobStatus.Scheduled, priority: 5);
|
||||
var lowPriorityJob = await CreateJobAsync("scan", JobStatus.Scheduled, priority: 1);
|
||||
|
||||
// Act - Query multiple times
|
||||
var results1 = await _jobRepository.GetScheduledJobsAsync(_tenantId, ["scan"], limit: 10);
|
||||
var results2 = await _jobRepository.GetScheduledJobsAsync(_tenantId, ["scan"], limit: 10);
|
||||
var results3 = await _jobRepository.GetScheduledJobsAsync(_tenantId, ["scan"], limit: 10);
|
||||
|
||||
// Assert - All queries should return same order (priority descending)
|
||||
var ids1 = results1.Select(j => j.Id).ToList();
|
||||
var ids2 = results2.Select(j => j.Id).ToList();
|
||||
var ids3 = results3.Select(j => j.Id).ToList();
|
||||
|
||||
ids1.Should().Equal(ids2);
|
||||
ids2.Should().Equal(ids3);
|
||||
|
||||
// Verify priority ordering (highest first)
|
||||
if (results1.Count >= 2)
|
||||
{
|
||||
results1[0].Priority.Should().BeGreaterThanOrEqualTo(results1[1].Priority);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetScheduledJobs_SamePriority_OrderByCreatedAt()
|
||||
{
|
||||
// Arrange - Create jobs with same priority
|
||||
var job1 = await CreateJobAsync("scan", JobStatus.Scheduled, priority: 5);
|
||||
await Task.Delay(10); // Small delay to ensure different created_at
|
||||
var job2 = await CreateJobAsync("scan", JobStatus.Scheduled, priority: 5);
|
||||
await Task.Delay(10);
|
||||
var job3 = await CreateJobAsync("scan", JobStatus.Scheduled, priority: 5);
|
||||
|
||||
// Act - Query multiple times
|
||||
var results1 = await _jobRepository.GetScheduledJobsAsync(_tenantId, ["scan"], limit: 10);
|
||||
var results2 = await _jobRepository.GetScheduledJobsAsync(_tenantId, ["scan"], limit: 10);
|
||||
|
||||
// Assert - Order should be consistent
|
||||
var ids1 = results1.Select(j => j.Id).ToList();
|
||||
var ids2 = results2.Select(j => j.Id).ToList();
|
||||
|
||||
ids1.Should().Equal(ids2);
|
||||
|
||||
// Verify created_at ordering for same priority
|
||||
for (int i = 0; i < results1.Count - 1; i++)
|
||||
{
|
||||
results1[i].CreatedAt.Should().BeLessOrEqualTo(results1[i + 1].CreatedAt);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ConcurrentQueries_SameStatus_AllReturnIdenticalResults()
|
||||
{
|
||||
// Arrange
|
||||
for (int i = 0; i < 5; i++)
|
||||
{
|
||||
await CreateJobAsync($"job-type-{i}", JobStatus.Scheduled, priority: 5);
|
||||
}
|
||||
|
||||
// Act - 20 concurrent queries
|
||||
var tasks = Enumerable.Range(0, 20)
|
||||
.Select(_ => _jobRepository.GetByStatusAsync(_tenantId, JobStatus.Scheduled))
|
||||
.ToList();
|
||||
|
||||
var results = await Task.WhenAll(tasks);
|
||||
|
||||
// Assert - All should return identical order
|
||||
var firstOrder = results[0].Select(j => j.Id).ToList();
|
||||
results.Should().AllSatisfy(r =>
|
||||
{
|
||||
r.Select(j => j.Id).ToList().Should().Equal(firstOrder);
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetById_MultipleQueries_ReturnsConsistentResult()
|
||||
{
|
||||
// Arrange
|
||||
var job = await CreateJobAsync("test-job", JobStatus.Scheduled, priority: 5);
|
||||
|
||||
// Act - Query multiple times
|
||||
var results = new List<JobEntity?>();
|
||||
for (int i = 0; i < 10; i++)
|
||||
{
|
||||
results.Add(await _jobRepository.GetByIdAsync(_tenantId, job.Id));
|
||||
}
|
||||
|
||||
// Assert - All should return identical job
|
||||
results.Should().AllSatisfy(r =>
|
||||
{
|
||||
r.Should().NotBeNull();
|
||||
r!.Id.Should().Be(job.Id);
|
||||
r.JobType.Should().Be("test-job");
|
||||
r.Priority.Should().Be(5);
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetByIdempotencyKey_MultipleQueries_ReturnsConsistentResult()
|
||||
{
|
||||
// Arrange
|
||||
var idempotencyKey = $"idem-{Guid.NewGuid():N}";
|
||||
var job = await CreateJobAsync("test-job", JobStatus.Scheduled, idempotencyKey: idempotencyKey);
|
||||
|
||||
// Act - Query multiple times
|
||||
var results = new List<JobEntity?>();
|
||||
for (int i = 0; i < 10; i++)
|
||||
{
|
||||
results.Add(await _jobRepository.GetByIdempotencyKeyAsync(_tenantId, idempotencyKey));
|
||||
}
|
||||
|
||||
// Assert - All should return same job
|
||||
results.Should().AllSatisfy(r =>
|
||||
{
|
||||
r.Should().NotBeNull();
|
||||
r!.Id.Should().Be(job.Id);
|
||||
r.IdempotencyKey.Should().Be(idempotencyKey);
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EmptyTenant_GetByStatus_ReturnsEmptyConsistently()
|
||||
{
|
||||
// Arrange
|
||||
var emptyTenantId = Guid.NewGuid().ToString();
|
||||
|
||||
// Act - Query empty tenant multiple times
|
||||
var results = new List<IReadOnlyList<JobEntity>>();
|
||||
for (int i = 0; i < 5; i++)
|
||||
{
|
||||
results.Add(await _jobRepository.GetByStatusAsync(emptyTenantId, JobStatus.Scheduled));
|
||||
}
|
||||
|
||||
// Assert - All should return empty
|
||||
results.Should().AllSatisfy(r => r.Should().BeEmpty());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TenantIsolation_JobsInDifferentTenants_QueriesReturnOnlyOwnTenant()
|
||||
{
|
||||
// Arrange
|
||||
var tenant1 = Guid.NewGuid().ToString();
|
||||
var tenant2 = Guid.NewGuid().ToString();
|
||||
|
||||
await CreateJobAsync("job-type-1", JobStatus.Scheduled, tenantId: tenant1);
|
||||
await CreateJobAsync("job-type-2", JobStatus.Scheduled, tenantId: tenant2);
|
||||
|
||||
// Act
|
||||
var tenant1Jobs = await _jobRepository.GetByStatusAsync(tenant1, JobStatus.Scheduled);
|
||||
var tenant2Jobs = await _jobRepository.GetByStatusAsync(tenant2, JobStatus.Scheduled);
|
||||
|
||||
// Assert
|
||||
tenant1Jobs.Should().HaveCount(1);
|
||||
tenant1Jobs[0].TenantId.Should().Be(tenant1);
|
||||
|
||||
tenant2Jobs.Should().HaveCount(1);
|
||||
tenant2Jobs[0].TenantId.Should().Be(tenant2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PaginatedQuery_OffsetAndLimit_DeterministicResults()
|
||||
{
|
||||
// Arrange - Create 10 jobs
|
||||
for (int i = 0; i < 10; i++)
|
||||
{
|
||||
await CreateJobAsync($"job-type-{i}", JobStatus.Scheduled, priority: 10 - i);
|
||||
}
|
||||
|
||||
// Act - Query with different offsets
|
||||
var page1a = await _jobRepository.GetByStatusAsync(_tenantId, JobStatus.Scheduled, limit: 3, offset: 0);
|
||||
var page2a = await _jobRepository.GetByStatusAsync(_tenantId, JobStatus.Scheduled, limit: 3, offset: 3);
|
||||
var page1b = await _jobRepository.GetByStatusAsync(_tenantId, JobStatus.Scheduled, limit: 3, offset: 0);
|
||||
var page2b = await _jobRepository.GetByStatusAsync(_tenantId, JobStatus.Scheduled, limit: 3, offset: 3);
|
||||
|
||||
// Assert - Same pages should return same results
|
||||
page1a.Select(j => j.Id).Should().Equal(page1b.Select(j => j.Id));
|
||||
page2a.Select(j => j.Id).Should().Equal(page2b.Select(j => j.Id));
|
||||
|
||||
// Pages should not overlap
|
||||
var page1Ids = page1a.Select(j => j.Id).ToHashSet();
|
||||
var page2Ids = page2a.Select(j => j.Id).ToHashSet();
|
||||
page1Ids.Intersect(page2Ids).Should().BeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task JobStatusTransitions_DoesNotAffectOrderOfOtherJobs()
|
||||
{
|
||||
// Arrange
|
||||
var job1 = await CreateJobAsync("scan", JobStatus.Scheduled, priority: 5);
|
||||
var job2 = await CreateJobAsync("scan", JobStatus.Scheduled, priority: 5);
|
||||
var job3 = await CreateJobAsync("scan", JobStatus.Scheduled, priority: 5);
|
||||
|
||||
var initialOrder = (await _jobRepository.GetByStatusAsync(_tenantId, JobStatus.Scheduled))
|
||||
.Select(j => j.Id).ToList();
|
||||
|
||||
// Act - Lease and complete job2
|
||||
var leased = await _jobRepository.TryLeaseJobAsync(_tenantId, job2.Id, "worker-1", TimeSpan.FromMinutes(5));
|
||||
if (leased != null)
|
||||
{
|
||||
await _jobRepository.CompleteAsync(_tenantId, job2.Id, leased.LeaseId!.Value);
|
||||
}
|
||||
|
||||
// Query remaining scheduled jobs
|
||||
var afterTransition = await _jobRepository.GetByStatusAsync(_tenantId, JobStatus.Scheduled);
|
||||
|
||||
// Assert - Remaining jobs should maintain their relative order
|
||||
var remainingIds = afterTransition.Select(j => j.Id).ToList();
|
||||
var expectedOrder = initialOrder.Where(id => id != job2.Id).ToList();
|
||||
remainingIds.Should().Equal(expectedOrder);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MultipleJobTypes_FilteringMaintainsOrder()
|
||||
{
|
||||
// Arrange
|
||||
await CreateJobAsync("scan", JobStatus.Scheduled, priority: 10);
|
||||
await CreateJobAsync("build", JobStatus.Scheduled, priority: 8);
|
||||
await CreateJobAsync("scan", JobStatus.Scheduled, priority: 6);
|
||||
await CreateJobAsync("deploy", JobStatus.Scheduled, priority: 4);
|
||||
await CreateJobAsync("scan", JobStatus.Scheduled, priority: 2);
|
||||
|
||||
// Act - Query only scan jobs multiple times
|
||||
var scanJobs1 = await _jobRepository.GetScheduledJobsAsync(_tenantId, ["scan"], limit: 10);
|
||||
var scanJobs2 = await _jobRepository.GetScheduledJobsAsync(_tenantId, ["scan"], limit: 10);
|
||||
|
||||
// Assert
|
||||
scanJobs1.Should().HaveCount(3);
|
||||
scanJobs1.Select(j => j.Id).Should().Equal(scanJobs2.Select(j => j.Id));
|
||||
|
||||
// Verify priority ordering
|
||||
scanJobs1[0].Priority.Should().BeGreaterThanOrEqualTo(scanJobs1[1].Priority);
|
||||
scanJobs1[1].Priority.Should().BeGreaterThanOrEqualTo(scanJobs1[2].Priority);
|
||||
}
|
||||
|
||||
private async Task<JobEntity> CreateJobAsync(
|
||||
string jobType,
|
||||
JobStatus status,
|
||||
int priority = 0,
|
||||
string? tenantId = null,
|
||||
string? idempotencyKey = null)
|
||||
{
|
||||
var job = new JobEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
TenantId = tenantId ?? _tenantId,
|
||||
JobType = jobType,
|
||||
Status = status,
|
||||
Priority = priority,
|
||||
Payload = """{"test": true}""",
|
||||
PayloadDigest = $"sha256:{Guid.NewGuid():N}",
|
||||
IdempotencyKey = idempotencyKey ?? $"idem-{Guid.NewGuid():N}",
|
||||
MaxAttempts = 3
|
||||
};
|
||||
return await _jobRepository.CreateAsync(job);
|
||||
}
|
||||
}
|
||||
@@ -11,8 +11,20 @@
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Dapper" Version="2.1.35" />
|
||||
<PackageReference Include="FluentAssertions" Version="6.12.0" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
|
||||
<PackageReference Include="Moq" Version="4.20.70" />
|
||||
<PackageReference Include="Testcontainers.PostgreSql" Version="4.3.0" />
|
||||
<PackageReference Include="xunit" Version="2.9.2" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
<PackageReference Include="coverlet.collector" Version="6.0.4">
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
Reference in New Issue
Block a user