// ----------------------------------------------------------------------------- // JobIdempotencyTests.cs // Sprint: SPRINT_5100_0009_0008_scheduler_tests // Task: SCHEDULER-5100-006 // Description: Model S1 idempotency tests for Scheduler job storage // ----------------------------------------------------------------------------- using FluentAssertions; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using StellaOps.Scheduler.Persistence.Postgres.Models; using StellaOps.Scheduler.Persistence.Postgres.Repositories; using StellaOps.TestKit; using Xunit; namespace StellaOps.Scheduler.Persistence.Postgres.Tests; /// /// Idempotency tests for Scheduler job storage operations. /// Implements Model S1 (Storage/Postgres) test requirements: /// - Same job enqueued twice → single execution /// - Idempotency key uniqueness enforced per tenant /// - Duplicate insertions handled gracefully /// [Collection(SchedulerPostgresCollection.Name)] [Trait("Category", TestCategories.Integration)] [Trait("Category", "StorageIdempotency")] public sealed class JobIdempotencyTests : IAsyncLifetime { private readonly SchedulerPostgresFixture _fixture; private SchedulerDataSource _dataSource = null!; private JobRepository _jobRepository = null!; private readonly string _tenantId = Guid.NewGuid().ToString(); public JobIdempotencyTests(SchedulerPostgresFixture fixture) { _fixture = fixture; } public async Task InitializeAsync() { await _fixture.TruncateAllTablesAsync(); var options = _fixture.Fixture.CreateOptions(); options.SchemaName = SchedulerDataSource.DefaultSchemaName; _dataSource = new SchedulerDataSource(Options.Create(options), NullLogger.Instance); _jobRepository = new JobRepository(_dataSource, NullLogger.Instance); } public Task DisposeAsync() => Task.CompletedTask; [Fact] public async Task CreateJob_SameIdempotencyKey_SecondInsertFails() { // Arrange var idempotencyKey = $"idem-{Guid.NewGuid():N}"; var job1 = CreateJob("job-type-1", idempotencyKey); var job2 = CreateJob("job-type-1", idempotencyKey); // Act await _jobRepository.CreateAsync(job1); var createAgain = async () => await _jobRepository.CreateAsync(job2); // Assert - Second insert should fail due to unique constraint on idempotency_key await createAgain.Should().ThrowAsync( "duplicate idempotency_key should be rejected"); } [Fact] public async Task GetByIdempotencyKey_ReturnsExistingJob() { // Arrange var idempotencyKey = $"idem-{Guid.NewGuid():N}"; var job = CreateJob("job-type-1", idempotencyKey); await _jobRepository.CreateAsync(job); // Act var existing = await _jobRepository.GetByIdempotencyKeyAsync(_tenantId, idempotencyKey); // Assert existing.Should().NotBeNull(); existing!.Id.Should().Be(job.Id); existing.IdempotencyKey.Should().Be(idempotencyKey); } [Fact] public async Task GetByIdempotencyKey_DifferentTenant_ReturnsNull() { // Arrange var idempotencyKey = $"idem-{Guid.NewGuid():N}"; var job = CreateJob("job-type-1", idempotencyKey); await _jobRepository.CreateAsync(job); // Act - Query with different tenant var otherTenant = Guid.NewGuid().ToString(); var existing = await _jobRepository.GetByIdempotencyKeyAsync(otherTenant, idempotencyKey); // Assert - Should not find job from different tenant existing.Should().BeNull(); } [Fact] public async Task SameIdempotencyKey_DifferentTenants_BothSucceed() { // Arrange var idempotencyKey = $"shared-idem-{Guid.NewGuid():N}"; var tenant1 = Guid.NewGuid().ToString(); var tenant2 = Guid.NewGuid().ToString(); var job1 = CreateJob("job-type-1", idempotencyKey, tenant1); var job2 = CreateJob("job-type-1", idempotencyKey, tenant2); // Act var created1 = await _jobRepository.CreateAsync(job1); var created2 = await _jobRepository.CreateAsync(job2); // Assert - Both should succeed (different tenants) created1.Should().NotBeNull(); created2.Should().NotBeNull(); created1.Id.Should().NotBe(created2.Id); } [Fact] public async Task MultipleJobs_UniqueIdempotencyKeys_AllCreated() { // Arrange var jobs = Enumerable.Range(1, 10) .Select(i => CreateJob($"job-type-{i}", $"idem-{i}-{Guid.NewGuid():N}")) .ToList(); // Act foreach (var job in jobs) { await _jobRepository.CreateAsync(job); } // Assert - All jobs should be created foreach (var job in jobs) { var fetched = await _jobRepository.GetByIdAsync(_tenantId, job.Id); fetched.Should().NotBeNull(); } } [Fact] public async Task JobIdempotency_PayloadDigestMatchesExpected() { // Arrange var idempotencyKey = $"idem-{Guid.NewGuid():N}"; var payloadDigest = $"sha256:{Guid.NewGuid():N}"; var job = CreateJob("job-type-1", idempotencyKey, payloadDigest: payloadDigest); // Act await _jobRepository.CreateAsync(job); var fetched = await _jobRepository.GetByIdempotencyKeyAsync(_tenantId, idempotencyKey); // Assert fetched.Should().NotBeNull(); fetched!.PayloadDigest.Should().Be(payloadDigest); } [Fact] public async Task CompletedJob_SameIdempotencyKey_StillRejectsNewInsert() { // Arrange var idempotencyKey = $"idem-{Guid.NewGuid():N}"; var job = CreateJob("job-type-1", idempotencyKey); var created = await _jobRepository.CreateAsync(job); // Complete the job var leased = await _jobRepository.TryLeaseJobAsync( _tenantId, created.Id, "worker-1", TimeSpan.FromMinutes(5)); if (leased != null) { await _jobRepository.CompleteAsync(_tenantId, created.Id, leased.LeaseId!.Value); } // Act - Try to create another job with same idempotency key var newJob = CreateJob("job-type-1", idempotencyKey); var createAgain = async () => await _jobRepository.CreateAsync(newJob); // Assert - Should still fail (idempotency key persists after completion) await createAgain.Should().ThrowAsync( "completed job's idempotency_key should still block new inserts"); } [Fact] public async Task FailedJob_SameIdempotencyKey_StillRejectsNewInsert() { // Arrange var idempotencyKey = $"idem-{Guid.NewGuid():N}"; var job = CreateJob("job-type-1", idempotencyKey); var created = await _jobRepository.CreateAsync(job); // Fail the job var leased = await _jobRepository.TryLeaseJobAsync( _tenantId, created.Id, "worker-1", TimeSpan.FromMinutes(5)); if (leased != null) { await _jobRepository.FailAsync(_tenantId, created.Id, leased.LeaseId!.Value, "test failure", retry: false); } // Act - Try to create another job with same idempotency key var newJob = CreateJob("job-type-1", idempotencyKey); var createAgain = async () => await _jobRepository.CreateAsync(newJob); // Assert - Should still fail await createAgain.Should().ThrowAsync( "failed job's idempotency_key should still block new inserts"); } [Fact] public async Task CanceledJob_SameIdempotencyKey_StillRejectsNewInsert() { // Arrange var idempotencyKey = $"idem-{Guid.NewGuid():N}"; var job = CreateJob("job-type-1", idempotencyKey); var created = await _jobRepository.CreateAsync(job); // Cancel the job await _jobRepository.CancelAsync(_tenantId, created.Id, "test cancellation"); // Act - Try to create another job with same idempotency key var newJob = CreateJob("job-type-1", idempotencyKey); var createAgain = async () => await _jobRepository.CreateAsync(newJob); // Assert - Should still fail await createAgain.Should().ThrowAsync( "canceled job's idempotency_key should still block new inserts"); } [Fact] public async Task TenantIsolation_JobsOnlyVisibleToOwnTenant() { // Arrange var tenant1 = Guid.NewGuid().ToString(); var tenant2 = Guid.NewGuid().ToString(); var job1 = CreateJob("job-type-1", $"idem-1-{Guid.NewGuid():N}", tenant1); var job2 = CreateJob("job-type-2", $"idem-2-{Guid.NewGuid():N}", tenant2); await _jobRepository.CreateAsync(job1); await _jobRepository.CreateAsync(job2); // Act var tenant1Jobs = await _jobRepository.GetByStatusAsync(tenant1, JobStatus.Scheduled, limit: 100); var tenant2Jobs = await _jobRepository.GetByStatusAsync(tenant2, JobStatus.Scheduled, limit: 100); // Assert tenant1Jobs.Should().NotContain(j => j.TenantId == tenant2); tenant2Jobs.Should().NotContain(j => j.TenantId == tenant1); } private JobEntity CreateJob(string jobType, string idempotencyKey, string? tenantId = null, string? payloadDigest = null) { return new JobEntity { Id = Guid.NewGuid(), TenantId = tenantId ?? _tenantId, JobType = jobType, Status = JobStatus.Scheduled, Priority = 0, Payload = """{"test": true}""", PayloadDigest = payloadDigest ?? $"sha256:{Guid.NewGuid():N}", IdempotencyKey = idempotencyKey, MaxAttempts = 3 }; } }