git.stella-ops.org/src/Scheduler/__Tests/StellaOps.Scheduler.Persistence.Tests/JobIdempotencyTests.cs

// -----------------------------------------------------------------------------
// JobIdempotencyTests.cs
// Sprint: SPRINT_5100_0009_0008_scheduler_tests
// Task: SCHEDULER-5100-006
// Description: Model S1 idempotency tests for Scheduler job storage
// -----------------------------------------------------------------------------

using FluentAssertions;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using StellaOps.Scheduler.Persistence.Postgres.Models;
using StellaOps.Scheduler.Persistence.Postgres.Repositories;
using StellaOps.TestKit;
using Xunit;

namespace StellaOps.Scheduler.Persistence.Postgres.Tests;

/// <summary>
/// Idempotency tests for Scheduler job storage operations.
/// Implements Model S1 (Storage/Postgres) test requirements:
/// - Same job enqueued twice → single execution
/// - Idempotency key uniqueness enforced per tenant
/// - Duplicate insertions handled gracefully
/// </summary>
[Collection(SchedulerPostgresCollection.Name)]
[Trait("Category", TestCategories.Integration)]
[Trait("Category", "StorageIdempotency")]
public sealed class JobIdempotencyTests : IAsyncLifetime
{
    private readonly SchedulerPostgresFixture _fixture;
    private SchedulerDataSource _dataSource = null!;
    private JobRepository _jobRepository = null!;
    private readonly string _tenantId = Guid.NewGuid().ToString();

    public JobIdempotencyTests(SchedulerPostgresFixture fixture)
    {
        _fixture = fixture;
    }

    public async Task InitializeAsync()
    {
        await _fixture.TruncateAllTablesAsync();

        var options = _fixture.Fixture.CreateOptions();
        options.SchemaName = SchedulerDataSource.DefaultSchemaName;
        _dataSource = new SchedulerDataSource(Options.Create(options), NullLogger<SchedulerDataSource>.Instance);
        _jobRepository = new JobRepository(_dataSource, NullLogger<JobRepository>.Instance);
    }

    public Task DisposeAsync() => Task.CompletedTask;

    [Fact]
    public async Task CreateJob_SameIdempotencyKey_SecondInsertFails()
    {
        // Arrange
        var idempotencyKey = $"idem-{Guid.NewGuid():N}";
        var job1 = CreateJob("job-type-1", idempotencyKey);
        var job2 = CreateJob("job-type-1", idempotencyKey);

        // Act
        await _jobRepository.CreateAsync(job1);
        var createAgain = async () => await _jobRepository.CreateAsync(job2);

        // Assert - Second insert should fail due to unique constraint on idempotency_key
        await createAgain.Should().ThrowAsync<Exception>(
            "duplicate idempotency_key should be rejected");
    }

    [Fact]
    public async Task GetByIdempotencyKey_ReturnsExistingJob()
    {
        // Arrange
        var idempotencyKey = $"idem-{Guid.NewGuid():N}";
        var job = CreateJob("job-type-1", idempotencyKey);
        await _jobRepository.CreateAsync(job);

        // Act
        var existing = await _jobRepository.GetByIdempotencyKeyAsync(_tenantId, idempotencyKey);

        // Assert
        existing.Should().NotBeNull();
        existing!.Id.Should().Be(job.Id);
        existing.IdempotencyKey.Should().Be(idempotencyKey);
    }

    [Fact]
    public async Task GetByIdempotencyKey_DifferentTenant_ReturnsNull()
    {
        // Arrange
        var idempotencyKey = $"idem-{Guid.NewGuid():N}";
        var job = CreateJob("job-type-1", idempotencyKey);
        await _jobRepository.CreateAsync(job);

        // Act - Query with different tenant
        var otherTenant = Guid.NewGuid().ToString();
        var existing = await _jobRepository.GetByIdempotencyKeyAsync(otherTenant, idempotencyKey);

        // Assert - Should not find job from different tenant
        existing.Should().BeNull();
    }

    [Fact]
    public async Task SameIdempotencyKey_DifferentTenants_BothSucceed()
    {
        // Arrange
        var idempotencyKey = $"shared-idem-{Guid.NewGuid():N}";
        var tenant1 = Guid.NewGuid().ToString();
        var tenant2 = Guid.NewGuid().ToString();

        var job1 = CreateJob("job-type-1", idempotencyKey, tenant1);
        var job2 = CreateJob("job-type-1", idempotencyKey, tenant2);

        // Act
        var created1 = await _jobRepository.CreateAsync(job1);
        var created2 = await _jobRepository.CreateAsync(job2);

        // Assert - Both should succeed (different tenants)
        created1.Should().NotBeNull();
        created2.Should().NotBeNull();
        created1.Id.Should().NotBe(created2.Id);
    }

    [Fact]
    public async Task MultipleJobs_UniqueIdempotencyKeys_AllCreated()
    {
        // Arrange
        var jobs = Enumerable.Range(1, 10)
            .Select(i => CreateJob($"job-type-{i}", $"idem-{i}-{Guid.NewGuid():N}"))
            .ToList();

        // Act
        foreach (var job in jobs)
        {
            await _jobRepository.CreateAsync(job);
        }

        // Assert - All jobs should be created
        foreach (var job in jobs)
        {
            var fetched = await _jobRepository.GetByIdAsync(_tenantId, job.Id);
            fetched.Should().NotBeNull();
        }
    }

    [Fact]
    public async Task JobIdempotency_PayloadDigestMatchesExpected()
    {
        // Arrange
        var idempotencyKey = $"idem-{Guid.NewGuid():N}";
        var payloadDigest = $"sha256:{Guid.NewGuid():N}";
        var job = CreateJob("job-type-1", idempotencyKey, payloadDigest: payloadDigest);

        // Act
        await _jobRepository.CreateAsync(job);
        var fetched = await _jobRepository.GetByIdempotencyKeyAsync(_tenantId, idempotencyKey);

        // Assert
        fetched.Should().NotBeNull();
        fetched!.PayloadDigest.Should().Be(payloadDigest);
    }

    [Fact]
    public async Task CompletedJob_SameIdempotencyKey_StillRejectsNewInsert()
    {
        // Arrange
        var idempotencyKey = $"idem-{Guid.NewGuid():N}";
        var job = CreateJob("job-type-1", idempotencyKey);
        var created = await _jobRepository.CreateAsync(job);

        // Complete the job
        var leased = await _jobRepository.TryLeaseJobAsync(
            _tenantId, created.Id, "worker-1", TimeSpan.FromMinutes(5));
        if (leased != null)
        {
            await _jobRepository.CompleteAsync(_tenantId, created.Id, leased.LeaseId!.Value);
        }

        // Act - Try to create another job with same idempotency key
        var newJob = CreateJob("job-type-1", idempotencyKey);
        var createAgain = async () => await _jobRepository.CreateAsync(newJob);

        // Assert - Should still fail (idempotency key persists after completion)
        await createAgain.Should().ThrowAsync<Exception>(
            "completed job's idempotency_key should still block new inserts");
    }

    [Fact]
    public async Task FailedJob_SameIdempotencyKey_StillRejectsNewInsert()
    {
        // Arrange
        var idempotencyKey = $"idem-{Guid.NewGuid():N}";
        var job = CreateJob("job-type-1", idempotencyKey);
        var created = await _jobRepository.CreateAsync(job);

        // Fail the job
        var leased = await _jobRepository.TryLeaseJobAsync(
            _tenantId, created.Id, "worker-1", TimeSpan.FromMinutes(5));
        if (leased != null)
        {
            await _jobRepository.FailAsync(_tenantId, created.Id, leased.LeaseId!.Value, "test failure", retry: false);
        }

        // Act - Try to create another job with same idempotency key
        var newJob = CreateJob("job-type-1", idempotencyKey);
        var createAgain = async () => await _jobRepository.CreateAsync(newJob);

        // Assert - Should still fail
        await createAgain.Should().ThrowAsync<Exception>(
            "failed job's idempotency_key should still block new inserts");
    }

    [Fact]
    public async Task CanceledJob_SameIdempotencyKey_StillRejectsNewInsert()
    {
        // Arrange
        var idempotencyKey = $"idem-{Guid.NewGuid():N}";
        var job = CreateJob("job-type-1", idempotencyKey);
        var created = await _jobRepository.CreateAsync(job);

        // Cancel the job
        await _jobRepository.CancelAsync(_tenantId, created.Id, "test cancellation");

        // Act - Try to create another job with same idempotency key
        var newJob = CreateJob("job-type-1", idempotencyKey);
        var createAgain = async () => await _jobRepository.CreateAsync(newJob);

        // Assert - Should still fail
        await createAgain.Should().ThrowAsync<Exception>(
            "canceled job's idempotency_key should still block new inserts");
    }

    [Fact]
    public async Task TenantIsolation_JobsOnlyVisibleToOwnTenant()
    {
        // Arrange
        var tenant1 = Guid.NewGuid().ToString();
        var tenant2 = Guid.NewGuid().ToString();

        var job1 = CreateJob("job-type-1", $"idem-1-{Guid.NewGuid():N}", tenant1);
        var job2 = CreateJob("job-type-2", $"idem-2-{Guid.NewGuid():N}", tenant2);

        await _jobRepository.CreateAsync(job1);
        await _jobRepository.CreateAsync(job2);

        // Act
        var tenant1Jobs = await _jobRepository.GetByStatusAsync(tenant1, JobStatus.Scheduled, limit: 100);
        var tenant2Jobs = await _jobRepository.GetByStatusAsync(tenant2, JobStatus.Scheduled, limit: 100);

        // Assert
        tenant1Jobs.Should().NotContain(j => j.TenantId == tenant2);
        tenant2Jobs.Should().NotContain(j => j.TenantId == tenant1);
    }

    private JobEntity CreateJob(string jobType, string idempotencyKey, string? tenantId = null, string? payloadDigest = null)
    {
        return new JobEntity
        {
            Id = Guid.NewGuid(),
            TenantId = tenantId ?? _tenantId,
            JobType = jobType,
            Status = JobStatus.Scheduled,
            Priority = 0,
            Payload = """{"test": true}""",
            PayloadDigest = payloadDigest ?? $"sha256:{Guid.NewGuid():N}",
            IdempotencyKey = idempotencyKey,
            MaxAttempts = 3
        };
    }
}