Add determinism tests for verdict artifact generation and update SHA256 sums script
- Implemented comprehensive tests for verdict artifact generation to ensure deterministic outputs across various scenarios, including identical inputs, parallel execution, and change ordering. - Created helper methods for generating sample verdict inputs and computing canonical hashes. - Added tests to validate the stability of canonical hashes, proof spine ordering, and summary statistics. - Introduced a new PowerShell script to update SHA256 sums for files, ensuring accurate hash generation and file integrity checks.
This commit is contained in:
StellaOps Bot
@@ -0,0 +1,307 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// DeliveryIdempotencyTests.cs
|
||||
// Sprint: SPRINT_5100_0009_0009_notify_tests
|
||||
// Task: NOTIFY-5100-010
|
||||
// Description: Model S1 idempotency tests for Notify delivery storage
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notify.Storage.Postgres.Models;
|
||||
using StellaOps.Notify.Storage.Postgres.Repositories;
|
||||
using StellaOps.TestKit;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Notify.Storage.Postgres.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Idempotency tests for Notify delivery storage operations.
|
||||
/// Implements Model S1 (Storage/Postgres) test requirements:
|
||||
/// - Same notification ID enqueued twice → single delivery
|
||||
/// - Correlation ID uniqueness enforced per tenant
|
||||
/// - Duplicate deliveries handled gracefully
|
||||
/// </summary>
|
||||
[Collection(NotifyPostgresCollection.Name)]
|
||||
[Trait("Category", TestCategories.Integration)]
|
||||
[Trait("Category", "StorageIdempotency")]
|
||||
public sealed class DeliveryIdempotencyTests : IAsyncLifetime
|
||||
{
|
||||
private readonly NotifyPostgresFixture _fixture;
|
||||
private NotifyDataSource _dataSource = null!;
|
||||
private DeliveryRepository _deliveryRepository = null!;
|
||||
private ChannelRepository _channelRepository = null!;
|
||||
private readonly string _tenantId = Guid.NewGuid().ToString();
|
||||
private Guid _channelId;
|
||||
|
||||
public DeliveryIdempotencyTests(NotifyPostgresFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
}
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
await _fixture.ExecuteSqlAsync(
|
||||
"TRUNCATE TABLE notify.audit, notify.deliveries, notify.digests, notify.channels RESTART IDENTITY CASCADE;");
|
||||
|
||||
var options = _fixture.Fixture.CreateOptions();
|
||||
options.SchemaName = _fixture.SchemaName;
|
||||
_dataSource = new NotifyDataSource(Options.Create(options), NullLogger<NotifyDataSource>.Instance);
|
||||
_deliveryRepository = new DeliveryRepository(_dataSource, NullLogger<DeliveryRepository>.Instance);
|
||||
_channelRepository = new ChannelRepository(_dataSource, NullLogger<ChannelRepository>.Instance);
|
||||
|
||||
// Create a channel for deliveries
|
||||
_channelId = Guid.NewGuid();
|
||||
await _channelRepository.CreateAsync(new ChannelEntity
|
||||
{
|
||||
Id = _channelId,
|
||||
TenantId = _tenantId,
|
||||
Name = "test-email",
|
||||
ChannelType = ChannelType.Email,
|
||||
Enabled = true
|
||||
});
|
||||
}
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
[Fact]
|
||||
public async Task CreateDelivery_SameId_SecondInsertFails()
|
||||
{
|
||||
// Arrange
|
||||
var deliveryId = Guid.NewGuid();
|
||||
var delivery1 = CreateDelivery(deliveryId, "user1@example.com");
|
||||
var delivery2 = CreateDelivery(deliveryId, "user2@example.com");
|
||||
|
||||
// Act
|
||||
await _deliveryRepository.CreateAsync(delivery1);
|
||||
var createAgain = async () => await _deliveryRepository.CreateAsync(delivery2);
|
||||
|
||||
// Assert - Second insert should fail due to unique constraint
|
||||
await createAgain.Should().ThrowAsync<Exception>(
|
||||
"duplicate delivery ID should be rejected");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetByCorrelationId_ReturnsExistingDelivery()
|
||||
{
|
||||
// Arrange
|
||||
var correlationId = $"corr-{Guid.NewGuid():N}";
|
||||
var delivery = CreateDelivery(correlationId: correlationId);
|
||||
await _deliveryRepository.CreateAsync(delivery);
|
||||
|
||||
// Act
|
||||
var existing = await _deliveryRepository.GetByCorrelationIdAsync(_tenantId, correlationId);
|
||||
|
||||
// Assert
|
||||
existing.Should().HaveCount(1);
|
||||
existing[0].Id.Should().Be(delivery.Id);
|
||||
existing[0].CorrelationId.Should().Be(correlationId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetByCorrelationId_DifferentTenant_ReturnsEmpty()
|
||||
{
|
||||
// Arrange
|
||||
var correlationId = $"corr-{Guid.NewGuid():N}";
|
||||
var delivery = CreateDelivery(correlationId: correlationId);
|
||||
await _deliveryRepository.CreateAsync(delivery);
|
||||
|
||||
// Act - Query with different tenant
|
||||
var otherTenant = Guid.NewGuid().ToString();
|
||||
var existing = await _deliveryRepository.GetByCorrelationIdAsync(otherTenant, correlationId);
|
||||
|
||||
// Assert - Should not find delivery from different tenant
|
||||
existing.Should().BeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MultipleDeliveriesWithSameCorrelation_AllReturned()
|
||||
{
|
||||
// Arrange
|
||||
var correlationId = $"corr-{Guid.NewGuid():N}";
|
||||
|
||||
// Create multiple deliveries for same correlation (e.g., email + Slack)
|
||||
var emailDelivery = CreateDelivery(
|
||||
correlationId: correlationId,
|
||||
recipient: "user@example.com");
|
||||
var slackDelivery = CreateDelivery(
|
||||
correlationId: correlationId,
|
||||
recipient: "@user-slack");
|
||||
|
||||
await _deliveryRepository.CreateAsync(emailDelivery);
|
||||
await _deliveryRepository.CreateAsync(slackDelivery);
|
||||
|
||||
// Act
|
||||
var deliveries = await _deliveryRepository.GetByCorrelationIdAsync(_tenantId, correlationId);
|
||||
|
||||
// Assert - Both deliveries should be returned
|
||||
deliveries.Should().HaveCount(2);
|
||||
deliveries.Select(d => d.Id).Should().Contain(emailDelivery.Id);
|
||||
deliveries.Select(d => d.Id).Should().Contain(slackDelivery.Id);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MultipleDeliveries_UniqueIds_AllCreated()
|
||||
{
|
||||
// Arrange
|
||||
var deliveries = Enumerable.Range(1, 10)
|
||||
.Select(i => CreateDelivery(recipient: $"user{i}@example.com"))
|
||||
.ToList();
|
||||
|
||||
// Act
|
||||
foreach (var delivery in deliveries)
|
||||
{
|
||||
await _deliveryRepository.CreateAsync(delivery);
|
||||
}
|
||||
|
||||
// Assert - All deliveries should be created
|
||||
foreach (var delivery in deliveries)
|
||||
{
|
||||
var fetched = await _deliveryRepository.GetByIdAsync(_tenantId, delivery.Id);
|
||||
fetched.Should().NotBeNull();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeliveredNotification_SameIdCannotBeRecreated()
|
||||
{
|
||||
// Arrange
|
||||
var delivery = CreateDelivery();
|
||||
await _deliveryRepository.CreateAsync(delivery);
|
||||
|
||||
// Mark as delivered
|
||||
await _deliveryRepository.MarkQueuedAsync(_tenantId, delivery.Id);
|
||||
await _deliveryRepository.MarkSentAsync(_tenantId, delivery.Id);
|
||||
await _deliveryRepository.MarkDeliveredAsync(_tenantId, delivery.Id);
|
||||
|
||||
// Act - Try to create another delivery with same ID
|
||||
var newDelivery = CreateDelivery(delivery.Id, "different@example.com");
|
||||
var createAgain = async () => await _deliveryRepository.CreateAsync(newDelivery);
|
||||
|
||||
// Assert - Should still fail
|
||||
await createAgain.Should().ThrowAsync<Exception>(
|
||||
"delivered notification's ID should still block new inserts");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FailedNotification_SameIdCannotBeRecreated()
|
||||
{
|
||||
// Arrange
|
||||
var delivery = CreateDelivery(maxAttempts: 1);
|
||||
await _deliveryRepository.CreateAsync(delivery);
|
||||
|
||||
// Mark as failed
|
||||
await _deliveryRepository.MarkFailedAsync(_tenantId, delivery.Id, "Test failure", TimeSpan.Zero);
|
||||
|
||||
// Verify it's actually failed
|
||||
var fetched = await _deliveryRepository.GetByIdAsync(_tenantId, delivery.Id);
|
||||
if (fetched!.Status == DeliveryStatus.Failed)
|
||||
{
|
||||
// Act - Try to create another delivery with same ID
|
||||
var newDelivery = CreateDelivery(delivery.Id, "different@example.com");
|
||||
var createAgain = async () => await _deliveryRepository.CreateAsync(newDelivery);
|
||||
|
||||
// Assert - Should still fail
|
||||
await createAgain.Should().ThrowAsync<Exception>(
|
||||
"failed notification's ID should still block new inserts");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TenantIsolation_DeliveriesOnlyVisibleToOwnTenant()
|
||||
{
|
||||
// Arrange
|
||||
var tenant1 = _tenantId;
|
||||
var tenant2 = Guid.NewGuid().ToString();
|
||||
|
||||
// Create channel for tenant2
|
||||
var tenant2ChannelId = Guid.NewGuid();
|
||||
await _channelRepository.CreateAsync(new ChannelEntity
|
||||
{
|
||||
Id = tenant2ChannelId,
|
||||
TenantId = tenant2,
|
||||
Name = "test-email-t2",
|
||||
ChannelType = ChannelType.Email,
|
||||
Enabled = true
|
||||
});
|
||||
|
||||
var delivery1 = CreateDelivery(recipient: "user1@example.com");
|
||||
var delivery2 = new DeliveryEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
TenantId = tenant2,
|
||||
ChannelId = tenant2ChannelId,
|
||||
Recipient = "user2@example.com",
|
||||
EventType = "test.event",
|
||||
Status = DeliveryStatus.Pending
|
||||
};
|
||||
|
||||
await _deliveryRepository.CreateAsync(delivery1);
|
||||
await _deliveryRepository.CreateAsync(delivery2);
|
||||
|
||||
// Act
|
||||
var tenant1Deliveries = await _deliveryRepository.GetByStatusAsync(tenant1, DeliveryStatus.Pending);
|
||||
var tenant2Deliveries = await _deliveryRepository.GetByStatusAsync(tenant2, DeliveryStatus.Pending);
|
||||
|
||||
// Assert
|
||||
tenant1Deliveries.Should().NotContain(d => d.TenantId == tenant2);
|
||||
tenant2Deliveries.Should().NotContain(d => d.TenantId == tenant1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetPending_SameDeliveryNotReturnedTwice()
|
||||
{
|
||||
// Arrange
|
||||
var delivery = CreateDelivery();
|
||||
await _deliveryRepository.CreateAsync(delivery);
|
||||
|
||||
// Act - Query pending multiple times
|
||||
var pending1 = await _deliveryRepository.GetPendingAsync(_tenantId);
|
||||
var pending2 = await _deliveryRepository.GetPendingAsync(_tenantId);
|
||||
|
||||
// Assert - Same delivery should appear in both (idempotent read)
|
||||
pending1.Should().ContainSingle(d => d.Id == delivery.Id);
|
||||
pending2.Should().ContainSingle(d => d.Id == delivery.Id);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MarkOperations_AreIdempotent()
|
||||
{
|
||||
// Arrange
|
||||
var delivery = CreateDelivery();
|
||||
await _deliveryRepository.CreateAsync(delivery);
|
||||
|
||||
// Act - Mark as queued twice
|
||||
var result1 = await _deliveryRepository.MarkQueuedAsync(_tenantId, delivery.Id);
|
||||
var result2 = await _deliveryRepository.MarkQueuedAsync(_tenantId, delivery.Id);
|
||||
|
||||
// Assert - First should succeed, second may return false but shouldn't throw
|
||||
result1.Should().BeTrue();
|
||||
// Second call behavior depends on implementation (may be true or false)
|
||||
|
||||
// Verify delivery is still in correct state
|
||||
var fetched = await _deliveryRepository.GetByIdAsync(_tenantId, delivery.Id);
|
||||
fetched.Should().NotBeNull();
|
||||
fetched!.Status.Should().Be(DeliveryStatus.Queued);
|
||||
}
|
||||
|
||||
private DeliveryEntity CreateDelivery(
|
||||
Guid? id = null,
|
||||
string? recipient = null,
|
||||
string? correlationId = null,
|
||||
int maxAttempts = 3)
|
||||
{
|
||||
return new DeliveryEntity
|
||||
{
|
||||
Id = id ?? Guid.NewGuid(),
|
||||
TenantId = _tenantId,
|
||||
ChannelId = _channelId,
|
||||
Recipient = recipient ?? "test@example.com",
|
||||
EventType = "test.event",
|
||||
EventPayload = """{"test": true}""",
|
||||
Status = DeliveryStatus.Pending,
|
||||
CorrelationId = correlationId,
|
||||
MaxAttempts = maxAttempts
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,320 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// NotifyMigrationTests.cs
|
||||
// Sprint: SPRINT_5100_0009_0009_notify_tests
|
||||
// Task: NOTIFY-5100-009
|
||||
// Description: Model S1 migration tests for Notify.Storage
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Reflection;
|
||||
using Dapper;
|
||||
using FluentAssertions;
|
||||
using Npgsql;
|
||||
using StellaOps.TestKit;
|
||||
using Testcontainers.PostgreSql;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Notify.Storage.Postgres.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Migration tests for Notify.Storage.
|
||||
/// Implements Model S1 (Storage/Postgres) migration test requirements:
|
||||
/// - Apply all migrations from scratch (fresh database)
|
||||
/// - Apply migrations from N-1 (incremental application)
|
||||
/// - Verify migration idempotency (apply twice → no error)
|
||||
/// </summary>
|
||||
[Trait("Category", TestCategories.Integration)]
|
||||
[Trait("Category", "StorageMigration")]
|
||||
public sealed class NotifyMigrationTests : IAsyncLifetime
|
||||
{
|
||||
private PostgreSqlContainer _container = null!;
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
_container = new PostgreSqlBuilder()
|
||||
.WithImage("postgres:16-alpine")
|
||||
.WithDatabase("notify_migration_test")
|
||||
.WithUsername("postgres")
|
||||
.WithPassword("postgres")
|
||||
.Build();
|
||||
|
||||
await _container.StartAsync();
|
||||
}
|
||||
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
await _container.DisposeAsync();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyMigrations_FromScratch_AllTablesCreated()
|
||||
{
|
||||
// Arrange
|
||||
var connectionString = _container.GetConnectionString();
|
||||
|
||||
// Act - Apply all migrations from scratch
|
||||
await ApplyAllMigrationsAsync(connectionString);
|
||||
|
||||
// Assert - Verify Notify tables exist
|
||||
await using var connection = new NpgsqlConnection(connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
var tables = await connection.QueryAsync<string>(
|
||||
@"SELECT table_name FROM information_schema.tables
|
||||
WHERE table_schema = 'notify'
|
||||
ORDER BY table_name");
|
||||
|
||||
var tableList = tables.ToList();
|
||||
|
||||
// Verify core Notify tables exist
|
||||
tableList.Should().Contain("deliveries", "deliveries table should exist");
|
||||
tableList.Should().Contain("channels", "channels table should exist");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyMigrations_FromScratch_AllMigrationsRecorded()
|
||||
{
|
||||
// Arrange
|
||||
var connectionString = _container.GetConnectionString();
|
||||
await ApplyAllMigrationsAsync(connectionString);
|
||||
|
||||
// Assert - Verify migrations are recorded
|
||||
await using var connection = new NpgsqlConnection(connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
var migrationsApplied = await connection.QueryAsync<string>(
|
||||
"SELECT migration_id FROM __migrations ORDER BY applied_at");
|
||||
|
||||
var migrationList = migrationsApplied.ToList();
|
||||
migrationList.Should().NotBeEmpty("migrations should be tracked");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyMigrations_Twice_IsIdempotent()
|
||||
{
|
||||
// Arrange
|
||||
var connectionString = _container.GetConnectionString();
|
||||
|
||||
// Act - Apply migrations twice
|
||||
await ApplyAllMigrationsAsync(connectionString);
|
||||
var applyAgain = async () => await ApplyAllMigrationsAsync(connectionString);
|
||||
|
||||
// Assert - Second application should not throw
|
||||
await applyAgain.Should().NotThrowAsync(
|
||||
"applying migrations twice should be idempotent");
|
||||
|
||||
// Verify migrations are not duplicated
|
||||
await using var connection = new NpgsqlConnection(connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
var migrationCount = await connection.ExecuteScalarAsync<int>(
|
||||
"SELECT COUNT(*) FROM __migrations");
|
||||
|
||||
// Count unique migrations
|
||||
var uniqueMigrations = await connection.ExecuteScalarAsync<int>(
|
||||
"SELECT COUNT(DISTINCT migration_id) FROM __migrations");
|
||||
|
||||
migrationCount.Should().Be(uniqueMigrations,
|
||||
"each migration should only be recorded once");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyMigrations_VerifySchemaIntegrity()
|
||||
{
|
||||
// Arrange
|
||||
var connectionString = _container.GetConnectionString();
|
||||
await ApplyAllMigrationsAsync(connectionString);
|
||||
|
||||
// Assert - Verify indexes exist
|
||||
await using var connection = new NpgsqlConnection(connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
var indexes = await connection.QueryAsync<string>(
|
||||
@"SELECT indexname FROM pg_indexes
|
||||
WHERE schemaname = 'notify'
|
||||
ORDER BY indexname");
|
||||
|
||||
var indexList = indexes.ToList();
|
||||
indexList.Should().NotBeEmpty("notify schema should have indexes");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyMigrations_IndividualMigrationsCanRollForward()
|
||||
{
|
||||
// Arrange
|
||||
var connectionString = _container.GetConnectionString();
|
||||
|
||||
// Act - Apply migrations in sequence
|
||||
var migrationFiles = GetMigrationFiles();
|
||||
|
||||
await using var connection = new NpgsqlConnection(connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
// Create migration tracking table first
|
||||
await connection.ExecuteAsync(@"
|
||||
CREATE TABLE IF NOT EXISTS __migrations (
|
||||
id SERIAL PRIMARY KEY,
|
||||
migration_id TEXT NOT NULL UNIQUE,
|
||||
applied_at TIMESTAMPTZ DEFAULT NOW()
|
||||
)");
|
||||
|
||||
// Apply each migration in order
|
||||
int appliedCount = 0;
|
||||
foreach (var migrationFile in migrationFiles.OrderBy(f => f))
|
||||
{
|
||||
var migrationId = Path.GetFileName(migrationFile);
|
||||
|
||||
// Check if already applied
|
||||
var alreadyApplied = await connection.ExecuteScalarAsync<int>(
|
||||
"SELECT COUNT(*) FROM __migrations WHERE migration_id = @Id",
|
||||
new { Id = migrationId });
|
||||
|
||||
if (alreadyApplied > 0)
|
||||
continue;
|
||||
|
||||
// Apply migration
|
||||
var sql = GetMigrationContent(migrationFile);
|
||||
if (!string.IsNullOrWhiteSpace(sql))
|
||||
{
|
||||
await connection.ExecuteAsync(sql);
|
||||
await connection.ExecuteAsync(
|
||||
"INSERT INTO __migrations (migration_id) VALUES (@Id)",
|
||||
new { Id = migrationId });
|
||||
appliedCount++;
|
||||
}
|
||||
}
|
||||
|
||||
// Assert - Migrations should be applied (if any exist)
|
||||
var totalMigrations = await connection.ExecuteScalarAsync<int>(
|
||||
"SELECT COUNT(*) FROM __migrations");
|
||||
|
||||
totalMigrations.Should().BeGreaterThanOrEqualTo(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyMigrations_ForeignKeyConstraintsValid()
|
||||
{
|
||||
// Arrange
|
||||
var connectionString = _container.GetConnectionString();
|
||||
await ApplyAllMigrationsAsync(connectionString);
|
||||
|
||||
// Assert - Verify foreign key constraints exist and are valid
|
||||
await using var connection = new NpgsqlConnection(connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
var foreignKeys = await connection.QueryAsync<string>(
|
||||
@"SELECT tc.constraint_name
|
||||
FROM information_schema.table_constraints tc
|
||||
WHERE tc.constraint_type = 'FOREIGN KEY'
|
||||
AND tc.table_schema = 'notify'
|
||||
ORDER BY tc.constraint_name");
|
||||
|
||||
var fkList = foreignKeys.ToList();
|
||||
// Foreign keys may or may not exist depending on schema design
|
||||
fkList.Should().NotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyMigrations_DeliveriesTableHasRequiredColumns()
|
||||
{
|
||||
// Arrange
|
||||
var connectionString = _container.GetConnectionString();
|
||||
await ApplyAllMigrationsAsync(connectionString);
|
||||
|
||||
// Assert - Verify deliveries table has required columns
|
||||
await using var connection = new NpgsqlConnection(connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
var columns = await connection.QueryAsync<string>(
|
||||
@"SELECT column_name FROM information_schema.columns
|
||||
WHERE table_name = 'deliveries' AND table_schema = 'notify'
|
||||
ORDER BY ordinal_position");
|
||||
|
||||
var columnList = columns.ToList();
|
||||
|
||||
if (columnList.Any())
|
||||
{
|
||||
columnList.Should().Contain("id", "deliveries table should have id column");
|
||||
columnList.Should().Contain("tenant_id", "deliveries table should have tenant_id column");
|
||||
columnList.Should().Contain("status", "deliveries table should have status column");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyMigrations_NotifySchemaExists()
|
||||
{
|
||||
// Arrange
|
||||
var connectionString = _container.GetConnectionString();
|
||||
await ApplyAllMigrationsAsync(connectionString);
|
||||
|
||||
// Assert - Verify notify schema exists
|
||||
await using var connection = new NpgsqlConnection(connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
var schemaExists = await connection.ExecuteScalarAsync<int>(
|
||||
@"SELECT COUNT(*) FROM information_schema.schemata
|
||||
WHERE schema_name = 'notify'");
|
||||
|
||||
schemaExists.Should().Be(1, "notify schema should exist");
|
||||
}
|
||||
|
||||
private async Task ApplyAllMigrationsAsync(string connectionString)
|
||||
{
|
||||
await using var connection = new NpgsqlConnection(connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
// Create migration tracking table
|
||||
await connection.ExecuteAsync(@"
|
||||
CREATE TABLE IF NOT EXISTS __migrations (
|
||||
id SERIAL PRIMARY KEY,
|
||||
migration_id TEXT NOT NULL UNIQUE,
|
||||
applied_at TIMESTAMPTZ DEFAULT NOW()
|
||||
)");
|
||||
|
||||
// Get and apply all migrations
|
||||
var migrationFiles = GetMigrationFiles();
|
||||
|
||||
foreach (var migrationFile in migrationFiles.OrderBy(f => f))
|
||||
{
|
||||
var migrationId = Path.GetFileName(migrationFile);
|
||||
|
||||
// Skip if already applied
|
||||
var alreadyApplied = await connection.ExecuteScalarAsync<int>(
|
||||
"SELECT COUNT(*) FROM __migrations WHERE migration_id = @Id",
|
||||
new { Id = migrationId });
|
||||
|
||||
if (alreadyApplied > 0)
|
||||
continue;
|
||||
|
||||
// Apply migration
|
||||
var sql = GetMigrationContent(migrationFile);
|
||||
if (!string.IsNullOrWhiteSpace(sql))
|
||||
{
|
||||
await connection.ExecuteAsync(sql);
|
||||
await connection.ExecuteAsync(
|
||||
"INSERT INTO __migrations (migration_id) VALUES (@Id)",
|
||||
new { Id = migrationId });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static IEnumerable<string> GetMigrationFiles()
|
||||
{
|
||||
var assembly = typeof(NotifyDataSource).Assembly;
|
||||
var resourceNames = assembly.GetManifestResourceNames()
|
||||
.Where(n => n.Contains("Migrations") && n.EndsWith(".sql"))
|
||||
.OrderBy(n => n);
|
||||
|
||||
return resourceNames;
|
||||
}
|
||||
|
||||
private static string GetMigrationContent(string resourceName)
|
||||
{
|
||||
var assembly = typeof(NotifyDataSource).Assembly;
|
||||
using var stream = assembly.GetManifestResourceStream(resourceName);
|
||||
if (stream == null)
|
||||
return string.Empty;
|
||||
|
||||
using var reader = new StreamReader(stream);
|
||||
return reader.ReadToEnd();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// RetryStatePersistenceTests.cs
|
||||
// Sprint: SPRINT_5100_0009_0009_notify_tests
|
||||
// Task: NOTIFY-5100-011
|
||||
// Description: Model S1 retry state persistence tests for Notify delivery storage
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notify.Storage.Postgres.Models;
|
||||
using StellaOps.Notify.Storage.Postgres.Repositories;
|
||||
using StellaOps.TestKit;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Notify.Storage.Postgres.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Retry state persistence tests for Notify delivery storage operations.
|
||||
/// Implements Model S1 (Storage/Postgres) test requirements:
|
||||
/// - Failed notification → retry state saved
|
||||
/// - Retry state persists across queries
|
||||
/// - Retry on next poll behavior
|
||||
/// </summary>
|
||||
[Collection(NotifyPostgresCollection.Name)]
|
||||
[Trait("Category", TestCategories.Integration)]
|
||||
[Trait("Category", "RetryPersistence")]
|
||||
public sealed class RetryStatePersistenceTests : IAsyncLifetime
|
||||
{
|
||||
private readonly NotifyPostgresFixture _fixture;
|
||||
private NotifyDataSource _dataSource = null!;
|
||||
private DeliveryRepository _deliveryRepository = null!;
|
||||
private ChannelRepository _channelRepository = null!;
|
||||
private readonly string _tenantId = Guid.NewGuid().ToString();
|
||||
private Guid _channelId;
|
||||
|
||||
public RetryStatePersistenceTests(NotifyPostgresFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
}
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
await _fixture.ExecuteSqlAsync(
|
||||
"TRUNCATE TABLE notify.audit, notify.deliveries, notify.digests, notify.channels RESTART IDENTITY CASCADE;");
|
||||
|
||||
var options = _fixture.Fixture.CreateOptions();
|
||||
options.SchemaName = _fixture.SchemaName;
|
||||
_dataSource = new NotifyDataSource(Options.Create(options), NullLogger<NotifyDataSource>.Instance);
|
||||
_deliveryRepository = new DeliveryRepository(_dataSource, NullLogger<DeliveryRepository>.Instance);
|
||||
_channelRepository = new ChannelRepository(_dataSource, NullLogger<ChannelRepository>.Instance);
|
||||
|
||||
// Create a channel for deliveries
|
||||
_channelId = Guid.NewGuid();
|
||||
await _channelRepository.CreateAsync(new ChannelEntity
|
||||
{
|
||||
Id = _channelId,
|
||||
TenantId = _tenantId,
|
||||
Name = "test-email",
|
||||
ChannelType = ChannelType.Email,
|
||||
Enabled = true
|
||||
});
|
||||
}
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
[Fact]
|
||||
public async Task MarkFailed_WithRetry_SavesNextRetryTime()
|
||||
{
|
||||
// Arrange
|
||||
var delivery = CreateDelivery(maxAttempts: 3);
|
||||
await _deliveryRepository.CreateAsync(delivery);
|
||||
|
||||
var retryDelay = TimeSpan.FromMinutes(5);
|
||||
|
||||
// Act
|
||||
await _deliveryRepository.MarkFailedAsync(_tenantId, delivery.Id, "Connection timeout", retryDelay);
|
||||
|
||||
// Assert
|
||||
var fetched = await _deliveryRepository.GetByIdAsync(_tenantId, delivery.Id);
|
||||
fetched.Should().NotBeNull();
|
||||
fetched!.Attempt.Should().BeGreaterThan(0);
|
||||
fetched.ErrorMessage.Should().Be("Connection timeout");
|
||||
|
||||
// If retry is scheduled (not yet exhausted attempts), should have next retry time
|
||||
if (fetched.Status == DeliveryStatus.Pending)
|
||||
{
|
||||
fetched.NextRetryAt.Should().NotBeNull();
|
||||
fetched.NextRetryAt.Should().BeAfter(DateTimeOffset.UtcNow);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MarkFailed_AttemptsExhausted_StatusIsFailed()
|
||||
{
|
||||
// Arrange
|
||||
var delivery = CreateDelivery(maxAttempts: 1);
|
||||
await _deliveryRepository.CreateAsync(delivery);
|
||||
|
||||
// Act - Fail once (exhausts max attempts)
|
||||
await _deliveryRepository.MarkFailedAsync(_tenantId, delivery.Id, "Permanent failure", TimeSpan.Zero);
|
||||
|
||||
// Assert
|
||||
var fetched = await _deliveryRepository.GetByIdAsync(_tenantId, delivery.Id);
|
||||
fetched.Should().NotBeNull();
|
||||
fetched!.Status.Should().Be(DeliveryStatus.Failed);
|
||||
fetched.FailedAt.Should().NotBeNull();
|
||||
fetched.ErrorMessage.Should().Be("Permanent failure");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RetryState_PersistsAcrossQueries()
|
||||
{
|
||||
// Arrange
|
||||
var delivery = CreateDelivery(maxAttempts: 5);
|
||||
await _deliveryRepository.CreateAsync(delivery);
|
||||
|
||||
var retryDelay = TimeSpan.FromMinutes(10);
|
||||
await _deliveryRepository.MarkFailedAsync(_tenantId, delivery.Id, "Transient error", retryDelay);
|
||||
|
||||
// Act - Query multiple times
|
||||
var fetched1 = await _deliveryRepository.GetByIdAsync(_tenantId, delivery.Id);
|
||||
var fetched2 = await _deliveryRepository.GetByIdAsync(_tenantId, delivery.Id);
|
||||
var fetched3 = await _deliveryRepository.GetByIdAsync(_tenantId, delivery.Id);
|
||||
|
||||
// Assert - All queries should return same retry state
|
||||
fetched1!.Attempt.Should().Be(fetched2!.Attempt);
|
||||
fetched2.Attempt.Should().Be(fetched3!.Attempt);
|
||||
|
||||
fetched1.ErrorMessage.Should().Be(fetched2.ErrorMessage);
|
||||
fetched2.ErrorMessage.Should().Be(fetched3.ErrorMessage);
|
||||
|
||||
fetched1.NextRetryAt.Should().Be(fetched2.NextRetryAt);
|
||||
fetched2.NextRetryAt.Should().Be(fetched3.NextRetryAt);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MultipleFailures_AttemptCountIncreases()
|
||||
{
|
||||
// Arrange
|
||||
var delivery = CreateDelivery(maxAttempts: 5);
|
||||
await _deliveryRepository.CreateAsync(delivery);
|
||||
|
||||
// Act - Fail multiple times
|
||||
await _deliveryRepository.MarkFailedAsync(_tenantId, delivery.Id, "Error 1", TimeSpan.FromSeconds(1));
|
||||
var afterFirst = await _deliveryRepository.GetByIdAsync(_tenantId, delivery.Id);
|
||||
|
||||
// Wait for retry window if needed
|
||||
if (afterFirst!.Status == DeliveryStatus.Pending && afterFirst.NextRetryAt.HasValue)
|
||||
{
|
||||
await _deliveryRepository.MarkFailedAsync(_tenantId, delivery.Id, "Error 2", TimeSpan.FromSeconds(1));
|
||||
}
|
||||
|
||||
// Assert
|
||||
var fetched = await _deliveryRepository.GetByIdAsync(_tenantId, delivery.Id);
|
||||
fetched.Should().NotBeNull();
|
||||
fetched!.Attempt.Should().BeGreaterThanOrEqualTo(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetPending_IncludesRetryableDeliveries()
|
||||
{
|
||||
// Arrange
|
||||
var delivery = CreateDelivery(maxAttempts: 3);
|
||||
await _deliveryRepository.CreateAsync(delivery);
|
||||
|
||||
// Fail with short retry delay
|
||||
await _deliveryRepository.MarkFailedAsync(_tenantId, delivery.Id, "Transient error", TimeSpan.Zero);
|
||||
|
||||
var fetched = await _deliveryRepository.GetByIdAsync(_tenantId, delivery.Id);
|
||||
|
||||
// Act - Query pending deliveries (should include retryable ones)
|
||||
if (fetched!.Status == DeliveryStatus.Pending)
|
||||
{
|
||||
var pending = await _deliveryRepository.GetPendingAsync(_tenantId);
|
||||
|
||||
// Assert - Delivery should be in pending list if retry time has passed
|
||||
if (fetched.NextRetryAt == null || fetched.NextRetryAt <= DateTimeOffset.UtcNow)
|
||||
{
|
||||
pending.Should().Contain(d => d.Id == delivery.Id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ErrorMessage_PreservedAfterRetry()
|
||||
{
|
||||
// Arrange
|
||||
var delivery = CreateDelivery(maxAttempts: 3);
|
||||
await _deliveryRepository.CreateAsync(delivery);
|
||||
|
||||
var errorMessage = "SMTP connection refused: 10.0.0.1:587";
|
||||
|
||||
// Act
|
||||
await _deliveryRepository.MarkFailedAsync(_tenantId, delivery.Id, errorMessage, TimeSpan.FromMinutes(5));
|
||||
|
||||
// Assert
|
||||
var fetched = await _deliveryRepository.GetByIdAsync(_tenantId, delivery.Id);
|
||||
fetched.Should().NotBeNull();
|
||||
fetched!.ErrorMessage.Should().Be(errorMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SuccessfulDelivery_ClearsRetryState()
|
||||
{
|
||||
// Arrange
|
||||
var delivery = CreateDelivery(maxAttempts: 3);
|
||||
await _deliveryRepository.CreateAsync(delivery);
|
||||
|
||||
// Fail first
|
||||
await _deliveryRepository.MarkFailedAsync(_tenantId, delivery.Id, "Transient error", TimeSpan.FromMinutes(1));
|
||||
|
||||
// Then succeed
|
||||
await _deliveryRepository.MarkQueuedAsync(_tenantId, delivery.Id);
|
||||
await _deliveryRepository.MarkSentAsync(_tenantId, delivery.Id);
|
||||
await _deliveryRepository.MarkDeliveredAsync(_tenantId, delivery.Id);
|
||||
|
||||
// Assert
|
||||
var fetched = await _deliveryRepository.GetByIdAsync(_tenantId, delivery.Id);
|
||||
fetched.Should().NotBeNull();
|
||||
fetched!.Status.Should().Be(DeliveryStatus.Delivered);
|
||||
fetched.DeliveredAt.Should().NotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MultipleDeliveries_RetryStatesAreIndependent()
|
||||
{
|
||||
// Arrange
|
||||
var delivery1 = CreateDelivery(maxAttempts: 3, recipient: "user1@example.com");
|
||||
var delivery2 = CreateDelivery(maxAttempts: 3, recipient: "user2@example.com");
|
||||
|
||||
await _deliveryRepository.CreateAsync(delivery1);
|
||||
await _deliveryRepository.CreateAsync(delivery2);
|
||||
|
||||
// Act - Fail delivery1, succeed delivery2
|
||||
await _deliveryRepository.MarkFailedAsync(_tenantId, delivery1.Id, "Error for user1", TimeSpan.FromMinutes(5));
|
||||
await _deliveryRepository.MarkQueuedAsync(_tenantId, delivery2.Id);
|
||||
await _deliveryRepository.MarkSentAsync(_tenantId, delivery2.Id);
|
||||
await _deliveryRepository.MarkDeliveredAsync(_tenantId, delivery2.Id);
|
||||
|
||||
// Assert
|
||||
var fetched1 = await _deliveryRepository.GetByIdAsync(_tenantId, delivery1.Id);
|
||||
var fetched2 = await _deliveryRepository.GetByIdAsync(_tenantId, delivery2.Id);
|
||||
|
||||
fetched1!.ErrorMessage.Should().Be("Error for user1");
|
||||
fetched2!.Status.Should().Be(DeliveryStatus.Delivered);
|
||||
fetched2.ErrorMessage.Should().BeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetByStatus_ReturnsPendingWithRetries()
|
||||
{
|
||||
// Arrange
|
||||
var delivery = CreateDelivery(maxAttempts: 5);
|
||||
await _deliveryRepository.CreateAsync(delivery);
|
||||
|
||||
await _deliveryRepository.MarkFailedAsync(_tenantId, delivery.Id, "Transient", TimeSpan.Zero);
|
||||
|
||||
var fetched = await _deliveryRepository.GetByIdAsync(_tenantId, delivery.Id);
|
||||
|
||||
// Act - Query by pending status
|
||||
if (fetched!.Status == DeliveryStatus.Pending)
|
||||
{
|
||||
var pending = await _deliveryRepository.GetByStatusAsync(_tenantId, DeliveryStatus.Pending);
|
||||
|
||||
// Assert
|
||||
pending.Should().Contain(d => d.Id == delivery.Id);
|
||||
var foundDelivery = pending.First(d => d.Id == delivery.Id);
|
||||
foundDelivery.Attempt.Should().BeGreaterThan(0);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetByStatus_ReturnsFailed()
|
||||
{
|
||||
// Arrange
|
||||
var delivery = CreateDelivery(maxAttempts: 1);
|
||||
await _deliveryRepository.CreateAsync(delivery);
|
||||
|
||||
await _deliveryRepository.MarkFailedAsync(_tenantId, delivery.Id, "Permanent failure", TimeSpan.Zero);
|
||||
|
||||
// Act
|
||||
var failed = await _deliveryRepository.GetByStatusAsync(_tenantId, DeliveryStatus.Failed);
|
||||
|
||||
// Assert
|
||||
failed.Should().Contain(d => d.Id == delivery.Id);
|
||||
var foundDelivery = failed.First(d => d.Id == delivery.Id);
|
||||
foundDelivery.FailedAt.Should().NotBeNull();
|
||||
foundDelivery.ErrorMessage.Should().Be("Permanent failure");
|
||||
}
|
||||
|
||||
private DeliveryEntity CreateDelivery(int maxAttempts = 3, string? recipient = null)
|
||||
{
|
||||
return new DeliveryEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
TenantId = _tenantId,
|
||||
ChannelId = _channelId,
|
||||
Recipient = recipient ?? "test@example.com",
|
||||
EventType = "test.event",
|
||||
EventPayload = """{"test": true}""",
|
||||
Status = DeliveryStatus.Pending,
|
||||
MaxAttempts = maxAttempts
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -12,9 +12,11 @@
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Dapper" Version="2.1.35" />
|
||||
<PackageReference Include="FluentAssertions" Version="6.12.0" />
|
||||
<PackageReference Include="Moq" Version="4.20.70" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.0" />
|
||||
<PackageReference Include="Testcontainers.PostgreSql" Version="4.3.0" />
|
||||
<PackageReference Include="xunit" Version="2.9.2" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
|
||||
Reference in New Issue
Block a user