up
This commit is contained in:
28
src/StellaOps.Notify.Engine/INotifyRuleEvaluator.cs
Normal file
28
src/StellaOps.Notify.Engine/INotifyRuleEvaluator.cs
Normal file
@@ -0,0 +1,28 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using StellaOps.Notify.Models;
|
||||
|
||||
namespace StellaOps.Notify.Engine;
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates Notify rules against platform events.
|
||||
/// </summary>
|
||||
public interface INotifyRuleEvaluator
|
||||
{
|
||||
/// <summary>
|
||||
/// Evaluates a single rule against an event and returns the match outcome.
|
||||
/// </summary>
|
||||
NotifyRuleEvaluationOutcome Evaluate(
|
||||
NotifyRule rule,
|
||||
NotifyEvent @event,
|
||||
DateTimeOffset? evaluationTimestamp = null);
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates a collection of rules against an event.
|
||||
/// </summary>
|
||||
ImmutableArray<NotifyRuleEvaluationOutcome> Evaluate(
|
||||
IEnumerable<NotifyRule> rules,
|
||||
NotifyEvent @event,
|
||||
DateTimeOffset? evaluationTimestamp = null);
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| NOTIFY-ENGINE-15-301 | TODO | Notify Engine Guild | NOTIFY-MODELS-15-101 | Rules evaluation core: tenant/kind filters, severity/delta gates, VEX gating, throttling, idempotency key generation. | Unit tests cover rule permutations; idempotency keys deterministic; documentation updated. |
|
||||
| NOTIFY-ENGINE-15-301 | DOING (2025-10-24) | Notify Engine Guild | NOTIFY-MODELS-15-101 | Rules evaluation core: tenant/kind filters, severity/delta gates, VEX gating, throttling, idempotency key generation. | Unit tests cover rule permutations; idempotency keys deterministic; documentation updated. |
|
||||
| NOTIFY-ENGINE-15-302 | TODO | Notify Engine Guild | NOTIFY-ENGINE-15-301 | Action planner + digest coalescer with window management and dedupe per architecture §4. | Digest windows tested; throttles and digests recorded; metrics counters exposed. |
|
||||
| NOTIFY-ENGINE-15-303 | TODO | Notify Engine Guild | NOTIFY-ENGINE-15-302 | Template rendering engine (Slack, Teams, Email, Webhook) with helpers and i18n support. | Rendering fixtures validated; helpers documented; deterministic output proven via golden tests. |
|
||||
| NOTIFY-ENGINE-15-304 | TODO | Notify Engine Guild | NOTIFY-ENGINE-15-303 | Test-send sandbox + preview utilities for WebService. | Preview/test functions validated; sample outputs returned; no state persisted. |
|
||||
|
||||
223
src/StellaOps.Notify.Queue.Tests/NatsNotifyDeliveryQueueTests.cs
Normal file
223
src/StellaOps.Notify.Queue.Tests/NatsNotifyDeliveryQueueTests.cs
Normal file
@@ -0,0 +1,223 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.Json.Nodes;
|
||||
using System.Threading.Tasks;
|
||||
using DotNet.Testcontainers.Builders;
|
||||
using DotNet.Testcontainers.Containers;
|
||||
using DotNet.Testcontainers.Configurations;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NATS.Client.Core;
|
||||
using NATS.Client.JetStream;
|
||||
using NATS.Client.JetStream.Models;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notify.Queue;
|
||||
using StellaOps.Notify.Queue.Nats;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Notify.Queue.Tests;
|
||||
|
||||
public sealed class NatsNotifyDeliveryQueueTests : IAsyncLifetime
|
||||
{
|
||||
private readonly TestcontainersContainer _nats;
|
||||
private string? _skipReason;
|
||||
|
||||
public NatsNotifyDeliveryQueueTests()
|
||||
{
|
||||
_nats = new TestcontainersBuilder<TestcontainersContainer>()
|
||||
.WithImage("nats:2.10-alpine")
|
||||
.WithCleanUp(true)
|
||||
.WithName($"nats-notify-delivery-{Guid.NewGuid():N}")
|
||||
.WithPortBinding(4222, true)
|
||||
.WithCommand("--jetstream")
|
||||
.WithWaitStrategy(Wait.ForUnixContainer().UntilPortIsAvailable(4222))
|
||||
.Build();
|
||||
}
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
await _nats.StartAsync();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_skipReason = $"NATS-backed delivery tests skipped: {ex.Message}";
|
||||
}
|
||||
}
|
||||
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
if (_skipReason is not null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _nats.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Publish_ShouldDeduplicate_ByDeliveryId()
|
||||
{
|
||||
if (SkipIfUnavailable())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var options = CreateOptions();
|
||||
await using var queue = CreateQueue(options);
|
||||
|
||||
var delivery = TestData.CreateDelivery("tenant-a");
|
||||
var message = new NotifyDeliveryQueueMessage(
|
||||
delivery,
|
||||
channelId: "chan-a",
|
||||
channelType: NotifyChannelType.Slack);
|
||||
|
||||
var first = await queue.PublishAsync(message);
|
||||
first.Deduplicated.Should().BeFalse();
|
||||
|
||||
var second = await queue.PublishAsync(message);
|
||||
second.Deduplicated.Should().BeTrue();
|
||||
second.MessageId.Should().Be(first.MessageId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Release_Retry_ShouldReschedule()
|
||||
{
|
||||
if (SkipIfUnavailable())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var options = CreateOptions();
|
||||
await using var queue = CreateQueue(options);
|
||||
|
||||
await queue.PublishAsync(new NotifyDeliveryQueueMessage(
|
||||
TestData.CreateDelivery(),
|
||||
channelId: "chan-retry",
|
||||
channelType: NotifyChannelType.Teams));
|
||||
|
||||
var lease = (await queue.LeaseAsync(new NotifyQueueLeaseRequest("worker-retry", 1, TimeSpan.FromSeconds(2)))).Single();
|
||||
|
||||
await lease.ReleaseAsync(NotifyQueueReleaseDisposition.Retry);
|
||||
|
||||
var retried = (await queue.LeaseAsync(new NotifyQueueLeaseRequest("worker-retry", 1, TimeSpan.FromSeconds(2)))).Single();
|
||||
retried.Attempt.Should().BeGreaterThan(lease.Attempt);
|
||||
|
||||
await retried.AcknowledgeAsync();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Release_RetryBeyondMax_ShouldDeadLetter()
|
||||
{
|
||||
if (SkipIfUnavailable())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var options = CreateOptions(static opts =>
|
||||
{
|
||||
opts.MaxDeliveryAttempts = 2;
|
||||
opts.Nats.DeadLetterStream = "NOTIFY_DELIVERY_DEAD_TEST";
|
||||
opts.Nats.DeadLetterSubject = "notify.delivery.dead.test";
|
||||
});
|
||||
|
||||
await using var queue = CreateQueue(options);
|
||||
|
||||
await queue.PublishAsync(new NotifyDeliveryQueueMessage(
|
||||
TestData.CreateDelivery(),
|
||||
channelId: "chan-dead",
|
||||
channelType: NotifyChannelType.Webhook));
|
||||
|
||||
var lease = (await queue.LeaseAsync(new NotifyQueueLeaseRequest("worker-dead", 1, TimeSpan.FromSeconds(2)))).Single();
|
||||
await lease.ReleaseAsync(NotifyQueueReleaseDisposition.Retry);
|
||||
|
||||
var second = (await queue.LeaseAsync(new NotifyQueueLeaseRequest("worker-dead", 1, TimeSpan.FromSeconds(2)))).Single();
|
||||
await second.ReleaseAsync(NotifyQueueReleaseDisposition.Retry);
|
||||
|
||||
await Task.Delay(200);
|
||||
|
||||
await using var connection = new NatsConnection(new NatsOpts { Url = options.Nats.Url! });
|
||||
await connection.ConnectAsync();
|
||||
var js = new NatsJSContext(connection);
|
||||
|
||||
var consumerConfig = new ConsumerConfig
|
||||
{
|
||||
DurableName = "notify-delivery-dead-test",
|
||||
DeliverPolicy = ConsumerConfigDeliverPolicy.All,
|
||||
AckPolicy = ConsumerConfigAckPolicy.Explicit
|
||||
};
|
||||
|
||||
var consumer = await js.CreateConsumerAsync(options.Nats.DeadLetterStream, consumerConfig);
|
||||
var fetchOpts = new NatsJSFetchOpts { MaxMsgs = 1, Expires = TimeSpan.FromSeconds(1) };
|
||||
|
||||
NatsJSMsg<byte[]>? dlqMsg = null;
|
||||
await foreach (var msg in consumer.FetchAsync(NatsRawSerializer<byte[]>.Default, fetchOpts))
|
||||
{
|
||||
dlqMsg = msg;
|
||||
await msg.AckAsync(new AckOpts());
|
||||
break;
|
||||
}
|
||||
|
||||
dlqMsg.Should().NotBeNull();
|
||||
}
|
||||
|
||||
private NatsNotifyDeliveryQueue CreateQueue(NotifyDeliveryQueueOptions options)
|
||||
{
|
||||
return new NatsNotifyDeliveryQueue(
|
||||
options,
|
||||
options.Nats,
|
||||
NullLogger<NatsNotifyDeliveryQueue>.Instance,
|
||||
TimeProvider.System);
|
||||
}
|
||||
|
||||
private NotifyDeliveryQueueOptions CreateOptions(Action<NotifyDeliveryQueueOptions>? configure = null)
|
||||
{
|
||||
var url = $"nats://{_nats.Hostname}:{_nats.GetMappedPublicPort(4222)}";
|
||||
|
||||
var opts = new NotifyDeliveryQueueOptions
|
||||
{
|
||||
Transport = NotifyQueueTransportKind.Nats,
|
||||
DefaultLeaseDuration = TimeSpan.FromSeconds(2),
|
||||
MaxDeliveryAttempts = 3,
|
||||
RetryInitialBackoff = TimeSpan.FromMilliseconds(20),
|
||||
RetryMaxBackoff = TimeSpan.FromMilliseconds(200),
|
||||
Nats = new NotifyNatsDeliveryQueueOptions
|
||||
{
|
||||
Url = url,
|
||||
Stream = "NOTIFY_DELIVERY_TEST",
|
||||
Subject = "notify.delivery.test",
|
||||
DeadLetterStream = "NOTIFY_DELIVERY_TEST_DEAD",
|
||||
DeadLetterSubject = "notify.delivery.test.dead",
|
||||
DurableConsumer = "notify-delivery-tests",
|
||||
MaxAckPending = 32,
|
||||
AckWait = TimeSpan.FromSeconds(2),
|
||||
RetryDelay = TimeSpan.FromMilliseconds(100),
|
||||
IdleHeartbeat = TimeSpan.FromMilliseconds(200)
|
||||
}
|
||||
};
|
||||
|
||||
configure?.Invoke(opts);
|
||||
return opts;
|
||||
}
|
||||
|
||||
private bool SkipIfUnavailable()
|
||||
=> _skipReason is not null;
|
||||
|
||||
private static class TestData
|
||||
{
|
||||
public static NotifyDelivery CreateDelivery(string tenantId = "tenant-1")
|
||||
{
|
||||
return NotifyDelivery.Create(
|
||||
deliveryId: Guid.NewGuid().ToString("n"),
|
||||
tenantId: tenantId,
|
||||
ruleId: "rule-1",
|
||||
actionId: "action-1",
|
||||
eventId: Guid.NewGuid(),
|
||||
kind: "scanner.report.ready",
|
||||
status: NotifyDeliveryStatus.Pending,
|
||||
createdAt: DateTimeOffset.UtcNow);
|
||||
}
|
||||
}
|
||||
}
|
||||
225
src/StellaOps.Notify.Queue.Tests/NatsNotifyEventQueueTests.cs
Normal file
225
src/StellaOps.Notify.Queue.Tests/NatsNotifyEventQueueTests.cs
Normal file
@@ -0,0 +1,225 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.Json.Nodes;
|
||||
using System.Threading.Tasks;
|
||||
using DotNet.Testcontainers.Builders;
|
||||
using DotNet.Testcontainers.Containers;
|
||||
using DotNet.Testcontainers.Configurations;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notify.Queue;
|
||||
using StellaOps.Notify.Queue.Nats;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Notify.Queue.Tests;
|
||||
|
||||
public sealed class NatsNotifyEventQueueTests : IAsyncLifetime
|
||||
{
|
||||
private readonly TestcontainersContainer _nats;
|
||||
private string? _skipReason;
|
||||
|
||||
public NatsNotifyEventQueueTests()
|
||||
{
|
||||
_nats = new TestcontainersBuilder<TestcontainersContainer>()
|
||||
.WithImage("nats:2.10-alpine")
|
||||
.WithCleanUp(true)
|
||||
.WithName($"nats-notify-tests-{Guid.NewGuid():N}")
|
||||
.WithPortBinding(4222, true)
|
||||
.WithCommand("--jetstream")
|
||||
.WithWaitStrategy(Wait.ForUnixContainer().UntilPortIsAvailable(4222))
|
||||
.Build();
|
||||
}
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
await _nats.StartAsync();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_skipReason = $"NATS-backed tests skipped: {ex.Message}";
|
||||
}
|
||||
}
|
||||
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
if (_skipReason is not null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _nats.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Publish_ShouldDeduplicate_ByIdempotencyKey()
|
||||
{
|
||||
if (SkipIfUnavailable())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var options = CreateOptions();
|
||||
await using var queue = CreateQueue(options);
|
||||
|
||||
var notifyEvent = TestData.CreateEvent("tenant-a");
|
||||
var message = new NotifyQueueEventMessage(
|
||||
notifyEvent,
|
||||
options.Nats.Subject,
|
||||
traceId: "trace-1");
|
||||
|
||||
var first = await queue.PublishAsync(message);
|
||||
first.Deduplicated.Should().BeFalse();
|
||||
|
||||
var second = await queue.PublishAsync(message);
|
||||
second.Deduplicated.Should().BeTrue();
|
||||
second.MessageId.Should().Be(first.MessageId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Lease_Acknowledge_ShouldRemoveMessage()
|
||||
{
|
||||
if (SkipIfUnavailable())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var options = CreateOptions();
|
||||
await using var queue = CreateQueue(options);
|
||||
|
||||
var notifyEvent = TestData.CreateEvent("tenant-b");
|
||||
var message = new NotifyQueueEventMessage(
|
||||
notifyEvent,
|
||||
options.Nats.Subject,
|
||||
traceId: "trace-xyz",
|
||||
attributes: new Dictionary<string, string> { { "source", "scanner" } });
|
||||
|
||||
await queue.PublishAsync(message);
|
||||
|
||||
var leases = await queue.LeaseAsync(new NotifyQueueLeaseRequest("worker-1", 1, TimeSpan.FromSeconds(2)));
|
||||
leases.Should().ContainSingle();
|
||||
|
||||
var lease = leases[0];
|
||||
lease.Attempt.Should().BeGreaterThanOrEqualTo(1);
|
||||
lease.Message.Event.EventId.Should().Be(notifyEvent.EventId);
|
||||
lease.TraceId.Should().Be("trace-xyz");
|
||||
lease.Attributes.Should().ContainKey("source").WhoseValue.Should().Be("scanner");
|
||||
|
||||
await lease.AcknowledgeAsync();
|
||||
|
||||
var afterAck = await queue.LeaseAsync(new NotifyQueueLeaseRequest("worker-1", 1, TimeSpan.FromSeconds(1)));
|
||||
afterAck.Should().BeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Lease_ShouldPreserveOrdering()
|
||||
{
|
||||
if (SkipIfUnavailable())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var options = CreateOptions();
|
||||
await using var queue = CreateQueue(options);
|
||||
|
||||
var first = TestData.CreateEvent();
|
||||
var second = TestData.CreateEvent();
|
||||
|
||||
await queue.PublishAsync(new NotifyQueueEventMessage(first, options.Nats.Subject));
|
||||
await queue.PublishAsync(new NotifyQueueEventMessage(second, options.Nats.Subject));
|
||||
|
||||
var leases = await queue.LeaseAsync(new NotifyQueueLeaseRequest("worker-order", 2, TimeSpan.FromSeconds(2)));
|
||||
leases.Should().HaveCount(2);
|
||||
|
||||
leases.Select(x => x.Message.Event.EventId)
|
||||
.Should()
|
||||
.ContainInOrder(first.EventId, second.EventId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ClaimExpired_ShouldReassignLease()
|
||||
{
|
||||
if (SkipIfUnavailable())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var options = CreateOptions();
|
||||
await using var queue = CreateQueue(options);
|
||||
|
||||
var notifyEvent = TestData.CreateEvent();
|
||||
await queue.PublishAsync(new NotifyQueueEventMessage(notifyEvent, options.Nats.Subject));
|
||||
|
||||
var leases = await queue.LeaseAsync(new NotifyQueueLeaseRequest("worker-initial", 1, TimeSpan.FromMilliseconds(500)));
|
||||
leases.Should().ContainSingle();
|
||||
|
||||
await Task.Delay(200);
|
||||
|
||||
var claimed = await queue.ClaimExpiredAsync(new NotifyQueueClaimOptions("worker-reclaim", 1, TimeSpan.FromMilliseconds(100)));
|
||||
claimed.Should().ContainSingle();
|
||||
|
||||
var lease = claimed[0];
|
||||
lease.Consumer.Should().Be("worker-reclaim");
|
||||
lease.Message.Event.EventId.Should().Be(notifyEvent.EventId);
|
||||
|
||||
await lease.AcknowledgeAsync();
|
||||
}
|
||||
|
||||
private NatsNotifyEventQueue CreateQueue(NotifyEventQueueOptions options)
|
||||
{
|
||||
return new NatsNotifyEventQueue(
|
||||
options,
|
||||
options.Nats,
|
||||
NullLogger<NatsNotifyEventQueue>.Instance,
|
||||
TimeProvider.System);
|
||||
}
|
||||
|
||||
private NotifyEventQueueOptions CreateOptions()
|
||||
{
|
||||
var connectionUrl = $"nats://{_nats.Hostname}:{_nats.GetMappedPublicPort(4222)}";
|
||||
|
||||
return new NotifyEventQueueOptions
|
||||
{
|
||||
Transport = NotifyQueueTransportKind.Nats,
|
||||
DefaultLeaseDuration = TimeSpan.FromSeconds(2),
|
||||
MaxDeliveryAttempts = 3,
|
||||
RetryInitialBackoff = TimeSpan.FromMilliseconds(50),
|
||||
RetryMaxBackoff = TimeSpan.FromSeconds(1),
|
||||
Nats = new NotifyNatsEventQueueOptions
|
||||
{
|
||||
Url = connectionUrl,
|
||||
Stream = "NOTIFY_TEST",
|
||||
Subject = "notify.test.events",
|
||||
DeadLetterStream = "NOTIFY_TEST_DEAD",
|
||||
DeadLetterSubject = "notify.test.events.dead",
|
||||
DurableConsumer = "notify-test-consumer",
|
||||
MaxAckPending = 32,
|
||||
AckWait = TimeSpan.FromSeconds(2),
|
||||
RetryDelay = TimeSpan.FromMilliseconds(100),
|
||||
IdleHeartbeat = TimeSpan.FromMilliseconds(100)
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private bool SkipIfUnavailable()
|
||||
=> _skipReason is not null;
|
||||
|
||||
private static class TestData
|
||||
{
|
||||
public static NotifyEvent CreateEvent(string tenant = "tenant-1")
|
||||
{
|
||||
return NotifyEvent.Create(
|
||||
Guid.NewGuid(),
|
||||
kind: "scanner.report.ready",
|
||||
tenant: tenant,
|
||||
ts: DateTimeOffset.UtcNow,
|
||||
payload: new JsonObject
|
||||
{
|
||||
["summary"] = "event"
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.Json.Nodes;
|
||||
using System.Threading.Tasks;
|
||||
using DotNet.Testcontainers.Builders;
|
||||
using DotNet.Testcontainers.Containers;
|
||||
using DotNet.Testcontainers.Configurations;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using StackExchange.Redis;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notify.Queue;
|
||||
using StellaOps.Notify.Queue.Redis;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Notify.Queue.Tests;
|
||||
|
||||
public sealed class RedisNotifyDeliveryQueueTests : IAsyncLifetime
|
||||
{
|
||||
private readonly RedisTestcontainer _redis;
|
||||
private string? _skipReason;
|
||||
|
||||
public RedisNotifyDeliveryQueueTests()
|
||||
{
|
||||
var configuration = new RedisTestcontainerConfiguration();
|
||||
_redis = new TestcontainersBuilder<RedisTestcontainer>()
|
||||
.WithDatabase(configuration)
|
||||
.Build();
|
||||
}
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
await _redis.StartAsync();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_skipReason = $"Redis-backed delivery tests skipped: {ex.Message}";
|
||||
}
|
||||
}
|
||||
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
if (_skipReason is not null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _redis.DisposeAsync().AsTask();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Publish_ShouldDeduplicate_ByDeliveryId()
|
||||
{
|
||||
if (SkipIfUnavailable())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var options = CreateOptions();
|
||||
await using var queue = CreateQueue(options);
|
||||
|
||||
var delivery = TestData.CreateDelivery();
|
||||
var message = new NotifyDeliveryQueueMessage(
|
||||
delivery,
|
||||
channelId: "channel-1",
|
||||
channelType: NotifyChannelType.Slack);
|
||||
|
||||
var first = await queue.PublishAsync(message);
|
||||
first.Deduplicated.Should().BeFalse();
|
||||
|
||||
var second = await queue.PublishAsync(message);
|
||||
second.Deduplicated.Should().BeTrue();
|
||||
second.MessageId.Should().Be(first.MessageId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Release_Retry_ShouldRescheduleDelivery()
|
||||
{
|
||||
if (SkipIfUnavailable())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var options = CreateOptions();
|
||||
await using var queue = CreateQueue(options);
|
||||
|
||||
await queue.PublishAsync(new NotifyDeliveryQueueMessage(
|
||||
TestData.CreateDelivery(),
|
||||
channelId: "channel-retry",
|
||||
channelType: NotifyChannelType.Teams));
|
||||
|
||||
var lease = (await queue.LeaseAsync(new NotifyQueueLeaseRequest("worker-retry", 1, TimeSpan.FromSeconds(1)))).Single();
|
||||
lease.Attempt.Should().Be(1);
|
||||
|
||||
await lease.ReleaseAsync(NotifyQueueReleaseDisposition.Retry);
|
||||
|
||||
var retried = (await queue.LeaseAsync(new NotifyQueueLeaseRequest("worker-retry", 1, TimeSpan.FromSeconds(1)))).Single();
|
||||
retried.Attempt.Should().Be(2);
|
||||
|
||||
await retried.AcknowledgeAsync();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Release_RetryBeyondMax_ShouldDeadLetter()
|
||||
{
|
||||
if (SkipIfUnavailable())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var options = CreateOptions(static opts =>
|
||||
{
|
||||
opts.MaxDeliveryAttempts = 2;
|
||||
opts.Redis.DeadLetterStreamName = "notify:deliveries:testdead";
|
||||
});
|
||||
|
||||
await using var queue = CreateQueue(options);
|
||||
|
||||
await queue.PublishAsync(new NotifyDeliveryQueueMessage(
|
||||
TestData.CreateDelivery(),
|
||||
channelId: "channel-dead",
|
||||
channelType: NotifyChannelType.Email));
|
||||
|
||||
var first = (await queue.LeaseAsync(new NotifyQueueLeaseRequest("worker-dead", 1, TimeSpan.FromSeconds(1)))).Single();
|
||||
await first.ReleaseAsync(NotifyQueueReleaseDisposition.Retry);
|
||||
|
||||
var second = (await queue.LeaseAsync(new NotifyQueueLeaseRequest("worker-dead", 1, TimeSpan.FromSeconds(1)))).Single();
|
||||
await second.ReleaseAsync(NotifyQueueReleaseDisposition.Retry);
|
||||
|
||||
await Task.Delay(100);
|
||||
|
||||
var mux = await ConnectionMultiplexer.ConnectAsync(_redis.ConnectionString);
|
||||
var db = mux.GetDatabase();
|
||||
var deadLetters = await db.StreamReadAsync(options.Redis.DeadLetterStreamName, "0-0");
|
||||
deadLetters.Should().NotBeEmpty();
|
||||
}
|
||||
|
||||
private RedisNotifyDeliveryQueue CreateQueue(NotifyDeliveryQueueOptions options)
|
||||
{
|
||||
return new RedisNotifyDeliveryQueue(
|
||||
options,
|
||||
options.Redis,
|
||||
NullLogger<RedisNotifyDeliveryQueue>.Instance,
|
||||
TimeProvider.System,
|
||||
async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
|
||||
}
|
||||
|
||||
private NotifyDeliveryQueueOptions CreateOptions(Action<NotifyDeliveryQueueOptions>? configure = null)
|
||||
{
|
||||
var opts = new NotifyDeliveryQueueOptions
|
||||
{
|
||||
Transport = NotifyQueueTransportKind.Redis,
|
||||
DefaultLeaseDuration = TimeSpan.FromSeconds(1),
|
||||
MaxDeliveryAttempts = 3,
|
||||
RetryInitialBackoff = TimeSpan.FromMilliseconds(10),
|
||||
RetryMaxBackoff = TimeSpan.FromMilliseconds(50),
|
||||
ClaimIdleThreshold = TimeSpan.FromSeconds(1),
|
||||
Redis = new NotifyRedisDeliveryQueueOptions
|
||||
{
|
||||
ConnectionString = _redis.ConnectionString,
|
||||
StreamName = "notify:deliveries:test",
|
||||
ConsumerGroup = "notify-delivery-tests",
|
||||
IdempotencyKeyPrefix = "notify:deliveries:test:idemp:"
|
||||
}
|
||||
};
|
||||
|
||||
configure?.Invoke(opts);
|
||||
return opts;
|
||||
}
|
||||
|
||||
private bool SkipIfUnavailable()
|
||||
=> _skipReason is not null;
|
||||
|
||||
private static class TestData
|
||||
{
|
||||
public static NotifyDelivery CreateDelivery()
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
return NotifyDelivery.Create(
|
||||
deliveryId: Guid.NewGuid().ToString("n"),
|
||||
tenantId: "tenant-1",
|
||||
ruleId: "rule-1",
|
||||
actionId: "action-1",
|
||||
eventId: Guid.NewGuid(),
|
||||
kind: "scanner.report.ready",
|
||||
status: NotifyDeliveryStatus.Pending,
|
||||
createdAt: now,
|
||||
metadata: new Dictionary<string, string>
|
||||
{
|
||||
["integration"] = "tests"
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
220
src/StellaOps.Notify.Queue.Tests/RedisNotifyEventQueueTests.cs
Normal file
220
src/StellaOps.Notify.Queue.Tests/RedisNotifyEventQueueTests.cs
Normal file
@@ -0,0 +1,220 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.Json.Nodes;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using DotNet.Testcontainers.Builders;
|
||||
using DotNet.Testcontainers.Containers;
|
||||
using DotNet.Testcontainers.Configurations;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using StackExchange.Redis;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notify.Queue;
|
||||
using StellaOps.Notify.Queue.Redis;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Notify.Queue.Tests;
|
||||
|
||||
public sealed class RedisNotifyEventQueueTests : IAsyncLifetime
|
||||
{
|
||||
private readonly RedisTestcontainer _redis;
|
||||
private string? _skipReason;
|
||||
|
||||
public RedisNotifyEventQueueTests()
|
||||
{
|
||||
var configuration = new RedisTestcontainerConfiguration();
|
||||
_redis = new TestcontainersBuilder<RedisTestcontainer>()
|
||||
.WithDatabase(configuration)
|
||||
.Build();
|
||||
}
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
await _redis.StartAsync();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_skipReason = $"Redis-backed tests skipped: {ex.Message}";
|
||||
}
|
||||
}
|
||||
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
if (_skipReason is not null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _redis.DisposeAsync().AsTask();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Publish_ShouldDeduplicate_ByIdempotencyKey()
|
||||
{
|
||||
if (SkipIfUnavailable())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var options = CreateOptions();
|
||||
await using var queue = CreateQueue(options);
|
||||
|
||||
var notifyEvent = TestData.CreateEvent(tenant: "tenant-a");
|
||||
var message = new NotifyQueueEventMessage(notifyEvent, options.Redis.Streams[0].Stream);
|
||||
|
||||
var first = await queue.PublishAsync(message);
|
||||
first.Deduplicated.Should().BeFalse();
|
||||
|
||||
var second = await queue.PublishAsync(message);
|
||||
second.Deduplicated.Should().BeTrue();
|
||||
second.MessageId.Should().Be(first.MessageId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Lease_Acknowledge_ShouldRemoveMessage()
|
||||
{
|
||||
if (SkipIfUnavailable())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var options = CreateOptions();
|
||||
await using var queue = CreateQueue(options);
|
||||
|
||||
var notifyEvent = TestData.CreateEvent(tenant: "tenant-b");
|
||||
var message = new NotifyQueueEventMessage(
|
||||
notifyEvent,
|
||||
options.Redis.Streams[0].Stream,
|
||||
traceId: "trace-123",
|
||||
attributes: new Dictionary<string, string> { { "source", "scanner" } });
|
||||
|
||||
await queue.PublishAsync(message);
|
||||
|
||||
var leases = await queue.LeaseAsync(new NotifyQueueLeaseRequest("worker-1", 1, TimeSpan.FromSeconds(5)));
|
||||
leases.Should().ContainSingle();
|
||||
|
||||
var lease = leases[0];
|
||||
lease.Attempt.Should().Be(1);
|
||||
lease.Message.Event.EventId.Should().Be(notifyEvent.EventId);
|
||||
lease.TraceId.Should().Be("trace-123");
|
||||
lease.Attributes.Should().ContainKey("source").WhoseValue.Should().Be("scanner");
|
||||
|
||||
await lease.AcknowledgeAsync();
|
||||
|
||||
var afterAck = await queue.LeaseAsync(new NotifyQueueLeaseRequest("worker-1", 1, TimeSpan.FromSeconds(5)));
|
||||
afterAck.Should().BeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Lease_ShouldPreserveOrdering()
|
||||
{
|
||||
if (SkipIfUnavailable())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var options = CreateOptions();
|
||||
await using var queue = CreateQueue(options);
|
||||
|
||||
var stream = options.Redis.Streams[0].Stream;
|
||||
var firstEvent = TestData.CreateEvent();
|
||||
var secondEvent = TestData.CreateEvent();
|
||||
|
||||
await queue.PublishAsync(new NotifyQueueEventMessage(firstEvent, stream));
|
||||
await queue.PublishAsync(new NotifyQueueEventMessage(secondEvent, stream));
|
||||
|
||||
var leases = await queue.LeaseAsync(new NotifyQueueLeaseRequest("worker-order", 2, TimeSpan.FromSeconds(5)));
|
||||
leases.Should().HaveCount(2);
|
||||
|
||||
leases.Select(l => l.Message.Event.EventId)
|
||||
.Should()
|
||||
.ContainInOrder(new[] { firstEvent.EventId, secondEvent.EventId });
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ClaimExpired_ShouldReassignLease()
|
||||
{
|
||||
if (SkipIfUnavailable())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var options = CreateOptions();
|
||||
await using var queue = CreateQueue(options);
|
||||
|
||||
var notifyEvent = TestData.CreateEvent();
|
||||
await queue.PublishAsync(new NotifyQueueEventMessage(notifyEvent, options.Redis.Streams[0].Stream));
|
||||
|
||||
var leases = await queue.LeaseAsync(new NotifyQueueLeaseRequest("worker-initial", 1, TimeSpan.FromSeconds(1)));
|
||||
leases.Should().ContainSingle();
|
||||
|
||||
// Ensure the message has been pending long enough for claim.
|
||||
await Task.Delay(50);
|
||||
|
||||
var claimed = await queue.ClaimExpiredAsync(new NotifyQueueClaimOptions("worker-reclaim", 1, TimeSpan.Zero));
|
||||
claimed.Should().ContainSingle();
|
||||
|
||||
var lease = claimed[0];
|
||||
lease.Consumer.Should().Be("worker-reclaim");
|
||||
lease.Message.Event.EventId.Should().Be(notifyEvent.EventId);
|
||||
|
||||
await lease.AcknowledgeAsync();
|
||||
}
|
||||
|
||||
private RedisNotifyEventQueue CreateQueue(NotifyEventQueueOptions options)
|
||||
{
|
||||
return new RedisNotifyEventQueue(
|
||||
options,
|
||||
options.Redis,
|
||||
NullLogger<RedisNotifyEventQueue>.Instance,
|
||||
TimeProvider.System,
|
||||
async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false));
|
||||
}
|
||||
|
||||
private NotifyEventQueueOptions CreateOptions()
|
||||
{
|
||||
var streamOptions = new NotifyRedisEventStreamOptions
|
||||
{
|
||||
Stream = "notify:test:events",
|
||||
ConsumerGroup = "notify-test-consumers",
|
||||
IdempotencyKeyPrefix = "notify:test:idemp:",
|
||||
ApproximateMaxLength = 1024
|
||||
};
|
||||
|
||||
var redisOptions = new NotifyRedisEventQueueOptions
|
||||
{
|
||||
ConnectionString = _redis.ConnectionString,
|
||||
Streams = new List<NotifyRedisEventStreamOptions> { streamOptions }
|
||||
};
|
||||
|
||||
return new NotifyEventQueueOptions
|
||||
{
|
||||
Transport = NotifyQueueTransportKind.Redis,
|
||||
DefaultLeaseDuration = TimeSpan.FromSeconds(5),
|
||||
Redis = redisOptions
|
||||
};
|
||||
}
|
||||
|
||||
private bool SkipIfUnavailable()
|
||||
=> _skipReason is not null;
|
||||
|
||||
private static class TestData
|
||||
{
|
||||
public static NotifyEvent CreateEvent(string tenant = "tenant-1")
|
||||
{
|
||||
return NotifyEvent.Create(
|
||||
Guid.NewGuid(),
|
||||
kind: "scanner.report.ready",
|
||||
tenant: tenant,
|
||||
ts: DateTimeOffset.UtcNow,
|
||||
payload: new JsonObject
|
||||
{
|
||||
["summary"] = "event"
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<IsPackable>false</IsPackable>
|
||||
<UseConcelierTestInfra>false</UseConcelierTestInfra>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
|
||||
<PackageReference Include="DotNet.Testcontainers" Version="1.7.0-beta.2269" />
|
||||
<PackageReference Include="FluentAssertions" Version="6.12.0" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.0" />
|
||||
<PackageReference Include="xunit" Version="2.9.2" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
<PackageReference Include="coverlet.collector" Version="6.0.4">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Notify.Models\StellaOps.Notify.Models.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Notify.Queue\StellaOps.Notify.Queue.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
80
src/StellaOps.Notify.Queue/Nats/NatsNotifyDeliveryLease.cs
Normal file
80
src/StellaOps.Notify.Queue/Nats/NatsNotifyDeliveryLease.cs
Normal file
@@ -0,0 +1,80 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using NATS.Client.JetStream;
|
||||
|
||||
namespace StellaOps.Notify.Queue.Nats;
|
||||
|
||||
internal sealed class NatsNotifyDeliveryLease : INotifyQueueLease<NotifyDeliveryQueueMessage>
|
||||
{
|
||||
private readonly NatsNotifyDeliveryQueue _queue;
|
||||
private readonly NatsJSMsg<byte[]> _message;
|
||||
private int _completed;
|
||||
|
||||
internal NatsNotifyDeliveryLease(
|
||||
NatsNotifyDeliveryQueue queue,
|
||||
NatsJSMsg<byte[]> message,
|
||||
string messageId,
|
||||
NotifyDeliveryQueueMessage payload,
|
||||
int attempt,
|
||||
string consumer,
|
||||
DateTimeOffset enqueuedAt,
|
||||
DateTimeOffset leaseExpiresAt,
|
||||
string idempotencyKey)
|
||||
{
|
||||
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
|
||||
_message = message;
|
||||
MessageId = messageId ?? throw new ArgumentNullException(nameof(messageId));
|
||||
Message = payload ?? throw new ArgumentNullException(nameof(payload));
|
||||
Attempt = attempt;
|
||||
Consumer = consumer ?? throw new ArgumentNullException(nameof(consumer));
|
||||
EnqueuedAt = enqueuedAt;
|
||||
LeaseExpiresAt = leaseExpiresAt;
|
||||
IdempotencyKey = idempotencyKey ?? payload.IdempotencyKey;
|
||||
}
|
||||
|
||||
public string MessageId { get; }
|
||||
|
||||
public int Attempt { get; internal set; }
|
||||
|
||||
public DateTimeOffset EnqueuedAt { get; }
|
||||
|
||||
public DateTimeOffset LeaseExpiresAt { get; private set; }
|
||||
|
||||
public string Consumer { get; }
|
||||
|
||||
public string Stream => Message.Stream;
|
||||
|
||||
public string TenantId => Message.TenantId;
|
||||
|
||||
public string? PartitionKey => Message.PartitionKey;
|
||||
|
||||
public string IdempotencyKey { get; }
|
||||
|
||||
public string? TraceId => Message.TraceId;
|
||||
|
||||
public IReadOnlyDictionary<string, string> Attributes => Message.Attributes;
|
||||
|
||||
public NotifyDeliveryQueueMessage Message { get; }
|
||||
|
||||
internal NatsJSMsg<byte[]> RawMessage => _message;
|
||||
|
||||
public Task AcknowledgeAsync(CancellationToken cancellationToken = default)
|
||||
=> _queue.AcknowledgeAsync(this, cancellationToken);
|
||||
|
||||
public Task RenewAsync(TimeSpan leaseDuration, CancellationToken cancellationToken = default)
|
||||
=> _queue.RenewLeaseAsync(this, leaseDuration, cancellationToken);
|
||||
|
||||
public Task ReleaseAsync(NotifyQueueReleaseDisposition disposition, CancellationToken cancellationToken = default)
|
||||
=> _queue.ReleaseAsync(this, disposition, cancellationToken);
|
||||
|
||||
public Task DeadLetterAsync(string reason, CancellationToken cancellationToken = default)
|
||||
=> _queue.DeadLetterAsync(this, reason, cancellationToken);
|
||||
|
||||
internal bool TryBeginCompletion()
|
||||
=> Interlocked.CompareExchange(ref _completed, 1, 0) == 0;
|
||||
|
||||
internal void RefreshLease(DateTimeOffset expiresAt)
|
||||
=> LeaseExpiresAt = expiresAt;
|
||||
}
|
||||
697
src/StellaOps.Notify.Queue/Nats/NatsNotifyDeliveryQueue.cs
Normal file
697
src/StellaOps.Notify.Queue/Nats/NatsNotifyDeliveryQueue.cs
Normal file
@@ -0,0 +1,697 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.ObjectModel;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using NATS.Client.Core;
|
||||
using NATS.Client.JetStream;
|
||||
using NATS.Client.JetStream.Models;
|
||||
using StellaOps.Notify.Models;
|
||||
|
||||
namespace StellaOps.Notify.Queue.Nats;
|
||||
|
||||
internal sealed class NatsNotifyDeliveryQueue : INotifyDeliveryQueue, IAsyncDisposable
|
||||
{
|
||||
private const string TransportName = "nats";
|
||||
|
||||
private static readonly INatsSerializer<byte[]> PayloadSerializer = NatsRawSerializer<byte[]>.Default;
|
||||
|
||||
private readonly NotifyDeliveryQueueOptions _queueOptions;
|
||||
private readonly NotifyNatsDeliveryQueueOptions _options;
|
||||
private readonly ILogger<NatsNotifyDeliveryQueue> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly SemaphoreSlim _connectionGate = new(1, 1);
|
||||
private readonly Func<NatsOpts, CancellationToken, ValueTask<NatsConnection>> _connectionFactory;
|
||||
|
||||
private NatsConnection? _connection;
|
||||
private NatsJSContext? _jsContext;
|
||||
private INatsJSConsumer? _consumer;
|
||||
private bool _disposed;
|
||||
|
||||
public NatsNotifyDeliveryQueue(
|
||||
NotifyDeliveryQueueOptions queueOptions,
|
||||
NotifyNatsDeliveryQueueOptions options,
|
||||
ILogger<NatsNotifyDeliveryQueue> logger,
|
||||
TimeProvider timeProvider,
|
||||
Func<NatsOpts, CancellationToken, ValueTask<NatsConnection>>? connectionFactory = null)
|
||||
{
|
||||
_queueOptions = queueOptions ?? throw new ArgumentNullException(nameof(queueOptions));
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_connectionFactory = connectionFactory ?? ((opts, token) => new ValueTask<NatsConnection>(new NatsConnection(opts)));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(_options.Url))
|
||||
{
|
||||
throw new InvalidOperationException("NATS connection URL must be configured for the Notify delivery queue.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(_options.Stream) || string.IsNullOrWhiteSpace(_options.Subject))
|
||||
{
|
||||
throw new InvalidOperationException("NATS stream and subject must be configured for the Notify delivery queue.");
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<NotifyQueueEnqueueResult> PublishAsync(
|
||||
NotifyDeliveryQueueMessage message,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(message);
|
||||
|
||||
var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
await EnsureStreamAndConsumerAsync(js, cancellationToken).ConfigureAwait(false);
|
||||
await EnsureDeadLetterStreamAsync(js, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var payload = Encoding.UTF8.GetBytes(NotifyCanonicalJsonSerializer.Serialize(message.Delivery));
|
||||
var headers = BuildHeaders(message);
|
||||
|
||||
var publishOpts = new NatsJSPubOpts
|
||||
{
|
||||
MsgId = message.IdempotencyKey,
|
||||
RetryAttempts = 0
|
||||
};
|
||||
|
||||
var ack = await js.PublishAsync(
|
||||
_options.Subject,
|
||||
payload,
|
||||
PayloadSerializer,
|
||||
publishOpts,
|
||||
headers,
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (ack.Duplicate)
|
||||
{
|
||||
NotifyQueueMetrics.RecordDeduplicated(TransportName, _options.Stream);
|
||||
_logger.LogDebug(
|
||||
"Duplicate Notify delivery enqueue detected for delivery {DeliveryId}.",
|
||||
message.Delivery.DeliveryId);
|
||||
|
||||
return new NotifyQueueEnqueueResult(ack.Seq.ToString(), true);
|
||||
}
|
||||
|
||||
NotifyQueueMetrics.RecordEnqueued(TransportName, _options.Stream);
|
||||
_logger.LogDebug(
|
||||
"Enqueued Notify delivery {DeliveryId} into NATS stream {Stream} (sequence {Sequence}).",
|
||||
message.Delivery.DeliveryId,
|
||||
ack.Stream,
|
||||
ack.Seq);
|
||||
|
||||
return new NotifyQueueEnqueueResult(ack.Seq.ToString(), false);
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<INotifyQueueLease<NotifyDeliveryQueueMessage>>> LeaseAsync(
|
||||
NotifyQueueLeaseRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
var consumer = await EnsureStreamAndConsumerAsync(js, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var fetchOpts = new NatsJSFetchOpts
|
||||
{
|
||||
MaxMsgs = request.BatchSize,
|
||||
Expires = request.LeaseDuration,
|
||||
IdleHeartbeat = _options.IdleHeartbeat
|
||||
};
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var leases = new List<INotifyQueueLease<NotifyDeliveryQueueMessage>>(request.BatchSize);
|
||||
|
||||
await foreach (var msg in consumer.FetchAsync(PayloadSerializer, fetchOpts, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var lease = CreateLease(msg, request.Consumer, now, request.LeaseDuration);
|
||||
if (lease is null)
|
||||
{
|
||||
await msg.AckAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
leases.Add(lease);
|
||||
}
|
||||
|
||||
return leases;
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<INotifyQueueLease<NotifyDeliveryQueueMessage>>> ClaimExpiredAsync(
|
||||
NotifyQueueClaimOptions options,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
var consumer = await EnsureStreamAndConsumerAsync(js, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var fetchOpts = new NatsJSFetchOpts
|
||||
{
|
||||
MaxMsgs = options.BatchSize,
|
||||
Expires = options.MinIdleTime,
|
||||
IdleHeartbeat = _options.IdleHeartbeat
|
||||
};
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var leases = new List<INotifyQueueLease<NotifyDeliveryQueueMessage>>(options.BatchSize);
|
||||
|
||||
await foreach (var msg in consumer.FetchAsync(PayloadSerializer, fetchOpts, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var deliveries = (int)(msg.Metadata?.NumDelivered ?? 1);
|
||||
if (deliveries <= 1)
|
||||
{
|
||||
await msg.NakAsync(new AckOpts(), TimeSpan.Zero, cancellationToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
var lease = CreateLease(msg, options.ClaimantConsumer, now, _queueOptions.DefaultLeaseDuration);
|
||||
if (lease is null)
|
||||
{
|
||||
await msg.AckAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
leases.Add(lease);
|
||||
}
|
||||
|
||||
return leases;
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_disposed = true;
|
||||
|
||||
if (_connection is not null)
|
||||
{
|
||||
await _connection.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
_connectionGate.Dispose();
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
internal async Task AcknowledgeAsync(
|
||||
NatsNotifyDeliveryLease lease,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!lease.TryBeginCompletion())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await lease.RawMessage.AckAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
|
||||
NotifyQueueMetrics.RecordAck(TransportName, _options.Stream);
|
||||
|
||||
_logger.LogDebug(
|
||||
"Acknowledged Notify delivery {DeliveryId} (sequence {Sequence}).",
|
||||
lease.Message.Delivery.DeliveryId,
|
||||
lease.MessageId);
|
||||
}
|
||||
|
||||
internal async Task RenewLeaseAsync(
|
||||
NatsNotifyDeliveryLease lease,
|
||||
TimeSpan leaseDuration,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await lease.RawMessage.AckProgressAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
|
||||
var expires = _timeProvider.GetUtcNow().Add(leaseDuration);
|
||||
lease.RefreshLease(expires);
|
||||
|
||||
_logger.LogDebug(
|
||||
"Renewed NATS lease for Notify delivery {DeliveryId} until {Expires:u}.",
|
||||
lease.Message.Delivery.DeliveryId,
|
||||
expires);
|
||||
}
|
||||
|
||||
internal async Task ReleaseAsync(
|
||||
NatsNotifyDeliveryLease lease,
|
||||
NotifyQueueReleaseDisposition disposition,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (disposition == NotifyQueueReleaseDisposition.Retry
|
||||
&& lease.Attempt >= _queueOptions.MaxDeliveryAttempts)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Notify delivery {DeliveryId} reached max delivery attempts ({Attempts}); moving to dead-letter stream.",
|
||||
lease.Message.Delivery.DeliveryId,
|
||||
lease.Attempt);
|
||||
|
||||
await DeadLetterAsync(
|
||||
lease,
|
||||
$"max-delivery-attempts:{lease.Attempt}",
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!lease.TryBeginCompletion())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (disposition == NotifyQueueReleaseDisposition.Retry)
|
||||
{
|
||||
var delay = CalculateBackoff(lease.Attempt);
|
||||
await lease.RawMessage.NakAsync(new AckOpts(), delay, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
NotifyQueueMetrics.RecordRetry(TransportName, _options.Stream);
|
||||
_logger.LogInformation(
|
||||
"Scheduled Notify delivery {DeliveryId} for retry with delay {Delay} (attempt {Attempt}).",
|
||||
lease.Message.Delivery.DeliveryId,
|
||||
delay,
|
||||
lease.Attempt);
|
||||
}
|
||||
else
|
||||
{
|
||||
await lease.RawMessage.AckTerminateAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
|
||||
NotifyQueueMetrics.RecordAck(TransportName, _options.Stream);
|
||||
_logger.LogInformation(
|
||||
"Abandoned Notify delivery {DeliveryId} after {Attempt} attempt(s).",
|
||||
lease.Message.Delivery.DeliveryId,
|
||||
lease.Attempt);
|
||||
}
|
||||
}
|
||||
|
||||
internal async Task DeadLetterAsync(
|
||||
NatsNotifyDeliveryLease lease,
|
||||
string reason,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!lease.TryBeginCompletion())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await lease.RawMessage.AckTerminateAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
await EnsureDeadLetterStreamAsync(js, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var payload = Encoding.UTF8.GetBytes(NotifyCanonicalJsonSerializer.Serialize(lease.Message.Delivery));
|
||||
var headers = BuildDeadLetterHeaders(lease, reason);
|
||||
|
||||
await js.PublishAsync(
|
||||
_options.DeadLetterSubject,
|
||||
payload,
|
||||
PayloadSerializer,
|
||||
new NatsJSPubOpts(),
|
||||
headers,
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
NotifyQueueMetrics.RecordDeadLetter(TransportName, _options.DeadLetterStream);
|
||||
_logger.LogError(
|
||||
"Dead-lettered Notify delivery {DeliveryId} (attempt {Attempt}): {Reason}",
|
||||
lease.Message.Delivery.DeliveryId,
|
||||
lease.Attempt,
|
||||
reason);
|
||||
}
|
||||
|
||||
internal async Task PingAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var connection = await EnsureConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await connection.PingAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<NatsJSContext> GetJetStreamAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_jsContext is not null)
|
||||
{
|
||||
return _jsContext;
|
||||
}
|
||||
|
||||
var connection = await EnsureConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await _connectionGate.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
_jsContext ??= new NatsJSContext(connection);
|
||||
return _jsContext;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_connectionGate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async ValueTask<INatsJSConsumer> EnsureStreamAndConsumerAsync(
|
||||
NatsJSContext js,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (_consumer is not null)
|
||||
{
|
||||
return _consumer;
|
||||
}
|
||||
|
||||
await _connectionGate.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_consumer is not null)
|
||||
{
|
||||
return _consumer;
|
||||
}
|
||||
|
||||
await EnsureStreamAsync(js, cancellationToken).ConfigureAwait(false);
|
||||
await EnsureDeadLetterStreamAsync(js, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var consumerConfig = new ConsumerConfig
|
||||
{
|
||||
DurableName = _options.DurableConsumer,
|
||||
AckPolicy = ConsumerConfigAckPolicy.Explicit,
|
||||
ReplayPolicy = ConsumerConfigReplayPolicy.Instant,
|
||||
DeliverPolicy = ConsumerConfigDeliverPolicy.All,
|
||||
AckWait = ToNanoseconds(_options.AckWait),
|
||||
MaxAckPending = _options.MaxAckPending,
|
||||
MaxDeliver = Math.Max(1, _queueOptions.MaxDeliveryAttempts),
|
||||
FilterSubjects = new[] { _options.Subject }
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
_consumer = await js.CreateConsumerAsync(
|
||||
_options.Stream,
|
||||
consumerConfig,
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (NatsJSApiException apiEx)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
apiEx,
|
||||
"CreateConsumerAsync failed with code {Code}; attempting to fetch existing durable consumer {Durable}.",
|
||||
apiEx.Error?.Code,
|
||||
_options.DurableConsumer);
|
||||
|
||||
_consumer = await js.GetConsumerAsync(
|
||||
_options.Stream,
|
||||
_options.DurableConsumer,
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return _consumer;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_connectionGate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<NatsConnection> EnsureConnectionAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_connection is not null)
|
||||
{
|
||||
return _connection;
|
||||
}
|
||||
|
||||
await _connectionGate.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_connection is not null)
|
||||
{
|
||||
return _connection;
|
||||
}
|
||||
|
||||
var opts = new NatsOpts
|
||||
{
|
||||
Url = _options.Url!,
|
||||
Name = "stellaops-notify-delivery",
|
||||
CommandTimeout = TimeSpan.FromSeconds(10),
|
||||
RequestTimeout = TimeSpan.FromSeconds(20),
|
||||
PingInterval = TimeSpan.FromSeconds(30)
|
||||
};
|
||||
|
||||
_connection = await _connectionFactory(opts, cancellationToken).ConfigureAwait(false);
|
||||
await _connection.ConnectAsync().ConfigureAwait(false);
|
||||
return _connection;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_connectionGate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task EnsureStreamAsync(NatsJSContext js, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await js.GetStreamAsync(_options.Stream, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (NatsJSApiException ex) when (ex.Error?.Code == 404)
|
||||
{
|
||||
var config = new StreamConfig(name: _options.Stream, subjects: new[] { _options.Subject })
|
||||
{
|
||||
Retention = StreamConfigRetention.Workqueue,
|
||||
Storage = StreamConfigStorage.File,
|
||||
MaxConsumers = -1,
|
||||
MaxMsgs = -1,
|
||||
MaxBytes = -1
|
||||
};
|
||||
|
||||
await js.CreateStreamAsync(config, cancellationToken).ConfigureAwait(false);
|
||||
_logger.LogInformation("Created NATS Notify delivery stream {Stream} ({Subject}).", _options.Stream, _options.Subject);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task EnsureDeadLetterStreamAsync(NatsJSContext js, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await js.GetStreamAsync(_options.DeadLetterStream, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (NatsJSApiException ex) when (ex.Error?.Code == 404)
|
||||
{
|
||||
var config = new StreamConfig(name: _options.DeadLetterStream, subjects: new[] { _options.DeadLetterSubject })
|
||||
{
|
||||
Retention = StreamConfigRetention.Workqueue,
|
||||
Storage = StreamConfigStorage.File,
|
||||
MaxConsumers = -1,
|
||||
MaxMsgs = -1,
|
||||
MaxBytes = -1
|
||||
};
|
||||
|
||||
await js.CreateStreamAsync(config, cancellationToken).ConfigureAwait(false);
|
||||
_logger.LogInformation("Created NATS Notify delivery dead-letter stream {Stream} ({Subject}).", _options.DeadLetterStream, _options.DeadLetterSubject);
|
||||
}
|
||||
}
|
||||
|
||||
private NatsNotifyDeliveryLease? CreateLease(
|
||||
NatsJSMsg<byte[]> message,
|
||||
string consumer,
|
||||
DateTimeOffset now,
|
||||
TimeSpan leaseDuration)
|
||||
{
|
||||
var payloadBytes = message.Data ?? Array.Empty<byte>();
|
||||
if (payloadBytes.Length == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
NotifyDelivery delivery;
|
||||
try
|
||||
{
|
||||
var json = Encoding.UTF8.GetString(payloadBytes);
|
||||
delivery = NotifyCanonicalJsonSerializer.Deserialize<NotifyDelivery>(json);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
ex,
|
||||
"Failed to deserialize Notify delivery payload for NATS message {Sequence}.",
|
||||
message.Metadata?.Sequence.Stream);
|
||||
return null;
|
||||
}
|
||||
|
||||
var headers = message.Headers ?? new NatsHeaders();
|
||||
|
||||
var deliveryId = TryGetHeader(headers, NotifyQueueFields.DeliveryId) ?? delivery.DeliveryId;
|
||||
var channelId = TryGetHeader(headers, NotifyQueueFields.ChannelId);
|
||||
var channelTypeRaw = TryGetHeader(headers, NotifyQueueFields.ChannelType);
|
||||
if (channelId is null || channelTypeRaw is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!Enum.TryParse<NotifyChannelType>(channelTypeRaw, ignoreCase: true, out var channelType))
|
||||
{
|
||||
_logger.LogWarning("Unknown channel type '{ChannelType}' for delivery {DeliveryId}.", channelTypeRaw, deliveryId);
|
||||
return null;
|
||||
}
|
||||
|
||||
var traceId = TryGetHeader(headers, NotifyQueueFields.TraceId);
|
||||
var partitionKey = TryGetHeader(headers, NotifyQueueFields.PartitionKey) ?? channelId;
|
||||
var idempotencyKey = TryGetHeader(headers, NotifyQueueFields.IdempotencyKey) ?? delivery.DeliveryId;
|
||||
|
||||
var enqueuedAt = TryGetHeader(headers, NotifyQueueFields.EnqueuedAt) is { } enqueuedRaw
|
||||
&& long.TryParse(enqueuedRaw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var unix)
|
||||
? DateTimeOffset.FromUnixTimeMilliseconds(unix)
|
||||
: now;
|
||||
|
||||
var attempt = TryGetHeader(headers, NotifyQueueFields.Attempt) is { } attemptRaw
|
||||
&& int.TryParse(attemptRaw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedAttempt)
|
||||
? parsedAttempt
|
||||
: 1;
|
||||
|
||||
if (message.Metadata?.NumDelivered is ulong delivered && delivered > 0)
|
||||
{
|
||||
var deliveredInt = delivered > int.MaxValue ? int.MaxValue : (int)delivered;
|
||||
if (deliveredInt > attempt)
|
||||
{
|
||||
attempt = deliveredInt;
|
||||
}
|
||||
}
|
||||
|
||||
var attributes = ExtractAttributes(headers);
|
||||
var leaseExpires = now.Add(leaseDuration);
|
||||
var messageId = message.Metadata?.Sequence.Stream.ToString() ?? Guid.NewGuid().ToString("n");
|
||||
|
||||
var queueMessage = new NotifyDeliveryQueueMessage(
|
||||
delivery,
|
||||
channelId,
|
||||
channelType,
|
||||
_options.Subject,
|
||||
traceId,
|
||||
attributes);
|
||||
|
||||
return new NatsNotifyDeliveryLease(
|
||||
this,
|
||||
message,
|
||||
messageId,
|
||||
queueMessage,
|
||||
attempt,
|
||||
consumer,
|
||||
enqueuedAt,
|
||||
leaseExpires,
|
||||
idempotencyKey);
|
||||
}
|
||||
|
||||
private NatsHeaders BuildHeaders(NotifyDeliveryQueueMessage message)
|
||||
{
|
||||
var headers = new NatsHeaders
|
||||
{
|
||||
{ NotifyQueueFields.DeliveryId, message.Delivery.DeliveryId },
|
||||
{ NotifyQueueFields.ChannelId, message.ChannelId },
|
||||
{ NotifyQueueFields.ChannelType, message.ChannelType.ToString() },
|
||||
{ NotifyQueueFields.Tenant, message.Delivery.TenantId },
|
||||
{ NotifyQueueFields.Attempt, "1" },
|
||||
{ NotifyQueueFields.EnqueuedAt, _timeProvider.GetUtcNow().ToUnixTimeMilliseconds().ToString(CultureInfo.InvariantCulture) },
|
||||
{ NotifyQueueFields.IdempotencyKey, message.IdempotencyKey },
|
||||
{ NotifyQueueFields.PartitionKey, message.PartitionKey }
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(message.TraceId))
|
||||
{
|
||||
headers.Add(NotifyQueueFields.TraceId, message.TraceId!);
|
||||
}
|
||||
|
||||
foreach (var kvp in message.Attributes)
|
||||
{
|
||||
headers.Add(NotifyQueueFields.AttributePrefix + kvp.Key, kvp.Value);
|
||||
}
|
||||
|
||||
return headers;
|
||||
}
|
||||
|
||||
private NatsHeaders BuildDeadLetterHeaders(NatsNotifyDeliveryLease lease, string reason)
|
||||
{
|
||||
var headers = new NatsHeaders
|
||||
{
|
||||
{ NotifyQueueFields.DeliveryId, lease.Message.Delivery.DeliveryId },
|
||||
{ NotifyQueueFields.ChannelId, lease.Message.ChannelId },
|
||||
{ NotifyQueueFields.ChannelType, lease.Message.ChannelType.ToString() },
|
||||
{ NotifyQueueFields.Tenant, lease.Message.Delivery.TenantId },
|
||||
{ NotifyQueueFields.Attempt, lease.Attempt.ToString(CultureInfo.InvariantCulture) },
|
||||
{ NotifyQueueFields.IdempotencyKey, lease.Message.IdempotencyKey },
|
||||
{ "deadletter-reason", reason }
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(lease.Message.TraceId))
|
||||
{
|
||||
headers.Add(NotifyQueueFields.TraceId, lease.Message.TraceId!);
|
||||
}
|
||||
|
||||
foreach (var kvp in lease.Message.Attributes)
|
||||
{
|
||||
headers.Add(NotifyQueueFields.AttributePrefix + kvp.Key, kvp.Value);
|
||||
}
|
||||
|
||||
return headers;
|
||||
}
|
||||
|
||||
private static string? TryGetHeader(NatsHeaders headers, string key)
|
||||
{
|
||||
if (headers.TryGetValue(key, out var values) && values.Count > 0)
|
||||
{
|
||||
var value = values[0];
|
||||
return string.IsNullOrWhiteSpace(value) ? null : value;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, string> ExtractAttributes(NatsHeaders headers)
|
||||
{
|
||||
var attributes = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var key in headers.Keys)
|
||||
{
|
||||
if (!key.StartsWith(NotifyQueueFields.AttributePrefix, StringComparison.Ordinal))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (headers.TryGetValue(key, out var values) && values.Count > 0)
|
||||
{
|
||||
attributes[key[NotifyQueueFields.AttributePrefix.Length..]] = values[0]!;
|
||||
}
|
||||
}
|
||||
|
||||
return attributes.Count == 0
|
||||
? EmptyReadOnlyDictionary<string, string>.Instance
|
||||
: new ReadOnlyDictionary<string, string>(attributes);
|
||||
}
|
||||
|
||||
private TimeSpan CalculateBackoff(int attempt)
|
||||
{
|
||||
var initial = _queueOptions.RetryInitialBackoff > TimeSpan.Zero
|
||||
? _queueOptions.RetryInitialBackoff
|
||||
: _options.RetryDelay;
|
||||
|
||||
if (initial <= TimeSpan.Zero)
|
||||
{
|
||||
return TimeSpan.Zero;
|
||||
}
|
||||
|
||||
if (attempt <= 1)
|
||||
{
|
||||
return initial;
|
||||
}
|
||||
|
||||
var max = _queueOptions.RetryMaxBackoff > TimeSpan.Zero
|
||||
? _queueOptions.RetryMaxBackoff
|
||||
: initial;
|
||||
|
||||
var exponent = attempt - 1;
|
||||
var scaledTicks = initial.Ticks * Math.Pow(2, exponent - 1);
|
||||
var cappedTicks = Math.Min(max.Ticks, scaledTicks);
|
||||
var resultTicks = Math.Max(initial.Ticks, (long)cappedTicks);
|
||||
return TimeSpan.FromTicks(resultTicks);
|
||||
}
|
||||
|
||||
private static long ToNanoseconds(TimeSpan value)
|
||||
=> value <= TimeSpan.Zero ? 0 : value.Ticks * 100L;
|
||||
|
||||
private static class EmptyReadOnlyDictionary<TKey, TValue>
|
||||
where TKey : notnull
|
||||
{
|
||||
public static readonly IReadOnlyDictionary<TKey, TValue> Instance =
|
||||
new ReadOnlyDictionary<TKey, TValue>(new Dictionary<TKey, TValue>(0, EqualityComparer<TKey>.Default));
|
||||
}
|
||||
}
|
||||
83
src/StellaOps.Notify.Queue/Nats/NatsNotifyEventLease.cs
Normal file
83
src/StellaOps.Notify.Queue/Nats/NatsNotifyEventLease.cs
Normal file
@@ -0,0 +1,83 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using NATS.Client.JetStream;
|
||||
|
||||
namespace StellaOps.Notify.Queue.Nats;
|
||||
|
||||
internal sealed class NatsNotifyEventLease : INotifyQueueLease<NotifyQueueEventMessage>
|
||||
{
|
||||
private readonly NatsNotifyEventQueue _queue;
|
||||
private readonly NatsJSMsg<byte[]> _message;
|
||||
private int _completed;
|
||||
|
||||
internal NatsNotifyEventLease(
|
||||
NatsNotifyEventQueue queue,
|
||||
NatsJSMsg<byte[]> message,
|
||||
string messageId,
|
||||
NotifyQueueEventMessage payload,
|
||||
int attempt,
|
||||
string consumer,
|
||||
DateTimeOffset enqueuedAt,
|
||||
DateTimeOffset leaseExpiresAt)
|
||||
{
|
||||
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
|
||||
if (EqualityComparer<NatsJSMsg<byte[]>>.Default.Equals(message, default))
|
||||
{
|
||||
throw new ArgumentException("Message must be provided.", nameof(message));
|
||||
}
|
||||
|
||||
_message = message;
|
||||
MessageId = messageId ?? throw new ArgumentNullException(nameof(messageId));
|
||||
Message = payload ?? throw new ArgumentNullException(nameof(payload));
|
||||
Attempt = attempt;
|
||||
Consumer = consumer ?? throw new ArgumentNullException(nameof(consumer));
|
||||
EnqueuedAt = enqueuedAt;
|
||||
LeaseExpiresAt = leaseExpiresAt;
|
||||
}
|
||||
|
||||
public string MessageId { get; }
|
||||
|
||||
public int Attempt { get; internal set; }
|
||||
|
||||
public DateTimeOffset EnqueuedAt { get; }
|
||||
|
||||
public DateTimeOffset LeaseExpiresAt { get; private set; }
|
||||
|
||||
public string Consumer { get; }
|
||||
|
||||
public string Stream => Message.Stream;
|
||||
|
||||
public string TenantId => Message.TenantId;
|
||||
|
||||
public string? PartitionKey => Message.PartitionKey;
|
||||
|
||||
public string IdempotencyKey => Message.IdempotencyKey;
|
||||
|
||||
public string? TraceId => Message.TraceId;
|
||||
|
||||
public IReadOnlyDictionary<string, string> Attributes => Message.Attributes;
|
||||
|
||||
public NotifyQueueEventMessage Message { get; }
|
||||
|
||||
internal NatsJSMsg<byte[]> RawMessage => _message;
|
||||
|
||||
public Task AcknowledgeAsync(CancellationToken cancellationToken = default)
|
||||
=> _queue.AcknowledgeAsync(this, cancellationToken);
|
||||
|
||||
public Task RenewAsync(TimeSpan leaseDuration, CancellationToken cancellationToken = default)
|
||||
=> _queue.RenewLeaseAsync(this, leaseDuration, cancellationToken);
|
||||
|
||||
public Task ReleaseAsync(NotifyQueueReleaseDisposition disposition, CancellationToken cancellationToken = default)
|
||||
=> _queue.ReleaseAsync(this, disposition, cancellationToken);
|
||||
|
||||
public Task DeadLetterAsync(string reason, CancellationToken cancellationToken = default)
|
||||
=> _queue.DeadLetterAsync(this, reason, cancellationToken);
|
||||
|
||||
internal bool TryBeginCompletion()
|
||||
=> Interlocked.CompareExchange(ref _completed, 1, 0) == 0;
|
||||
|
||||
internal void RefreshLease(DateTimeOffset expiresAt)
|
||||
=> LeaseExpiresAt = expiresAt;
|
||||
}
|
||||
698
src/StellaOps.Notify.Queue/Nats/NatsNotifyEventQueue.cs
Normal file
698
src/StellaOps.Notify.Queue/Nats/NatsNotifyEventQueue.cs
Normal file
@@ -0,0 +1,698 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.ObjectModel;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using NATS.Client.Core;
|
||||
using NATS.Client.JetStream;
|
||||
using NATS.Client.JetStream.Models;
|
||||
using StellaOps.Notify.Models;
|
||||
|
||||
namespace StellaOps.Notify.Queue.Nats;
|
||||
|
||||
internal sealed class NatsNotifyEventQueue : INotifyEventQueue, IAsyncDisposable
|
||||
{
|
||||
private const string TransportName = "nats";
|
||||
|
||||
private static readonly INatsSerializer<byte[]> PayloadSerializer = NatsRawSerializer<byte[]>.Default;
|
||||
|
||||
private readonly NotifyEventQueueOptions _queueOptions;
|
||||
private readonly NotifyNatsEventQueueOptions _options;
|
||||
private readonly ILogger<NatsNotifyEventQueue> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly SemaphoreSlim _connectionGate = new(1, 1);
|
||||
private readonly Func<NatsOpts, CancellationToken, ValueTask<NatsConnection>> _connectionFactory;
|
||||
|
||||
private NatsConnection? _connection;
|
||||
private NatsJSContext? _jsContext;
|
||||
private INatsJSConsumer? _consumer;
|
||||
private bool _disposed;
|
||||
|
||||
public NatsNotifyEventQueue(
|
||||
NotifyEventQueueOptions queueOptions,
|
||||
NotifyNatsEventQueueOptions options,
|
||||
ILogger<NatsNotifyEventQueue> logger,
|
||||
TimeProvider timeProvider,
|
||||
Func<NatsOpts, CancellationToken, ValueTask<NatsConnection>>? connectionFactory = null)
|
||||
{
|
||||
_queueOptions = queueOptions ?? throw new ArgumentNullException(nameof(queueOptions));
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_connectionFactory = connectionFactory ?? ((opts, cancellationToken) => new ValueTask<NatsConnection>(new NatsConnection(opts)));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(_options.Url))
|
||||
{
|
||||
throw new InvalidOperationException("NATS connection URL must be configured for the Notify event queue.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(_options.Stream) || string.IsNullOrWhiteSpace(_options.Subject))
|
||||
{
|
||||
throw new InvalidOperationException("NATS stream and subject must be configured for the Notify event queue.");
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<NotifyQueueEnqueueResult> PublishAsync(
|
||||
NotifyQueueEventMessage message,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(message);
|
||||
|
||||
var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
await EnsureStreamAndConsumerAsync(js, cancellationToken).ConfigureAwait(false);
|
||||
await EnsureDeadLetterStreamAsync(js, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var idempotencyKey = string.IsNullOrWhiteSpace(message.IdempotencyKey)
|
||||
? message.Event.EventId.ToString("N")
|
||||
: message.IdempotencyKey;
|
||||
|
||||
var payload = Encoding.UTF8.GetBytes(NotifyCanonicalJsonSerializer.Serialize(message.Event));
|
||||
var headers = BuildHeaders(message, idempotencyKey);
|
||||
|
||||
var publishOpts = new NatsJSPubOpts
|
||||
{
|
||||
MsgId = idempotencyKey,
|
||||
RetryAttempts = 0
|
||||
};
|
||||
|
||||
var ack = await js.PublishAsync(
|
||||
_options.Subject,
|
||||
payload,
|
||||
PayloadSerializer,
|
||||
publishOpts,
|
||||
headers,
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (ack.Duplicate)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Duplicate Notify event enqueue detected for idempotency token {Token}.",
|
||||
idempotencyKey);
|
||||
|
||||
NotifyQueueMetrics.RecordDeduplicated(TransportName, _options.Stream);
|
||||
return new NotifyQueueEnqueueResult(ack.Seq.ToString(), true);
|
||||
}
|
||||
|
||||
NotifyQueueMetrics.RecordEnqueued(TransportName, _options.Stream);
|
||||
_logger.LogDebug(
|
||||
"Enqueued Notify event {EventId} into NATS stream {Stream} (sequence {Sequence}).",
|
||||
message.Event.EventId,
|
||||
ack.Stream,
|
||||
ack.Seq);
|
||||
|
||||
return new NotifyQueueEnqueueResult(ack.Seq.ToString(), false);
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<INotifyQueueLease<NotifyQueueEventMessage>>> LeaseAsync(
|
||||
NotifyQueueLeaseRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
var consumer = await EnsureStreamAndConsumerAsync(js, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var fetchOpts = new NatsJSFetchOpts
|
||||
{
|
||||
MaxMsgs = request.BatchSize,
|
||||
Expires = request.LeaseDuration,
|
||||
IdleHeartbeat = _options.IdleHeartbeat
|
||||
};
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var leases = new List<INotifyQueueLease<NotifyQueueEventMessage>>(request.BatchSize);
|
||||
|
||||
await foreach (var msg in consumer.FetchAsync(PayloadSerializer, fetchOpts, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var lease = CreateLease(msg, request.Consumer, now, request.LeaseDuration);
|
||||
if (lease is null)
|
||||
{
|
||||
await msg.AckAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
leases.Add(lease);
|
||||
}
|
||||
|
||||
return leases;
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<INotifyQueueLease<NotifyQueueEventMessage>>> ClaimExpiredAsync(
|
||||
NotifyQueueClaimOptions options,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
var consumer = await EnsureStreamAndConsumerAsync(js, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var fetchOpts = new NatsJSFetchOpts
|
||||
{
|
||||
MaxMsgs = options.BatchSize,
|
||||
Expires = options.MinIdleTime,
|
||||
IdleHeartbeat = _options.IdleHeartbeat
|
||||
};
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var leases = new List<INotifyQueueLease<NotifyQueueEventMessage>>(options.BatchSize);
|
||||
|
||||
await foreach (var msg in consumer.FetchAsync(PayloadSerializer, fetchOpts, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var deliveries = (int)(msg.Metadata?.NumDelivered ?? 1);
|
||||
if (deliveries <= 1)
|
||||
{
|
||||
await msg.NakAsync(new AckOpts(), TimeSpan.Zero, cancellationToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
var lease = CreateLease(msg, options.ClaimantConsumer, now, _queueOptions.DefaultLeaseDuration);
|
||||
if (lease is null)
|
||||
{
|
||||
await msg.AckAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
leases.Add(lease);
|
||||
}
|
||||
|
||||
return leases;
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_disposed = true;
|
||||
|
||||
if (_connection is not null)
|
||||
{
|
||||
await _connection.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
_connectionGate.Dispose();
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
internal async Task AcknowledgeAsync(
|
||||
NatsNotifyEventLease lease,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!lease.TryBeginCompletion())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await lease.RawMessage.AckAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
|
||||
NotifyQueueMetrics.RecordAck(TransportName, _options.Stream);
|
||||
|
||||
_logger.LogDebug(
|
||||
"Acknowledged Notify event {EventId} (sequence {Sequence}).",
|
||||
lease.Message.Event.EventId,
|
||||
lease.MessageId);
|
||||
}
|
||||
|
||||
internal async Task RenewLeaseAsync(
|
||||
NatsNotifyEventLease lease,
|
||||
TimeSpan leaseDuration,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await lease.RawMessage.AckProgressAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var expires = _timeProvider.GetUtcNow().Add(leaseDuration);
|
||||
lease.RefreshLease(expires);
|
||||
|
||||
_logger.LogDebug(
|
||||
"Renewed NATS lease for Notify event {EventId} until {Expires:u}.",
|
||||
lease.Message.Event.EventId,
|
||||
expires);
|
||||
}
|
||||
|
||||
internal async Task ReleaseAsync(
|
||||
NatsNotifyEventLease lease,
|
||||
NotifyQueueReleaseDisposition disposition,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (disposition == NotifyQueueReleaseDisposition.Retry
|
||||
&& lease.Attempt >= _queueOptions.MaxDeliveryAttempts)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Notify event {EventId} reached max delivery attempts ({Attempts}); moving to dead-letter stream.",
|
||||
lease.Message.Event.EventId,
|
||||
lease.Attempt);
|
||||
|
||||
await DeadLetterAsync(
|
||||
lease,
|
||||
$"max-delivery-attempts:{lease.Attempt}",
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!lease.TryBeginCompletion())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (disposition == NotifyQueueReleaseDisposition.Retry)
|
||||
{
|
||||
var delay = CalculateBackoff(lease.Attempt);
|
||||
await lease.RawMessage.NakAsync(new AckOpts(), delay, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
NotifyQueueMetrics.RecordRetry(TransportName, _options.Stream);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Scheduled Notify event {EventId} for retry with delay {Delay} (attempt {Attempt}).",
|
||||
lease.Message.Event.EventId,
|
||||
delay,
|
||||
lease.Attempt);
|
||||
}
|
||||
else
|
||||
{
|
||||
await lease.RawMessage.AckTerminateAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
|
||||
NotifyQueueMetrics.RecordAck(TransportName, _options.Stream);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Abandoned Notify event {EventId} after {Attempt} attempt(s).",
|
||||
lease.Message.Event.EventId,
|
||||
lease.Attempt);
|
||||
}
|
||||
}
|
||||
|
||||
internal async Task DeadLetterAsync(
|
||||
NatsNotifyEventLease lease,
|
||||
string reason,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!lease.TryBeginCompletion())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await lease.RawMessage.AckTerminateAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
await EnsureDeadLetterStreamAsync(js, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var headers = BuildDeadLetterHeaders(lease, reason);
|
||||
var payload = Encoding.UTF8.GetBytes(NotifyCanonicalJsonSerializer.Serialize(lease.Message.Event));
|
||||
|
||||
await js.PublishAsync(
|
||||
_options.DeadLetterSubject,
|
||||
payload,
|
||||
PayloadSerializer,
|
||||
new NatsJSPubOpts(),
|
||||
headers,
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
NotifyQueueMetrics.RecordDeadLetter(TransportName, _options.DeadLetterStream);
|
||||
|
||||
_logger.LogError(
|
||||
"Dead-lettered Notify event {EventId} (attempt {Attempt}): {Reason}",
|
||||
lease.Message.Event.EventId,
|
||||
lease.Attempt,
|
||||
reason);
|
||||
}
|
||||
|
||||
internal async Task PingAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var connection = await EnsureConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await connection.PingAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<NatsJSContext> GetJetStreamAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_jsContext is not null)
|
||||
{
|
||||
return _jsContext;
|
||||
}
|
||||
|
||||
var connection = await EnsureConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await _connectionGate.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
_jsContext ??= new NatsJSContext(connection);
|
||||
return _jsContext;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_connectionGate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async ValueTask<INatsJSConsumer> EnsureStreamAndConsumerAsync(
|
||||
NatsJSContext js,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (_consumer is not null)
|
||||
{
|
||||
return _consumer;
|
||||
}
|
||||
|
||||
await _connectionGate.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_consumer is not null)
|
||||
{
|
||||
return _consumer;
|
||||
}
|
||||
|
||||
await EnsureStreamAsync(js, cancellationToken).ConfigureAwait(false);
|
||||
await EnsureDeadLetterStreamAsync(js, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var consumerConfig = new ConsumerConfig
|
||||
{
|
||||
DurableName = _options.DurableConsumer,
|
||||
AckPolicy = ConsumerConfigAckPolicy.Explicit,
|
||||
ReplayPolicy = ConsumerConfigReplayPolicy.Instant,
|
||||
DeliverPolicy = ConsumerConfigDeliverPolicy.All,
|
||||
AckWait = ToNanoseconds(_options.AckWait),
|
||||
MaxAckPending = _options.MaxAckPending,
|
||||
MaxDeliver = Math.Max(1, _queueOptions.MaxDeliveryAttempts),
|
||||
FilterSubjects = new[] { _options.Subject }
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
_consumer = await js.CreateConsumerAsync(
|
||||
_options.Stream,
|
||||
consumerConfig,
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (NatsJSApiException apiEx)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
apiEx,
|
||||
"CreateConsumerAsync failed with code {Code}; attempting to fetch existing durable consumer {Durable}.",
|
||||
apiEx.Error?.Code,
|
||||
_options.DurableConsumer);
|
||||
|
||||
_consumer = await js.GetConsumerAsync(
|
||||
_options.Stream,
|
||||
_options.DurableConsumer,
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return _consumer;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_connectionGate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<NatsConnection> EnsureConnectionAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_connection is not null)
|
||||
{
|
||||
return _connection;
|
||||
}
|
||||
|
||||
await _connectionGate.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_connection is not null)
|
||||
{
|
||||
return _connection;
|
||||
}
|
||||
|
||||
var opts = new NatsOpts
|
||||
{
|
||||
Url = _options.Url!,
|
||||
Name = "stellaops-notify-queue",
|
||||
CommandTimeout = TimeSpan.FromSeconds(10),
|
||||
RequestTimeout = TimeSpan.FromSeconds(20),
|
||||
PingInterval = TimeSpan.FromSeconds(30)
|
||||
};
|
||||
|
||||
_connection = await _connectionFactory(opts, cancellationToken).ConfigureAwait(false);
|
||||
await _connection.ConnectAsync().ConfigureAwait(false);
|
||||
return _connection;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_connectionGate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task EnsureStreamAsync(NatsJSContext js, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await js.GetStreamAsync(_options.Stream, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (NatsJSApiException ex) when (ex.Error?.Code == 404)
|
||||
{
|
||||
var config = new StreamConfig(name: _options.Stream, subjects: new[] { _options.Subject })
|
||||
{
|
||||
Retention = StreamConfigRetention.Workqueue,
|
||||
Storage = StreamConfigStorage.File,
|
||||
MaxConsumers = -1,
|
||||
MaxMsgs = -1,
|
||||
MaxBytes = -1
|
||||
};
|
||||
|
||||
await js.CreateStreamAsync(config, cancellationToken).ConfigureAwait(false);
|
||||
_logger.LogInformation("Created NATS Notify stream {Stream} ({Subject}).", _options.Stream, _options.Subject);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task EnsureDeadLetterStreamAsync(NatsJSContext js, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await js.GetStreamAsync(_options.DeadLetterStream, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (NatsJSApiException ex) when (ex.Error?.Code == 404)
|
||||
{
|
||||
var config = new StreamConfig(name: _options.DeadLetterStream, subjects: new[] { _options.DeadLetterSubject })
|
||||
{
|
||||
Retention = StreamConfigRetention.Workqueue,
|
||||
Storage = StreamConfigStorage.File,
|
||||
MaxConsumers = -1,
|
||||
MaxMsgs = -1,
|
||||
MaxBytes = -1
|
||||
};
|
||||
|
||||
await js.CreateStreamAsync(config, cancellationToken).ConfigureAwait(false);
|
||||
_logger.LogInformation("Created NATS Notify dead-letter stream {Stream} ({Subject}).", _options.DeadLetterStream, _options.DeadLetterSubject);
|
||||
}
|
||||
}
|
||||
|
||||
private NatsNotifyEventLease? CreateLease(
|
||||
NatsJSMsg<byte[]> message,
|
||||
string consumer,
|
||||
DateTimeOffset now,
|
||||
TimeSpan leaseDuration)
|
||||
{
|
||||
var payloadBytes = message.Data ?? Array.Empty<byte>();
|
||||
if (payloadBytes.Length == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
NotifyEvent notifyEvent;
|
||||
try
|
||||
{
|
||||
var json = Encoding.UTF8.GetString(payloadBytes);
|
||||
notifyEvent = NotifyCanonicalJsonSerializer.Deserialize<NotifyEvent>(json);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
ex,
|
||||
"Failed to deserialize Notify event payload for NATS message {Sequence}.",
|
||||
message.Metadata?.Sequence.Stream);
|
||||
return null;
|
||||
}
|
||||
|
||||
var headers = message.Headers ?? new NatsHeaders();
|
||||
|
||||
var idempotencyKey = TryGetHeader(headers, NotifyQueueFields.IdempotencyKey)
|
||||
?? notifyEvent.EventId.ToString("N");
|
||||
|
||||
var partitionKey = TryGetHeader(headers, NotifyQueueFields.PartitionKey);
|
||||
var traceId = TryGetHeader(headers, NotifyQueueFields.TraceId);
|
||||
var enqueuedAt = TryGetHeader(headers, NotifyQueueFields.EnqueuedAt) is { } enqueuedRaw
|
||||
&& long.TryParse(enqueuedRaw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var unix)
|
||||
? DateTimeOffset.FromUnixTimeMilliseconds(unix)
|
||||
: now;
|
||||
|
||||
var attempt = TryGetHeader(headers, NotifyQueueFields.Attempt) is { } attemptRaw
|
||||
&& int.TryParse(attemptRaw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedAttempt)
|
||||
? parsedAttempt
|
||||
: 1;
|
||||
|
||||
if (message.Metadata?.NumDelivered is ulong delivered && delivered > 0)
|
||||
{
|
||||
var deliveredInt = delivered > int.MaxValue ? int.MaxValue : (int)delivered;
|
||||
if (deliveredInt > attempt)
|
||||
{
|
||||
attempt = deliveredInt;
|
||||
}
|
||||
}
|
||||
|
||||
var attributes = ExtractAttributes(headers);
|
||||
var leaseExpires = now.Add(leaseDuration);
|
||||
var messageId = message.Metadata?.Sequence.Stream.ToString() ?? Guid.NewGuid().ToString("n");
|
||||
|
||||
var queueMessage = new NotifyQueueEventMessage(
|
||||
notifyEvent,
|
||||
_options.Subject,
|
||||
idempotencyKey,
|
||||
partitionKey,
|
||||
traceId,
|
||||
attributes);
|
||||
|
||||
return new NatsNotifyEventLease(
|
||||
this,
|
||||
message,
|
||||
messageId,
|
||||
queueMessage,
|
||||
attempt,
|
||||
consumer,
|
||||
enqueuedAt,
|
||||
leaseExpires);
|
||||
}
|
||||
|
||||
private NatsHeaders BuildHeaders(NotifyQueueEventMessage message, string idempotencyKey)
|
||||
{
|
||||
var headers = new NatsHeaders
|
||||
{
|
||||
{ NotifyQueueFields.EventId, message.Event.EventId.ToString("D") },
|
||||
{ NotifyQueueFields.Tenant, message.TenantId },
|
||||
{ NotifyQueueFields.Kind, message.Event.Kind },
|
||||
{ NotifyQueueFields.Attempt, "1" },
|
||||
{ NotifyQueueFields.EnqueuedAt, _timeProvider.GetUtcNow().ToUnixTimeMilliseconds().ToString(CultureInfo.InvariantCulture) },
|
||||
{ NotifyQueueFields.IdempotencyKey, idempotencyKey }
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(message.TraceId))
|
||||
{
|
||||
headers.Add(NotifyQueueFields.TraceId, message.TraceId!);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(message.PartitionKey))
|
||||
{
|
||||
headers.Add(NotifyQueueFields.PartitionKey, message.PartitionKey!);
|
||||
}
|
||||
|
||||
foreach (var kvp in message.Attributes)
|
||||
{
|
||||
headers.Add(NotifyQueueFields.AttributePrefix + kvp.Key, kvp.Value);
|
||||
}
|
||||
|
||||
return headers;
|
||||
}
|
||||
|
||||
private NatsHeaders BuildDeadLetterHeaders(NatsNotifyEventLease lease, string reason)
|
||||
{
|
||||
var headers = new NatsHeaders
|
||||
{
|
||||
{ NotifyQueueFields.EventId, lease.Message.Event.EventId.ToString("D") },
|
||||
{ NotifyQueueFields.Tenant, lease.Message.TenantId },
|
||||
{ NotifyQueueFields.Kind, lease.Message.Event.Kind },
|
||||
{ NotifyQueueFields.Attempt, lease.Attempt.ToString(CultureInfo.InvariantCulture) },
|
||||
{ NotifyQueueFields.IdempotencyKey, lease.Message.IdempotencyKey },
|
||||
{ "deadletter-reason", reason }
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(lease.Message.TraceId))
|
||||
{
|
||||
headers.Add(NotifyQueueFields.TraceId, lease.Message.TraceId!);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(lease.Message.PartitionKey))
|
||||
{
|
||||
headers.Add(NotifyQueueFields.PartitionKey, lease.Message.PartitionKey!);
|
||||
}
|
||||
|
||||
foreach (var kvp in lease.Message.Attributes)
|
||||
{
|
||||
headers.Add(NotifyQueueFields.AttributePrefix + kvp.Key, kvp.Value);
|
||||
}
|
||||
|
||||
return headers;
|
||||
}
|
||||
|
||||
private static string? TryGetHeader(NatsHeaders headers, string key)
|
||||
{
|
||||
if (headers.TryGetValue(key, out var values) && values.Count > 0)
|
||||
{
|
||||
var value = values[0];
|
||||
return string.IsNullOrWhiteSpace(value) ? null : value;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, string> ExtractAttributes(NatsHeaders headers)
|
||||
{
|
||||
var attributes = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var key in headers.Keys)
|
||||
{
|
||||
if (!key.StartsWith(NotifyQueueFields.AttributePrefix, StringComparison.Ordinal))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (headers.TryGetValue(key, out var values) && values.Count > 0)
|
||||
{
|
||||
attributes[key[NotifyQueueFields.AttributePrefix.Length..]] = values[0]!;
|
||||
}
|
||||
}
|
||||
|
||||
return attributes.Count == 0
|
||||
? EmptyReadOnlyDictionary<string, string>.Instance
|
||||
: new ReadOnlyDictionary<string, string>(attributes);
|
||||
}
|
||||
|
||||
private TimeSpan CalculateBackoff(int attempt)
|
||||
{
|
||||
var initial = _queueOptions.RetryInitialBackoff > TimeSpan.Zero
|
||||
? _queueOptions.RetryInitialBackoff
|
||||
: _options.RetryDelay;
|
||||
|
||||
if (initial <= TimeSpan.Zero)
|
||||
{
|
||||
return TimeSpan.Zero;
|
||||
}
|
||||
|
||||
if (attempt <= 1)
|
||||
{
|
||||
return initial;
|
||||
}
|
||||
|
||||
var max = _queueOptions.RetryMaxBackoff > TimeSpan.Zero
|
||||
? _queueOptions.RetryMaxBackoff
|
||||
: initial;
|
||||
|
||||
var exponent = attempt - 1;
|
||||
var scaledTicks = initial.Ticks * Math.Pow(2, exponent - 1);
|
||||
var cappedTicks = Math.Min(max.Ticks, scaledTicks);
|
||||
var resultTicks = Math.Max(initial.Ticks, (long)cappedTicks);
|
||||
return TimeSpan.FromTicks(resultTicks);
|
||||
}
|
||||
|
||||
private static long ToNanoseconds(TimeSpan value)
|
||||
=> value <= TimeSpan.Zero ? 0 : value.Ticks * 100L;
|
||||
|
||||
private static class EmptyReadOnlyDictionary<TKey, TValue>
|
||||
where TKey : notnull
|
||||
{
|
||||
public static readonly IReadOnlyDictionary<TKey, TValue> Instance =
|
||||
new ReadOnlyDictionary<TKey, TValue>(new Dictionary<TKey, TValue>(0, EqualityComparer<TKey>.Default));
|
||||
}
|
||||
}
|
||||
55
src/StellaOps.Notify.Queue/NotifyDeliveryQueueHealthCheck.cs
Normal file
55
src/StellaOps.Notify.Queue/NotifyDeliveryQueueHealthCheck.cs
Normal file
@@ -0,0 +1,55 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Notify.Queue.Nats;
|
||||
using StellaOps.Notify.Queue.Redis;
|
||||
|
||||
namespace StellaOps.Notify.Queue;
|
||||
|
||||
public sealed class NotifyDeliveryQueueHealthCheck : IHealthCheck
|
||||
{
|
||||
private readonly INotifyDeliveryQueue _queue;
|
||||
private readonly ILogger<NotifyDeliveryQueueHealthCheck> _logger;
|
||||
|
||||
public NotifyDeliveryQueueHealthCheck(
|
||||
INotifyDeliveryQueue queue,
|
||||
ILogger<NotifyDeliveryQueueHealthCheck> logger)
|
||||
{
|
||||
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<HealthCheckResult> CheckHealthAsync(
|
||||
HealthCheckContext context,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
try
|
||||
{
|
||||
switch (_queue)
|
||||
{
|
||||
case RedisNotifyDeliveryQueue redisQueue:
|
||||
await redisQueue.PingAsync(cancellationToken).ConfigureAwait(false);
|
||||
return HealthCheckResult.Healthy("Redis Notify delivery queue reachable.");
|
||||
|
||||
case NatsNotifyDeliveryQueue natsQueue:
|
||||
await natsQueue.PingAsync(cancellationToken).ConfigureAwait(false);
|
||||
return HealthCheckResult.Healthy("NATS Notify delivery queue reachable.");
|
||||
|
||||
default:
|
||||
return HealthCheckResult.Healthy("Notify delivery queue transport without dedicated ping returned healthy.");
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Notify delivery queue health check failed.");
|
||||
return new HealthCheckResult(
|
||||
context.Registration.FailureStatus,
|
||||
"Notify delivery queue transport unreachable.",
|
||||
ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
69
src/StellaOps.Notify.Queue/NotifyDeliveryQueueOptions.cs
Normal file
69
src/StellaOps.Notify.Queue/NotifyDeliveryQueueOptions.cs
Normal file
@@ -0,0 +1,69 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Notify.Queue;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration options for the Notify delivery queue abstraction.
|
||||
/// </summary>
|
||||
public sealed class NotifyDeliveryQueueOptions
|
||||
{
|
||||
public NotifyQueueTransportKind Transport { get; set; } = NotifyQueueTransportKind.Redis;
|
||||
|
||||
public NotifyRedisDeliveryQueueOptions Redis { get; set; } = new();
|
||||
|
||||
public NotifyNatsDeliveryQueueOptions Nats { get; set; } = new();
|
||||
|
||||
public TimeSpan DefaultLeaseDuration { get; set; } = TimeSpan.FromMinutes(5);
|
||||
|
||||
public int MaxDeliveryAttempts { get; set; } = 5;
|
||||
|
||||
public TimeSpan RetryInitialBackoff { get; set; } = TimeSpan.FromSeconds(5);
|
||||
|
||||
public TimeSpan RetryMaxBackoff { get; set; } = TimeSpan.FromMinutes(2);
|
||||
|
||||
public TimeSpan ClaimIdleThreshold { get; set; } = TimeSpan.FromMinutes(5);
|
||||
}
|
||||
|
||||
public sealed class NotifyRedisDeliveryQueueOptions
|
||||
{
|
||||
public string? ConnectionString { get; set; }
|
||||
|
||||
public int? Database { get; set; }
|
||||
|
||||
public TimeSpan InitializationTimeout { get; set; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
public string StreamName { get; set; } = "notify:deliveries";
|
||||
|
||||
public string ConsumerGroup { get; set; } = "notify-deliveries";
|
||||
|
||||
public string IdempotencyKeyPrefix { get; set; } = "notify:deliveries:idemp:";
|
||||
|
||||
public int? ApproximateMaxLength { get; set; }
|
||||
|
||||
public string DeadLetterStreamName { get; set; } = "notify:deliveries:dead";
|
||||
|
||||
public TimeSpan DeadLetterRetention { get; set; } = TimeSpan.FromDays(7);
|
||||
}
|
||||
|
||||
public sealed class NotifyNatsDeliveryQueueOptions
|
||||
{
|
||||
public string? Url { get; set; }
|
||||
|
||||
public string Stream { get; set; } = "NOTIFY_DELIVERIES";
|
||||
|
||||
public string Subject { get; set; } = "notify.deliveries";
|
||||
|
||||
public string DurableConsumer { get; set; } = "notify-deliveries";
|
||||
|
||||
public string DeadLetterStream { get; set; } = "NOTIFY_DELIVERIES_DEAD";
|
||||
|
||||
public string DeadLetterSubject { get; set; } = "notify.deliveries.dead";
|
||||
|
||||
public int MaxAckPending { get; set; } = 128;
|
||||
|
||||
public TimeSpan AckWait { get; set; } = TimeSpan.FromMinutes(5);
|
||||
|
||||
public TimeSpan RetryDelay { get; set; } = TimeSpan.FromSeconds(10);
|
||||
|
||||
public TimeSpan IdleHeartbeat { get; set; } = TimeSpan.FromSeconds(30);
|
||||
}
|
||||
177
src/StellaOps.Notify.Queue/NotifyEventQueueOptions.cs
Normal file
177
src/StellaOps.Notify.Queue/NotifyEventQueueOptions.cs
Normal file
@@ -0,0 +1,177 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StellaOps.Notify.Queue;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration options for the Notify event queue abstraction.
|
||||
/// </summary>
|
||||
public sealed class NotifyEventQueueOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Transport backing the queue.
|
||||
/// </summary>
|
||||
public NotifyQueueTransportKind Transport { get; set; } = NotifyQueueTransportKind.Redis;
|
||||
|
||||
/// <summary>
|
||||
/// Redis-specific configuration.
|
||||
/// </summary>
|
||||
public NotifyRedisEventQueueOptions Redis { get; set; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// NATS JetStream-specific configuration.
|
||||
/// </summary>
|
||||
public NotifyNatsEventQueueOptions Nats { get; set; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Default lease duration to use when consumers do not specify one explicitly.
|
||||
/// </summary>
|
||||
public TimeSpan DefaultLeaseDuration { get; set; } = TimeSpan.FromMinutes(5);
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of deliveries before a message should be considered failed.
|
||||
/// </summary>
|
||||
public int MaxDeliveryAttempts { get; set; } = 5;
|
||||
|
||||
/// <summary>
|
||||
/// Initial retry backoff applied when a message is released for retry.
|
||||
/// </summary>
|
||||
public TimeSpan RetryInitialBackoff { get; set; } = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>
|
||||
/// Cap applied to exponential retry backoff.
|
||||
/// </summary>
|
||||
public TimeSpan RetryMaxBackoff { get; set; } = TimeSpan.FromMinutes(2);
|
||||
|
||||
/// <summary>
|
||||
/// Minimum idle window before a pending message becomes eligible for claim.
|
||||
/// </summary>
|
||||
public TimeSpan ClaimIdleThreshold { get; set; } = TimeSpan.FromMinutes(5);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Redis transport options for the Notify event queue.
|
||||
/// </summary>
|
||||
public sealed class NotifyRedisEventQueueOptions
|
||||
{
|
||||
private IReadOnlyList<NotifyRedisEventStreamOptions> _streams = new List<NotifyRedisEventStreamOptions>
|
||||
{
|
||||
NotifyRedisEventStreamOptions.ForDefaultStream()
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Connection string for the Redis instance.
|
||||
/// </summary>
|
||||
public string? ConnectionString { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional logical database to select when connecting.
|
||||
/// </summary>
|
||||
public int? Database { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Time allowed for initial connection/consumer-group creation.
|
||||
/// </summary>
|
||||
public TimeSpan InitializationTimeout { get; set; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>
|
||||
/// TTL applied to idempotency keys stored alongside events.
|
||||
/// </summary>
|
||||
public TimeSpan IdempotencyWindow { get; set; } = TimeSpan.FromHours(12);
|
||||
|
||||
/// <summary>
|
||||
/// Streams consumed by Notify. Ordering is preserved during leasing.
|
||||
/// </summary>
|
||||
public IReadOnlyList<NotifyRedisEventStreamOptions> Streams
|
||||
{
|
||||
get => _streams;
|
||||
set => _streams = value is null || value.Count == 0
|
||||
? new List<NotifyRedisEventStreamOptions> { NotifyRedisEventStreamOptions.ForDefaultStream() }
|
||||
: value;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Per-Redis-stream options for the Notify event queue.
|
||||
/// </summary>
|
||||
public sealed class NotifyRedisEventStreamOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the Redis stream containing events.
|
||||
/// </summary>
|
||||
public string Stream { get; set; } = "notify:events";
|
||||
|
||||
/// <summary>
|
||||
/// Consumer group used by Notify workers.
|
||||
/// </summary>
|
||||
public string ConsumerGroup { get; set; } = "notify-workers";
|
||||
|
||||
/// <summary>
|
||||
/// Prefix used when storing idempotency keys in Redis.
|
||||
/// </summary>
|
||||
public string IdempotencyKeyPrefix { get; set; } = "notify:events:idemp:";
|
||||
|
||||
/// <summary>
|
||||
/// Approximate maximum length for the stream; when set Redis will trim entries.
|
||||
/// </summary>
|
||||
public int? ApproximateMaxLength { get; set; }
|
||||
|
||||
public static NotifyRedisEventStreamOptions ForDefaultStream()
|
||||
=> new();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// NATS JetStream options for the Notify event queue.
|
||||
/// </summary>
|
||||
public sealed class NotifyNatsEventQueueOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// URL for the JetStream-enabled NATS cluster.
|
||||
/// </summary>
|
||||
public string? Url { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Stream name carrying Notify events.
|
||||
/// </summary>
|
||||
public string Stream { get; set; } = "NOTIFY_EVENTS";
|
||||
|
||||
/// <summary>
|
||||
/// Subject that producers publish Notify events to.
|
||||
/// </summary>
|
||||
public string Subject { get; set; } = "notify.events";
|
||||
|
||||
/// <summary>
|
||||
/// Durable consumer identifier for Notify workers.
|
||||
/// </summary>
|
||||
public string DurableConsumer { get; set; } = "notify-workers";
|
||||
|
||||
/// <summary>
|
||||
/// Dead-letter stream name used when deliveries exhaust retry budget.
|
||||
/// </summary>
|
||||
public string DeadLetterStream { get; set; } = "NOTIFY_EVENTS_DEAD";
|
||||
|
||||
/// <summary>
|
||||
/// Subject used for dead-letter publications.
|
||||
/// </summary>
|
||||
public string DeadLetterSubject { get; set; } = "notify.events.dead";
|
||||
|
||||
/// <summary>
|
||||
/// Maximum pending messages before backpressure is applied.
|
||||
/// </summary>
|
||||
public int MaxAckPending { get; set; } = 256;
|
||||
|
||||
/// <summary>
|
||||
/// Visibility timeout applied to leased events.
|
||||
/// </summary>
|
||||
public TimeSpan AckWait { get; set; } = TimeSpan.FromMinutes(5);
|
||||
|
||||
/// <summary>
|
||||
/// Delay applied when releasing a message for retry.
|
||||
/// </summary>
|
||||
public TimeSpan RetryDelay { get; set; } = TimeSpan.FromSeconds(10);
|
||||
|
||||
/// <summary>
|
||||
/// Idle heartbeat emitted by the server to detect consumer disconnects.
|
||||
/// </summary>
|
||||
public TimeSpan IdleHeartbeat { get; set; } = TimeSpan.FromSeconds(30);
|
||||
}
|
||||
231
src/StellaOps.Notify.Queue/NotifyQueueContracts.cs
Normal file
231
src/StellaOps.Notify.Queue/NotifyQueueContracts.cs
Normal file
@@ -0,0 +1,231 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.ObjectModel;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Notify.Models;
|
||||
|
||||
namespace StellaOps.Notify.Queue;
|
||||
|
||||
/// <summary>
|
||||
/// Message queued for Notify event processing.
|
||||
/// </summary>
|
||||
public sealed class NotifyQueueEventMessage
|
||||
{
|
||||
private readonly NotifyEvent _event;
|
||||
private readonly IReadOnlyDictionary<string, string> _attributes;
|
||||
|
||||
public NotifyQueueEventMessage(
|
||||
NotifyEvent @event,
|
||||
string stream,
|
||||
string? idempotencyKey = null,
|
||||
string? partitionKey = null,
|
||||
string? traceId = null,
|
||||
IReadOnlyDictionary<string, string>? attributes = null)
|
||||
{
|
||||
_event = @event ?? throw new ArgumentNullException(nameof(@event));
|
||||
if (string.IsNullOrWhiteSpace(stream))
|
||||
{
|
||||
throw new ArgumentException("Stream must be provided.", nameof(stream));
|
||||
}
|
||||
|
||||
Stream = stream;
|
||||
IdempotencyKey = string.IsNullOrWhiteSpace(idempotencyKey)
|
||||
? @event.EventId.ToString("N")
|
||||
: idempotencyKey!;
|
||||
PartitionKey = string.IsNullOrWhiteSpace(partitionKey) ? null : partitionKey.Trim();
|
||||
TraceId = string.IsNullOrWhiteSpace(traceId) ? null : traceId.Trim();
|
||||
_attributes = attributes is null
|
||||
? EmptyReadOnlyDictionary<string, string>.Instance
|
||||
: new ReadOnlyDictionary<string, string>(new Dictionary<string, string>(attributes, StringComparer.Ordinal));
|
||||
}
|
||||
|
||||
public NotifyEvent Event => _event;
|
||||
|
||||
public string Stream { get; }
|
||||
|
||||
public string IdempotencyKey { get; }
|
||||
|
||||
public string TenantId => _event.Tenant;
|
||||
|
||||
public string? PartitionKey { get; }
|
||||
|
||||
public string? TraceId { get; }
|
||||
|
||||
public IReadOnlyDictionary<string, string> Attributes => _attributes;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Message queued for channel delivery execution.
|
||||
/// </summary>
|
||||
public sealed class NotifyDeliveryQueueMessage
|
||||
{
|
||||
public const string DefaultStream = "notify:deliveries";
|
||||
|
||||
private readonly IReadOnlyDictionary<string, string> _attributes;
|
||||
|
||||
public NotifyDeliveryQueueMessage(
|
||||
NotifyDelivery delivery,
|
||||
string channelId,
|
||||
NotifyChannelType channelType,
|
||||
string? stream = null,
|
||||
string? traceId = null,
|
||||
IReadOnlyDictionary<string, string>? attributes = null)
|
||||
{
|
||||
Delivery = delivery ?? throw new ArgumentNullException(nameof(delivery));
|
||||
ChannelId = NotifyValidation.EnsureNotNullOrWhiteSpace(channelId, nameof(channelId));
|
||||
ChannelType = channelType;
|
||||
Stream = string.IsNullOrWhiteSpace(stream) ? DefaultStream : stream!.Trim();
|
||||
TraceId = string.IsNullOrWhiteSpace(traceId) ? null : traceId.Trim();
|
||||
_attributes = attributes is null
|
||||
? EmptyReadOnlyDictionary<string, string>.Instance
|
||||
: new ReadOnlyDictionary<string, string>(new Dictionary<string, string>(attributes, StringComparer.Ordinal));
|
||||
}
|
||||
|
||||
public NotifyDelivery Delivery { get; }
|
||||
|
||||
public string ChannelId { get; }
|
||||
|
||||
public NotifyChannelType ChannelType { get; }
|
||||
|
||||
public string Stream { get; }
|
||||
|
||||
public string? TraceId { get; }
|
||||
|
||||
public string TenantId => Delivery.TenantId;
|
||||
|
||||
public string IdempotencyKey => Delivery.DeliveryId;
|
||||
|
||||
public string PartitionKey => ChannelId;
|
||||
|
||||
public IReadOnlyDictionary<string, string> Attributes => _attributes;
|
||||
}
|
||||
|
||||
public readonly record struct NotifyQueueEnqueueResult(string MessageId, bool Deduplicated);
|
||||
|
||||
public sealed class NotifyQueueLeaseRequest
|
||||
{
|
||||
public NotifyQueueLeaseRequest(string consumer, int batchSize, TimeSpan leaseDuration)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(consumer))
|
||||
{
|
||||
throw new ArgumentException("Consumer must be provided.", nameof(consumer));
|
||||
}
|
||||
|
||||
if (batchSize <= 0)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(batchSize), batchSize, "Batch size must be positive.");
|
||||
}
|
||||
|
||||
if (leaseDuration <= TimeSpan.Zero)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(leaseDuration), leaseDuration, "Lease duration must be positive.");
|
||||
}
|
||||
|
||||
Consumer = consumer;
|
||||
BatchSize = batchSize;
|
||||
LeaseDuration = leaseDuration;
|
||||
}
|
||||
|
||||
public string Consumer { get; }
|
||||
|
||||
public int BatchSize { get; }
|
||||
|
||||
public TimeSpan LeaseDuration { get; }
|
||||
}
|
||||
|
||||
public sealed class NotifyQueueClaimOptions
|
||||
{
|
||||
public NotifyQueueClaimOptions(string claimantConsumer, int batchSize, TimeSpan minIdleTime)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(claimantConsumer))
|
||||
{
|
||||
throw new ArgumentException("Consumer must be provided.", nameof(claimantConsumer));
|
||||
}
|
||||
|
||||
if (batchSize <= 0)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(batchSize), batchSize, "Batch size must be positive.");
|
||||
}
|
||||
|
||||
if (minIdleTime < TimeSpan.Zero)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(minIdleTime), minIdleTime, "Minimum idle time cannot be negative.");
|
||||
}
|
||||
|
||||
ClaimantConsumer = claimantConsumer;
|
||||
BatchSize = batchSize;
|
||||
MinIdleTime = minIdleTime;
|
||||
}
|
||||
|
||||
public string ClaimantConsumer { get; }
|
||||
|
||||
public int BatchSize { get; }
|
||||
|
||||
public TimeSpan MinIdleTime { get; }
|
||||
}
|
||||
|
||||
public enum NotifyQueueReleaseDisposition
|
||||
{
|
||||
Retry,
|
||||
Abandon
|
||||
}
|
||||
|
||||
public interface INotifyQueue<TMessage>
|
||||
{
|
||||
ValueTask<NotifyQueueEnqueueResult> PublishAsync(TMessage message, CancellationToken cancellationToken = default);
|
||||
|
||||
ValueTask<IReadOnlyList<INotifyQueueLease<TMessage>>> LeaseAsync(NotifyQueueLeaseRequest request, CancellationToken cancellationToken = default);
|
||||
|
||||
ValueTask<IReadOnlyList<INotifyQueueLease<TMessage>>> ClaimExpiredAsync(NotifyQueueClaimOptions options, CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
public interface INotifyQueueLease<out TMessage>
|
||||
{
|
||||
string MessageId { get; }
|
||||
|
||||
int Attempt { get; }
|
||||
|
||||
DateTimeOffset EnqueuedAt { get; }
|
||||
|
||||
DateTimeOffset LeaseExpiresAt { get; }
|
||||
|
||||
string Consumer { get; }
|
||||
|
||||
string Stream { get; }
|
||||
|
||||
string TenantId { get; }
|
||||
|
||||
string? PartitionKey { get; }
|
||||
|
||||
string IdempotencyKey { get; }
|
||||
|
||||
string? TraceId { get; }
|
||||
|
||||
IReadOnlyDictionary<string, string> Attributes { get; }
|
||||
|
||||
TMessage Message { get; }
|
||||
|
||||
Task AcknowledgeAsync(CancellationToken cancellationToken = default);
|
||||
|
||||
Task RenewAsync(TimeSpan leaseDuration, CancellationToken cancellationToken = default);
|
||||
|
||||
Task ReleaseAsync(NotifyQueueReleaseDisposition disposition, CancellationToken cancellationToken = default);
|
||||
|
||||
Task DeadLetterAsync(string reason, CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
public interface INotifyEventQueue : INotifyQueue<NotifyQueueEventMessage>
|
||||
{
|
||||
}
|
||||
|
||||
public interface INotifyDeliveryQueue : INotifyQueue<NotifyDeliveryQueueMessage>
|
||||
{
|
||||
}
|
||||
|
||||
internal static class EmptyReadOnlyDictionary<TKey, TValue>
|
||||
where TKey : notnull
|
||||
{
|
||||
public static readonly IReadOnlyDictionary<TKey, TValue> Instance =
|
||||
new ReadOnlyDictionary<TKey, TValue>(new Dictionary<TKey, TValue>(0, EqualityComparer<TKey>.Default));
|
||||
}
|
||||
18
src/StellaOps.Notify.Queue/NotifyQueueFields.cs
Normal file
18
src/StellaOps.Notify.Queue/NotifyQueueFields.cs
Normal file
@@ -0,0 +1,18 @@
|
||||
namespace StellaOps.Notify.Queue;
|
||||
|
||||
internal static class NotifyQueueFields
|
||||
{
|
||||
public const string Payload = "payload";
|
||||
public const string EventId = "eventId";
|
||||
public const string DeliveryId = "deliveryId";
|
||||
public const string Tenant = "tenant";
|
||||
public const string Kind = "kind";
|
||||
public const string Attempt = "attempt";
|
||||
public const string EnqueuedAt = "enqueuedAt";
|
||||
public const string TraceId = "traceId";
|
||||
public const string PartitionKey = "partitionKey";
|
||||
public const string ChannelId = "channelId";
|
||||
public const string ChannelType = "channelType";
|
||||
public const string IdempotencyKey = "idempotency";
|
||||
public const string AttributePrefix = "attr:";
|
||||
}
|
||||
55
src/StellaOps.Notify.Queue/NotifyQueueHealthCheck.cs
Normal file
55
src/StellaOps.Notify.Queue/NotifyQueueHealthCheck.cs
Normal file
@@ -0,0 +1,55 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Notify.Queue.Nats;
|
||||
using StellaOps.Notify.Queue.Redis;
|
||||
|
||||
namespace StellaOps.Notify.Queue;
|
||||
|
||||
public sealed class NotifyQueueHealthCheck : IHealthCheck
|
||||
{
|
||||
private readonly INotifyEventQueue _queue;
|
||||
private readonly ILogger<NotifyQueueHealthCheck> _logger;
|
||||
|
||||
public NotifyQueueHealthCheck(
|
||||
INotifyEventQueue queue,
|
||||
ILogger<NotifyQueueHealthCheck> logger)
|
||||
{
|
||||
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<HealthCheckResult> CheckHealthAsync(
|
||||
HealthCheckContext context,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
try
|
||||
{
|
||||
switch (_queue)
|
||||
{
|
||||
case RedisNotifyEventQueue redisQueue:
|
||||
await redisQueue.PingAsync(cancellationToken).ConfigureAwait(false);
|
||||
return HealthCheckResult.Healthy("Redis Notify queue reachable.");
|
||||
|
||||
case NatsNotifyEventQueue natsQueue:
|
||||
await natsQueue.PingAsync(cancellationToken).ConfigureAwait(false);
|
||||
return HealthCheckResult.Healthy("NATS Notify queue reachable.");
|
||||
|
||||
default:
|
||||
return HealthCheckResult.Healthy("Notify queue transport without dedicated ping returned healthy.");
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Notify queue health check failed.");
|
||||
return new HealthCheckResult(
|
||||
context.Registration.FailureStatus,
|
||||
"Notify queue transport unreachable.",
|
||||
ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
39
src/StellaOps.Notify.Queue/NotifyQueueMetrics.cs
Normal file
39
src/StellaOps.Notify.Queue/NotifyQueueMetrics.cs
Normal file
@@ -0,0 +1,39 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Notify.Queue;
|
||||
|
||||
internal static class NotifyQueueMetrics
|
||||
{
|
||||
private const string TransportTag = "transport";
|
||||
private const string StreamTag = "stream";
|
||||
|
||||
private static readonly Meter Meter = new("StellaOps.Notify.Queue");
|
||||
private static readonly Counter<long> EnqueuedCounter = Meter.CreateCounter<long>("notify_queue_enqueued_total");
|
||||
private static readonly Counter<long> DeduplicatedCounter = Meter.CreateCounter<long>("notify_queue_deduplicated_total");
|
||||
private static readonly Counter<long> AckCounter = Meter.CreateCounter<long>("notify_queue_ack_total");
|
||||
private static readonly Counter<long> RetryCounter = Meter.CreateCounter<long>("notify_queue_retry_total");
|
||||
private static readonly Counter<long> DeadLetterCounter = Meter.CreateCounter<long>("notify_queue_deadletter_total");
|
||||
|
||||
public static void RecordEnqueued(string transport, string stream)
|
||||
=> EnqueuedCounter.Add(1, BuildTags(transport, stream));
|
||||
|
||||
public static void RecordDeduplicated(string transport, string stream)
|
||||
=> DeduplicatedCounter.Add(1, BuildTags(transport, stream));
|
||||
|
||||
public static void RecordAck(string transport, string stream)
|
||||
=> AckCounter.Add(1, BuildTags(transport, stream));
|
||||
|
||||
public static void RecordRetry(string transport, string stream)
|
||||
=> RetryCounter.Add(1, BuildTags(transport, stream));
|
||||
|
||||
public static void RecordDeadLetter(string transport, string stream)
|
||||
=> DeadLetterCounter.Add(1, BuildTags(transport, stream));
|
||||
|
||||
private static KeyValuePair<string, object?>[] BuildTags(string transport, string stream)
|
||||
=> new[]
|
||||
{
|
||||
new KeyValuePair<string, object?>(TransportTag, transport),
|
||||
new KeyValuePair<string, object?>(StreamTag, stream)
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Notify.Queue.Nats;
|
||||
using StellaOps.Notify.Queue.Redis;
|
||||
|
||||
namespace StellaOps.Notify.Queue;
|
||||
|
||||
public static class NotifyQueueServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddNotifyEventQueue(
|
||||
this IServiceCollection services,
|
||||
IConfiguration configuration,
|
||||
string sectionName = "notify:queue")
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
var eventOptions = new NotifyEventQueueOptions();
|
||||
configuration.GetSection(sectionName).Bind(eventOptions);
|
||||
|
||||
services.TryAddSingleton(TimeProvider.System);
|
||||
services.AddSingleton(eventOptions);
|
||||
|
||||
services.AddSingleton<INotifyEventQueue>(sp =>
|
||||
{
|
||||
var loggerFactory = sp.GetRequiredService<ILoggerFactory>();
|
||||
var timeProvider = sp.GetService<TimeProvider>() ?? TimeProvider.System;
|
||||
var opts = sp.GetRequiredService<NotifyEventQueueOptions>();
|
||||
|
||||
return opts.Transport switch
|
||||
{
|
||||
NotifyQueueTransportKind.Redis => new RedisNotifyEventQueue(
|
||||
opts,
|
||||
opts.Redis,
|
||||
loggerFactory.CreateLogger<RedisNotifyEventQueue>(),
|
||||
timeProvider),
|
||||
NotifyQueueTransportKind.Nats => new NatsNotifyEventQueue(
|
||||
opts,
|
||||
opts.Nats,
|
||||
loggerFactory.CreateLogger<NatsNotifyEventQueue>(),
|
||||
timeProvider),
|
||||
_ => throw new InvalidOperationException($"Unsupported Notify queue transport kind '{opts.Transport}'.")
|
||||
};
|
||||
});
|
||||
|
||||
services.AddSingleton<NotifyQueueHealthCheck>();
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
public static IServiceCollection AddNotifyDeliveryQueue(
|
||||
this IServiceCollection services,
|
||||
IConfiguration configuration,
|
||||
string sectionName = "notify:deliveryQueue")
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
var deliveryOptions = new NotifyDeliveryQueueOptions();
|
||||
configuration.GetSection(sectionName).Bind(deliveryOptions);
|
||||
|
||||
services.AddSingleton(deliveryOptions);
|
||||
|
||||
services.AddSingleton<INotifyDeliveryQueue>(sp =>
|
||||
{
|
||||
var loggerFactory = sp.GetRequiredService<ILoggerFactory>();
|
||||
var timeProvider = sp.GetService<TimeProvider>() ?? TimeProvider.System;
|
||||
var opts = sp.GetRequiredService<NotifyDeliveryQueueOptions>();
|
||||
var eventOpts = sp.GetService<NotifyEventQueueOptions>();
|
||||
|
||||
ApplyDeliveryFallbacks(opts, eventOpts);
|
||||
|
||||
return opts.Transport switch
|
||||
{
|
||||
NotifyQueueTransportKind.Redis => new RedisNotifyDeliveryQueue(
|
||||
opts,
|
||||
opts.Redis,
|
||||
loggerFactory.CreateLogger<RedisNotifyDeliveryQueue>(),
|
||||
timeProvider),
|
||||
NotifyQueueTransportKind.Nats => new NatsNotifyDeliveryQueue(
|
||||
opts,
|
||||
opts.Nats,
|
||||
loggerFactory.CreateLogger<NatsNotifyDeliveryQueue>(),
|
||||
timeProvider),
|
||||
_ => throw new InvalidOperationException($"Unsupported Notify delivery queue transport kind '{opts.Transport}'.")
|
||||
};
|
||||
});
|
||||
|
||||
services.AddSingleton<NotifyDeliveryQueueHealthCheck>();
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
public static IHealthChecksBuilder AddNotifyQueueHealthCheck(
|
||||
this IHealthChecksBuilder builder)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(builder);
|
||||
|
||||
builder.Services.TryAddSingleton<NotifyQueueHealthCheck>();
|
||||
builder.AddCheck<NotifyQueueHealthCheck>(
|
||||
name: "notify-queue",
|
||||
failureStatus: HealthStatus.Unhealthy,
|
||||
tags: new[] { "notify", "queue" });
|
||||
|
||||
return builder;
|
||||
}
|
||||
|
||||
public static IHealthChecksBuilder AddNotifyDeliveryQueueHealthCheck(
|
||||
this IHealthChecksBuilder builder)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(builder);
|
||||
|
||||
builder.Services.TryAddSingleton<NotifyDeliveryQueueHealthCheck>();
|
||||
builder.AddCheck<NotifyDeliveryQueueHealthCheck>(
|
||||
name: "notify-delivery-queue",
|
||||
failureStatus: HealthStatus.Unhealthy,
|
||||
tags: new[] { "notify", "queue", "delivery" });
|
||||
|
||||
return builder;
|
||||
}
|
||||
|
||||
private static void ApplyDeliveryFallbacks(
|
||||
NotifyDeliveryQueueOptions deliveryOptions,
|
||||
NotifyEventQueueOptions? eventOptions)
|
||||
{
|
||||
if (eventOptions is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(deliveryOptions.Redis.ConnectionString))
|
||||
{
|
||||
deliveryOptions.Redis.ConnectionString = eventOptions.Redis.ConnectionString;
|
||||
deliveryOptions.Redis.Database ??= eventOptions.Redis.Database;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(deliveryOptions.Nats.Url))
|
||||
{
|
||||
deliveryOptions.Nats.Url = eventOptions.Nats.Url;
|
||||
}
|
||||
}
|
||||
}
|
||||
10
src/StellaOps.Notify.Queue/NotifyQueueTransportKind.cs
Normal file
10
src/StellaOps.Notify.Queue/NotifyQueueTransportKind.cs
Normal file
@@ -0,0 +1,10 @@
|
||||
namespace StellaOps.Notify.Queue;
|
||||
|
||||
/// <summary>
|
||||
/// Supported transports for the Notify event queue.
|
||||
/// </summary>
|
||||
public enum NotifyQueueTransportKind
|
||||
{
|
||||
Redis,
|
||||
Nats
|
||||
}
|
||||
3
src/StellaOps.Notify.Queue/Properties/AssemblyInfo.cs
Normal file
3
src/StellaOps.Notify.Queue/Properties/AssemblyInfo.cs
Normal file
@@ -0,0 +1,3 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("StellaOps.Notify.Queue.Tests")]
|
||||
76
src/StellaOps.Notify.Queue/Redis/RedisNotifyDeliveryLease.cs
Normal file
76
src/StellaOps.Notify.Queue/Redis/RedisNotifyDeliveryLease.cs
Normal file
@@ -0,0 +1,76 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StellaOps.Notify.Queue.Redis;
|
||||
|
||||
internal sealed class RedisNotifyDeliveryLease : INotifyQueueLease<NotifyDeliveryQueueMessage>
|
||||
{
|
||||
private readonly RedisNotifyDeliveryQueue _queue;
|
||||
private int _completed;
|
||||
|
||||
internal RedisNotifyDeliveryLease(
|
||||
RedisNotifyDeliveryQueue queue,
|
||||
string messageId,
|
||||
NotifyDeliveryQueueMessage message,
|
||||
int attempt,
|
||||
DateTimeOffset enqueuedAt,
|
||||
DateTimeOffset leaseExpiresAt,
|
||||
string consumer,
|
||||
string? idempotencyKey,
|
||||
string partitionKey)
|
||||
{
|
||||
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
|
||||
MessageId = messageId ?? throw new ArgumentNullException(nameof(messageId));
|
||||
Message = message ?? throw new ArgumentNullException(nameof(message));
|
||||
Attempt = attempt;
|
||||
EnqueuedAt = enqueuedAt;
|
||||
LeaseExpiresAt = leaseExpiresAt;
|
||||
Consumer = consumer ?? throw new ArgumentNullException(nameof(consumer));
|
||||
IdempotencyKey = idempotencyKey ?? message.IdempotencyKey;
|
||||
PartitionKey = partitionKey ?? message.ChannelId;
|
||||
}
|
||||
|
||||
public string MessageId { get; }
|
||||
|
||||
public int Attempt { get; internal set; }
|
||||
|
||||
public DateTimeOffset EnqueuedAt { get; }
|
||||
|
||||
public DateTimeOffset LeaseExpiresAt { get; private set; }
|
||||
|
||||
public string Consumer { get; }
|
||||
|
||||
public string Stream => Message.Stream;
|
||||
|
||||
public string TenantId => Message.TenantId;
|
||||
|
||||
public string PartitionKey { get; }
|
||||
|
||||
public string IdempotencyKey { get; }
|
||||
|
||||
public string? TraceId => Message.TraceId;
|
||||
|
||||
public IReadOnlyDictionary<string, string> Attributes => Message.Attributes;
|
||||
|
||||
public NotifyDeliveryQueueMessage Message { get; }
|
||||
|
||||
public Task AcknowledgeAsync(CancellationToken cancellationToken = default)
|
||||
=> _queue.AcknowledgeAsync(this, cancellationToken);
|
||||
|
||||
public Task RenewAsync(TimeSpan leaseDuration, CancellationToken cancellationToken = default)
|
||||
=> _queue.RenewLeaseAsync(this, leaseDuration, cancellationToken);
|
||||
|
||||
public Task ReleaseAsync(NotifyQueueReleaseDisposition disposition, CancellationToken cancellationToken = default)
|
||||
=> _queue.ReleaseAsync(this, disposition, cancellationToken);
|
||||
|
||||
public Task DeadLetterAsync(string reason, CancellationToken cancellationToken = default)
|
||||
=> _queue.DeadLetterAsync(this, reason, cancellationToken);
|
||||
|
||||
internal bool TryBeginCompletion()
|
||||
=> Interlocked.CompareExchange(ref _completed, 1, 0) == 0;
|
||||
|
||||
internal void RefreshLease(DateTimeOffset expiresAt)
|
||||
=> LeaseExpiresAt = expiresAt;
|
||||
}
|
||||
788
src/StellaOps.Notify.Queue/Redis/RedisNotifyDeliveryQueue.cs
Normal file
788
src/StellaOps.Notify.Queue/Redis/RedisNotifyDeliveryQueue.cs
Normal file
@@ -0,0 +1,788 @@
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.ObjectModel;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StackExchange.Redis;
|
||||
using StellaOps.Notify.Models;
|
||||
|
||||
namespace StellaOps.Notify.Queue.Redis;
|
||||
|
||||
internal sealed class RedisNotifyDeliveryQueue : INotifyDeliveryQueue, IAsyncDisposable
|
||||
{
|
||||
private const string TransportName = "redis";
|
||||
|
||||
private readonly NotifyDeliveryQueueOptions _options;
|
||||
private readonly NotifyRedisDeliveryQueueOptions _redisOptions;
|
||||
private readonly ILogger<RedisNotifyDeliveryQueue> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly Func<ConfigurationOptions, Task<IConnectionMultiplexer>> _connectionFactory;
|
||||
private readonly SemaphoreSlim _connectionLock = new(1, 1);
|
||||
private readonly SemaphoreSlim _groupLock = new(1, 1);
|
||||
private readonly ConcurrentDictionary<string, bool> _streamInitialized = new(StringComparer.Ordinal);
|
||||
|
||||
private IConnectionMultiplexer? _connection;
|
||||
private bool _disposed;
|
||||
|
||||
public RedisNotifyDeliveryQueue(
|
||||
NotifyDeliveryQueueOptions options,
|
||||
NotifyRedisDeliveryQueueOptions redisOptions,
|
||||
ILogger<RedisNotifyDeliveryQueue> logger,
|
||||
TimeProvider timeProvider,
|
||||
Func<ConfigurationOptions, Task<IConnectionMultiplexer>>? connectionFactory = null)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_redisOptions = redisOptions ?? throw new ArgumentNullException(nameof(redisOptions));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_connectionFactory = connectionFactory ?? (async config =>
|
||||
{
|
||||
var connection = await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false);
|
||||
return (IConnectionMultiplexer)connection;
|
||||
});
|
||||
|
||||
if (string.IsNullOrWhiteSpace(_redisOptions.ConnectionString))
|
||||
{
|
||||
throw new InvalidOperationException("Redis connection string must be configured for the Notify delivery queue.");
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<NotifyQueueEnqueueResult> PublishAsync(
|
||||
NotifyDeliveryQueueMessage message,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(message);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
||||
await EnsureConsumerGroupAsync(db, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var attempt = 1;
|
||||
var entries = BuildEntries(message, now, attempt);
|
||||
|
||||
var messageId = await AddToStreamAsync(
|
||||
db,
|
||||
_redisOptions.StreamName,
|
||||
entries)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var idempotencyKey = BuildIdempotencyKey(message.IdempotencyKey);
|
||||
var stored = await db.StringSetAsync(
|
||||
idempotencyKey,
|
||||
messageId,
|
||||
when: When.NotExists,
|
||||
expiry: _options.ClaimIdleThreshold)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (!stored)
|
||||
{
|
||||
await db.StreamDeleteAsync(
|
||||
_redisOptions.StreamName,
|
||||
new RedisValue[] { messageId })
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var existing = await db.StringGetAsync(idempotencyKey).ConfigureAwait(false);
|
||||
var duplicateId = existing.IsNullOrEmpty ? messageId : existing;
|
||||
|
||||
NotifyQueueMetrics.RecordDeduplicated(TransportName, _redisOptions.StreamName);
|
||||
_logger.LogDebug(
|
||||
"Duplicate Notify delivery enqueue detected for delivery {DeliveryId}.",
|
||||
message.Delivery.DeliveryId);
|
||||
|
||||
return new NotifyQueueEnqueueResult(duplicateId.ToString()!, true);
|
||||
}
|
||||
|
||||
NotifyQueueMetrics.RecordEnqueued(TransportName, _redisOptions.StreamName);
|
||||
_logger.LogDebug(
|
||||
"Enqueued Notify delivery {DeliveryId} (channel {ChannelId}) into stream {Stream}.",
|
||||
message.Delivery.DeliveryId,
|
||||
message.ChannelId,
|
||||
_redisOptions.StreamName);
|
||||
|
||||
return new NotifyQueueEnqueueResult(messageId.ToString()!, false);
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<INotifyQueueLease<NotifyDeliveryQueueMessage>>> LeaseAsync(
|
||||
NotifyQueueLeaseRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
||||
await EnsureConsumerGroupAsync(db, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var entries = await db.StreamReadGroupAsync(
|
||||
_redisOptions.StreamName,
|
||||
_redisOptions.ConsumerGroup,
|
||||
request.Consumer,
|
||||
StreamPosition.NewMessages,
|
||||
request.BatchSize)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (entries is null || entries.Length == 0)
|
||||
{
|
||||
return Array.Empty<INotifyQueueLease<NotifyDeliveryQueueMessage>>();
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var leases = new List<INotifyQueueLease<NotifyDeliveryQueueMessage>>(entries.Length);
|
||||
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
var lease = TryMapLease(entry, request.Consumer, now, request.LeaseDuration, attemptOverride: null);
|
||||
if (lease is null)
|
||||
{
|
||||
await AckPoisonAsync(db, entry.Id).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
leases.Add(lease);
|
||||
}
|
||||
|
||||
return leases;
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<INotifyQueueLease<NotifyDeliveryQueueMessage>>> ClaimExpiredAsync(
|
||||
NotifyQueueClaimOptions options,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
||||
await EnsureConsumerGroupAsync(db, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var pending = await db.StreamPendingMessagesAsync(
|
||||
_redisOptions.StreamName,
|
||||
_redisOptions.ConsumerGroup,
|
||||
options.BatchSize,
|
||||
RedisValue.Null,
|
||||
(long)options.MinIdleTime.TotalMilliseconds)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (pending is null || pending.Length == 0)
|
||||
{
|
||||
return Array.Empty<INotifyQueueLease<NotifyDeliveryQueueMessage>>();
|
||||
}
|
||||
|
||||
var eligible = pending
|
||||
.Where(p => p.IdleTimeInMilliseconds >= options.MinIdleTime.TotalMilliseconds)
|
||||
.ToArray();
|
||||
|
||||
if (eligible.Length == 0)
|
||||
{
|
||||
return Array.Empty<INotifyQueueLease<NotifyDeliveryQueueMessage>>();
|
||||
}
|
||||
|
||||
var messageIds = eligible
|
||||
.Select(static p => (RedisValue)p.MessageId)
|
||||
.ToArray();
|
||||
|
||||
var entries = await db.StreamClaimAsync(
|
||||
_redisOptions.StreamName,
|
||||
_redisOptions.ConsumerGroup,
|
||||
options.ClaimantConsumer,
|
||||
0,
|
||||
messageIds)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (entries is null || entries.Length == 0)
|
||||
{
|
||||
return Array.Empty<INotifyQueueLease<NotifyDeliveryQueueMessage>>();
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var attemptLookup = eligible
|
||||
.Where(static info => !info.MessageId.IsNullOrEmpty)
|
||||
.ToDictionary(
|
||||
info => info.MessageId!.ToString(),
|
||||
info => (int)Math.Max(1, info.DeliveryCount),
|
||||
StringComparer.Ordinal);
|
||||
|
||||
var leases = new List<INotifyQueueLease<NotifyDeliveryQueueMessage>>(entries.Length);
|
||||
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
attemptLookup.TryGetValue(entry.Id.ToString(), out var attempt);
|
||||
var lease = TryMapLease(entry, options.ClaimantConsumer, now, _options.DefaultLeaseDuration, attempt == 0 ? null : attempt);
|
||||
if (lease is null)
|
||||
{
|
||||
await AckPoisonAsync(db, entry.Id).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
leases.Add(lease);
|
||||
}
|
||||
|
||||
return leases;
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_disposed = true;
|
||||
if (_connection is not null)
|
||||
{
|
||||
await _connection.CloseAsync().ConfigureAwait(false);
|
||||
_connection.Dispose();
|
||||
}
|
||||
|
||||
_connectionLock.Dispose();
|
||||
_groupLock.Dispose();
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
internal async Task AcknowledgeAsync(
|
||||
RedisNotifyDeliveryLease lease,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!lease.TryBeginCompletion())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await db.StreamAcknowledgeAsync(
|
||||
_redisOptions.StreamName,
|
||||
_redisOptions.ConsumerGroup,
|
||||
new RedisValue[] { lease.MessageId })
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await db.StreamDeleteAsync(
|
||||
_redisOptions.StreamName,
|
||||
new RedisValue[] { lease.MessageId })
|
||||
.ConfigureAwait(false);
|
||||
|
||||
NotifyQueueMetrics.RecordAck(TransportName, _redisOptions.StreamName);
|
||||
_logger.LogDebug(
|
||||
"Acknowledged Notify delivery {DeliveryId} (message {MessageId}).",
|
||||
lease.Message.Delivery.DeliveryId,
|
||||
lease.MessageId);
|
||||
}
|
||||
|
||||
internal async Task RenewLeaseAsync(
|
||||
RedisNotifyDeliveryLease lease,
|
||||
TimeSpan leaseDuration,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await db.StreamClaimAsync(
|
||||
_redisOptions.StreamName,
|
||||
_redisOptions.ConsumerGroup,
|
||||
lease.Consumer,
|
||||
0,
|
||||
new RedisValue[] { lease.MessageId })
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var expires = _timeProvider.GetUtcNow().Add(leaseDuration);
|
||||
lease.RefreshLease(expires);
|
||||
|
||||
_logger.LogDebug(
|
||||
"Renewed Notify delivery lease {DeliveryId} until {Expires:u}.",
|
||||
lease.Message.Delivery.DeliveryId,
|
||||
expires);
|
||||
}
|
||||
|
||||
internal async Task ReleaseAsync(
|
||||
RedisNotifyDeliveryLease lease,
|
||||
NotifyQueueReleaseDisposition disposition,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (disposition == NotifyQueueReleaseDisposition.Retry
|
||||
&& lease.Attempt >= _options.MaxDeliveryAttempts)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Notify delivery {DeliveryId} reached max delivery attempts ({Attempts}); moving to dead-letter stream.",
|
||||
lease.Message.Delivery.DeliveryId,
|
||||
lease.Attempt);
|
||||
|
||||
await DeadLetterAsync(
|
||||
lease,
|
||||
$"max-delivery-attempts:{lease.Attempt}",
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (!lease.TryBeginCompletion())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
||||
await db.StreamAcknowledgeAsync(
|
||||
_redisOptions.StreamName,
|
||||
_redisOptions.ConsumerGroup,
|
||||
new RedisValue[] { lease.MessageId })
|
||||
.ConfigureAwait(false);
|
||||
await db.StreamDeleteAsync(
|
||||
_redisOptions.StreamName,
|
||||
new RedisValue[] { lease.MessageId })
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (disposition == NotifyQueueReleaseDisposition.Retry)
|
||||
{
|
||||
NotifyQueueMetrics.RecordRetry(TransportName, _redisOptions.StreamName);
|
||||
|
||||
var delay = CalculateBackoff(lease.Attempt);
|
||||
if (delay > TimeSpan.Zero)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (TaskCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var entries = BuildEntries(lease.Message, now, lease.Attempt + 1);
|
||||
|
||||
await AddToStreamAsync(
|
||||
db,
|
||||
_redisOptions.StreamName,
|
||||
entries)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
NotifyQueueMetrics.RecordEnqueued(TransportName, _redisOptions.StreamName);
|
||||
_logger.LogInformation(
|
||||
"Retrying Notify delivery {DeliveryId} (attempt {Attempt}).",
|
||||
lease.Message.Delivery.DeliveryId,
|
||||
lease.Attempt + 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
NotifyQueueMetrics.RecordAck(TransportName, _redisOptions.StreamName);
|
||||
_logger.LogInformation(
|
||||
"Abandoned Notify delivery {DeliveryId} after {Attempt} attempt(s).",
|
||||
lease.Message.Delivery.DeliveryId,
|
||||
lease.Attempt);
|
||||
}
|
||||
}
|
||||
|
||||
internal async Task DeadLetterAsync(
|
||||
RedisNotifyDeliveryLease lease,
|
||||
string reason,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!lease.TryBeginCompletion())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await db.StreamAcknowledgeAsync(
|
||||
_redisOptions.StreamName,
|
||||
_redisOptions.ConsumerGroup,
|
||||
new RedisValue[] { lease.MessageId })
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await db.StreamDeleteAsync(
|
||||
_redisOptions.StreamName,
|
||||
new RedisValue[] { lease.MessageId })
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await EnsureDeadLetterStreamAsync(db, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var entries = BuildDeadLetterEntries(lease, reason);
|
||||
await AddToStreamAsync(
|
||||
db,
|
||||
_redisOptions.DeadLetterStreamName,
|
||||
entries)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
NotifyQueueMetrics.RecordDeadLetter(TransportName, _redisOptions.DeadLetterStreamName);
|
||||
_logger.LogError(
|
||||
"Dead-lettered Notify delivery {DeliveryId} (attempt {Attempt}): {Reason}",
|
||||
lease.Message.Delivery.DeliveryId,
|
||||
lease.Attempt,
|
||||
reason);
|
||||
}
|
||||
|
||||
internal async ValueTask PingAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
||||
_ = await db.PingAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<IDatabase> GetDatabaseAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_connection is { IsConnected: true })
|
||||
{
|
||||
return _connection.GetDatabase(_redisOptions.Database ?? -1);
|
||||
}
|
||||
|
||||
await _connectionLock.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_connection is { IsConnected: true })
|
||||
{
|
||||
return _connection.GetDatabase(_redisOptions.Database ?? -1);
|
||||
}
|
||||
|
||||
var configuration = ConfigurationOptions.Parse(_redisOptions.ConnectionString!);
|
||||
configuration.AbortOnConnectFail = false;
|
||||
if (_redisOptions.Database.HasValue)
|
||||
{
|
||||
configuration.DefaultDatabase = _redisOptions.Database.Value;
|
||||
}
|
||||
|
||||
using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
timeoutCts.CancelAfter(_redisOptions.InitializationTimeout);
|
||||
|
||||
_connection = await _connectionFactory(configuration).WaitAsync(timeoutCts.Token).ConfigureAwait(false);
|
||||
return _connection.GetDatabase(_redisOptions.Database ?? -1);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_connectionLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task EnsureConsumerGroupAsync(
|
||||
IDatabase database,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (_streamInitialized.ContainsKey(_redisOptions.StreamName))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _groupLock.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_streamInitialized.ContainsKey(_redisOptions.StreamName))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await database.StreamCreateConsumerGroupAsync(
|
||||
_redisOptions.StreamName,
|
||||
_redisOptions.ConsumerGroup,
|
||||
StreamPosition.Beginning,
|
||||
createStream: true)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (RedisServerException ex) when (ex.Message.Contains("BUSYGROUP", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
// group already exists
|
||||
}
|
||||
|
||||
_streamInitialized[_redisOptions.StreamName] = true;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_groupLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task EnsureDeadLetterStreamAsync(
|
||||
IDatabase database,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (_streamInitialized.ContainsKey(_redisOptions.DeadLetterStreamName))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _groupLock.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_streamInitialized.ContainsKey(_redisOptions.DeadLetterStreamName))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await database.StreamCreateConsumerGroupAsync(
|
||||
_redisOptions.DeadLetterStreamName,
|
||||
_redisOptions.ConsumerGroup,
|
||||
StreamPosition.Beginning,
|
||||
createStream: true)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (RedisServerException ex) when (ex.Message.Contains("BUSYGROUP", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
// ignore
|
||||
}
|
||||
|
||||
_streamInitialized[_redisOptions.DeadLetterStreamName] = true;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_groupLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private NameValueEntry[] BuildEntries(
|
||||
NotifyDeliveryQueueMessage message,
|
||||
DateTimeOffset enqueuedAt,
|
||||
int attempt)
|
||||
{
|
||||
var json = NotifyCanonicalJsonSerializer.Serialize(message.Delivery);
|
||||
var attributeCount = message.Attributes.Count;
|
||||
|
||||
var entries = ArrayPool<NameValueEntry>.Shared.Rent(8 + attributeCount);
|
||||
var index = 0;
|
||||
|
||||
entries[index++] = new NameValueEntry(NotifyQueueFields.Payload, json);
|
||||
entries[index++] = new NameValueEntry(NotifyQueueFields.DeliveryId, message.Delivery.DeliveryId);
|
||||
entries[index++] = new NameValueEntry(NotifyQueueFields.ChannelId, message.ChannelId);
|
||||
entries[index++] = new NameValueEntry(NotifyQueueFields.ChannelType, message.ChannelType.ToString());
|
||||
entries[index++] = new NameValueEntry(NotifyQueueFields.Tenant, message.Delivery.TenantId);
|
||||
entries[index++] = new NameValueEntry(NotifyQueueFields.Attempt, attempt);
|
||||
entries[index++] = new NameValueEntry(NotifyQueueFields.EnqueuedAt, enqueuedAt.ToUnixTimeMilliseconds());
|
||||
entries[index++] = new NameValueEntry(NotifyQueueFields.IdempotencyKey, message.IdempotencyKey);
|
||||
entries[index++] = new NameValueEntry(NotifyQueueFields.TraceId, message.TraceId ?? string.Empty);
|
||||
entries[index++] = new NameValueEntry(NotifyQueueFields.PartitionKey, message.PartitionKey);
|
||||
|
||||
if (attributeCount > 0)
|
||||
{
|
||||
foreach (var kvp in message.Attributes)
|
||||
{
|
||||
entries[index++] = new NameValueEntry(
|
||||
NotifyQueueFields.AttributePrefix + kvp.Key,
|
||||
kvp.Value);
|
||||
}
|
||||
}
|
||||
|
||||
return entries.AsSpan(0, index).ToArray();
|
||||
}
|
||||
|
||||
private NameValueEntry[] BuildDeadLetterEntries(RedisNotifyDeliveryLease lease, string reason)
|
||||
{
|
||||
var json = NotifyCanonicalJsonSerializer.Serialize(lease.Message.Delivery);
|
||||
var attributes = lease.Message.Attributes;
|
||||
var attributeCount = attributes.Count;
|
||||
|
||||
var entries = ArrayPool<NameValueEntry>.Shared.Rent(9 + attributeCount);
|
||||
var index = 0;
|
||||
|
||||
entries[index++] = new NameValueEntry(NotifyQueueFields.Payload, json);
|
||||
entries[index++] = new NameValueEntry(NotifyQueueFields.DeliveryId, lease.Message.Delivery.DeliveryId);
|
||||
entries[index++] = new NameValueEntry(NotifyQueueFields.ChannelId, lease.Message.ChannelId);
|
||||
entries[index++] = new NameValueEntry(NotifyQueueFields.ChannelType, lease.Message.ChannelType.ToString());
|
||||
entries[index++] = new NameValueEntry(NotifyQueueFields.Tenant, lease.Message.Delivery.TenantId);
|
||||
entries[index++] = new NameValueEntry(NotifyQueueFields.Attempt, lease.Attempt);
|
||||
entries[index++] = new NameValueEntry(NotifyQueueFields.IdempotencyKey, lease.Message.IdempotencyKey);
|
||||
entries[index++] = new NameValueEntry("deadletter-reason", reason);
|
||||
entries[index++] = new NameValueEntry(NotifyQueueFields.TraceId, lease.Message.TraceId ?? string.Empty);
|
||||
|
||||
foreach (var kvp in attributes)
|
||||
{
|
||||
entries[index++] = new NameValueEntry(
|
||||
NotifyQueueFields.AttributePrefix + kvp.Key,
|
||||
kvp.Value);
|
||||
}
|
||||
|
||||
return entries.AsSpan(0, index).ToArray();
|
||||
}
|
||||
|
||||
private RedisNotifyDeliveryLease? TryMapLease(
|
||||
StreamEntry entry,
|
||||
string consumer,
|
||||
DateTimeOffset now,
|
||||
TimeSpan leaseDuration,
|
||||
int? attemptOverride)
|
||||
{
|
||||
if (entry.Values is null || entry.Values.Length == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
string? payload = null;
|
||||
string? deliveryId = null;
|
||||
string? channelId = null;
|
||||
string? channelTypeRaw = null;
|
||||
string? traceId = null;
|
||||
string? idempotency = null;
|
||||
string? partitionKey = null;
|
||||
long? enqueuedAtUnix = null;
|
||||
var attempt = attemptOverride ?? 1;
|
||||
var attributes = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var value in entry.Values)
|
||||
{
|
||||
var name = value.Name.ToString();
|
||||
var data = value.Value;
|
||||
if (name.Equals(NotifyQueueFields.Payload, StringComparison.Ordinal))
|
||||
{
|
||||
payload = data.ToString();
|
||||
}
|
||||
else if (name.Equals(NotifyQueueFields.DeliveryId, StringComparison.Ordinal))
|
||||
{
|
||||
deliveryId = data.ToString();
|
||||
}
|
||||
else if (name.Equals(NotifyQueueFields.ChannelId, StringComparison.Ordinal))
|
||||
{
|
||||
channelId = data.ToString();
|
||||
}
|
||||
else if (name.Equals(NotifyQueueFields.ChannelType, StringComparison.Ordinal))
|
||||
{
|
||||
channelTypeRaw = data.ToString();
|
||||
}
|
||||
else if (name.Equals(NotifyQueueFields.Attempt, StringComparison.Ordinal))
|
||||
{
|
||||
if (int.TryParse(data.ToString(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed))
|
||||
{
|
||||
attempt = Math.Max(parsed, attempt);
|
||||
}
|
||||
}
|
||||
else if (name.Equals(NotifyQueueFields.EnqueuedAt, StringComparison.Ordinal))
|
||||
{
|
||||
if (long.TryParse(data.ToString(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var unix))
|
||||
{
|
||||
enqueuedAtUnix = unix;
|
||||
}
|
||||
}
|
||||
else if (name.Equals(NotifyQueueFields.IdempotencyKey, StringComparison.Ordinal))
|
||||
{
|
||||
idempotency = data.ToString();
|
||||
}
|
||||
else if (name.Equals(NotifyQueueFields.TraceId, StringComparison.Ordinal))
|
||||
{
|
||||
var text = data.ToString();
|
||||
traceId = string.IsNullOrWhiteSpace(text) ? null : text;
|
||||
}
|
||||
else if (name.Equals(NotifyQueueFields.PartitionKey, StringComparison.Ordinal))
|
||||
{
|
||||
partitionKey = data.ToString();
|
||||
}
|
||||
else if (name.StartsWith(NotifyQueueFields.AttributePrefix, StringComparison.Ordinal))
|
||||
{
|
||||
attributes[name[NotifyQueueFields.AttributePrefix.Length..]] = data.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
if (payload is null || deliveryId is null || channelId is null || channelTypeRaw is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
NotifyDelivery delivery;
|
||||
try
|
||||
{
|
||||
delivery = NotifyCanonicalJsonSerializer.Deserialize<NotifyDelivery>(payload);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
ex,
|
||||
"Failed to deserialize Notify delivery payload for entry {EntryId}.",
|
||||
entry.Id.ToString());
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!Enum.TryParse<NotifyChannelType>(channelTypeRaw, ignoreCase: true, out var channelType))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Unknown channel type '{ChannelType}' for delivery {DeliveryId}; acknowledging as poison.",
|
||||
channelTypeRaw,
|
||||
deliveryId);
|
||||
return null;
|
||||
}
|
||||
|
||||
var attributeView = attributes.Count == 0
|
||||
? EmptyReadOnlyDictionary<string, string>.Instance
|
||||
: new ReadOnlyDictionary<string, string>(attributes);
|
||||
|
||||
var enqueuedAt = enqueuedAtUnix is null
|
||||
? now
|
||||
: DateTimeOffset.FromUnixTimeMilliseconds(enqueuedAtUnix.Value);
|
||||
|
||||
var message = new NotifyDeliveryQueueMessage(
|
||||
delivery,
|
||||
channelId,
|
||||
channelType,
|
||||
_redisOptions.StreamName,
|
||||
traceId,
|
||||
attributeView);
|
||||
|
||||
var leaseExpires = now.Add(leaseDuration);
|
||||
|
||||
return new RedisNotifyDeliveryLease(
|
||||
this,
|
||||
entry.Id.ToString(),
|
||||
message,
|
||||
attempt,
|
||||
enqueuedAt,
|
||||
leaseExpires,
|
||||
consumer,
|
||||
idempotency,
|
||||
partitionKey ?? channelId);
|
||||
}
|
||||
|
||||
private async Task AckPoisonAsync(IDatabase database, RedisValue messageId)
|
||||
{
|
||||
await database.StreamAcknowledgeAsync(
|
||||
_redisOptions.StreamName,
|
||||
_redisOptions.ConsumerGroup,
|
||||
new RedisValue[] { messageId })
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await database.StreamDeleteAsync(
|
||||
_redisOptions.StreamName,
|
||||
new RedisValue[] { messageId })
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static async Task<RedisValue> AddToStreamAsync(
|
||||
IDatabase database,
|
||||
string stream,
|
||||
IReadOnlyList<NameValueEntry> entries)
|
||||
{
|
||||
return await database.StreamAddAsync(
|
||||
stream,
|
||||
entries.ToArray())
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private string BuildIdempotencyKey(string token)
|
||||
=> string.Concat(_redisOptions.IdempotencyKeyPrefix, token);
|
||||
|
||||
private TimeSpan CalculateBackoff(int attempt)
|
||||
{
|
||||
var initial = _options.RetryInitialBackoff > TimeSpan.Zero
|
||||
? _options.RetryInitialBackoff
|
||||
: TimeSpan.FromSeconds(1);
|
||||
|
||||
if (initial <= TimeSpan.Zero)
|
||||
{
|
||||
return TimeSpan.Zero;
|
||||
}
|
||||
|
||||
if (attempt <= 1)
|
||||
{
|
||||
return initial;
|
||||
}
|
||||
|
||||
var max = _options.RetryMaxBackoff > TimeSpan.Zero
|
||||
? _options.RetryMaxBackoff
|
||||
: initial;
|
||||
|
||||
var exponent = attempt - 1;
|
||||
var scaledTicks = initial.Ticks * Math.Pow(2, exponent - 1);
|
||||
var cappedTicks = Math.Min(max.Ticks, scaledTicks);
|
||||
var resultTicks = Math.Max(initial.Ticks, (long)cappedTicks);
|
||||
return TimeSpan.FromTicks(resultTicks);
|
||||
}
|
||||
}
|
||||
76
src/StellaOps.Notify.Queue/Redis/RedisNotifyEventLease.cs
Normal file
76
src/StellaOps.Notify.Queue/Redis/RedisNotifyEventLease.cs
Normal file
@@ -0,0 +1,76 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StellaOps.Notify.Queue.Redis;
|
||||
|
||||
internal sealed class RedisNotifyEventLease : INotifyQueueLease<NotifyQueueEventMessage>
|
||||
{
|
||||
private readonly RedisNotifyEventQueue _queue;
|
||||
private int _completed;
|
||||
|
||||
internal RedisNotifyEventLease(
|
||||
RedisNotifyEventQueue queue,
|
||||
NotifyRedisEventStreamOptions streamOptions,
|
||||
string messageId,
|
||||
NotifyQueueEventMessage message,
|
||||
int attempt,
|
||||
string consumer,
|
||||
DateTimeOffset enqueuedAt,
|
||||
DateTimeOffset leaseExpiresAt)
|
||||
{
|
||||
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
|
||||
StreamOptions = streamOptions ?? throw new ArgumentNullException(nameof(streamOptions));
|
||||
MessageId = messageId ?? throw new ArgumentNullException(nameof(messageId));
|
||||
Message = message ?? throw new ArgumentNullException(nameof(message));
|
||||
Attempt = attempt;
|
||||
Consumer = consumer ?? throw new ArgumentNullException(nameof(consumer));
|
||||
EnqueuedAt = enqueuedAt;
|
||||
LeaseExpiresAt = leaseExpiresAt;
|
||||
}
|
||||
|
||||
internal NotifyRedisEventStreamOptions StreamOptions { get; }
|
||||
|
||||
public string MessageId { get; }
|
||||
|
||||
public int Attempt { get; }
|
||||
|
||||
public DateTimeOffset EnqueuedAt { get; }
|
||||
|
||||
public DateTimeOffset LeaseExpiresAt { get; private set; }
|
||||
|
||||
public string Consumer { get; }
|
||||
|
||||
public string Stream => StreamOptions.Stream;
|
||||
|
||||
public string TenantId => Message.TenantId;
|
||||
|
||||
public string? PartitionKey => Message.PartitionKey;
|
||||
|
||||
public string IdempotencyKey => Message.IdempotencyKey;
|
||||
|
||||
public string? TraceId => Message.TraceId;
|
||||
|
||||
public IReadOnlyDictionary<string, string> Attributes => Message.Attributes;
|
||||
|
||||
public NotifyQueueEventMessage Message { get; }
|
||||
|
||||
public Task AcknowledgeAsync(CancellationToken cancellationToken = default)
|
||||
=> _queue.AcknowledgeAsync(this, cancellationToken);
|
||||
|
||||
public Task RenewAsync(TimeSpan leaseDuration, CancellationToken cancellationToken = default)
|
||||
=> _queue.RenewLeaseAsync(this, leaseDuration, cancellationToken);
|
||||
|
||||
public Task ReleaseAsync(NotifyQueueReleaseDisposition disposition, CancellationToken cancellationToken = default)
|
||||
=> _queue.ReleaseAsync(this, disposition, cancellationToken);
|
||||
|
||||
public Task DeadLetterAsync(string reason, CancellationToken cancellationToken = default)
|
||||
=> _queue.DeadLetterAsync(this, reason, cancellationToken);
|
||||
|
||||
internal bool TryBeginCompletion()
|
||||
=> Interlocked.CompareExchange(ref _completed, 1, 0) == 0;
|
||||
|
||||
internal void RefreshLease(DateTimeOffset expiresAt)
|
||||
=> LeaseExpiresAt = expiresAt;
|
||||
}
|
||||
655
src/StellaOps.Notify.Queue/Redis/RedisNotifyEventQueue.cs
Normal file
655
src/StellaOps.Notify.Queue/Redis/RedisNotifyEventQueue.cs
Normal file
@@ -0,0 +1,655 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.ObjectModel;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StackExchange.Redis;
|
||||
using StellaOps.Notify.Models;
|
||||
|
||||
namespace StellaOps.Notify.Queue.Redis;
|
||||
|
||||
internal sealed class RedisNotifyEventQueue : INotifyEventQueue, IAsyncDisposable
|
||||
{
|
||||
private const string TransportName = "redis";
|
||||
|
||||
private readonly NotifyEventQueueOptions _options;
|
||||
private readonly NotifyRedisEventQueueOptions _redisOptions;
|
||||
private readonly ILogger<RedisNotifyEventQueue> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly Func<ConfigurationOptions, Task<IConnectionMultiplexer>> _connectionFactory;
|
||||
private readonly SemaphoreSlim _connectionLock = new(1, 1);
|
||||
private readonly SemaphoreSlim _groupInitLock = new(1, 1);
|
||||
private readonly IReadOnlyDictionary<string, NotifyRedisEventStreamOptions> _streamsByName;
|
||||
private readonly ConcurrentDictionary<string, bool> _initializedStreams = new(StringComparer.Ordinal);
|
||||
|
||||
private IConnectionMultiplexer? _connection;
|
||||
private bool _disposed;
|
||||
|
||||
public RedisNotifyEventQueue(
|
||||
NotifyEventQueueOptions options,
|
||||
NotifyRedisEventQueueOptions redisOptions,
|
||||
ILogger<RedisNotifyEventQueue> logger,
|
||||
TimeProvider timeProvider,
|
||||
Func<ConfigurationOptions, Task<IConnectionMultiplexer>>? connectionFactory = null)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_redisOptions = redisOptions ?? throw new ArgumentNullException(nameof(redisOptions));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_connectionFactory = connectionFactory ?? (async config =>
|
||||
{
|
||||
var connection = await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false);
|
||||
return (IConnectionMultiplexer)connection;
|
||||
});
|
||||
|
||||
if (string.IsNullOrWhiteSpace(_redisOptions.ConnectionString))
|
||||
{
|
||||
throw new InvalidOperationException("Redis connection string must be configured for Notify event queue.");
|
||||
}
|
||||
|
||||
_streamsByName = _redisOptions.Streams.ToDictionary(
|
||||
stream => stream.Stream,
|
||||
stream => stream,
|
||||
StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
public async ValueTask<NotifyQueueEnqueueResult> PublishAsync(
|
||||
NotifyQueueEventMessage message,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(message);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var streamOptions = GetStreamOptions(message.Stream);
|
||||
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
||||
await EnsureStreamInitializedAsync(db, streamOptions, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var entries = BuildEntries(message, now, attempt: 1);
|
||||
|
||||
var messageId = await AddToStreamAsync(
|
||||
db,
|
||||
streamOptions,
|
||||
entries)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var idempotencyToken = string.IsNullOrWhiteSpace(message.IdempotencyKey)
|
||||
? message.Event.EventId.ToString("N")
|
||||
: message.IdempotencyKey;
|
||||
|
||||
var idempotencyKey = streamOptions.IdempotencyKeyPrefix + idempotencyToken;
|
||||
var stored = await db.StringSetAsync(
|
||||
idempotencyKey,
|
||||
messageId,
|
||||
when: When.NotExists,
|
||||
expiry: _redisOptions.IdempotencyWindow)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (!stored)
|
||||
{
|
||||
await db.StreamDeleteAsync(
|
||||
streamOptions.Stream,
|
||||
new RedisValue[] { messageId })
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var existing = await db.StringGetAsync(idempotencyKey).ConfigureAwait(false);
|
||||
var duplicateId = existing.IsNullOrEmpty ? messageId : existing;
|
||||
|
||||
_logger.LogDebug(
|
||||
"Duplicate Notify event enqueue detected for idempotency token {Token}; returning existing stream id {StreamId}.",
|
||||
idempotencyToken,
|
||||
duplicateId.ToString());
|
||||
|
||||
NotifyQueueMetrics.RecordDeduplicated(TransportName, streamOptions.Stream);
|
||||
return new NotifyQueueEnqueueResult(duplicateId.ToString()!, true);
|
||||
}
|
||||
|
||||
NotifyQueueMetrics.RecordEnqueued(TransportName, streamOptions.Stream);
|
||||
|
||||
_logger.LogDebug(
|
||||
"Enqueued Notify event {EventId} for tenant {Tenant} on stream {Stream} (id {StreamId}).",
|
||||
message.Event.EventId,
|
||||
message.TenantId,
|
||||
streamOptions.Stream,
|
||||
messageId.ToString());
|
||||
|
||||
return new NotifyQueueEnqueueResult(messageId.ToString()!, false);
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<INotifyQueueLease<NotifyQueueEventMessage>>> LeaseAsync(
|
||||
NotifyQueueLeaseRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var leases = new List<INotifyQueueLease<NotifyQueueEventMessage>>(request.BatchSize);
|
||||
|
||||
foreach (var streamOptions in _streamsByName.Values)
|
||||
{
|
||||
await EnsureStreamInitializedAsync(db, streamOptions, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var remaining = request.BatchSize - leases.Count;
|
||||
if (remaining <= 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
var entries = await db.StreamReadGroupAsync(
|
||||
streamOptions.Stream,
|
||||
streamOptions.ConsumerGroup,
|
||||
request.Consumer,
|
||||
StreamPosition.NewMessages,
|
||||
remaining)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (entries is null || entries.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
var lease = TryMapLease(
|
||||
streamOptions,
|
||||
entry,
|
||||
request.Consumer,
|
||||
now,
|
||||
request.LeaseDuration,
|
||||
attemptOverride: null);
|
||||
|
||||
if (lease is null)
|
||||
{
|
||||
await AckPoisonAsync(db, streamOptions, entry.Id).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
leases.Add(lease);
|
||||
|
||||
if (leases.Count >= request.BatchSize)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return leases;
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<INotifyQueueLease<NotifyQueueEventMessage>>> ClaimExpiredAsync(
|
||||
NotifyQueueClaimOptions options,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var leases = new List<INotifyQueueLease<NotifyQueueEventMessage>>(options.BatchSize);
|
||||
|
||||
foreach (var streamOptions in _streamsByName.Values)
|
||||
{
|
||||
await EnsureStreamInitializedAsync(db, streamOptions, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var pending = await db.StreamPendingMessagesAsync(
|
||||
streamOptions.Stream,
|
||||
streamOptions.ConsumerGroup,
|
||||
options.BatchSize,
|
||||
RedisValue.Null,
|
||||
(long)options.MinIdleTime.TotalMilliseconds)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (pending is null || pending.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var eligible = pending
|
||||
.Where(p => p.IdleTimeInMilliseconds >= options.MinIdleTime.TotalMilliseconds)
|
||||
.ToArray();
|
||||
|
||||
if (eligible.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var messageIds = eligible
|
||||
.Select(static p => (RedisValue)p.MessageId)
|
||||
.ToArray();
|
||||
|
||||
var entries = await db.StreamClaimAsync(
|
||||
streamOptions.Stream,
|
||||
streamOptions.ConsumerGroup,
|
||||
options.ClaimantConsumer,
|
||||
0,
|
||||
messageIds)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (entries is null || entries.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var attemptById = eligible
|
||||
.Where(static info => !info.MessageId.IsNullOrEmpty)
|
||||
.ToDictionary(
|
||||
info => info.MessageId!.ToString(),
|
||||
info => (int)Math.Max(1, info.DeliveryCount),
|
||||
StringComparer.Ordinal);
|
||||
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
var entryId = entry.Id.ToString();
|
||||
attemptById.TryGetValue(entryId, out var attempt);
|
||||
|
||||
var lease = TryMapLease(
|
||||
streamOptions,
|
||||
entry,
|
||||
options.ClaimantConsumer,
|
||||
now,
|
||||
_options.DefaultLeaseDuration,
|
||||
attempt == 0 ? null : attempt);
|
||||
|
||||
if (lease is null)
|
||||
{
|
||||
await AckPoisonAsync(db, streamOptions, entry.Id).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
leases.Add(lease);
|
||||
if (leases.Count >= options.BatchSize)
|
||||
{
|
||||
return leases;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return leases;
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_disposed = true;
|
||||
if (_connection is not null)
|
||||
{
|
||||
await _connection.CloseAsync();
|
||||
_connection.Dispose();
|
||||
}
|
||||
|
||||
_connectionLock.Dispose();
|
||||
_groupInitLock.Dispose();
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
internal async Task AcknowledgeAsync(
|
||||
RedisNotifyEventLease lease,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!lease.TryBeginCompletion())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
||||
var streamOptions = lease.StreamOptions;
|
||||
|
||||
await db.StreamAcknowledgeAsync(
|
||||
streamOptions.Stream,
|
||||
streamOptions.ConsumerGroup,
|
||||
new RedisValue[] { lease.MessageId })
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await db.StreamDeleteAsync(
|
||||
streamOptions.Stream,
|
||||
new RedisValue[] { lease.MessageId })
|
||||
.ConfigureAwait(false);
|
||||
|
||||
NotifyQueueMetrics.RecordAck(TransportName, streamOptions.Stream);
|
||||
|
||||
_logger.LogDebug(
|
||||
"Acknowledged Notify event {EventId} on consumer {Consumer} (stream {Stream}, id {MessageId}).",
|
||||
lease.Message.Event.EventId,
|
||||
lease.Consumer,
|
||||
streamOptions.Stream,
|
||||
lease.MessageId);
|
||||
}
|
||||
|
||||
internal async Task RenewLeaseAsync(
|
||||
RedisNotifyEventLease lease,
|
||||
TimeSpan leaseDuration,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
||||
var streamOptions = lease.StreamOptions;
|
||||
|
||||
await db.StreamClaimAsync(
|
||||
streamOptions.Stream,
|
||||
streamOptions.ConsumerGroup,
|
||||
lease.Consumer,
|
||||
0,
|
||||
new RedisValue[] { lease.MessageId })
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var expires = _timeProvider.GetUtcNow().Add(leaseDuration);
|
||||
lease.RefreshLease(expires);
|
||||
|
||||
_logger.LogDebug(
|
||||
"Renewed Notify event lease for {EventId} until {Expires:u}.",
|
||||
lease.Message.Event.EventId,
|
||||
expires);
|
||||
}
|
||||
|
||||
internal Task ReleaseAsync(
|
||||
RedisNotifyEventLease lease,
|
||||
NotifyQueueReleaseDisposition disposition,
|
||||
CancellationToken cancellationToken)
|
||||
=> Task.FromException(new NotSupportedException("Retry/abandon is not supported for Notify event streams."));
|
||||
|
||||
internal async Task DeadLetterAsync(
|
||||
RedisNotifyEventLease lease,
|
||||
string reason,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!lease.TryBeginCompletion())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
||||
var streamOptions = lease.StreamOptions;
|
||||
|
||||
await db.StreamAcknowledgeAsync(
|
||||
streamOptions.Stream,
|
||||
streamOptions.ConsumerGroup,
|
||||
new RedisValue[] { lease.MessageId })
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await db.StreamDeleteAsync(
|
||||
streamOptions.Stream,
|
||||
new RedisValue[] { lease.MessageId })
|
||||
.ConfigureAwait(false);
|
||||
|
||||
_logger.LogWarning(
|
||||
"Dead-lettered Notify event {EventId} on stream {Stream} with reason '{Reason}'.",
|
||||
lease.Message.Event.EventId,
|
||||
streamOptions.Stream,
|
||||
reason);
|
||||
}
|
||||
|
||||
internal async ValueTask PingAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
||||
_ = await db.PingAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private NotifyRedisEventStreamOptions GetStreamOptions(string stream)
|
||||
{
|
||||
if (!_streamsByName.TryGetValue(stream, out var options))
|
||||
{
|
||||
throw new InvalidOperationException($"Stream '{stream}' is not configured for the Notify event queue.");
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
private async Task<IDatabase> GetDatabaseAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_connection is { IsConnected: true })
|
||||
{
|
||||
return _connection.GetDatabase(_redisOptions.Database ?? -1);
|
||||
}
|
||||
|
||||
await _connectionLock.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_connection is { IsConnected: true })
|
||||
{
|
||||
return _connection.GetDatabase(_redisOptions.Database ?? -1);
|
||||
}
|
||||
|
||||
var configuration = ConfigurationOptions.Parse(_redisOptions.ConnectionString!);
|
||||
configuration.AbortOnConnectFail = false;
|
||||
if (_redisOptions.Database.HasValue)
|
||||
{
|
||||
configuration.DefaultDatabase = _redisOptions.Database;
|
||||
}
|
||||
|
||||
using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
timeoutCts.CancelAfter(_redisOptions.InitializationTimeout);
|
||||
|
||||
_connection = await _connectionFactory(configuration).WaitAsync(timeoutCts.Token).ConfigureAwait(false);
|
||||
return _connection.GetDatabase(_redisOptions.Database ?? -1);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_connectionLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task EnsureStreamInitializedAsync(
|
||||
IDatabase database,
|
||||
NotifyRedisEventStreamOptions streamOptions,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (_initializedStreams.ContainsKey(streamOptions.Stream))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _groupInitLock.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_initializedStreams.ContainsKey(streamOptions.Stream))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await database.StreamCreateConsumerGroupAsync(
|
||||
streamOptions.Stream,
|
||||
streamOptions.ConsumerGroup,
|
||||
StreamPosition.Beginning,
|
||||
createStream: true)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (RedisServerException ex) when (ex.Message.Contains("BUSYGROUP", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
// Consumer group already exists — nothing to do.
|
||||
}
|
||||
|
||||
_initializedStreams[streamOptions.Stream] = true;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_groupInitLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<RedisValue> AddToStreamAsync(
|
||||
IDatabase database,
|
||||
NotifyRedisEventStreamOptions streamOptions,
|
||||
IReadOnlyList<NameValueEntry> entries)
|
||||
{
|
||||
return await database.StreamAddAsync(
|
||||
streamOptions.Stream,
|
||||
entries.ToArray(),
|
||||
maxLength: streamOptions.ApproximateMaxLength,
|
||||
useApproximateMaxLength: streamOptions.ApproximateMaxLength is not null)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private IReadOnlyList<NameValueEntry> BuildEntries(
|
||||
NotifyQueueEventMessage message,
|
||||
DateTimeOffset enqueuedAt,
|
||||
int attempt)
|
||||
{
|
||||
var payload = NotifyCanonicalJsonSerializer.Serialize(message.Event);
|
||||
|
||||
var entries = new List<NameValueEntry>(8 + message.Attributes.Count)
|
||||
{
|
||||
new(NotifyQueueFields.Payload, payload),
|
||||
new(NotifyQueueFields.EventId, message.Event.EventId.ToString("D")),
|
||||
new(NotifyQueueFields.Tenant, message.TenantId),
|
||||
new(NotifyQueueFields.Kind, message.Event.Kind),
|
||||
new(NotifyQueueFields.Attempt, attempt),
|
||||
new(NotifyQueueFields.EnqueuedAt, enqueuedAt.ToUnixTimeMilliseconds()),
|
||||
new(NotifyQueueFields.IdempotencyKey, message.IdempotencyKey),
|
||||
new(NotifyQueueFields.PartitionKey, message.PartitionKey ?? string.Empty),
|
||||
new(NotifyQueueFields.TraceId, message.TraceId ?? string.Empty)
|
||||
};
|
||||
|
||||
foreach (var kvp in message.Attributes)
|
||||
{
|
||||
entries.Add(new NameValueEntry(
|
||||
NotifyQueueFields.AttributePrefix + kvp.Key,
|
||||
kvp.Value));
|
||||
}
|
||||
|
||||
return entries;
|
||||
}
|
||||
|
||||
private RedisNotifyEventLease? TryMapLease(
|
||||
NotifyRedisEventStreamOptions streamOptions,
|
||||
StreamEntry entry,
|
||||
string consumer,
|
||||
DateTimeOffset now,
|
||||
TimeSpan leaseDuration,
|
||||
int? attemptOverride)
|
||||
{
|
||||
if (entry.Values is null || entry.Values.Length == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
string? payloadJson = null;
|
||||
string? eventIdRaw = null;
|
||||
long? enqueuedAtUnix = null;
|
||||
string? idempotency = null;
|
||||
string? partitionKey = null;
|
||||
string? traceId = null;
|
||||
var attempt = attemptOverride ?? 1;
|
||||
var attributes = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var field in entry.Values)
|
||||
{
|
||||
var name = field.Name.ToString();
|
||||
var value = field.Value;
|
||||
if (name.Equals(NotifyQueueFields.Payload, StringComparison.Ordinal))
|
||||
{
|
||||
payloadJson = value.ToString();
|
||||
}
|
||||
else if (name.Equals(NotifyQueueFields.EventId, StringComparison.Ordinal))
|
||||
{
|
||||
eventIdRaw = value.ToString();
|
||||
}
|
||||
else if (name.Equals(NotifyQueueFields.Attempt, StringComparison.Ordinal))
|
||||
{
|
||||
if (int.TryParse(value.ToString(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed))
|
||||
{
|
||||
attempt = Math.Max(parsed, attempt);
|
||||
}
|
||||
}
|
||||
else if (name.Equals(NotifyQueueFields.EnqueuedAt, StringComparison.Ordinal))
|
||||
{
|
||||
if (long.TryParse(value.ToString(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var unix))
|
||||
{
|
||||
enqueuedAtUnix = unix;
|
||||
}
|
||||
}
|
||||
else if (name.Equals(NotifyQueueFields.IdempotencyKey, StringComparison.Ordinal))
|
||||
{
|
||||
var text = value.ToString();
|
||||
idempotency = string.IsNullOrWhiteSpace(text) ? null : text;
|
||||
}
|
||||
else if (name.Equals(NotifyQueueFields.PartitionKey, StringComparison.Ordinal))
|
||||
{
|
||||
var text = value.ToString();
|
||||
partitionKey = string.IsNullOrWhiteSpace(text) ? null : text;
|
||||
}
|
||||
else if (name.Equals(NotifyQueueFields.TraceId, StringComparison.Ordinal))
|
||||
{
|
||||
var text = value.ToString();
|
||||
traceId = string.IsNullOrWhiteSpace(text) ? null : text;
|
||||
}
|
||||
else if (name.StartsWith(NotifyQueueFields.AttributePrefix, StringComparison.Ordinal))
|
||||
{
|
||||
var key = name[NotifyQueueFields.AttributePrefix.Length..];
|
||||
attributes[key] = value.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
if (payloadJson is null || enqueuedAtUnix is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
NotifyEvent notifyEvent;
|
||||
try
|
||||
{
|
||||
notifyEvent = NotifyCanonicalJsonSerializer.Deserialize<NotifyEvent>(payloadJson);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
ex,
|
||||
"Failed to deserialize Notify event payload for stream {Stream} entry {EntryId}.",
|
||||
streamOptions.Stream,
|
||||
entry.Id.ToString());
|
||||
return null;
|
||||
}
|
||||
|
||||
var attributeView = attributes.Count == 0
|
||||
? EmptyReadOnlyDictionary<string, string>.Instance
|
||||
: new ReadOnlyDictionary<string, string>(attributes);
|
||||
|
||||
var message = new NotifyQueueEventMessage(
|
||||
notifyEvent,
|
||||
streamOptions.Stream,
|
||||
idempotencyKey: idempotency ?? notifyEvent.EventId.ToString("N"),
|
||||
partitionKey: partitionKey,
|
||||
traceId: traceId,
|
||||
attributes: attributeView);
|
||||
|
||||
var enqueuedAt = DateTimeOffset.FromUnixTimeMilliseconds(enqueuedAtUnix.Value);
|
||||
var leaseExpiresAt = now.Add(leaseDuration);
|
||||
|
||||
return new RedisNotifyEventLease(
|
||||
this,
|
||||
streamOptions,
|
||||
entry.Id.ToString(),
|
||||
message,
|
||||
attempt,
|
||||
consumer,
|
||||
enqueuedAt,
|
||||
leaseExpiresAt);
|
||||
}
|
||||
|
||||
private async Task AckPoisonAsync(
|
||||
IDatabase database,
|
||||
NotifyRedisEventStreamOptions streamOptions,
|
||||
RedisValue messageId)
|
||||
{
|
||||
await database.StreamAcknowledgeAsync(
|
||||
streamOptions.Stream,
|
||||
streamOptions.ConsumerGroup,
|
||||
new RedisValue[] { messageId })
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await database.StreamDeleteAsync(
|
||||
streamOptions.Stream,
|
||||
new RedisValue[] { messageId })
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,23 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
</Project>
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="8.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.1" />
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks" Version="8.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks.Abstractions" Version="8.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.1" />
|
||||
<PackageReference Include="NATS.Client.Core" Version="2.0.0" />
|
||||
<PackageReference Include="NATS.Client.JetStream" Version="2.0.0" />
|
||||
<PackageReference Include="StackExchange.Redis" Version="2.7.33" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Notify.Models\StellaOps.Notify.Models.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
@@ -2,6 +2,6 @@
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| NOTIFY-QUEUE-15-401 | TODO | Notify Queue Guild | NOTIFY-MODELS-15-101 | Build queue abstraction + Redis Streams adapter with ack/claim APIs, idempotency tokens, serialization contracts. | Adapter integration tests cover enqueue/dequeue/ack; ordering preserved; idempotency tokens supported. |
|
||||
| NOTIFY-QUEUE-15-402 | TODO | Notify Queue Guild | NOTIFY-QUEUE-15-401 | Add NATS JetStream adapter with configuration binding, health probes, failover. | Health endpoints verified; failover documented; integration tests exercise both adapters. |
|
||||
| NOTIFY-QUEUE-15-403 | TODO | Notify Queue Guild | NOTIFY-QUEUE-15-401 | Delivery queue for channel actions with retry schedules, poison queues, and metrics instrumentation. | Delivery queue integration tests cover retries/dead-letter; metrics/logging emitted per spec. |
|
||||
| NOTIFY-QUEUE-15-401 | DONE (2025-10-23) | Notify Queue Guild | NOTIFY-MODELS-15-101 | Build queue abstraction + Redis Streams adapter with ack/claim APIs, idempotency tokens, serialization contracts. | Adapter integration tests cover enqueue/dequeue/ack; ordering preserved; idempotency tokens supported. |
|
||||
| NOTIFY-QUEUE-15-402 | DONE (2025-10-23) | Notify Queue Guild | NOTIFY-QUEUE-15-401 | Add NATS JetStream adapter with configuration binding, health probes, failover. | Health endpoints verified; failover documented; integration tests exercise both adapters. |
|
||||
| NOTIFY-QUEUE-15-403 | DONE (2025-10-23) | Notify Queue Guild | NOTIFY-QUEUE-15-401 | Delivery queue for channel actions with retry schedules, poison queues, and metrics instrumentation. | Delivery queue integration tests cover retries/dead-letter; metrics/logging emitted per spec. |
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notify.Models;
|
||||
using StellaOps.Notify.Queue;
|
||||
using StellaOps.Notify.Worker;
|
||||
using StellaOps.Notify.Worker.Handlers;
|
||||
using StellaOps.Notify.Worker.Processing;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Notify.Worker.Tests;
|
||||
|
||||
public sealed class NotifyEventLeaseProcessorTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task ProcessOnce_ShouldAcknowledgeSuccessfulLease()
|
||||
{
|
||||
var lease = new FakeLease();
|
||||
var queue = new FakeEventQueue(lease);
|
||||
var handler = new TestHandler();
|
||||
var options = Options.Create(new NotifyWorkerOptions { LeaseBatchSize = 1, LeaseDuration = TimeSpan.FromSeconds(5) });
|
||||
var processor = new NotifyEventLeaseProcessor(queue, handler, options, NullLogger<NotifyEventLeaseProcessor>.Instance, TimeProvider.System);
|
||||
|
||||
var processed = await processor.ProcessOnceAsync(CancellationToken.None);
|
||||
|
||||
processed.Should().Be(1);
|
||||
lease.AcknowledgeCount.Should().Be(1);
|
||||
lease.ReleaseCount.Should().Be(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProcessOnce_ShouldRetryOnHandlerFailure()
|
||||
{
|
||||
var lease = new FakeLease();
|
||||
var queue = new FakeEventQueue(lease);
|
||||
var handler = new TestHandler(shouldThrow: true);
|
||||
var options = Options.Create(new NotifyWorkerOptions { LeaseBatchSize = 1, LeaseDuration = TimeSpan.FromSeconds(5) });
|
||||
var processor = new NotifyEventLeaseProcessor(queue, handler, options, NullLogger<NotifyEventLeaseProcessor>.Instance, TimeProvider.System);
|
||||
|
||||
var processed = await processor.ProcessOnceAsync(CancellationToken.None);
|
||||
|
||||
processed.Should().Be(1);
|
||||
lease.AcknowledgeCount.Should().Be(0);
|
||||
lease.ReleaseCount.Should().Be(1);
|
||||
lease.LastDisposition.Should().Be(NotifyQueueReleaseDisposition.Retry);
|
||||
}
|
||||
|
||||
private sealed class FakeEventQueue : INotifyEventQueue
|
||||
{
|
||||
private readonly Queue<INotifyQueueLease<NotifyQueueEventMessage>> _leases;
|
||||
|
||||
public FakeEventQueue(params INotifyQueueLease<NotifyQueueEventMessage>[] leases)
|
||||
{
|
||||
_leases = new Queue<INotifyQueueLease<NotifyQueueEventMessage>>(leases);
|
||||
}
|
||||
|
||||
public ValueTask<NotifyQueueEnqueueResult> PublishAsync(NotifyQueueEventMessage message, CancellationToken cancellationToken = default)
|
||||
=> throw new NotSupportedException();
|
||||
|
||||
public ValueTask<IReadOnlyList<INotifyQueueLease<NotifyQueueEventMessage>>> LeaseAsync(NotifyQueueLeaseRequest request, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (_leases.Count == 0)
|
||||
{
|
||||
return ValueTask.FromResult<IReadOnlyList<INotifyQueueLease<NotifyQueueEventMessage>>>(Array.Empty<INotifyQueueLease<NotifyQueueEventMessage>>());
|
||||
}
|
||||
|
||||
return ValueTask.FromResult<IReadOnlyList<INotifyQueueLease<NotifyQueueEventMessage>>>(new[] { _leases.Dequeue() });
|
||||
}
|
||||
|
||||
public ValueTask<IReadOnlyList<INotifyQueueLease<NotifyQueueEventMessage>>> ClaimExpiredAsync(NotifyQueueClaimOptions options, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.FromResult<IReadOnlyList<INotifyQueueLease<NotifyQueueEventMessage>>>(Array.Empty<INotifyQueueLease<NotifyQueueEventMessage>>());
|
||||
}
|
||||
|
||||
private sealed class FakeLease : INotifyQueueLease<NotifyQueueEventMessage>
|
||||
{
|
||||
private readonly NotifyQueueEventMessage _message;
|
||||
|
||||
public FakeLease()
|
||||
{
|
||||
var notifyEvent = NotifyEvent.Create(
|
||||
Guid.NewGuid(),
|
||||
kind: "test.event",
|
||||
tenant: "tenant-1",
|
||||
ts: DateTimeOffset.UtcNow,
|
||||
payload: null);
|
||||
|
||||
_message = new NotifyQueueEventMessage(notifyEvent, "notify:events", traceId: "trace-123");
|
||||
}
|
||||
|
||||
public string MessageId { get; } = Guid.NewGuid().ToString("n");
|
||||
|
||||
public int Attempt { get; internal set; } = 1;
|
||||
|
||||
public DateTimeOffset EnqueuedAt { get; } = DateTimeOffset.UtcNow;
|
||||
|
||||
public DateTimeOffset LeaseExpiresAt { get; private set; } = DateTimeOffset.UtcNow.AddSeconds(30);
|
||||
|
||||
public string Consumer { get; } = "worker-1";
|
||||
|
||||
public string Stream => _message.Stream;
|
||||
|
||||
public string TenantId => _message.TenantId;
|
||||
|
||||
public string? PartitionKey => _message.PartitionKey;
|
||||
|
||||
public string IdempotencyKey => _message.IdempotencyKey;
|
||||
|
||||
public string? TraceId => _message.TraceId;
|
||||
|
||||
public IReadOnlyDictionary<string, string> Attributes => _message.Attributes;
|
||||
|
||||
public NotifyQueueEventMessage Message => _message;
|
||||
|
||||
public int AcknowledgeCount { get; private set; }
|
||||
|
||||
public int ReleaseCount { get; private set; }
|
||||
|
||||
public NotifyQueueReleaseDisposition? LastDisposition { get; private set; }
|
||||
|
||||
public Task AcknowledgeAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
AcknowledgeCount++;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task RenewAsync(TimeSpan leaseDuration, CancellationToken cancellationToken = default)
|
||||
{
|
||||
LeaseExpiresAt = DateTimeOffset.UtcNow.Add(leaseDuration);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task ReleaseAsync(NotifyQueueReleaseDisposition disposition, CancellationToken cancellationToken = default)
|
||||
{
|
||||
LastDisposition = disposition;
|
||||
ReleaseCount++;
|
||||
Attempt++;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task DeadLetterAsync(string reason, CancellationToken cancellationToken = default)
|
||||
=> Task.CompletedTask;
|
||||
}
|
||||
|
||||
private sealed class TestHandler : INotifyEventHandler
|
||||
{
|
||||
private readonly bool _shouldThrow;
|
||||
|
||||
public TestHandler(bool shouldThrow = false)
|
||||
{
|
||||
_shouldThrow = shouldThrow;
|
||||
}
|
||||
|
||||
public Task HandleAsync(NotifyQueueEventMessage message, CancellationToken cancellationToken)
|
||||
{
|
||||
if (_shouldThrow)
|
||||
{
|
||||
throw new InvalidOperationException("handler failure");
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<IsPackable>false</IsPackable>
|
||||
<UseConcelierTestInfra>false</UseConcelierTestInfra>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="FluentAssertions" Version="6.12.0" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.0" />
|
||||
<PackageReference Include="xunit" Version="2.9.2" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
<PackageReference Include="coverlet.collector" Version="6.0.4">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Notify.Worker\StellaOps.Notify.Worker.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Notify.Queue\StellaOps.Notify.Queue.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Notify.Models\StellaOps.Notify.Models.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
10
src/StellaOps.Notify.Worker/Handlers/INotifyEventHandler.cs
Normal file
10
src/StellaOps.Notify.Worker/Handlers/INotifyEventHandler.cs
Normal file
@@ -0,0 +1,10 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Notify.Queue;
|
||||
|
||||
namespace StellaOps.Notify.Worker.Handlers;
|
||||
|
||||
public interface INotifyEventHandler
|
||||
{
|
||||
Task HandleAsync(NotifyQueueEventMessage message, CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Notify.Queue;
|
||||
|
||||
namespace StellaOps.Notify.Worker.Handlers;
|
||||
|
||||
internal sealed class NoOpNotifyEventHandler : INotifyEventHandler
|
||||
{
|
||||
private readonly ILogger<NoOpNotifyEventHandler> _logger;
|
||||
|
||||
public NoOpNotifyEventHandler(ILogger<NoOpNotifyEventHandler> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public Task HandleAsync(NotifyQueueEventMessage message, CancellationToken cancellationToken)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"No-op handler acknowledged event {EventId} (tenant {TenantId}).",
|
||||
message.Event.EventId,
|
||||
message.TenantId);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
52
src/StellaOps.Notify.Worker/NotifyWorkerOptions.cs
Normal file
52
src/StellaOps.Notify.Worker/NotifyWorkerOptions.cs
Normal file
@@ -0,0 +1,52 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Notify.Worker;
|
||||
|
||||
public sealed class NotifyWorkerOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Worker identifier prefix; defaults to machine name.
|
||||
/// </summary>
|
||||
public string? WorkerId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Number of messages to lease per iteration.
|
||||
/// </summary>
|
||||
public int LeaseBatchSize { get; set; } = 16;
|
||||
|
||||
/// <summary>
|
||||
/// Duration a lease remains active before it becomes eligible for claim.
|
||||
/// </summary>
|
||||
public TimeSpan LeaseDuration { get; set; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>
|
||||
/// Delay applied when no work is available.
|
||||
/// </summary>
|
||||
public TimeSpan IdleDelay { get; set; } = TimeSpan.FromMilliseconds(250);
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of event leases processed concurrently.
|
||||
/// </summary>
|
||||
public int MaxConcurrency { get; set; } = 4;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of consecutive failures before the worker delays.
|
||||
/// </summary>
|
||||
public int FailureBackoffThreshold { get; set; } = 3;
|
||||
|
||||
/// <summary>
|
||||
/// Delay applied when the failure threshold is reached.
|
||||
/// </summary>
|
||||
public TimeSpan FailureBackoffDelay { get; set; } = TimeSpan.FromSeconds(5);
|
||||
|
||||
internal string ResolveWorkerId()
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(WorkerId))
|
||||
{
|
||||
return WorkerId!;
|
||||
}
|
||||
|
||||
var host = Environment.MachineName;
|
||||
return $"{host}-{Guid.NewGuid():n}";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Notify.Queue;
|
||||
using StellaOps.Notify.Worker.Handlers;
|
||||
|
||||
namespace StellaOps.Notify.Worker.Processing;
|
||||
|
||||
internal sealed class NotifyEventLeaseProcessor
|
||||
{
|
||||
private static readonly ActivitySource ActivitySource = new("StellaOps.Notify.Worker");
|
||||
|
||||
private readonly INotifyEventQueue _queue;
|
||||
private readonly INotifyEventHandler _handler;
|
||||
private readonly NotifyWorkerOptions _options;
|
||||
private readonly ILogger<NotifyEventLeaseProcessor> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly string _workerId;
|
||||
|
||||
public NotifyEventLeaseProcessor(
|
||||
INotifyEventQueue queue,
|
||||
INotifyEventHandler handler,
|
||||
IOptions<NotifyWorkerOptions> options,
|
||||
ILogger<NotifyEventLeaseProcessor> logger,
|
||||
TimeProvider timeProvider)
|
||||
{
|
||||
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
|
||||
_handler = handler ?? throw new ArgumentNullException(nameof(handler));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_workerId = _options.ResolveWorkerId();
|
||||
}
|
||||
|
||||
public async Task<int> ProcessOnceAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var leaseRequest = new NotifyQueueLeaseRequest(
|
||||
consumer: _workerId,
|
||||
batchSize: Math.Max(1, _options.LeaseBatchSize),
|
||||
leaseDuration: _options.LeaseDuration <= TimeSpan.Zero ? TimeSpan.FromSeconds(30) : _options.LeaseDuration);
|
||||
|
||||
IReadOnlyList<INotifyQueueLease<NotifyQueueEventMessage>> leases;
|
||||
try
|
||||
{
|
||||
leases = await _queue.LeaseAsync(leaseRequest, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to lease Notify events.");
|
||||
throw;
|
||||
}
|
||||
|
||||
if (leases.Count == 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var processed = 0;
|
||||
foreach (var lease in leases)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
processed++;
|
||||
await ProcessLeaseAsync(lease, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return processed;
|
||||
}
|
||||
|
||||
private async Task ProcessLeaseAsync(
|
||||
INotifyQueueLease<NotifyQueueEventMessage> lease,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var message = lease.Message;
|
||||
var correlationId = message.TraceId ?? message.Event.EventId.ToString("N");
|
||||
|
||||
using var scope = _logger.BeginScope(new Dictionary<string, object?>
|
||||
{
|
||||
["notifyTraceId"] = correlationId,
|
||||
["notifyTenantId"] = message.TenantId,
|
||||
["notifyEventId"] = message.Event.EventId,
|
||||
["notifyAttempt"] = lease.Attempt
|
||||
});
|
||||
|
||||
using var activity = ActivitySource.StartActivity("notify.event.process", ActivityKind.Consumer);
|
||||
activity?.SetTag("notify.tenant_id", message.TenantId);
|
||||
activity?.SetTag("notify.event_id", message.Event.EventId);
|
||||
activity?.SetTag("notify.attempt", lease.Attempt);
|
||||
activity?.SetTag("notify.worker_id", _workerId);
|
||||
|
||||
try
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Processing notify event {EventId} (tenant {TenantId}, attempt {Attempt}).",
|
||||
message.Event.EventId,
|
||||
message.TenantId,
|
||||
lease.Attempt);
|
||||
|
||||
await _handler.HandleAsync(message, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await lease.AcknowledgeAsync(cancellationToken).ConfigureAwait(false);
|
||||
_logger.LogInformation(
|
||||
"Acknowledged notify event {EventId} (tenant {TenantId}).",
|
||||
message.Event.EventId,
|
||||
message.TenantId);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Worker cancellation requested while processing event {EventId}; returning lease to queue.",
|
||||
message.Event.EventId);
|
||||
|
||||
await SafeReleaseAsync(lease, NotifyQueueReleaseDisposition.Retry, CancellationToken.None).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"Failed to process notify event {EventId}; scheduling retry.",
|
||||
message.Event.EventId);
|
||||
|
||||
await SafeReleaseAsync(lease, NotifyQueueReleaseDisposition.Retry, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task SafeReleaseAsync(
|
||||
INotifyQueueLease<NotifyQueueEventMessage> lease,
|
||||
NotifyQueueReleaseDisposition disposition,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await lease.ReleaseAsync(disposition, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// Suppress release errors during shutdown.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace StellaOps.Notify.Worker.Processing;
|
||||
|
||||
internal sealed class NotifyEventLeaseWorker : BackgroundService
|
||||
{
|
||||
private readonly NotifyEventLeaseProcessor _processor;
|
||||
private readonly NotifyWorkerOptions _options;
|
||||
private readonly ILogger<NotifyEventLeaseWorker> _logger;
|
||||
|
||||
public NotifyEventLeaseWorker(
|
||||
NotifyEventLeaseProcessor processor,
|
||||
IOptions<NotifyWorkerOptions> options,
|
||||
ILogger<NotifyEventLeaseWorker> logger)
|
||||
{
|
||||
_processor = processor ?? throw new ArgumentNullException(nameof(processor));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
var idleDelay = _options.IdleDelay <= TimeSpan.Zero
|
||||
? TimeSpan.FromMilliseconds(500)
|
||||
: _options.IdleDelay;
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
int processed;
|
||||
try
|
||||
{
|
||||
processed = await _processor.ProcessOnceAsync(stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Notify worker processing loop encountered an error.");
|
||||
await Task.Delay(_options.FailureBackoffDelay, stoppingToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (processed == 0)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.Delay(idleDelay, stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
33
src/StellaOps.Notify.Worker/Program.cs
Normal file
33
src/StellaOps.Notify.Worker/Program.cs
Normal file
@@ -0,0 +1,33 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Notify.Queue;
|
||||
using StellaOps.Notify.Worker;
|
||||
using StellaOps.Notify.Worker.Handlers;
|
||||
using StellaOps.Notify.Worker.Processing;
|
||||
|
||||
var builder = Host.CreateApplicationBuilder(args);
|
||||
|
||||
builder.Configuration
|
||||
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: true)
|
||||
.AddEnvironmentVariables(prefix: "NOTIFY_");
|
||||
|
||||
builder.Logging.ClearProviders();
|
||||
builder.Logging.AddSimpleConsole(options =>
|
||||
{
|
||||
options.TimestampFormat = "yyyy-MM-ddTHH:mm:ss.fffZ ";
|
||||
options.UseUtcTimestamp = true;
|
||||
});
|
||||
|
||||
builder.Services.Configure<NotifyWorkerOptions>(builder.Configuration.GetSection("notify:worker"));
|
||||
builder.Services.AddSingleton(TimeProvider.System);
|
||||
|
||||
builder.Services.AddNotifyEventQueue(builder.Configuration, "notify:queue");
|
||||
builder.Services.AddNotifyDeliveryQueue(builder.Configuration, "notify:deliveryQueue");
|
||||
|
||||
builder.Services.AddSingleton<INotifyEventHandler, NoOpNotifyEventHandler>();
|
||||
builder.Services.AddSingleton<NotifyEventLeaseProcessor>();
|
||||
builder.Services.AddHostedService<NotifyEventLeaseWorker>();
|
||||
|
||||
await builder.Build().RunAsync().ConfigureAwait(false);
|
||||
3
src/StellaOps.Notify.Worker/Properties/AssemblyInfo.cs
Normal file
3
src/StellaOps.Notify.Worker/Properties/AssemblyInfo.cs
Normal file
@@ -0,0 +1,3 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("StellaOps.Notify.Worker.Tests")]
|
||||
@@ -1,8 +1,24 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<OutputType>Exe</OutputType>
|
||||
</PropertyGroup>
|
||||
</Project>
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<OutputType>Exe</OutputType>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="8.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="8.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="8.0.0" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Notify.Queue\StellaOps.Notify.Queue.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Notify.Models\StellaOps.Notify.Models.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Update="appsettings.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| NOTIFY-WORKER-15-201 | TODO | Notify Worker Guild | NOTIFY-QUEUE-15-401 | Implement bus subscription + leasing loop with correlation IDs, backoff, dead-letter handling (§1–§5). | Worker consumes events from queue, ack/retry behaviour proven in integration tests; logs include correlation IDs. |
|
||||
| NOTIFY-WORKER-15-202 | TODO | Notify Worker Guild | NOTIFY-ENGINE-15-301 | Wire rules evaluation pipeline (tenant scoping, filters, throttles, digests, idempotency) with deterministic decisions. | Evaluation unit tests cover rule combinations; throttles/digests produce expected suppression; idempotency keys validated. |
|
||||
| NOTIFY-WORKER-15-203 | TODO | Notify Worker Guild | NOTIFY-ENGINE-15-302 | Channel dispatch orchestration: invoke connectors, manage retries/jitter, record delivery outcomes. | Connector mocks show retries/backoff; delivery results stored; metrics incremented per outcome. |
|
||||
| NOTIFY-WORKER-15-201 | DONE (2025-10-23) | Notify Worker Guild | NOTIFY-QUEUE-15-401 | Implement bus subscription + leasing loop with correlation IDs, backoff, dead-letter handling (§1–§5). | Worker consumes events from queue, ack/retry behaviour proven in integration tests; logs include correlation IDs. |
|
||||
| NOTIFY-WORKER-15-202 | TODO | Notify Worker Guild | NOTIFY-WORKER-15-201 | Wire rules evaluation pipeline (tenant scoping, filters, throttles, digests, idempotency) with deterministic decisions. | Evaluation unit tests cover rule combinations; throttles/digests produce expected suppression; idempotency keys validated. |
|
||||
| NOTIFY-WORKER-15-203 | TODO | Notify Worker Guild | NOTIFY-WORKER-15-202 | Channel dispatch orchestration: invoke connectors, manage retries/jitter, record delivery outcomes. | Connector mocks show retries/backoff; delivery results stored; metrics incremented per outcome. |
|
||||
| NOTIFY-WORKER-15-204 | TODO | Notify Worker Guild | NOTIFY-WORKER-15-203 | Metrics/telemetry: `notify.sent_total`, `notify.dropped_total`, latency histograms, tracing integration. | Metrics emitted per spec; OTLP spans annotated; dashboards documented. |
|
||||
|
||||
43
src/StellaOps.Notify.Worker/appsettings.json
Normal file
43
src/StellaOps.Notify.Worker/appsettings.json
Normal file
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"Logging": {
|
||||
"LogLevel": {
|
||||
"Default": "Information",
|
||||
"Microsoft": "Warning",
|
||||
"Microsoft.Hosting.Lifetime": "Information"
|
||||
}
|
||||
},
|
||||
"notify": {
|
||||
"worker": {
|
||||
"leaseBatchSize": 16,
|
||||
"leaseDuration": "00:00:30",
|
||||
"idleDelay": "00:00:00.250",
|
||||
"maxConcurrency": 4,
|
||||
"failureBackoffThreshold": 3,
|
||||
"failureBackoffDelay": "00:00:05"
|
||||
},
|
||||
"queue": {
|
||||
"transport": "Redis",
|
||||
"redis": {
|
||||
"connectionString": "localhost:6379",
|
||||
"streams": [
|
||||
{
|
||||
"stream": "notify:events",
|
||||
"consumerGroup": "notify-workers",
|
||||
"idempotencyKeyPrefix": "notify:events:idemp:",
|
||||
"approximateMaxLength": 100000
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"deliveryQueue": {
|
||||
"transport": "Redis",
|
||||
"redis": {
|
||||
"connectionString": "localhost:6379",
|
||||
"streamName": "notify:deliveries",
|
||||
"consumerGroup": "notify-delivery",
|
||||
"idempotencyKeyPrefix": "notify:deliveries:idemp:",
|
||||
"deadLetterStreamName": "notify:deliveries:dead"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,11 +2,11 @@
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| UI-AUTH-13-001 | TODO | UI Guild | AUTH-DPOP-11-001, AUTH-MTLS-11-002 | Integrate Authority OIDC + DPoP flows with session management. | Login/logout flows pass e2e tests; tokens refreshed; DPoP nonce handling validated. |
|
||||
| UI-AUTH-13-001 | DONE (2025-10-23) | UI Guild | AUTH-DPOP-11-001, AUTH-MTLS-11-002 | Integrate Authority OIDC + DPoP flows with session management. | Login/logout flows pass e2e tests; tokens refreshed; DPoP nonce handling validated. |
|
||||
| UI-SCANS-13-002 | TODO | UI Guild | SCANNER-WEB-09-102, SIGNER-API-11-101 | Build scans module (list/detail/SBOM/diff/attestation) with performance + accessibility targets. | Cypress tests cover SBOM/diff; performance budgets met; accessibility checks pass. |
|
||||
| UI-VEX-13-003 | TODO | UI Guild | EXCITITOR-CORE-02-001, EXCITITOR-EXPORT-01-005 | Implement VEX explorer + policy editor with preview integration. | VEX views render consensus/conflicts; staged policy preview works; accessibility checks pass. |
|
||||
| UI-ADMIN-13-004 | TODO | UI Guild | AUTH-MTLS-11-002 | Deliver admin area (tenants/clients/quotas/licensing) with RBAC + audit hooks. | Admin e2e tests pass; unauthorized access blocked; telemetry wired. |
|
||||
| UI-ATTEST-11-005 | TODO | UI Guild | SIGNER-API-11-101, ATTESTOR-API-11-201 | Attestation visibility (Rekor id, status) on Scan Detail. | UI shows Rekor UUID/status; mock attestation fixtures displayed; tests cover success/failure. |
|
||||
| UI-ATTEST-11-005 | DONE (2025-10-23) | UI Guild | SIGNER-API-11-101, ATTESTOR-API-11-201 | Attestation visibility (Rekor id, status) on Scan Detail. | UI shows Rekor UUID/status; mock attestation fixtures displayed; tests cover success/failure. |
|
||||
| UI-SCHED-13-005 | TODO | UI Guild | SCHED-WEB-16-101 | Scheduler panel: schedules CRUD, run history, dry-run preview using API/mocks. | Panel functional with mocked endpoints; UX signoff; integration tests added. |
|
||||
| UI-NOTIFY-13-006 | DOING (2025-10-19) | UI Guild | NOTIFY-WEB-15-101 | Notify panel: channels/rules CRUD, deliveries view, test send integration. | Panel interacts with mocked Notify API; tests cover rule lifecycle; docs updated. |
|
||||
| UI-POLICY-13-007 | TODO | UI Guild | POLICY-CORE-09-006, SCANNER-WEB-09-103 | Surface policy confidence metadata (band, age, quiet provenance) on preview and report views. | UI renders new columns/tooltips, accessibility and responsive checks pass, Cypress regression updated with confidence fixtures. |
|
||||
|
||||
@@ -33,6 +33,26 @@ Run `ng build` to build the project. The build artifacts will be stored in the `
|
||||
- `npm run test:watch` keeps Karma in watch mode for local development.
|
||||
|
||||
`verify:chromium` prints every location inspected (environment overrides, system paths, `.cache/chromium/`). Set `CHROME_BIN` or `STELLAOPS_CHROMIUM_BIN` if you host the binary in a non-standard path.
|
||||
|
||||
## Runtime configuration
|
||||
|
||||
The SPA loads environment details from `/config.json` at startup. During development we ship a stub configuration under `src/config/config.json`; adjust the issuer, client ID, and API base URLs to match your Authority instance. To reset, copy `src/config/config.sample.json` back to `src/config/config.json`:
|
||||
|
||||
```bash
|
||||
cp src/config/config.sample.json src/config/config.json
|
||||
```
|
||||
|
||||
When packaging for another environment, replace the file before building so the generated bundle contains the correct defaults. Gateways that rewrite `/config.json` at request time can override these settings without rebuilding.
|
||||
|
||||
## End-to-end tests
|
||||
|
||||
Playwright drives the high-level auth UX using the stub configuration above. Ensure the Angular dev server can bind to `127.0.0.1:4400`, then run:
|
||||
|
||||
```bash
|
||||
npm run test:e2e
|
||||
```
|
||||
|
||||
The Playwright config auto-starts `npm run serve:test` and intercepts Authority redirects, so no live IdP is required. For CI/offline nodes, pre-install the required browsers via `npx playwright install --with-deps` and cache the results alongside your npm cache.
|
||||
|
||||
## Running end-to-end tests
|
||||
|
||||
|
||||
@@ -6,3 +6,4 @@
|
||||
| WEB1.TRIVY-SETTINGS-TESTS | DONE (2025-10-21) | UX Specialist, Angular Eng | WEB1.TRIVY-SETTINGS | **DONE (2025-10-21)** – Added headless Karma harness (`ng test --watch=false`) wired to ChromeHeadless/CI launcher, created `karma.conf.cjs`, updated npm scripts + docs with Chromium prerequisites so CI/offline runners can execute specs deterministically. | Angular CLI available (npm scripts chained), Karma suite for Trivy DB components passing locally and in CI, docs note required prerequisites. |
|
||||
| WEB1.DEPS-13-001 | DONE (2025-10-21) | UX Specialist, Angular Eng, DevEx | WEB1.TRIVY-SETTINGS-TESTS | Stabilise Angular workspace dependencies for CI/offline nodes: refresh `package-lock.json`, ensure Puppeteer/Chromium binaries optional, document deterministic install workflow. | `npm install` completes without manual intervention on air-gapped nodes, `npm test` headless run succeeds from clean checkout, README updated with lockfile + cache steps. |
|
||||
| WEB-POLICY-FIXTURES-10-001 | DONE (2025-10-23) | Angular Eng | SAMPLES-13-004 | Wire policy preview/report doc fixtures into UI harness (test utility or Storybook substitute) with type bindings and validation guard so UI stays aligned with documented payloads. | JSON fixtures importable within Angular workspace, typed helpers exported for reuse, Karma spec validates critical fields (confidence band, unknown metrics, DSSE summary). |
|
||||
| UI-AUTH-13-001 | DONE (2025-10-23) | UI Guild | AUTH-DPOP-11-001, AUTH-MTLS-11-002 | Integrate Authority OIDC + DPoP flows with session management (Angular SPA). | APP_INITIALIZER loads runtime config; login/logout flows drive Authority code flow; DPoP proofs generated/stored, nonce retries handled; unit specs cover proof binding + session persistence. |
|
||||
|
||||
@@ -25,10 +25,15 @@
|
||||
],
|
||||
"tsConfig": "tsconfig.app.json",
|
||||
"inlineStyleLanguage": "scss",
|
||||
"assets": [
|
||||
"src/favicon.ico",
|
||||
"src/assets"
|
||||
],
|
||||
"assets": [
|
||||
"src/favicon.ico",
|
||||
"src/assets",
|
||||
{
|
||||
"glob": "config.json",
|
||||
"input": "src/config",
|
||||
"output": "."
|
||||
}
|
||||
],
|
||||
"styles": [
|
||||
"src/styles.scss"
|
||||
],
|
||||
@@ -88,7 +93,12 @@
|
||||
"inlineStyleLanguage": "scss",
|
||||
"assets": [
|
||||
"src/favicon.ico",
|
||||
"src/assets"
|
||||
"src/assets",
|
||||
{
|
||||
"glob": "config.json",
|
||||
"input": "src/config",
|
||||
"output": "."
|
||||
}
|
||||
],
|
||||
"styles": [
|
||||
"src/styles.scss"
|
||||
|
||||
63
src/StellaOps.Web/package-lock.json
generated
63
src/StellaOps.Web/package-lock.json
generated
@@ -24,6 +24,7 @@
|
||||
"@angular-devkit/build-angular": "^17.3.17",
|
||||
"@angular/cli": "^17.3.17",
|
||||
"@angular/compiler-cli": "^17.3.0",
|
||||
"@playwright/test": "^1.47.2",
|
||||
"@types/jasmine": "~5.1.0",
|
||||
"jasmine-core": "~5.1.0",
|
||||
"karma": "~6.4.0",
|
||||
@@ -5074,6 +5075,21 @@
|
||||
"node": ">=14"
|
||||
}
|
||||
},
|
||||
"node_modules/@playwright/test": {
|
||||
"version": "1.56.1",
|
||||
"resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.56.1.tgz",
|
||||
"integrity": "sha512-vSMYtL/zOcFpvJCW71Q/OEGQb7KYBPAdKh35WNSkaZA75JlAO8ED8UN6GUNTm3drWomcbcqRPFqQbLae8yBTdg==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"playwright": "1.56.1"
|
||||
},
|
||||
"bin": {
|
||||
"playwright": "cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@rollup/rollup-linux-x64-gnu": {
|
||||
"version": "4.52.5",
|
||||
"cpu": [
|
||||
@@ -5313,9 +5329,6 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/@types/node-forge": {
|
||||
"version": "1.3.14",
|
||||
"dev": true,
|
||||
@@ -8233,6 +8246,20 @@
|
||||
"dev": true,
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/fsevents": {
|
||||
"version": "2.3.2",
|
||||
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
|
||||
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
|
||||
"dev": true,
|
||||
"hasInstallScript": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/function-bind": {
|
||||
"version": "1.1.2",
|
||||
"dev": true,
|
||||
@@ -10928,6 +10955,36 @@
|
||||
"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/playwright": {
|
||||
"version": "1.56.1",
|
||||
"resolved": "https://registry.npmjs.org/playwright/-/playwright-1.56.1.tgz",
|
||||
"integrity": "sha512-aFi5B0WovBHTEvpM3DzXTUaeN6eN0qWnTkKx4NQaH4Wvcmc153PdaY2UBdSYKaGYw+UyWXSVyxDUg5DoPEttjw==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"playwright-core": "1.56.1"
|
||||
},
|
||||
"bin": {
|
||||
"playwright": "cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"fsevents": "2.3.2"
|
||||
}
|
||||
},
|
||||
"node_modules/playwright-core": {
|
||||
"version": "1.56.1",
|
||||
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.56.1.tgz",
|
||||
"integrity": "sha512-hutraynyn31F+Bifme+Ps9Vq59hKuUCz7H1kDOcBs+2oGguKkWTU50bBWrtz34OUWmIwpBTWDxaRPXrIXkgvmQ==",
|
||||
"dev": true,
|
||||
"bin": {
|
||||
"playwright-core": "cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/postcss": {
|
||||
"version": "8.4.35",
|
||||
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.35.tgz",
|
||||
|
||||
@@ -9,6 +9,8 @@
|
||||
"test": "npm run verify:chromium && ng test --watch=false",
|
||||
"test:watch": "ng test --watch",
|
||||
"test:ci": "npm run test",
|
||||
"test:e2e": "playwright test",
|
||||
"serve:test": "ng serve --configuration development --port 4400 --host 127.0.0.1",
|
||||
"verify:chromium": "node ./scripts/verify-chromium.js",
|
||||
"ci:install": "npm ci --prefer-offline --no-audit --no-fund"
|
||||
},
|
||||
@@ -33,7 +35,8 @@
|
||||
"devDependencies": {
|
||||
"@angular-devkit/build-angular": "^17.3.17",
|
||||
"@angular/cli": "^17.3.17",
|
||||
"@angular/compiler-cli": "^17.3.0",
|
||||
"@angular/compiler-cli": "^17.3.0",
|
||||
"@playwright/test": "^1.47.2",
|
||||
"@types/jasmine": "~5.1.0",
|
||||
"jasmine-core": "~5.1.0",
|
||||
"karma": "~6.4.0",
|
||||
|
||||
22
src/StellaOps.Web/playwright.config.ts
Normal file
22
src/StellaOps.Web/playwright.config.ts
Normal file
@@ -0,0 +1,22 @@
|
||||
import { defineConfig } from '@playwright/test';
|
||||
|
||||
const port = process.env.PLAYWRIGHT_PORT
|
||||
? Number.parseInt(process.env.PLAYWRIGHT_PORT, 10)
|
||||
: 4400;
|
||||
|
||||
export default defineConfig({
|
||||
testDir: 'tests/e2e',
|
||||
timeout: 30_000,
|
||||
retries: process.env.CI ? 1 : 0,
|
||||
use: {
|
||||
baseURL: process.env.PLAYWRIGHT_BASE_URL ?? `http://127.0.0.1:${port}`,
|
||||
trace: 'retain-on-failure',
|
||||
},
|
||||
webServer: {
|
||||
command: 'npm run serve:test',
|
||||
reuseExistingServer: !process.env.CI,
|
||||
url: `http://127.0.0.1:${port}`,
|
||||
stdout: 'ignore',
|
||||
stderr: 'ignore',
|
||||
},
|
||||
});
|
||||
@@ -5,7 +5,19 @@
|
||||
<a routerLink="/concelier/trivy-db-settings" routerLinkActive="active">
|
||||
Trivy DB Export
|
||||
</a>
|
||||
<a routerLink="/scans/scan-verified-001" routerLinkActive="active">
|
||||
Scan Detail
|
||||
</a>
|
||||
</nav>
|
||||
<div class="app-auth">
|
||||
<ng-container *ngIf="isAuthenticated(); else signIn">
|
||||
<span class="app-user" aria-live="polite">{{ displayName() }}</span>
|
||||
<button type="button" (click)="onSignOut()">Sign out</button>
|
||||
</ng-container>
|
||||
<ng-template #signIn>
|
||||
<button type="button" (click)="onSignIn()">Sign in</button>
|
||||
</ng-template>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<main class="app-content">
|
||||
|
||||
@@ -50,6 +50,36 @@
|
||||
}
|
||||
}
|
||||
|
||||
.app-auth {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.75rem;
|
||||
|
||||
.app-user {
|
||||
font-size: 0.9rem;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
button {
|
||||
appearance: none;
|
||||
border: none;
|
||||
border-radius: 9999px;
|
||||
padding: 0.35rem 0.9rem;
|
||||
font-size: 0.85rem;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
color: #0f172a;
|
||||
background-color: rgba(248, 250, 252, 0.9);
|
||||
transition: transform 0.2s ease, background-color 0.2s ease;
|
||||
|
||||
&:hover,
|
||||
&:focus-visible {
|
||||
background-color: #facc15;
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.app-content {
|
||||
flex: 1;
|
||||
padding: 2rem 1.5rem;
|
||||
|
||||
@@ -1,11 +1,22 @@
|
||||
import { TestBed } from '@angular/core/testing';
|
||||
import { RouterTestingModule } from '@angular/router/testing';
|
||||
import { AppComponent } from './app.component';
|
||||
import { AuthorityAuthService } from './core/auth/authority-auth.service';
|
||||
import { AuthSessionStore } from './core/auth/auth-session.store';
|
||||
|
||||
class AuthorityAuthServiceStub {
|
||||
beginLogin = jasmine.createSpy('beginLogin');
|
||||
logout = jasmine.createSpy('logout');
|
||||
}
|
||||
|
||||
describe('AppComponent', () => {
|
||||
beforeEach(async () => {
|
||||
await TestBed.configureTestingModule({
|
||||
imports: [AppComponent, RouterTestingModule],
|
||||
providers: [
|
||||
AuthSessionStore,
|
||||
{ provide: AuthorityAuthService, useClass: AuthorityAuthServiceStub },
|
||||
],
|
||||
}).compileComponents();
|
||||
});
|
||||
|
||||
|
||||
@@ -1,11 +1,51 @@
|
||||
import { Component } from '@angular/core';
|
||||
import { RouterLink, RouterLinkActive, RouterOutlet } from '@angular/router';
|
||||
import { CommonModule } from '@angular/common';
|
||||
import {
|
||||
ChangeDetectionStrategy,
|
||||
Component,
|
||||
computed,
|
||||
inject,
|
||||
} from '@angular/core';
|
||||
import { Router, RouterLink, RouterLinkActive, RouterOutlet } from '@angular/router';
|
||||
|
||||
import { AuthorityAuthService } from './core/auth/authority-auth.service';
|
||||
import { AuthSessionStore } from './core/auth/auth-session.store';
|
||||
|
||||
@Component({
|
||||
selector: 'app-root',
|
||||
standalone: true,
|
||||
imports: [RouterOutlet, RouterLink, RouterLinkActive],
|
||||
imports: [CommonModule, RouterOutlet, RouterLink, RouterLinkActive],
|
||||
templateUrl: './app.component.html',
|
||||
styleUrl: './app.component.scss'
|
||||
styleUrl: './app.component.scss',
|
||||
changeDetection: ChangeDetectionStrategy.OnPush,
|
||||
})
|
||||
export class AppComponent {}
|
||||
export class AppComponent {
|
||||
private readonly router = inject(Router);
|
||||
private readonly auth = inject(AuthorityAuthService);
|
||||
private readonly sessionStore = inject(AuthSessionStore);
|
||||
|
||||
readonly status = this.sessionStore.status;
|
||||
readonly identity = this.sessionStore.identity;
|
||||
readonly subjectHint = this.sessionStore.subjectHint;
|
||||
readonly isAuthenticated = this.sessionStore.isAuthenticated;
|
||||
|
||||
readonly displayName = computed(() => {
|
||||
const identity = this.identity();
|
||||
if (identity?.name) {
|
||||
return identity.name;
|
||||
}
|
||||
if (identity?.email) {
|
||||
return identity.email;
|
||||
}
|
||||
const hint = this.subjectHint();
|
||||
return hint ?? 'anonymous';
|
||||
});
|
||||
|
||||
onSignIn(): void {
|
||||
const returnUrl = this.router.url === '/' ? undefined : this.router.url;
|
||||
void this.auth.beginLogin(returnUrl);
|
||||
}
|
||||
|
||||
onSignOut(): void {
|
||||
void this.auth.logout();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,14 +1,28 @@
|
||||
import { provideHttpClient } from '@angular/common/http';
|
||||
import { ApplicationConfig } from '@angular/core';
|
||||
import { HTTP_INTERCEPTORS, provideHttpClient, withInterceptorsFromDi } from '@angular/common/http';
|
||||
import { APP_INITIALIZER, ApplicationConfig } from '@angular/core';
|
||||
import { provideRouter } from '@angular/router';
|
||||
|
||||
import { routes } from './app.routes';
|
||||
import { CONCELIER_EXPORTER_API_BASE_URL } from './core/api/concelier-exporter.client';
|
||||
import { AppConfigService } from './core/config/app-config.service';
|
||||
import { AuthHttpInterceptor } from './core/auth/auth-http.interceptor';
|
||||
|
||||
export const appConfig: ApplicationConfig = {
|
||||
providers: [
|
||||
provideRouter(routes),
|
||||
provideHttpClient(),
|
||||
provideHttpClient(withInterceptorsFromDi()),
|
||||
{
|
||||
provide: APP_INITIALIZER,
|
||||
multi: true,
|
||||
useFactory: (configService: AppConfigService) => () =>
|
||||
configService.load(),
|
||||
deps: [AppConfigService],
|
||||
},
|
||||
{
|
||||
provide: HTTP_INTERCEPTORS,
|
||||
useClass: AuthHttpInterceptor,
|
||||
multi: true,
|
||||
},
|
||||
{
|
||||
provide: CONCELIER_EXPORTER_API_BASE_URL,
|
||||
useValue: '/api/v1/concelier/exporters/trivy-db',
|
||||
|
||||
@@ -8,6 +8,20 @@ export const routes: Routes = [
|
||||
(m) => m.TrivyDbSettingsPageComponent
|
||||
),
|
||||
},
|
||||
{
|
||||
path: 'scans/:scanId',
|
||||
loadComponent: () =>
|
||||
import('./features/scans/scan-detail-page.component').then(
|
||||
(m) => m.ScanDetailPageComponent
|
||||
),
|
||||
},
|
||||
{
|
||||
path: 'auth/callback',
|
||||
loadComponent: () =>
|
||||
import('./features/auth/auth-callback.component').then(
|
||||
(m) => m.AuthCallbackComponent
|
||||
),
|
||||
},
|
||||
{
|
||||
path: '',
|
||||
pathMatch: 'full',
|
||||
|
||||
17
src/StellaOps.Web/src/app/core/api/scanner.models.ts
Normal file
17
src/StellaOps.Web/src/app/core/api/scanner.models.ts
Normal file
@@ -0,0 +1,17 @@
|
||||
export type ScanAttestationStatusKind = 'verified' | 'pending' | 'failed';
|
||||
|
||||
export interface ScanAttestationStatus {
|
||||
readonly uuid: string;
|
||||
readonly status: ScanAttestationStatusKind;
|
||||
readonly index?: number;
|
||||
readonly logUrl?: string;
|
||||
readonly checkedAt?: string;
|
||||
readonly statusMessage?: string;
|
||||
}
|
||||
|
||||
export interface ScanDetail {
|
||||
readonly scanId: string;
|
||||
readonly imageDigest: string;
|
||||
readonly completedAt: string;
|
||||
readonly attestation?: ScanAttestationStatus;
|
||||
}
|
||||
171
src/StellaOps.Web/src/app/core/auth/auth-http.interceptor.ts
Normal file
171
src/StellaOps.Web/src/app/core/auth/auth-http.interceptor.ts
Normal file
@@ -0,0 +1,171 @@
|
||||
import {
|
||||
HttpErrorResponse,
|
||||
HttpEvent,
|
||||
HttpHandler,
|
||||
HttpInterceptor,
|
||||
HttpRequest,
|
||||
} from '@angular/common/http';
|
||||
import { Injectable } from '@angular/core';
|
||||
import { Observable, firstValueFrom, from, throwError } from 'rxjs';
|
||||
import { catchError, switchMap } from 'rxjs/operators';
|
||||
|
||||
import { AppConfigService } from '../config/app-config.service';
|
||||
import { DpopService } from './dpop/dpop.service';
|
||||
import { AuthorityAuthService } from './authority-auth.service';
|
||||
|
||||
const RETRY_HEADER = 'X-StellaOps-DPoP-Retry';
|
||||
|
||||
@Injectable()
|
||||
export class AuthHttpInterceptor implements HttpInterceptor {
|
||||
private excludedOrigins: Set<string> | null = null;
|
||||
private tokenEndpoint: string | null = null;
|
||||
private authorityResolved = false;
|
||||
|
||||
constructor(
|
||||
private readonly auth: AuthorityAuthService,
|
||||
private readonly config: AppConfigService,
|
||||
private readonly dpop: DpopService
|
||||
) {
|
||||
// lazy resolve authority configuration in intercept to allow APP_INITIALIZER to run first
|
||||
}
|
||||
|
||||
intercept(
|
||||
request: HttpRequest<unknown>,
|
||||
next: HttpHandler
|
||||
): Observable<HttpEvent<unknown>> {
|
||||
this.ensureAuthorityInfo();
|
||||
|
||||
if (request.headers.has('Authorization') || this.shouldSkip(request.url)) {
|
||||
return next.handle(request);
|
||||
}
|
||||
|
||||
return from(
|
||||
this.auth.getAuthHeadersForRequest(
|
||||
this.resolveAbsoluteUrl(request.url),
|
||||
request.method
|
||||
)
|
||||
).pipe(
|
||||
switchMap((headers) => {
|
||||
if (!headers) {
|
||||
return next.handle(request);
|
||||
}
|
||||
const authorizedRequest = request.clone({
|
||||
setHeaders: {
|
||||
Authorization: headers.authorization,
|
||||
DPoP: headers.dpop,
|
||||
},
|
||||
headers: request.headers.set(RETRY_HEADER, '0'),
|
||||
});
|
||||
return next.handle(authorizedRequest);
|
||||
}),
|
||||
catchError((error: HttpErrorResponse) =>
|
||||
this.handleError(request, error, next)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
private handleError(
|
||||
request: HttpRequest<unknown>,
|
||||
error: HttpErrorResponse,
|
||||
next: HttpHandler
|
||||
): Observable<HttpEvent<unknown>> {
|
||||
if (error.status !== 401) {
|
||||
return throwError(() => error);
|
||||
}
|
||||
|
||||
const nonce = error.headers?.get('DPoP-Nonce');
|
||||
if (!nonce) {
|
||||
return throwError(() => error);
|
||||
}
|
||||
|
||||
if (request.headers.get(RETRY_HEADER) === '1') {
|
||||
return throwError(() => error);
|
||||
}
|
||||
|
||||
return from(this.retryWithNonce(request, nonce, next)).pipe(
|
||||
catchError(() => throwError(() => error))
|
||||
);
|
||||
}
|
||||
|
||||
private async retryWithNonce(
|
||||
request: HttpRequest<unknown>,
|
||||
nonce: string,
|
||||
next: HttpHandler
|
||||
): Promise<HttpEvent<unknown>> {
|
||||
await this.dpop.setNonce(nonce);
|
||||
const headers = await this.auth.getAuthHeadersForRequest(
|
||||
this.resolveAbsoluteUrl(request.url),
|
||||
request.method
|
||||
);
|
||||
if (!headers) {
|
||||
throw new Error('Unable to refresh authorization headers after nonce.');
|
||||
}
|
||||
|
||||
const retried = request.clone({
|
||||
setHeaders: {
|
||||
Authorization: headers.authorization,
|
||||
DPoP: headers.dpop,
|
||||
},
|
||||
headers: request.headers.set(RETRY_HEADER, '1'),
|
||||
});
|
||||
|
||||
return firstValueFrom(next.handle(retried));
|
||||
}
|
||||
|
||||
private shouldSkip(url: string): boolean {
|
||||
this.ensureAuthorityInfo();
|
||||
const absolute = this.resolveAbsoluteUrl(url);
|
||||
if (!absolute) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
const resolved = new URL(absolute);
|
||||
if (resolved.pathname.endsWith('/config.json')) {
|
||||
return true;
|
||||
}
|
||||
if (this.tokenEndpoint && absolute.startsWith(this.tokenEndpoint)) {
|
||||
return true;
|
||||
}
|
||||
const origin = resolved.origin;
|
||||
return this.excludedOrigins?.has(origin) ?? false;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private resolveAbsoluteUrl(url: string): string {
|
||||
try {
|
||||
if (url.startsWith('http://') || url.startsWith('https://')) {
|
||||
return url;
|
||||
}
|
||||
const base =
|
||||
typeof window !== 'undefined' && window.location
|
||||
? window.location.origin
|
||||
: undefined;
|
||||
return base ? new URL(url, base).toString() : url;
|
||||
} catch {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
private ensureAuthorityInfo(): void {
|
||||
if (this.authorityResolved) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const authority = this.config.authority;
|
||||
this.tokenEndpoint = new URL(
|
||||
authority.tokenEndpoint,
|
||||
authority.issuer
|
||||
).toString();
|
||||
this.excludedOrigins = new Set<string>([
|
||||
this.tokenEndpoint,
|
||||
new URL(authority.authorizeEndpoint, authority.issuer).origin,
|
||||
]);
|
||||
this.authorityResolved = true;
|
||||
} catch {
|
||||
// Configuration not yet loaded; interceptor will retry on the next request.
|
||||
}
|
||||
}
|
||||
}
|
||||
49
src/StellaOps.Web/src/app/core/auth/auth-session.model.ts
Normal file
49
src/StellaOps.Web/src/app/core/auth/auth-session.model.ts
Normal file
@@ -0,0 +1,49 @@
|
||||
export interface AuthTokens {
|
||||
readonly accessToken: string;
|
||||
readonly expiresAtEpochMs: number;
|
||||
readonly refreshToken?: string;
|
||||
readonly tokenType: 'Bearer';
|
||||
readonly scope: string;
|
||||
}
|
||||
|
||||
export interface AuthIdentity {
|
||||
readonly subject: string;
|
||||
readonly name?: string;
|
||||
readonly email?: string;
|
||||
readonly roles: readonly string[];
|
||||
readonly idToken?: string;
|
||||
}
|
||||
|
||||
export interface AuthSession {
|
||||
readonly tokens: AuthTokens;
|
||||
readonly identity: AuthIdentity;
|
||||
/**
|
||||
* SHA-256 JWK thumbprint of the active DPoP key pair.
|
||||
*/
|
||||
readonly dpopKeyThumbprint: string;
|
||||
readonly issuedAtEpochMs: number;
|
||||
}
|
||||
|
||||
export interface PersistedSessionMetadata {
|
||||
readonly subject: string;
|
||||
readonly expiresAtEpochMs: number;
|
||||
readonly issuedAtEpochMs: number;
|
||||
readonly dpopKeyThumbprint: string;
|
||||
}
|
||||
|
||||
export type AuthStatus =
|
||||
| 'unauthenticated'
|
||||
| 'authenticated'
|
||||
| 'refreshing'
|
||||
| 'loading';
|
||||
|
||||
export const ACCESS_TOKEN_REFRESH_THRESHOLD_MS = 60_000;
|
||||
|
||||
export const SESSION_STORAGE_KEY = 'stellaops.auth.session.info';
|
||||
|
||||
export type AuthErrorReason =
|
||||
| 'invalid_state'
|
||||
| 'token_exchange_failed'
|
||||
| 'refresh_failed'
|
||||
| 'dpop_generation_failed'
|
||||
| 'configuration_missing';
|
||||
@@ -0,0 +1,48 @@
|
||||
import { TestBed } from '@angular/core/testing';
|
||||
|
||||
import { AuthSession, AuthTokens, SESSION_STORAGE_KEY } from './auth-session.model';
|
||||
import { AuthSessionStore } from './auth-session.store';
|
||||
|
||||
describe('AuthSessionStore', () => {
|
||||
let store: AuthSessionStore;
|
||||
|
||||
beforeEach(() => {
|
||||
sessionStorage.clear();
|
||||
TestBed.configureTestingModule({
|
||||
providers: [AuthSessionStore],
|
||||
});
|
||||
store = TestBed.inject(AuthSessionStore);
|
||||
});
|
||||
|
||||
it('persists minimal metadata when session is set', () => {
|
||||
const tokens: AuthTokens = {
|
||||
accessToken: 'token-abc',
|
||||
expiresAtEpochMs: Date.now() + 120_000,
|
||||
refreshToken: 'refresh-xyz',
|
||||
scope: 'openid ui.read',
|
||||
tokenType: 'Bearer',
|
||||
};
|
||||
|
||||
const session: AuthSession = {
|
||||
tokens,
|
||||
identity: {
|
||||
subject: 'user-123',
|
||||
name: 'Alex Operator',
|
||||
roles: ['ui.read'],
|
||||
},
|
||||
dpopKeyThumbprint: 'thumbprint-1',
|
||||
issuedAtEpochMs: Date.now(),
|
||||
};
|
||||
|
||||
store.setSession(session);
|
||||
|
||||
const persisted = sessionStorage.getItem(SESSION_STORAGE_KEY);
|
||||
expect(persisted).toBeTruthy();
|
||||
const parsed = JSON.parse(persisted ?? '{}');
|
||||
expect(parsed.subject).toBe('user-123');
|
||||
expect(parsed.dpopKeyThumbprint).toBe('thumbprint-1');
|
||||
|
||||
store.clear();
|
||||
expect(sessionStorage.getItem(SESSION_STORAGE_KEY)).toBeNull();
|
||||
});
|
||||
});
|
||||
107
src/StellaOps.Web/src/app/core/auth/auth-session.store.ts
Normal file
107
src/StellaOps.Web/src/app/core/auth/auth-session.store.ts
Normal file
@@ -0,0 +1,107 @@
|
||||
import { Injectable, computed, signal } from '@angular/core';
|
||||
|
||||
import {
|
||||
AuthSession,
|
||||
AuthStatus,
|
||||
PersistedSessionMetadata,
|
||||
SESSION_STORAGE_KEY,
|
||||
} from './auth-session.model';
|
||||
|
||||
@Injectable({
|
||||
providedIn: 'root',
|
||||
})
|
||||
export class AuthSessionStore {
|
||||
private readonly sessionSignal = signal<AuthSession | null>(null);
|
||||
private readonly statusSignal = signal<AuthStatus>('unauthenticated');
|
||||
private readonly persistedSignal =
|
||||
signal<PersistedSessionMetadata | null>(this.readPersistedMetadata());
|
||||
|
||||
readonly session = computed(() => this.sessionSignal());
|
||||
readonly status = computed(() => this.statusSignal());
|
||||
|
||||
readonly identity = computed(() => this.sessionSignal()?.identity ?? null);
|
||||
readonly subjectHint = computed(
|
||||
() =>
|
||||
this.sessionSignal()?.identity.subject ??
|
||||
this.persistedSignal()?.subject ??
|
||||
null
|
||||
);
|
||||
|
||||
readonly expiresAtEpochMs = computed(
|
||||
() => this.sessionSignal()?.tokens.expiresAtEpochMs ?? null
|
||||
);
|
||||
|
||||
readonly isAuthenticated = computed(
|
||||
() => this.sessionSignal() !== null && this.statusSignal() !== 'loading'
|
||||
);
|
||||
|
||||
setStatus(status: AuthStatus): void {
|
||||
this.statusSignal.set(status);
|
||||
}
|
||||
|
||||
setSession(session: AuthSession | null): void {
|
||||
this.sessionSignal.set(session);
|
||||
if (!session) {
|
||||
this.statusSignal.set('unauthenticated');
|
||||
this.persistedSignal.set(null);
|
||||
this.clearPersistedMetadata();
|
||||
return;
|
||||
}
|
||||
|
||||
this.statusSignal.set('authenticated');
|
||||
const metadata: PersistedSessionMetadata = {
|
||||
subject: session.identity.subject,
|
||||
expiresAtEpochMs: session.tokens.expiresAtEpochMs,
|
||||
issuedAtEpochMs: session.issuedAtEpochMs,
|
||||
dpopKeyThumbprint: session.dpopKeyThumbprint,
|
||||
};
|
||||
this.persistedSignal.set(metadata);
|
||||
this.persistMetadata(metadata);
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.sessionSignal.set(null);
|
||||
this.statusSignal.set('unauthenticated');
|
||||
this.persistedSignal.set(null);
|
||||
this.clearPersistedMetadata();
|
||||
}
|
||||
|
||||
private readPersistedMetadata(): PersistedSessionMetadata | null {
|
||||
if (typeof sessionStorage === 'undefined') {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const raw = sessionStorage.getItem(SESSION_STORAGE_KEY);
|
||||
if (!raw) {
|
||||
return null;
|
||||
}
|
||||
const parsed = JSON.parse(raw) as PersistedSessionMetadata;
|
||||
if (
|
||||
typeof parsed.subject !== 'string' ||
|
||||
typeof parsed.expiresAtEpochMs !== 'number' ||
|
||||
typeof parsed.issuedAtEpochMs !== 'number' ||
|
||||
typeof parsed.dpopKeyThumbprint !== 'string'
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return parsed;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private persistMetadata(metadata: PersistedSessionMetadata): void {
|
||||
if (typeof sessionStorage === 'undefined') {
|
||||
return;
|
||||
}
|
||||
sessionStorage.setItem(SESSION_STORAGE_KEY, JSON.stringify(metadata));
|
||||
}
|
||||
|
||||
private clearPersistedMetadata(): void {
|
||||
if (typeof sessionStorage === 'undefined') {
|
||||
return;
|
||||
}
|
||||
sessionStorage.removeItem(SESSION_STORAGE_KEY);
|
||||
}
|
||||
}
|
||||
45
src/StellaOps.Web/src/app/core/auth/auth-storage.service.ts
Normal file
45
src/StellaOps.Web/src/app/core/auth/auth-storage.service.ts
Normal file
@@ -0,0 +1,45 @@
|
||||
import { Injectable } from '@angular/core';
|
||||
|
||||
const LOGIN_REQUEST_KEY = 'stellaops.auth.login.request';
|
||||
|
||||
export interface PendingLoginRequest {
|
||||
readonly state: string;
|
||||
readonly codeVerifier: string;
|
||||
readonly createdAtEpochMs: number;
|
||||
readonly returnUrl?: string;
|
||||
readonly nonce?: string;
|
||||
}
|
||||
|
||||
@Injectable({
|
||||
providedIn: 'root',
|
||||
})
|
||||
export class AuthStorageService {
|
||||
savePendingLogin(request: PendingLoginRequest): void {
|
||||
if (typeof sessionStorage === 'undefined') {
|
||||
return;
|
||||
}
|
||||
sessionStorage.setItem(LOGIN_REQUEST_KEY, JSON.stringify(request));
|
||||
}
|
||||
|
||||
consumePendingLogin(expectedState: string): PendingLoginRequest | null {
|
||||
if (typeof sessionStorage === 'undefined') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const raw = sessionStorage.getItem(LOGIN_REQUEST_KEY);
|
||||
if (!raw) {
|
||||
return null;
|
||||
}
|
||||
|
||||
sessionStorage.removeItem(LOGIN_REQUEST_KEY);
|
||||
try {
|
||||
const request = JSON.parse(raw) as PendingLoginRequest;
|
||||
if (request.state !== expectedState) {
|
||||
return null;
|
||||
}
|
||||
return request;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
430
src/StellaOps.Web/src/app/core/auth/authority-auth.service.ts
Normal file
430
src/StellaOps.Web/src/app/core/auth/authority-auth.service.ts
Normal file
@@ -0,0 +1,430 @@
|
||||
import { HttpClient, HttpHeaders, HttpResponse } from '@angular/common/http';
|
||||
import { Injectable } from '@angular/core';
|
||||
import { firstValueFrom } from 'rxjs';
|
||||
|
||||
import { AppConfigService } from '../config/app-config.service';
|
||||
import { AuthorityConfig } from '../config/app-config.model';
|
||||
import {
|
||||
ACCESS_TOKEN_REFRESH_THRESHOLD_MS,
|
||||
AuthErrorReason,
|
||||
AuthSession,
|
||||
AuthTokens,
|
||||
} from './auth-session.model';
|
||||
import { AuthSessionStore } from './auth-session.store';
|
||||
import {
|
||||
AuthStorageService,
|
||||
PendingLoginRequest,
|
||||
} from './auth-storage.service';
|
||||
import { DpopService } from './dpop/dpop.service';
|
||||
import { base64UrlDecode } from './dpop/jose-utilities';
|
||||
import { createPkcePair } from './pkce.util';
|
||||
|
||||
interface TokenResponse {
|
||||
readonly access_token: string;
|
||||
readonly token_type: string;
|
||||
readonly expires_in: number;
|
||||
readonly scope?: string;
|
||||
readonly refresh_token?: string;
|
||||
readonly id_token?: string;
|
||||
}
|
||||
|
||||
interface RefreshTokenResponse extends TokenResponse {}
|
||||
|
||||
export interface AuthorizationHeaders {
|
||||
readonly authorization: string;
|
||||
readonly dpop: string;
|
||||
}
|
||||
|
||||
export interface CompleteLoginResult {
|
||||
readonly returnUrl?: string;
|
||||
}
|
||||
|
||||
const TOKEN_CONTENT_TYPE = 'application/x-www-form-urlencoded';
|
||||
|
||||
@Injectable({
|
||||
providedIn: 'root',
|
||||
})
|
||||
export class AuthorityAuthService {
|
||||
private refreshTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private refreshInFlight: Promise<void> | null = null;
|
||||
private lastError: AuthErrorReason | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly http: HttpClient,
|
||||
private readonly config: AppConfigService,
|
||||
private readonly sessionStore: AuthSessionStore,
|
||||
private readonly storage: AuthStorageService,
|
||||
private readonly dpop: DpopService
|
||||
) {}
|
||||
|
||||
get error(): AuthErrorReason | null {
|
||||
return this.lastError;
|
||||
}
|
||||
|
||||
async beginLogin(returnUrl?: string): Promise<void> {
|
||||
const authority = this.config.authority;
|
||||
const pkce = await createPkcePair();
|
||||
const state = crypto.randomUUID ? crypto.randomUUID() : createRandomId();
|
||||
const nonce = crypto.randomUUID ? crypto.randomUUID() : createRandomId();
|
||||
|
||||
// Generate the DPoP key pair up-front so the same key is bound to the token.
|
||||
await this.dpop.getThumbprint();
|
||||
|
||||
const authorizeUrl = this.buildAuthorizeUrl(authority, {
|
||||
state,
|
||||
nonce,
|
||||
codeChallenge: pkce.challenge,
|
||||
codeChallengeMethod: pkce.method,
|
||||
returnUrl,
|
||||
});
|
||||
|
||||
const now = Date.now();
|
||||
this.storage.savePendingLogin({
|
||||
state,
|
||||
codeVerifier: pkce.verifier,
|
||||
createdAtEpochMs: now,
|
||||
returnUrl,
|
||||
nonce,
|
||||
});
|
||||
|
||||
window.location.assign(authorizeUrl);
|
||||
}
|
||||
|
||||
/**
|
||||
* Completes the authorization code flow after the Authority redirects back with ?code & ?state.
|
||||
*/
|
||||
async completeLoginFromRedirect(
|
||||
queryParams: URLSearchParams
|
||||
): Promise<CompleteLoginResult> {
|
||||
const code = queryParams.get('code');
|
||||
const state = queryParams.get('state');
|
||||
if (!code || !state) {
|
||||
throw new Error('Missing authorization code or state.');
|
||||
}
|
||||
|
||||
const pending = this.storage.consumePendingLogin(state);
|
||||
if (!pending) {
|
||||
this.lastError = 'invalid_state';
|
||||
throw new Error('State parameter did not match pending login request.');
|
||||
}
|
||||
|
||||
try {
|
||||
const tokenResponse = await this.exchangeCodeForTokens(
|
||||
code,
|
||||
pending.codeVerifier
|
||||
);
|
||||
await this.onTokenResponse(tokenResponse, pending.nonce ?? null);
|
||||
this.lastError = null;
|
||||
return { returnUrl: pending.returnUrl };
|
||||
} catch (error) {
|
||||
this.lastError = 'token_exchange_failed';
|
||||
this.sessionStore.clear();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async ensureValidAccessToken(): Promise<string | null> {
|
||||
const session = this.sessionStore.session();
|
||||
if (!session) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
if (now < session.tokens.expiresAtEpochMs - ACCESS_TOKEN_REFRESH_THRESHOLD_MS) {
|
||||
return session.tokens.accessToken;
|
||||
}
|
||||
|
||||
await this.refreshAccessToken();
|
||||
const refreshed = this.sessionStore.session();
|
||||
return refreshed?.tokens.accessToken ?? null;
|
||||
}
|
||||
|
||||
async getAuthHeadersForRequest(
|
||||
url: string,
|
||||
method: string
|
||||
): Promise<AuthorizationHeaders | null> {
|
||||
const accessToken = await this.ensureValidAccessToken();
|
||||
if (!accessToken) {
|
||||
return null;
|
||||
}
|
||||
const dpopProof = await this.dpop.createProof({
|
||||
htm: method,
|
||||
htu: url,
|
||||
accessToken,
|
||||
});
|
||||
return {
|
||||
authorization: `DPoP ${accessToken}`,
|
||||
dpop: dpopProof,
|
||||
};
|
||||
}
|
||||
|
||||
async refreshAccessToken(): Promise<void> {
|
||||
const session = this.sessionStore.session();
|
||||
const refreshToken = session?.tokens.refreshToken;
|
||||
if (!refreshToken) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.refreshInFlight) {
|
||||
await this.refreshInFlight;
|
||||
return;
|
||||
}
|
||||
|
||||
this.refreshInFlight = this.executeRefresh(refreshToken)
|
||||
.catch((error) => {
|
||||
this.lastError = 'refresh_failed';
|
||||
this.sessionStore.clear();
|
||||
throw error;
|
||||
})
|
||||
.finally(() => {
|
||||
this.refreshInFlight = null;
|
||||
});
|
||||
|
||||
await this.refreshInFlight;
|
||||
}
|
||||
|
||||
async logout(): Promise<void> {
|
||||
const session = this.sessionStore.session();
|
||||
this.cancelRefreshTimer();
|
||||
this.sessionStore.clear();
|
||||
await this.dpop.setNonce(null);
|
||||
|
||||
const authority = this.config.authority;
|
||||
if (!authority.logoutEndpoint) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (session?.identity.idToken) {
|
||||
const url = new URL(authority.logoutEndpoint, authority.issuer);
|
||||
url.searchParams.set('post_logout_redirect_uri', authority.postLogoutRedirectUri ?? authority.redirectUri);
|
||||
url.searchParams.set('id_token_hint', session.identity.idToken);
|
||||
window.location.assign(url.toString());
|
||||
} else {
|
||||
window.location.assign(authority.postLogoutRedirectUri ?? authority.redirectUri);
|
||||
}
|
||||
}
|
||||
|
||||
private async exchangeCodeForTokens(
|
||||
code: string,
|
||||
codeVerifier: string
|
||||
): Promise<HttpResponse<TokenResponse>> {
|
||||
const authority = this.config.authority;
|
||||
const tokenUrl = new URL(authority.tokenEndpoint, authority.issuer).toString();
|
||||
|
||||
const body = new URLSearchParams();
|
||||
body.set('grant_type', 'authorization_code');
|
||||
body.set('code', code);
|
||||
body.set('redirect_uri', authority.redirectUri);
|
||||
body.set('client_id', authority.clientId);
|
||||
body.set('code_verifier', codeVerifier);
|
||||
if (authority.audience) {
|
||||
body.set('audience', authority.audience);
|
||||
}
|
||||
|
||||
const dpopProof = await this.dpop.createProof({
|
||||
htm: 'POST',
|
||||
htu: tokenUrl,
|
||||
});
|
||||
|
||||
const headers = new HttpHeaders({
|
||||
'Content-Type': TOKEN_CONTENT_TYPE,
|
||||
DPoP: dpopProof,
|
||||
});
|
||||
|
||||
return firstValueFrom(
|
||||
this.http.post<TokenResponse>(tokenUrl, body.toString(), {
|
||||
headers,
|
||||
withCredentials: true,
|
||||
observe: 'response',
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
private async executeRefresh(refreshToken: string): Promise<void> {
|
||||
const authority = this.config.authority;
|
||||
const tokenUrl = new URL(authority.tokenEndpoint, authority.issuer).toString();
|
||||
const body = new URLSearchParams();
|
||||
body.set('grant_type', 'refresh_token');
|
||||
body.set('refresh_token', refreshToken);
|
||||
body.set('client_id', authority.clientId);
|
||||
if (authority.audience) {
|
||||
body.set('audience', authority.audience);
|
||||
}
|
||||
|
||||
const proof = await this.dpop.createProof({
|
||||
htm: 'POST',
|
||||
htu: tokenUrl,
|
||||
});
|
||||
|
||||
const headers = new HttpHeaders({
|
||||
'Content-Type': TOKEN_CONTENT_TYPE,
|
||||
DPoP: proof,
|
||||
});
|
||||
|
||||
const response = await firstValueFrom(
|
||||
this.http.post<RefreshTokenResponse>(tokenUrl, body.toString(), {
|
||||
headers,
|
||||
withCredentials: true,
|
||||
observe: 'response',
|
||||
})
|
||||
);
|
||||
|
||||
await this.onTokenResponse(response, null);
|
||||
}
|
||||
|
||||
private async onTokenResponse(
|
||||
response: HttpResponse<TokenResponse>,
|
||||
expectedNonce: string | null
|
||||
): Promise<void> {
|
||||
const nonce = response.headers.get('DPoP-Nonce');
|
||||
if (nonce) {
|
||||
await this.dpop.setNonce(nonce);
|
||||
}
|
||||
|
||||
const payload = response.body;
|
||||
if (!payload) {
|
||||
throw new Error('Token response did not include a body.');
|
||||
}
|
||||
|
||||
const tokens = this.toAuthTokens(payload);
|
||||
const identity = this.parseIdentity(payload.id_token ?? '', expectedNonce);
|
||||
const thumbprint = await this.dpop.getThumbprint();
|
||||
if (!thumbprint) {
|
||||
throw new Error('DPoP thumbprint unavailable.');
|
||||
}
|
||||
|
||||
const session: AuthSession = {
|
||||
tokens,
|
||||
identity,
|
||||
dpopKeyThumbprint: thumbprint,
|
||||
issuedAtEpochMs: Date.now(),
|
||||
};
|
||||
this.sessionStore.setSession(session);
|
||||
this.scheduleRefresh(tokens, this.config.authority);
|
||||
}
|
||||
|
||||
private toAuthTokens(payload: TokenResponse): AuthTokens {
|
||||
const expiresAtEpochMs = Date.now() + payload.expires_in * 1000;
|
||||
return {
|
||||
accessToken: payload.access_token,
|
||||
tokenType: (payload.token_type ?? 'Bearer') as 'Bearer',
|
||||
refreshToken: payload.refresh_token,
|
||||
scope: payload.scope ?? '',
|
||||
expiresAtEpochMs,
|
||||
};
|
||||
}
|
||||
|
||||
private parseIdentity(
|
||||
idToken: string,
|
||||
expectedNonce: string | null
|
||||
): AuthSession['identity'] {
|
||||
if (!idToken) {
|
||||
return {
|
||||
subject: 'unknown',
|
||||
roles: [],
|
||||
};
|
||||
}
|
||||
|
||||
const claims = decodeJwt(idToken);
|
||||
const nonceClaim = claims['nonce'];
|
||||
if (
|
||||
expectedNonce &&
|
||||
typeof nonceClaim === 'string' &&
|
||||
nonceClaim !== expectedNonce
|
||||
) {
|
||||
throw new Error('OIDC nonce mismatch.');
|
||||
}
|
||||
|
||||
const subjectClaim = claims['sub'];
|
||||
const nameClaim = claims['name'];
|
||||
const emailClaim = claims['email'];
|
||||
const rolesClaim = claims['role'];
|
||||
|
||||
return {
|
||||
subject: typeof subjectClaim === 'string' ? subjectClaim : 'unknown',
|
||||
name: typeof nameClaim === 'string' ? nameClaim : undefined,
|
||||
email: typeof emailClaim === 'string' ? emailClaim : undefined,
|
||||
roles: Array.isArray(rolesClaim)
|
||||
? rolesClaim.filter((entry: unknown): entry is string =>
|
||||
typeof entry === 'string'
|
||||
)
|
||||
: [],
|
||||
idToken,
|
||||
};
|
||||
}
|
||||
|
||||
private scheduleRefresh(tokens: AuthTokens, authority: AuthorityConfig): void {
|
||||
this.cancelRefreshTimer();
|
||||
const leeway =
|
||||
(authority.refreshLeewaySeconds ?? 60) * 1000 +
|
||||
ACCESS_TOKEN_REFRESH_THRESHOLD_MS;
|
||||
const now = Date.now();
|
||||
const ttl = Math.max(tokens.expiresAtEpochMs - now - leeway, 5_000);
|
||||
this.refreshTimer = setTimeout(() => {
|
||||
void this.refreshAccessToken();
|
||||
}, ttl);
|
||||
}
|
||||
|
||||
private cancelRefreshTimer(): void {
|
||||
if (this.refreshTimer) {
|
||||
clearTimeout(this.refreshTimer);
|
||||
this.refreshTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
private buildAuthorizeUrl(
|
||||
authority: AuthorityConfig,
|
||||
options: {
|
||||
state: string;
|
||||
nonce: string;
|
||||
codeChallenge: string;
|
||||
codeChallengeMethod: 'S256';
|
||||
returnUrl?: string;
|
||||
}
|
||||
): string {
|
||||
const authorizeUrl = new URL(
|
||||
authority.authorizeEndpoint,
|
||||
authority.issuer
|
||||
);
|
||||
authorizeUrl.searchParams.set('response_type', 'code');
|
||||
authorizeUrl.searchParams.set('client_id', authority.clientId);
|
||||
authorizeUrl.searchParams.set('redirect_uri', authority.redirectUri);
|
||||
authorizeUrl.searchParams.set('scope', authority.scope);
|
||||
authorizeUrl.searchParams.set('state', options.state);
|
||||
authorizeUrl.searchParams.set('nonce', options.nonce);
|
||||
authorizeUrl.searchParams.set('code_challenge', options.codeChallenge);
|
||||
authorizeUrl.searchParams.set(
|
||||
'code_challenge_method',
|
||||
options.codeChallengeMethod
|
||||
);
|
||||
if (authority.audience) {
|
||||
authorizeUrl.searchParams.set('audience', authority.audience);
|
||||
}
|
||||
if (options.returnUrl) {
|
||||
authorizeUrl.searchParams.set('ui_return', options.returnUrl);
|
||||
}
|
||||
return authorizeUrl.toString();
|
||||
}
|
||||
}
|
||||
|
||||
function decodeJwt(token: string): Record<string, unknown> {
|
||||
const parts = token.split('.');
|
||||
if (parts.length < 2) {
|
||||
return {};
|
||||
}
|
||||
const payload = base64UrlDecode(parts[1]);
|
||||
const json = new TextDecoder().decode(payload);
|
||||
try {
|
||||
return JSON.parse(json) as Record<string, unknown>;
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function createRandomId(): string {
|
||||
const array = new Uint8Array(16);
|
||||
crypto.getRandomValues(array);
|
||||
return Array.from(array, (value) =>
|
||||
value.toString(16).padStart(2, '0')
|
||||
).join('');
|
||||
}
|
||||
181
src/StellaOps.Web/src/app/core/auth/dpop/dpop-key-store.ts
Normal file
181
src/StellaOps.Web/src/app/core/auth/dpop/dpop-key-store.ts
Normal file
@@ -0,0 +1,181 @@
|
||||
import { Injectable } from '@angular/core';
|
||||
|
||||
import { DPoPAlgorithm } from '../../config/app-config.model';
|
||||
import { computeJwkThumbprint } from './jose-utilities';
|
||||
|
||||
const DB_NAME = 'stellaops-auth';
|
||||
const STORE_NAME = 'dpopKeys';
|
||||
const PRIMARY_KEY = 'primary';
|
||||
const DB_VERSION = 1;
|
||||
|
||||
interface PersistedKeyPair {
|
||||
readonly id: string;
|
||||
readonly algorithm: DPoPAlgorithm;
|
||||
readonly publicJwk: JsonWebKey;
|
||||
readonly privateJwk: JsonWebKey;
|
||||
readonly thumbprint: string;
|
||||
readonly createdAtIso: string;
|
||||
}
|
||||
|
||||
export interface LoadedDpopKeyPair {
|
||||
readonly algorithm: DPoPAlgorithm;
|
||||
readonly privateKey: CryptoKey;
|
||||
readonly publicKey: CryptoKey;
|
||||
readonly publicJwk: JsonWebKey;
|
||||
readonly thumbprint: string;
|
||||
}
|
||||
|
||||
@Injectable({
|
||||
providedIn: 'root',
|
||||
})
|
||||
export class DpopKeyStore {
|
||||
private dbPromise: Promise<IDBDatabase> | null = null;
|
||||
|
||||
async load(): Promise<LoadedDpopKeyPair | null> {
|
||||
const record = await this.read();
|
||||
if (!record) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const [privateKey, publicKey] = await Promise.all([
|
||||
crypto.subtle.importKey(
|
||||
'jwk',
|
||||
record.privateJwk,
|
||||
this.toKeyAlgorithm(record.algorithm),
|
||||
true,
|
||||
['sign']
|
||||
),
|
||||
crypto.subtle.importKey(
|
||||
'jwk',
|
||||
record.publicJwk,
|
||||
this.toKeyAlgorithm(record.algorithm),
|
||||
true,
|
||||
['verify']
|
||||
),
|
||||
]);
|
||||
|
||||
return {
|
||||
algorithm: record.algorithm,
|
||||
privateKey,
|
||||
publicKey,
|
||||
publicJwk: record.publicJwk,
|
||||
thumbprint: record.thumbprint,
|
||||
};
|
||||
}
|
||||
|
||||
async save(
|
||||
keyPair: CryptoKeyPair,
|
||||
algorithm: DPoPAlgorithm
|
||||
): Promise<LoadedDpopKeyPair> {
|
||||
const [publicJwk, privateJwk] = await Promise.all([
|
||||
crypto.subtle.exportKey('jwk', keyPair.publicKey),
|
||||
crypto.subtle.exportKey('jwk', keyPair.privateKey),
|
||||
]);
|
||||
|
||||
if (!publicJwk) {
|
||||
throw new Error('Failed to export public JWK for DPoP key pair.');
|
||||
}
|
||||
|
||||
const thumbprint = await computeJwkThumbprint(publicJwk);
|
||||
const record: PersistedKeyPair = {
|
||||
id: PRIMARY_KEY,
|
||||
algorithm,
|
||||
publicJwk,
|
||||
privateJwk,
|
||||
thumbprint,
|
||||
createdAtIso: new Date().toISOString(),
|
||||
};
|
||||
|
||||
await this.write(record);
|
||||
|
||||
return {
|
||||
algorithm,
|
||||
privateKey: keyPair.privateKey,
|
||||
publicKey: keyPair.publicKey,
|
||||
publicJwk,
|
||||
thumbprint,
|
||||
};
|
||||
}
|
||||
|
||||
async clear(): Promise<void> {
|
||||
const db = await this.openDb();
|
||||
await transactionPromise(db, STORE_NAME, 'readwrite', (store) =>
|
||||
store.delete(PRIMARY_KEY)
|
||||
);
|
||||
}
|
||||
|
||||
async generate(algorithm: DPoPAlgorithm): Promise<LoadedDpopKeyPair> {
|
||||
const algo = this.toKeyAlgorithm(algorithm);
|
||||
const keyPair = await crypto.subtle.generateKey(algo, true, [
|
||||
'sign',
|
||||
'verify',
|
||||
]);
|
||||
|
||||
const stored = await this.save(keyPair, algorithm);
|
||||
return stored;
|
||||
}
|
||||
|
||||
private async read(): Promise<PersistedKeyPair | null> {
|
||||
const db = await this.openDb();
|
||||
return transactionPromise(db, STORE_NAME, 'readonly', (store) =>
|
||||
store.get(PRIMARY_KEY)
|
||||
);
|
||||
}
|
||||
|
||||
private async write(record: PersistedKeyPair): Promise<void> {
|
||||
const db = await this.openDb();
|
||||
await transactionPromise(db, STORE_NAME, 'readwrite', (store) =>
|
||||
store.put(record)
|
||||
);
|
||||
}
|
||||
|
||||
private toKeyAlgorithm(algorithm: DPoPAlgorithm): EcKeyImportParams {
|
||||
switch (algorithm) {
|
||||
case 'ES384':
|
||||
return { name: 'ECDSA', namedCurve: 'P-384' };
|
||||
case 'EdDSA':
|
||||
throw new Error('EdDSA DPoP keys are not yet supported.');
|
||||
case 'ES256':
|
||||
default:
|
||||
return { name: 'ECDSA', namedCurve: 'P-256' };
|
||||
}
|
||||
}
|
||||
|
||||
private async openDb(): Promise<IDBDatabase> {
|
||||
if (typeof indexedDB === 'undefined') {
|
||||
throw new Error('IndexedDB is not available for DPoP key persistence.');
|
||||
}
|
||||
|
||||
if (!this.dbPromise) {
|
||||
this.dbPromise = new Promise<IDBDatabase>((resolve, reject) => {
|
||||
const request = indexedDB.open(DB_NAME, DB_VERSION);
|
||||
request.onupgradeneeded = () => {
|
||||
const db = request.result;
|
||||
if (!db.objectStoreNames.contains(STORE_NAME)) {
|
||||
db.createObjectStore(STORE_NAME, { keyPath: 'id' });
|
||||
}
|
||||
};
|
||||
request.onsuccess = () => resolve(request.result);
|
||||
request.onerror = () => reject(request.error);
|
||||
});
|
||||
}
|
||||
|
||||
return this.dbPromise;
|
||||
}
|
||||
}
|
||||
|
||||
function transactionPromise<T>(
|
||||
db: IDBDatabase,
|
||||
storeName: string,
|
||||
mode: IDBTransactionMode,
|
||||
executor: (store: IDBObjectStore) => IDBRequest<T>
|
||||
): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const transaction = db.transaction(storeName, mode);
|
||||
const store = transaction.objectStore(storeName);
|
||||
const request = executor(store);
|
||||
request.onsuccess = () => resolve(request.result);
|
||||
request.onerror = () => reject(request.error);
|
||||
transaction.onabort = () => reject(transaction.error);
|
||||
});
|
||||
}
|
||||
103
src/StellaOps.Web/src/app/core/auth/dpop/dpop.service.spec.ts
Normal file
103
src/StellaOps.Web/src/app/core/auth/dpop/dpop.service.spec.ts
Normal file
@@ -0,0 +1,103 @@
|
||||
import { HttpClientTestingModule } from '@angular/common/http/testing';
|
||||
import { TestBed } from '@angular/core/testing';
|
||||
|
||||
import { APP_CONFIG, AppConfig } from '../../config/app-config.model';
|
||||
import { AppConfigService } from '../../config/app-config.service';
|
||||
import { base64UrlDecode } from './jose-utilities';
|
||||
import { DpopKeyStore } from './dpop-key-store';
|
||||
import { DpopService } from './dpop.service';
|
||||
|
||||
describe('DpopService', () => {
|
||||
const originalTimeout = jasmine.DEFAULT_TIMEOUT_INTERVAL;
|
||||
const config: AppConfig = {
|
||||
authority: {
|
||||
issuer: 'https://auth.stellaops.test/',
|
||||
clientId: 'ui-client',
|
||||
authorizeEndpoint: 'https://auth.stellaops.test/connect/authorize',
|
||||
tokenEndpoint: 'https://auth.stellaops.test/connect/token',
|
||||
redirectUri: 'https://ui.stellaops.test/auth/callback',
|
||||
scope: 'openid profile ui.read',
|
||||
audience: 'https://scanner.stellaops.test',
|
||||
},
|
||||
apiBaseUrls: {
|
||||
authority: 'https://auth.stellaops.test',
|
||||
scanner: 'https://scanner.stellaops.test',
|
||||
policy: 'https://policy.stellaops.test',
|
||||
concelier: 'https://concelier.stellaops.test',
|
||||
attestor: 'https://attestor.stellaops.test',
|
||||
},
|
||||
};
|
||||
|
||||
beforeEach(async () => {
|
||||
jasmine.DEFAULT_TIMEOUT_INTERVAL = 20000;
|
||||
TestBed.configureTestingModule({
|
||||
imports: [HttpClientTestingModule],
|
||||
providers: [
|
||||
AppConfigService,
|
||||
DpopKeyStore,
|
||||
DpopService,
|
||||
{
|
||||
provide: APP_CONFIG,
|
||||
useValue: config,
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
jasmine.DEFAULT_TIMEOUT_INTERVAL = originalTimeout;
|
||||
const store = TestBed.inject(DpopKeyStore);
|
||||
try {
|
||||
await store.clear();
|
||||
} catch {
|
||||
// ignore cleanup issues in test environment
|
||||
}
|
||||
});
|
||||
|
||||
it('creates a DPoP proof with expected header values', async () => {
|
||||
const appConfig = TestBed.inject(AppConfigService);
|
||||
appConfig.setConfigForTesting(config);
|
||||
const service = TestBed.inject(DpopService);
|
||||
|
||||
const proof = await service.createProof({
|
||||
htm: 'get',
|
||||
htu: 'https://scanner.stellaops.test/api/v1/scans',
|
||||
});
|
||||
|
||||
const [rawHeader, rawPayload] = proof.split('.');
|
||||
const header = JSON.parse(
|
||||
new TextDecoder().decode(base64UrlDecode(rawHeader))
|
||||
);
|
||||
const payload = JSON.parse(
|
||||
new TextDecoder().decode(base64UrlDecode(rawPayload))
|
||||
);
|
||||
|
||||
expect(header.typ).toBe('dpop+jwt');
|
||||
expect(header.alg).toBe('ES256');
|
||||
expect(header.jwk.kty).toBe('EC');
|
||||
expect(payload.htm).toBe('GET');
|
||||
expect(payload.htu).toBe('https://scanner.stellaops.test/api/v1/scans');
|
||||
expect(typeof payload.iat).toBe('number');
|
||||
expect(typeof payload.jti).toBe('string');
|
||||
});
|
||||
|
||||
it('binds access token hash when provided', async () => {
|
||||
const appConfig = TestBed.inject(AppConfigService);
|
||||
appConfig.setConfigForTesting(config);
|
||||
const service = TestBed.inject(DpopService);
|
||||
|
||||
const accessToken = 'sample-access-token';
|
||||
const proof = await service.createProof({
|
||||
htm: 'post',
|
||||
htu: 'https://scanner.stellaops.test/api/v1/scans',
|
||||
accessToken,
|
||||
});
|
||||
|
||||
const payload = JSON.parse(
|
||||
new TextDecoder().decode(base64UrlDecode(proof.split('.')[1]))
|
||||
);
|
||||
|
||||
expect(payload.ath).toBeDefined();
|
||||
expect(typeof payload.ath).toBe('string');
|
||||
});
|
||||
});
|
||||
148
src/StellaOps.Web/src/app/core/auth/dpop/dpop.service.ts
Normal file
148
src/StellaOps.Web/src/app/core/auth/dpop/dpop.service.ts
Normal file
@@ -0,0 +1,148 @@
|
||||
import { Injectable, computed, signal } from '@angular/core';
|
||||
|
||||
import { AppConfigService } from '../../config/app-config.service';
|
||||
import { DPoPAlgorithm } from '../../config/app-config.model';
|
||||
import { sha256, base64UrlEncode, derToJoseSignature } from './jose-utilities';
|
||||
import { DpopKeyStore, LoadedDpopKeyPair } from './dpop-key-store';
|
||||
|
||||
export interface DpopProofOptions {
|
||||
readonly htm: string;
|
||||
readonly htu: string;
|
||||
readonly accessToken?: string;
|
||||
readonly nonce?: string | null;
|
||||
}
|
||||
|
||||
@Injectable({
|
||||
providedIn: 'root',
|
||||
})
|
||||
export class DpopService {
|
||||
private keyPairPromise: Promise<LoadedDpopKeyPair> | null = null;
|
||||
private readonly nonceSignal = signal<string | null>(null);
|
||||
readonly nonce = computed(() => this.nonceSignal());
|
||||
|
||||
constructor(
|
||||
private readonly config: AppConfigService,
|
||||
private readonly store: DpopKeyStore
|
||||
) {}
|
||||
|
||||
async setNonce(nonce: string | null): Promise<void> {
|
||||
this.nonceSignal.set(nonce);
|
||||
}
|
||||
|
||||
async getThumbprint(): Promise<string | null> {
|
||||
const key = await this.getOrCreateKeyPair();
|
||||
return key.thumbprint ?? null;
|
||||
}
|
||||
|
||||
async rotateKey(): Promise<void> {
|
||||
const algorithm = this.resolveAlgorithm();
|
||||
this.keyPairPromise = this.store.generate(algorithm);
|
||||
}
|
||||
|
||||
async createProof(options: DpopProofOptions): Promise<string> {
|
||||
const keyPair = await this.getOrCreateKeyPair();
|
||||
|
||||
const header = {
|
||||
typ: 'dpop+jwt',
|
||||
alg: keyPair.algorithm,
|
||||
jwk: keyPair.publicJwk,
|
||||
};
|
||||
|
||||
const nowSeconds = Math.floor(Date.now() / 1000);
|
||||
const payload: Record<string, unknown> = {
|
||||
htm: options.htm.toUpperCase(),
|
||||
htu: normalizeHtu(options.htu),
|
||||
iat: nowSeconds,
|
||||
jti: crypto.randomUUID ? crypto.randomUUID() : createRandomId(),
|
||||
};
|
||||
|
||||
const nonce = options.nonce ?? this.nonceSignal();
|
||||
if (nonce) {
|
||||
payload['nonce'] = nonce;
|
||||
}
|
||||
|
||||
if (options.accessToken) {
|
||||
const accessTokenHash = await sha256(
|
||||
new TextEncoder().encode(options.accessToken)
|
||||
);
|
||||
payload['ath'] = base64UrlEncode(accessTokenHash);
|
||||
}
|
||||
|
||||
const encodedHeader = base64UrlEncode(JSON.stringify(header));
|
||||
const encodedPayload = base64UrlEncode(JSON.stringify(payload));
|
||||
const signingInput = `${encodedHeader}.${encodedPayload}`;
|
||||
const signature = await crypto.subtle.sign(
|
||||
{
|
||||
name: 'ECDSA',
|
||||
hash: this.resolveHashAlgorithm(keyPair.algorithm),
|
||||
},
|
||||
keyPair.privateKey,
|
||||
new TextEncoder().encode(signingInput)
|
||||
);
|
||||
|
||||
const joseSignature = base64UrlEncode(derToJoseSignature(signature));
|
||||
return `${signingInput}.${joseSignature}`;
|
||||
}
|
||||
|
||||
private async getOrCreateKeyPair(): Promise<LoadedDpopKeyPair> {
|
||||
if (!this.keyPairPromise) {
|
||||
this.keyPairPromise = this.loadKeyPair();
|
||||
}
|
||||
try {
|
||||
return await this.keyPairPromise;
|
||||
} catch (error) {
|
||||
// Reset the memoized promise so a subsequent call can retry.
|
||||
this.keyPairPromise = null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async loadKeyPair(): Promise<LoadedDpopKeyPair> {
|
||||
const algorithm = this.resolveAlgorithm();
|
||||
try {
|
||||
const existing = await this.store.load();
|
||||
if (existing && existing.algorithm === algorithm) {
|
||||
return existing;
|
||||
}
|
||||
} catch {
|
||||
// fall through to regeneration
|
||||
}
|
||||
|
||||
return this.store.generate(algorithm);
|
||||
}
|
||||
|
||||
private resolveAlgorithm(): DPoPAlgorithm {
|
||||
const authority = this.config.authority;
|
||||
return authority.dpopAlgorithms?.[0] ?? 'ES256';
|
||||
}
|
||||
|
||||
private resolveHashAlgorithm(algorithm: DPoPAlgorithm): string {
|
||||
switch (algorithm) {
|
||||
case 'ES384':
|
||||
return 'SHA-384';
|
||||
case 'ES256':
|
||||
default:
|
||||
return 'SHA-256';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeHtu(value: string): string {
|
||||
try {
|
||||
const base =
|
||||
typeof window !== 'undefined' && window.location
|
||||
? window.location.origin
|
||||
: undefined;
|
||||
const url = base ? new URL(value, base) : new URL(value);
|
||||
url.hash = '';
|
||||
return url.toString();
|
||||
} catch {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
function createRandomId(): string {
|
||||
const array = new Uint8Array(16);
|
||||
crypto.getRandomValues(array);
|
||||
return base64UrlEncode(array);
|
||||
}
|
||||
123
src/StellaOps.Web/src/app/core/auth/dpop/jose-utilities.ts
Normal file
123
src/StellaOps.Web/src/app/core/auth/dpop/jose-utilities.ts
Normal file
@@ -0,0 +1,123 @@
|
||||
export async function sha256(data: Uint8Array): Promise<Uint8Array> {
|
||||
const digest = await crypto.subtle.digest('SHA-256', data);
|
||||
return new Uint8Array(digest);
|
||||
}
|
||||
|
||||
export function base64UrlEncode(
|
||||
input: ArrayBuffer | Uint8Array | string
|
||||
): string {
|
||||
let bytes: Uint8Array;
|
||||
if (typeof input === 'string') {
|
||||
bytes = new TextEncoder().encode(input);
|
||||
} else if (input instanceof Uint8Array) {
|
||||
bytes = input;
|
||||
} else {
|
||||
bytes = new Uint8Array(input);
|
||||
}
|
||||
|
||||
let binary = '';
|
||||
for (let i = 0; i < bytes.byteLength; i += 1) {
|
||||
binary += String.fromCharCode(bytes[i]);
|
||||
}
|
||||
|
||||
return btoa(binary).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
|
||||
}
|
||||
|
||||
export function base64UrlDecode(value: string): Uint8Array {
|
||||
const normalized = value.replace(/-/g, '+').replace(/_/g, '/');
|
||||
const padding = normalized.length % 4;
|
||||
const padded =
|
||||
padding === 0 ? normalized : normalized + '='.repeat(4 - padding);
|
||||
const binary = atob(padded);
|
||||
const bytes = new Uint8Array(binary.length);
|
||||
for (let i = 0; i < binary.length; i += 1) {
|
||||
bytes[i] = binary.charCodeAt(i);
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
export async function computeJwkThumbprint(jwk: JsonWebKey): Promise<string> {
|
||||
const canonical = canonicalizeJwk(jwk);
|
||||
const digest = await sha256(new TextEncoder().encode(canonical));
|
||||
return base64UrlEncode(digest);
|
||||
}
|
||||
|
||||
function canonicalizeJwk(jwk: JsonWebKey): string {
|
||||
if (!jwk.kty) {
|
||||
throw new Error('JWK must include "kty"');
|
||||
}
|
||||
|
||||
if (jwk.kty === 'EC') {
|
||||
const { crv, kty, x, y } = jwk;
|
||||
if (!crv || !x || !y) {
|
||||
throw new Error('EC JWK must include "crv", "x", and "y".');
|
||||
}
|
||||
return JSON.stringify({ crv, kty, x, y });
|
||||
}
|
||||
|
||||
if (jwk.kty === 'OKP') {
|
||||
const { crv, kty, x } = jwk;
|
||||
if (!crv || !x) {
|
||||
throw new Error('OKP JWK must include "crv" and "x".');
|
||||
}
|
||||
return JSON.stringify({ crv, kty, x });
|
||||
}
|
||||
|
||||
throw new Error(`Unsupported JWK key type: ${jwk.kty}`);
|
||||
}
|
||||
|
||||
export function derToJoseSignature(der: ArrayBuffer): Uint8Array {
|
||||
const bytes = new Uint8Array(der);
|
||||
if (bytes[0] !== 0x30) {
|
||||
// Some implementations already return raw (r || s) signature bytes.
|
||||
if (bytes.length === 64) {
|
||||
return bytes;
|
||||
}
|
||||
throw new Error('Invalid DER signature: expected sequence.');
|
||||
}
|
||||
|
||||
let offset = 2; // skip SEQUENCE header and length (assume short form)
|
||||
if (bytes[1] & 0x80) {
|
||||
const lengthBytes = bytes[1] & 0x7f;
|
||||
offset = 2 + lengthBytes;
|
||||
}
|
||||
|
||||
if (bytes[offset] !== 0x02) {
|
||||
throw new Error('Invalid DER signature: expected INTEGER for r.');
|
||||
}
|
||||
const rLength = bytes[offset + 1];
|
||||
let r = bytes.slice(offset + 2, offset + 2 + rLength);
|
||||
offset = offset + 2 + rLength;
|
||||
|
||||
if (bytes[offset] !== 0x02) {
|
||||
throw new Error('Invalid DER signature: expected INTEGER for s.');
|
||||
}
|
||||
const sLength = bytes[offset + 1];
|
||||
let s = bytes.slice(offset + 2, offset + 2 + sLength);
|
||||
|
||||
r = trimLeadingZeros(r);
|
||||
s = trimLeadingZeros(s);
|
||||
|
||||
const targetLength = 32;
|
||||
const signature = new Uint8Array(targetLength * 2);
|
||||
signature.set(padStart(r, targetLength), 0);
|
||||
signature.set(padStart(s, targetLength), targetLength);
|
||||
return signature;
|
||||
}
|
||||
|
||||
function trimLeadingZeros(bytes: Uint8Array): Uint8Array {
|
||||
let start = 0;
|
||||
while (start < bytes.length - 1 && bytes[start] === 0x00) {
|
||||
start += 1;
|
||||
}
|
||||
return bytes.subarray(start);
|
||||
}
|
||||
|
||||
function padStart(bytes: Uint8Array, length: number): Uint8Array {
|
||||
if (bytes.length >= length) {
|
||||
return bytes;
|
||||
}
|
||||
const padded = new Uint8Array(length);
|
||||
padded.set(bytes, length - bytes.length);
|
||||
return padded;
|
||||
}
|
||||
24
src/StellaOps.Web/src/app/core/auth/pkce.util.ts
Normal file
24
src/StellaOps.Web/src/app/core/auth/pkce.util.ts
Normal file
@@ -0,0 +1,24 @@
|
||||
import { base64UrlEncode, sha256 } from './dpop/jose-utilities';
|
||||
|
||||
export interface PkcePair {
|
||||
readonly verifier: string;
|
||||
readonly challenge: string;
|
||||
readonly method: 'S256';
|
||||
}
|
||||
|
||||
const VERIFIER_BYTE_LENGTH = 32;
|
||||
|
||||
export async function createPkcePair(): Promise<PkcePair> {
|
||||
const verifierBytes = new Uint8Array(VERIFIER_BYTE_LENGTH);
|
||||
crypto.getRandomValues(verifierBytes);
|
||||
|
||||
const verifier = base64UrlEncode(verifierBytes);
|
||||
const challengeBytes = await sha256(new TextEncoder().encode(verifier));
|
||||
const challenge = base64UrlEncode(challengeBytes);
|
||||
|
||||
return {
|
||||
verifier,
|
||||
challenge,
|
||||
method: 'S256',
|
||||
};
|
||||
}
|
||||
49
src/StellaOps.Web/src/app/core/config/app-config.model.ts
Normal file
49
src/StellaOps.Web/src/app/core/config/app-config.model.ts
Normal file
@@ -0,0 +1,49 @@
|
||||
import { InjectionToken } from '@angular/core';
|
||||
|
||||
export type DPoPAlgorithm = 'ES256' | 'ES384' | 'EdDSA';
|
||||
|
||||
export interface AuthorityConfig {
|
||||
readonly issuer: string;
|
||||
readonly clientId: string;
|
||||
readonly authorizeEndpoint: string;
|
||||
readonly tokenEndpoint: string;
|
||||
readonly logoutEndpoint?: string;
|
||||
readonly redirectUri: string;
|
||||
readonly postLogoutRedirectUri?: string;
|
||||
readonly scope: string;
|
||||
readonly audience: string;
|
||||
/**
|
||||
* Preferred algorithms for DPoP proofs, in order of preference.
|
||||
* Defaults to ES256 if omitted.
|
||||
*/
|
||||
readonly dpopAlgorithms?: readonly DPoPAlgorithm[];
|
||||
/**
|
||||
* Seconds of leeway before access token expiry that should trigger a proactive refresh.
|
||||
* Defaults to 60.
|
||||
*/
|
||||
readonly refreshLeewaySeconds?: number;
|
||||
}
|
||||
|
||||
export interface ApiBaseUrlConfig {
|
||||
readonly scanner: string;
|
||||
readonly policy: string;
|
||||
readonly concelier: string;
|
||||
readonly excitor?: string;
|
||||
readonly attestor: string;
|
||||
readonly authority: string;
|
||||
readonly notify?: string;
|
||||
readonly scheduler?: string;
|
||||
}
|
||||
|
||||
export interface TelemetryConfig {
|
||||
readonly otlpEndpoint?: string;
|
||||
readonly sampleRate?: number;
|
||||
}
|
||||
|
||||
export interface AppConfig {
|
||||
readonly authority: AuthorityConfig;
|
||||
readonly apiBaseUrls: ApiBaseUrlConfig;
|
||||
readonly telemetry?: TelemetryConfig;
|
||||
}
|
||||
|
||||
export const APP_CONFIG = new InjectionToken<AppConfig>('STELLAOPS_APP_CONFIG');
|
||||
99
src/StellaOps.Web/src/app/core/config/app-config.service.ts
Normal file
99
src/StellaOps.Web/src/app/core/config/app-config.service.ts
Normal file
@@ -0,0 +1,99 @@
|
||||
import { HttpClient } from '@angular/common/http';
|
||||
import {
|
||||
Inject,
|
||||
Injectable,
|
||||
Optional,
|
||||
computed,
|
||||
signal,
|
||||
} from '@angular/core';
|
||||
import { firstValueFrom } from 'rxjs';
|
||||
|
||||
import {
|
||||
APP_CONFIG,
|
||||
AppConfig,
|
||||
AuthorityConfig,
|
||||
DPoPAlgorithm,
|
||||
} from './app-config.model';
|
||||
|
||||
const DEFAULT_CONFIG_URL = '/config.json';
|
||||
const DEFAULT_DPOP_ALG: DPoPAlgorithm = 'ES256';
|
||||
const DEFAULT_REFRESH_LEEWAY_SECONDS = 60;
|
||||
|
||||
@Injectable({
|
||||
providedIn: 'root',
|
||||
})
|
||||
export class AppConfigService {
|
||||
private readonly configSignal = signal<AppConfig | null>(null);
|
||||
private readonly authoritySignal = computed<AuthorityConfig | null>(() => {
|
||||
const config = this.configSignal();
|
||||
return config?.authority ?? null;
|
||||
});
|
||||
|
||||
constructor(
|
||||
private readonly http: HttpClient,
|
||||
@Optional() @Inject(APP_CONFIG) private readonly staticConfig: AppConfig | null
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Loads application configuration either from the injected static value or via HTTP fetch.
|
||||
* Must be called during application bootstrap (see APP_INITIALIZER wiring).
|
||||
*/
|
||||
async load(configUrl: string = DEFAULT_CONFIG_URL): Promise<void> {
|
||||
if (this.configSignal()) {
|
||||
return;
|
||||
}
|
||||
|
||||
const config = this.staticConfig ?? (await this.fetchConfig(configUrl));
|
||||
this.configSignal.set(this.normalizeConfig(config));
|
||||
}
|
||||
|
||||
/**
|
||||
* Allows tests to short-circuit configuration loading.
|
||||
*/
|
||||
setConfigForTesting(config: AppConfig): void {
|
||||
this.configSignal.set(this.normalizeConfig(config));
|
||||
}
|
||||
|
||||
get config(): AppConfig {
|
||||
const current = this.configSignal();
|
||||
if (!current) {
|
||||
throw new Error('App configuration has not been loaded yet.');
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
get authority(): AuthorityConfig {
|
||||
const authority = this.authoritySignal();
|
||||
if (!authority) {
|
||||
throw new Error('Authority configuration has not been loaded yet.');
|
||||
}
|
||||
return authority;
|
||||
}
|
||||
|
||||
private async fetchConfig(configUrl: string): Promise<AppConfig> {
|
||||
const response = await firstValueFrom(
|
||||
this.http.get<AppConfig>(configUrl, {
|
||||
headers: { 'Cache-Control': 'no-cache' },
|
||||
withCredentials: false,
|
||||
})
|
||||
);
|
||||
return response;
|
||||
}
|
||||
|
||||
private normalizeConfig(config: AppConfig): AppConfig {
|
||||
const authority = {
|
||||
...config.authority,
|
||||
dpopAlgorithms:
|
||||
config.authority.dpopAlgorithms?.length ?? 0
|
||||
? config.authority.dpopAlgorithms
|
||||
: [DEFAULT_DPOP_ALG],
|
||||
refreshLeewaySeconds:
|
||||
config.authority.refreshLeewaySeconds ?? DEFAULT_REFRESH_LEEWAY_SECONDS,
|
||||
};
|
||||
|
||||
return {
|
||||
...config,
|
||||
authority,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import { CommonModule } from '@angular/common';
|
||||
import { Component, OnInit, inject, signal } from '@angular/core';
|
||||
import { ActivatedRoute, Router } from '@angular/router';
|
||||
|
||||
import { AuthorityAuthService } from '../../core/auth/authority-auth.service';
|
||||
|
||||
@Component({
|
||||
selector: 'app-auth-callback',
|
||||
standalone: true,
|
||||
imports: [CommonModule],
|
||||
template: `
|
||||
<section class="auth-callback">
|
||||
<p *ngIf="state() === 'processing'">Completing sign-in…</p>
|
||||
<p *ngIf="state() === 'error'" class="error">
|
||||
We were unable to complete the sign-in flow. Please try again.
|
||||
</p>
|
||||
</section>
|
||||
`,
|
||||
styles: [
|
||||
`
|
||||
.auth-callback {
|
||||
margin: 4rem auto;
|
||||
max-width: 420px;
|
||||
text-align: center;
|
||||
font-size: 1rem;
|
||||
color: #0f172a;
|
||||
}
|
||||
|
||||
.error {
|
||||
color: #dc2626;
|
||||
font-weight: 500;
|
||||
}
|
||||
`,
|
||||
],
|
||||
})
|
||||
export class AuthCallbackComponent implements OnInit {
|
||||
private readonly route = inject(ActivatedRoute);
|
||||
private readonly router = inject(Router);
|
||||
private readonly auth = inject(AuthorityAuthService);
|
||||
|
||||
readonly state = signal<'processing' | 'error'>('processing');
|
||||
|
||||
async ngOnInit(): Promise<void> {
|
||||
const params = this.route.snapshot.queryParamMap;
|
||||
const searchParams = new URLSearchParams();
|
||||
params.keys.forEach((key) => {
|
||||
const value = params.get(key);
|
||||
if (value != null) {
|
||||
searchParams.set(key, value);
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await this.auth.completeLoginFromRedirect(searchParams);
|
||||
const returnUrl = result.returnUrl ?? '/';
|
||||
await this.router.navigateByUrl(returnUrl, { replaceUrl: true });
|
||||
} catch {
|
||||
this.state.set('error');
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
<section class="attestation-panel" [attr.data-status]="statusClass">
|
||||
<header class="attestation-header">
|
||||
<h2>Attestation</h2>
|
||||
<span class="status-badge" [ngClass]="statusClass">
|
||||
{{ statusLabel }}
|
||||
</span>
|
||||
</header>
|
||||
|
||||
<dl class="attestation-meta">
|
||||
<div>
|
||||
<dt>Rekor UUID</dt>
|
||||
<dd><code>{{ attestation.uuid }}</code></dd>
|
||||
</div>
|
||||
<div *ngIf="attestation.index !== undefined">
|
||||
<dt>Log index</dt>
|
||||
<dd>{{ attestation.index }}</dd>
|
||||
</div>
|
||||
<div *ngIf="attestation.logUrl">
|
||||
<dt>Log URL</dt>
|
||||
<dd>
|
||||
<a
|
||||
[href]="attestation.logUrl"
|
||||
rel="noopener noreferrer"
|
||||
target="_blank"
|
||||
>
|
||||
{{ attestation.logUrl }}
|
||||
</a>
|
||||
</dd>
|
||||
</div>
|
||||
<div *ngIf="attestation.checkedAt">
|
||||
<dt>Last checked</dt>
|
||||
<dd>{{ attestation.checkedAt }}</dd>
|
||||
</div>
|
||||
<div *ngIf="attestation.statusMessage">
|
||||
<dt>Details</dt>
|
||||
<dd>{{ attestation.statusMessage }}</dd>
|
||||
</div>
|
||||
</dl>
|
||||
</section>
|
||||
@@ -0,0 +1,75 @@
|
||||
.attestation-panel {
|
||||
border: 1px solid #1f2933;
|
||||
border-radius: 8px;
|
||||
padding: 1.25rem;
|
||||
background: #111827;
|
||||
color: #f8fafc;
|
||||
display: grid;
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.attestation-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
}
|
||||
|
||||
.attestation-header h2 {
|
||||
margin: 0;
|
||||
font-size: 1.125rem;
|
||||
}
|
||||
|
||||
.status-badge {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
padding: 0.35rem 0.75rem;
|
||||
border-radius: 999px;
|
||||
font-size: 0.875rem;
|
||||
font-weight: 600;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.05em;
|
||||
}
|
||||
|
||||
.status-badge.verified {
|
||||
background-color: rgba(34, 197, 94, 0.2);
|
||||
color: #34d399;
|
||||
}
|
||||
|
||||
.status-badge.pending {
|
||||
background-color: rgba(234, 179, 8, 0.2);
|
||||
color: #eab308;
|
||||
}
|
||||
|
||||
.status-badge.failed {
|
||||
background-color: rgba(248, 113, 113, 0.2);
|
||||
color: #f87171;
|
||||
}
|
||||
|
||||
.attestation-meta {
|
||||
margin: 0;
|
||||
display: grid;
|
||||
gap: 0.75rem;
|
||||
}
|
||||
|
||||
.attestation-meta div {
|
||||
display: grid;
|
||||
gap: 0.25rem;
|
||||
}
|
||||
|
||||
.attestation-meta dt {
|
||||
font-size: 0.75rem;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.05em;
|
||||
color: #9ca3af;
|
||||
}
|
||||
|
||||
.attestation-meta dd {
|
||||
margin: 0;
|
||||
font-family: 'JetBrains Mono', 'Fira Code', 'SFMono-Regular', monospace;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.attestation-meta a {
|
||||
color: #60a5fa;
|
||||
text-decoration: underline;
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
import { ScanAttestationPanelComponent } from './scan-attestation-panel.component';
|
||||
|
||||
describe('ScanAttestationPanelComponent', () => {
|
||||
let component: ScanAttestationPanelComponent;
|
||||
let fixture: ComponentFixture<ScanAttestationPanelComponent>;
|
||||
|
||||
beforeEach(async () => {
|
||||
await TestBed.configureTestingModule({
|
||||
imports: [ScanAttestationPanelComponent],
|
||||
}).compileComponents();
|
||||
|
||||
fixture = TestBed.createComponent(ScanAttestationPanelComponent);
|
||||
component = fixture.componentInstance;
|
||||
});
|
||||
|
||||
it('renders verified attestation details', () => {
|
||||
component.attestation = {
|
||||
uuid: '1234',
|
||||
status: 'verified',
|
||||
index: 42,
|
||||
logUrl: 'https://rekor.example',
|
||||
checkedAt: '2025-10-23T10:05:00Z',
|
||||
statusMessage: 'Rekor transparency log inclusion proof verified.',
|
||||
};
|
||||
|
||||
fixture.detectChanges();
|
||||
|
||||
const element: HTMLElement = fixture.nativeElement;
|
||||
expect(element.querySelector('.status-badge')?.textContent?.trim()).toBe(
|
||||
'Verified'
|
||||
);
|
||||
expect(element.textContent).toContain('1234');
|
||||
expect(element.textContent).toContain('42');
|
||||
expect(element.textContent).toContain('https://rekor.example');
|
||||
});
|
||||
|
||||
it('renders failure message when attestation verification fails', () => {
|
||||
component.attestation = {
|
||||
uuid: 'abcd',
|
||||
status: 'failed',
|
||||
statusMessage: 'Verification failed: inclusion proof mismatch.',
|
||||
};
|
||||
|
||||
fixture.detectChanges();
|
||||
|
||||
const element: HTMLElement = fixture.nativeElement;
|
||||
expect(element.querySelector('.status-badge')?.textContent?.trim()).toBe(
|
||||
'Verification failed'
|
||||
);
|
||||
expect(element.textContent).toContain(
|
||||
'Verification failed: inclusion proof mismatch.'
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,42 @@
|
||||
import { CommonModule } from '@angular/common';
|
||||
import {
|
||||
ChangeDetectionStrategy,
|
||||
Component,
|
||||
Input,
|
||||
} from '@angular/core';
|
||||
import {
|
||||
ScanAttestationStatus,
|
||||
ScanAttestationStatusKind,
|
||||
} from '../../core/api/scanner.models';
|
||||
|
||||
@Component({
|
||||
selector: 'app-scan-attestation-panel',
|
||||
standalone: true,
|
||||
imports: [CommonModule],
|
||||
templateUrl: './scan-attestation-panel.component.html',
|
||||
styleUrls: ['./scan-attestation-panel.component.scss'],
|
||||
changeDetection: ChangeDetectionStrategy.OnPush,
|
||||
})
|
||||
export class ScanAttestationPanelComponent {
|
||||
@Input({ required: true }) attestation!: ScanAttestationStatus;
|
||||
|
||||
get statusLabel(): string {
|
||||
return this.toStatusLabel(this.attestation?.status);
|
||||
}
|
||||
|
||||
get statusClass(): string {
|
||||
return this.attestation?.status ?? 'pending';
|
||||
}
|
||||
|
||||
private toStatusLabel(status: ScanAttestationStatusKind | undefined): string {
|
||||
switch (status) {
|
||||
case 'verified':
|
||||
return 'Verified';
|
||||
case 'failed':
|
||||
return 'Verification failed';
|
||||
case 'pending':
|
||||
default:
|
||||
return 'Pending verification';
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
<section class="scan-detail">
|
||||
<header class="scan-detail__header">
|
||||
<h1>Scan Detail</h1>
|
||||
<div class="scenario-toggle" role="group" aria-label="Scenario selector">
|
||||
<button
|
||||
type="button"
|
||||
class="scenario-button"
|
||||
[class.active]="scenario() === 'verified'"
|
||||
(click)="onSelectScenario('verified')"
|
||||
data-scenario="verified"
|
||||
>
|
||||
Verified
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
class="scenario-button"
|
||||
[class.active]="scenario() === 'failed'"
|
||||
(click)="onSelectScenario('failed')"
|
||||
data-scenario="failed"
|
||||
>
|
||||
Failure
|
||||
</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<section class="scan-summary">
|
||||
<h2>Image</h2>
|
||||
<dl>
|
||||
<div>
|
||||
<dt>Scan ID</dt>
|
||||
<dd>{{ scan().scanId }}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Image digest</dt>
|
||||
<dd><code>{{ scan().imageDigest }}</code></dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Completed at</dt>
|
||||
<dd>{{ scan().completedAt }}</dd>
|
||||
</div>
|
||||
</dl>
|
||||
</section>
|
||||
|
||||
<app-scan-attestation-panel
|
||||
*ngIf="scan().attestation as attestation"
|
||||
[attestation]="attestation"
|
||||
/>
|
||||
|
||||
<p *ngIf="!scan().attestation" class="attestation-empty">
|
||||
No attestation has been recorded for this scan.
|
||||
</p>
|
||||
</section>
|
||||
@@ -0,0 +1,79 @@
|
||||
.scan-detail {
|
||||
display: grid;
|
||||
gap: 1.5rem;
|
||||
padding: 1.5rem;
|
||||
color: #e2e8f0;
|
||||
background: #0f172a;
|
||||
min-height: calc(100vh - 120px);
|
||||
}
|
||||
|
||||
.scan-detail__header {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.scan-detail__header h1 {
|
||||
margin: 0;
|
||||
font-size: 1.5rem;
|
||||
}
|
||||
|
||||
.scenario-toggle {
|
||||
display: inline-flex;
|
||||
border: 1px solid #1f2933;
|
||||
border-radius: 999px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.scenario-button {
|
||||
background: transparent;
|
||||
color: inherit;
|
||||
border: none;
|
||||
padding: 0.5rem 1.25rem;
|
||||
cursor: pointer;
|
||||
font-size: 0.9rem;
|
||||
letter-spacing: 0.03em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.scenario-button.active {
|
||||
background: #1d4ed8;
|
||||
color: #f8fafc;
|
||||
}
|
||||
|
||||
.scan-summary {
|
||||
border: 1px solid #1f2933;
|
||||
border-radius: 8px;
|
||||
padding: 1.25rem;
|
||||
background: #111827;
|
||||
}
|
||||
|
||||
.scan-summary h2 {
|
||||
margin: 0 0 0.75rem 0;
|
||||
font-size: 1.125rem;
|
||||
}
|
||||
|
||||
.scan-summary dl {
|
||||
margin: 0;
|
||||
display: grid;
|
||||
gap: 0.75rem;
|
||||
}
|
||||
|
||||
.scan-summary dt {
|
||||
font-size: 0.75rem;
|
||||
text-transform: uppercase;
|
||||
color: #94a3b8;
|
||||
}
|
||||
|
||||
.scan-summary dd {
|
||||
margin: 0;
|
||||
font-family: 'JetBrains Mono', 'Fira Code', 'SFMono-Regular', monospace;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.attestation-empty {
|
||||
font-style: italic;
|
||||
color: #94a3b8;
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
import { RouterTestingModule } from '@angular/router/testing';
|
||||
import { ScanDetailPageComponent } from './scan-detail-page.component';
|
||||
import {
|
||||
scanDetailWithFailedAttestation,
|
||||
scanDetailWithVerifiedAttestation,
|
||||
} from '../../testing/scan-fixtures';
|
||||
|
||||
describe('ScanDetailPageComponent', () => {
|
||||
let fixture: ComponentFixture<ScanDetailPageComponent>;
|
||||
let component: ScanDetailPageComponent;
|
||||
|
||||
beforeEach(async () => {
|
||||
await TestBed.configureTestingModule({
|
||||
imports: [RouterTestingModule, ScanDetailPageComponent],
|
||||
}).compileComponents();
|
||||
|
||||
fixture = TestBed.createComponent(ScanDetailPageComponent);
|
||||
component = fixture.componentInstance;
|
||||
});
|
||||
|
||||
it('shows the verified attestation scenario by default', () => {
|
||||
fixture.detectChanges();
|
||||
|
||||
const element: HTMLElement = fixture.nativeElement;
|
||||
expect(element.textContent).toContain(
|
||||
scanDetailWithVerifiedAttestation.attestation?.uuid ?? ''
|
||||
);
|
||||
expect(element.querySelector('.status-badge')?.textContent?.trim()).toBe(
|
||||
'Verified'
|
||||
);
|
||||
});
|
||||
|
||||
it('switches to failure scenario when toggle is clicked', () => {
|
||||
fixture.detectChanges();
|
||||
|
||||
const failureButton: HTMLButtonElement | null =
|
||||
fixture.nativeElement.querySelector('[data-scenario="failed"]');
|
||||
failureButton?.click();
|
||||
fixture.detectChanges();
|
||||
|
||||
const element: HTMLElement = fixture.nativeElement;
|
||||
expect(element.textContent).toContain(
|
||||
scanDetailWithFailedAttestation.attestation?.uuid ?? ''
|
||||
);
|
||||
expect(element.querySelector('.status-badge')?.textContent?.trim()).toBe(
|
||||
'Verification failed'
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,62 @@
|
||||
import { CommonModule } from '@angular/common';
|
||||
import {
|
||||
ChangeDetectionStrategy,
|
||||
Component,
|
||||
computed,
|
||||
inject,
|
||||
signal,
|
||||
} from '@angular/core';
|
||||
import { ActivatedRoute } from '@angular/router';
|
||||
import { ScanAttestationPanelComponent } from './scan-attestation-panel.component';
|
||||
import { ScanDetail } from '../../core/api/scanner.models';
|
||||
import {
|
||||
scanDetailWithFailedAttestation,
|
||||
scanDetailWithVerifiedAttestation,
|
||||
} from '../../testing/scan-fixtures';
|
||||
|
||||
type Scenario = 'verified' | 'failed';
|
||||
|
||||
const SCENARIO_MAP: Record<Scenario, ScanDetail> = {
|
||||
verified: scanDetailWithVerifiedAttestation,
|
||||
failed: scanDetailWithFailedAttestation,
|
||||
};
|
||||
|
||||
@Component({
|
||||
selector: 'app-scan-detail-page',
|
||||
standalone: true,
|
||||
imports: [CommonModule, ScanAttestationPanelComponent],
|
||||
templateUrl: './scan-detail-page.component.html',
|
||||
styleUrls: ['./scan-detail-page.component.scss'],
|
||||
changeDetection: ChangeDetectionStrategy.OnPush,
|
||||
})
|
||||
export class ScanDetailPageComponent {
|
||||
private readonly route = inject(ActivatedRoute);
|
||||
|
||||
readonly scenario = signal<Scenario>('verified');
|
||||
|
||||
readonly scan = computed<ScanDetail>(() => {
|
||||
const current = this.scenario();
|
||||
return SCENARIO_MAP[current];
|
||||
});
|
||||
|
||||
constructor() {
|
||||
const routeScenario =
|
||||
(this.route.snapshot.queryParamMap.get('scenario') as Scenario | null) ??
|
||||
null;
|
||||
if (routeScenario && routeScenario in SCENARIO_MAP) {
|
||||
this.scenario.set(routeScenario);
|
||||
return;
|
||||
}
|
||||
|
||||
const scanId = this.route.snapshot.paramMap.get('scanId');
|
||||
if (scanId === scanDetailWithFailedAttestation.scanId) {
|
||||
this.scenario.set('failed');
|
||||
} else {
|
||||
this.scenario.set('verified');
|
||||
}
|
||||
}
|
||||
|
||||
onSelectScenario(next: Scenario): void {
|
||||
this.scenario.set(next);
|
||||
}
|
||||
}
|
||||
@@ -28,12 +28,20 @@ describe('policy fixtures', () => {
|
||||
|
||||
it('aligns preview and report fixtures', () => {
|
||||
const preview = getPolicyPreviewFixture();
|
||||
const report = getPolicyReportFixture();
|
||||
const { reportResponse } = getPolicyReportFixture();
|
||||
|
||||
expect(report.report.policy.digest).toEqual(preview.previewResponse.policyDigest);
|
||||
expect(report.report.verdicts.length).toEqual(report.report.summary.total);
|
||||
expect(report.report.verdicts.length).toBeGreaterThan(0);
|
||||
expect(report.report.verdicts.some(v => v.confidenceBand != null)).toBeTrue();
|
||||
expect(reportResponse.report.policy.digest).toEqual(
|
||||
preview.previewResponse.policyDigest
|
||||
);
|
||||
expect(reportResponse.report.verdicts.length).toEqual(
|
||||
reportResponse.report.summary.total
|
||||
);
|
||||
expect(reportResponse.report.verdicts.length).toBeGreaterThan(0);
|
||||
expect(
|
||||
reportResponse.report.verdicts.some(
|
||||
(verdict) => verdict.confidenceBand != null
|
||||
)
|
||||
).toBeTrue();
|
||||
});
|
||||
|
||||
it('provides DSSE metadata for report fixture', () => {
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
import previewSample from '../../../../samples/policy/policy-preview-unknown.json';
|
||||
import reportSample from '../../../../samples/policy/policy-report-unknown.json';
|
||||
import previewSample from '../../../../../samples/policy/policy-preview-unknown.json';
|
||||
import reportSample from '../../../../../samples/policy/policy-report-unknown.json';
|
||||
import {
|
||||
PolicyPreviewSample,
|
||||
PolicyReportSample,
|
||||
} from '../core/api/policy-preview.models';
|
||||
|
||||
const previewFixture: PolicyPreviewSample = previewSample;
|
||||
const reportFixture: PolicyReportSample = reportSample;
|
||||
const previewFixture: PolicyPreviewSample =
|
||||
previewSample as unknown as PolicyPreviewSample;
|
||||
const reportFixture: PolicyReportSample =
|
||||
reportSample as unknown as PolicyReportSample;
|
||||
|
||||
export function getPolicyPreviewFixture(): PolicyPreviewSample {
|
||||
return clone(previewFixture);
|
||||
|
||||
30
src/StellaOps.Web/src/app/testing/scan-fixtures.ts
Normal file
30
src/StellaOps.Web/src/app/testing/scan-fixtures.ts
Normal file
@@ -0,0 +1,30 @@
|
||||
import { ScanDetail } from '../core/api/scanner.models';
|
||||
|
||||
export const scanDetailWithVerifiedAttestation: ScanDetail = {
|
||||
scanId: 'scan-verified-001',
|
||||
imageDigest:
|
||||
'sha256:9f92a8c39f8d4f7bb1a60f2be650b3019b9a1bb50d2da839efa9bf2a278a0071',
|
||||
completedAt: '2025-10-20T18:22:04Z',
|
||||
attestation: {
|
||||
uuid: '018ed91c-9b64-7edc-b9ac-0bada2f8d501',
|
||||
index: 412398,
|
||||
logUrl: 'https://rekor.sigstore.dev',
|
||||
status: 'verified',
|
||||
checkedAt: '2025-10-23T12:04:52Z',
|
||||
statusMessage: 'Rekor transparency log inclusion proof verified.',
|
||||
},
|
||||
};
|
||||
|
||||
export const scanDetailWithFailedAttestation: ScanDetail = {
|
||||
scanId: 'scan-failed-002',
|
||||
imageDigest:
|
||||
'sha256:b0d6865de537e45bdd9dd72cdac02bc6f459f0e546ed9134e2afc2fccd6298e0',
|
||||
completedAt: '2025-10-19T07:14:33Z',
|
||||
attestation: {
|
||||
uuid: '018ed91c-ffff-4882-9955-0027c0bbb090',
|
||||
status: 'failed',
|
||||
checkedAt: '2025-10-23T09:18:11Z',
|
||||
statusMessage:
|
||||
'Verification failed: inclusion proof leaf hash mismatch at depth 4.',
|
||||
},
|
||||
};
|
||||
26
src/StellaOps.Web/src/config/config.json
Normal file
26
src/StellaOps.Web/src/config/config.json
Normal file
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"authority": {
|
||||
"issuer": "https://authority.local",
|
||||
"clientId": "stellaops-ui",
|
||||
"authorizeEndpoint": "https://authority.local/connect/authorize",
|
||||
"tokenEndpoint": "https://authority.local/connect/token",
|
||||
"logoutEndpoint": "https://authority.local/connect/logout",
|
||||
"redirectUri": "http://localhost:4400/auth/callback",
|
||||
"postLogoutRedirectUri": "http://localhost:4400/",
|
||||
"scope": "openid profile ui.read",
|
||||
"audience": "https://scanner.local",
|
||||
"dpopAlgorithms": ["ES256"],
|
||||
"refreshLeewaySeconds": 60
|
||||
},
|
||||
"apiBaseUrls": {
|
||||
"authority": "https://authority.local",
|
||||
"scanner": "https://scanner.local",
|
||||
"policy": "https://scanner.local",
|
||||
"concelier": "https://concelier.local",
|
||||
"attestor": "https://attestor.local"
|
||||
},
|
||||
"telemetry": {
|
||||
"otlpEndpoint": "http://localhost:4318/v1/traces",
|
||||
"sampleRate": 0.1
|
||||
}
|
||||
}
|
||||
26
src/StellaOps.Web/src/config/config.sample.json
Normal file
26
src/StellaOps.Web/src/config/config.sample.json
Normal file
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"authority": {
|
||||
"issuer": "https://authority.example.dev",
|
||||
"clientId": "stellaops-ui",
|
||||
"authorizeEndpoint": "https://authority.example.dev/connect/authorize",
|
||||
"tokenEndpoint": "https://authority.example.dev/connect/token",
|
||||
"logoutEndpoint": "https://authority.example.dev/connect/logout",
|
||||
"redirectUri": "http://localhost:4400/auth/callback",
|
||||
"postLogoutRedirectUri": "http://localhost:4400/",
|
||||
"scope": "openid profile ui.read",
|
||||
"audience": "https://scanner.example.dev",
|
||||
"dpopAlgorithms": ["ES256"],
|
||||
"refreshLeewaySeconds": 60
|
||||
},
|
||||
"apiBaseUrls": {
|
||||
"authority": "https://authority.example.dev",
|
||||
"scanner": "https://scanner.example.dev",
|
||||
"policy": "https://scanner.example.dev",
|
||||
"concelier": "https://concelier.example.dev",
|
||||
"attestor": "https://attestor.example.dev"
|
||||
},
|
||||
"telemetry": {
|
||||
"otlpEndpoint": "",
|
||||
"sampleRate": 0
|
||||
}
|
||||
}
|
||||
4
src/StellaOps.Web/test-results/.last-run.json
Normal file
4
src/StellaOps.Web/test-results/.last-run.json
Normal file
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"status": "passed",
|
||||
"failedTests": []
|
||||
}
|
||||
78
src/StellaOps.Web/tests/e2e/auth.spec.ts
Normal file
78
src/StellaOps.Web/tests/e2e/auth.spec.ts
Normal file
@@ -0,0 +1,78 @@
|
||||
import { expect, test } from '@playwright/test';
|
||||
|
||||
const mockConfig = {
|
||||
authority: {
|
||||
issuer: 'https://authority.local',
|
||||
clientId: 'stellaops-ui',
|
||||
authorizeEndpoint: 'https://authority.local/connect/authorize',
|
||||
tokenEndpoint: 'https://authority.local/connect/token',
|
||||
logoutEndpoint: 'https://authority.local/connect/logout',
|
||||
redirectUri: 'http://127.0.0.1:4400/auth/callback',
|
||||
postLogoutRedirectUri: 'http://127.0.0.1:4400/',
|
||||
scope: 'openid profile ui.read',
|
||||
audience: 'https://scanner.local',
|
||||
dpopAlgorithms: ['ES256'],
|
||||
refreshLeewaySeconds: 60,
|
||||
},
|
||||
apiBaseUrls: {
|
||||
authority: 'https://authority.local',
|
||||
scanner: 'https://scanner.local',
|
||||
policy: 'https://scanner.local',
|
||||
concelier: 'https://concelier.local',
|
||||
attestor: 'https://attestor.local',
|
||||
},
|
||||
};
|
||||
|
||||
test.beforeEach(async ({ page }) => {
|
||||
page.on('console', (message) => {
|
||||
// bubble up browser logs for debugging
|
||||
console.log('[browser]', message.type(), message.text());
|
||||
});
|
||||
page.on('pageerror', (error) => {
|
||||
console.log('[pageerror]', error.message);
|
||||
});
|
||||
await page.addInitScript(() => {
|
||||
// Capture attempted redirects so the test can assert against them.
|
||||
(window as any).__stellaopsAssignedUrls = [];
|
||||
const originalAssign = window.location.assign.bind(window.location);
|
||||
window.location.assign = (url: string | URL) => {
|
||||
(window as any).__stellaopsAssignedUrls.push(url.toString());
|
||||
};
|
||||
|
||||
window.sessionStorage.clear();
|
||||
});
|
||||
await page.route('**/config.json', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify(mockConfig),
|
||||
})
|
||||
);
|
||||
await page.route('https://authority.local/**', (route) => route.abort());
|
||||
});
|
||||
|
||||
test('sign-in flow builds Authority authorization URL', async ({ page }) => {
|
||||
await page.goto('/');
|
||||
const signInButton = page.getByRole('button', { name: /sign in/i });
|
||||
await expect(signInButton).toBeVisible();
|
||||
const [request] = await Promise.all([
|
||||
page.waitForRequest('https://authority.local/connect/authorize*'),
|
||||
signInButton.click(),
|
||||
]);
|
||||
|
||||
const authorizeUrl = new URL(request.url());
|
||||
expect(authorizeUrl.origin).toBe('https://authority.local');
|
||||
expect(authorizeUrl.pathname).toBe('/connect/authorize');
|
||||
expect(authorizeUrl.searchParams.get('client_id')).toBe('stellaops-ui');
|
||||
|
||||
});
|
||||
|
||||
test('callback without pending state surfaces error message', async ({ page }) => {
|
||||
await page.route('https://authority.local/**', (route) =>
|
||||
route.fulfill({ status: 400, body: 'blocked' })
|
||||
);
|
||||
await page.goto('/auth/callback?code=test-code&state=missing');
|
||||
await expect(
|
||||
page.getByText('We were unable to complete the sign-in flow. Please try again.')
|
||||
).toBeVisible({ timeout: 10000 });
|
||||
});
|
||||
@@ -5,8 +5,7 @@ namespace StellaOps.Zastava.Core.Tests.Contracts;
|
||||
public sealed class ZastavaContractVersionsTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData("zastava.runtime.event@v1", "zastava.runtime.event", 1, 0)]
|
||||
[InlineData("zastava.runtime.event@v1.0", "zastava.runtime.event", 1, 0)]
|
||||
[InlineData("zastava.runtime.event@v1.0", "zastava.runtime.event", 1, 0)]
|
||||
[InlineData("zastava.admission.decision@v1.2", "zastava.admission.decision", 1, 2)]
|
||||
public void TryParse_ParsesCanonicalForms(string input, string schema, int major, int minor)
|
||||
{
|
||||
@@ -31,36 +30,72 @@ public sealed class ZastavaContractVersionsTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IsRuntimeEventSupported_RespectsMajorCompatibility()
|
||||
{
|
||||
Assert.True(ZastavaContractVersions.IsRuntimeEventSupported("zastava.runtime.event@v1"));
|
||||
Assert.True(ZastavaContractVersions.IsRuntimeEventSupported("zastava.runtime.event@v1.0"));
|
||||
Assert.False(ZastavaContractVersions.IsRuntimeEventSupported("zastava.runtime.event@v2.0"));
|
||||
Assert.False(ZastavaContractVersions.IsRuntimeEventSupported("zastava.admission.decision@v1"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NegotiateRuntimeEvent_PicksHighestCommonVersion()
|
||||
{
|
||||
var negotiated = ZastavaContractVersions.NegotiateRuntimeEvent(new[]
|
||||
{
|
||||
public void IsRuntimeEventSupported_RespectsMajorCompatibility()
|
||||
{
|
||||
Assert.True(ZastavaContractVersions.ContractVersion.TryParse("zastava.runtime.event@v1.0", out var candidate));
|
||||
Assert.True(candidate.IsCompatibleWith(ZastavaContractVersions.RuntimeEvent), $"Candidate {candidate} incompatible with {ZastavaContractVersions.RuntimeEvent}");
|
||||
Assert.True(ZastavaContractVersions.IsRuntimeEventSupported("zastava.runtime.event@v1.0"));
|
||||
Assert.False(ZastavaContractVersions.IsRuntimeEventSupported("zastava.runtime.event@v2.0"));
|
||||
Assert.False(ZastavaContractVersions.IsRuntimeEventSupported("zastava.admission.decision@v1"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IsAdmissionDecisionSupported_RespectsMajorCompatibility()
|
||||
{
|
||||
Assert.True(ZastavaContractVersions.ContractVersion.TryParse("zastava.admission.decision@v1.0", out var candidate));
|
||||
Assert.True(candidate.IsCompatibleWith(ZastavaContractVersions.AdmissionDecision), $"Candidate {candidate} incompatible with {ZastavaContractVersions.AdmissionDecision}");
|
||||
Assert.True(ZastavaContractVersions.IsAdmissionDecisionSupported("zastava.admission.decision@v1.0"));
|
||||
Assert.False(ZastavaContractVersions.IsAdmissionDecisionSupported("zastava.admission.decision@v0.9"));
|
||||
Assert.False(ZastavaContractVersions.IsAdmissionDecisionSupported("zastava.runtime.event@v1"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NegotiateRuntimeEvent_PicksHighestCommonVersion()
|
||||
{
|
||||
var negotiated = ZastavaContractVersions.NegotiateRuntimeEvent(new[]
|
||||
{
|
||||
"zastava.runtime.event@v1.0",
|
||||
"zastava.runtime.event@v0.9",
|
||||
"zastava.admission.decision@v1"
|
||||
});
|
||||
|
||||
Assert.Equal("zastava.runtime.event@v1.0", negotiated.ToString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NegotiateRuntimeEvent_FallsBackToLocalWhenNoMatch()
|
||||
{
|
||||
var negotiated = ZastavaContractVersions.NegotiateRuntimeEvent(new[]
|
||||
{
|
||||
});
|
||||
|
||||
Assert.Equal("zastava.runtime.event@v1.0", negotiated.ToString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NegotiateAdmissionDecision_PicksHighestCommonVersion()
|
||||
{
|
||||
var negotiated = ZastavaContractVersions.NegotiateAdmissionDecision(new[]
|
||||
{
|
||||
"zastava.admission.decision@v1.2",
|
||||
"zastava.admission.decision@v1.0",
|
||||
"zastava.runtime.event@v1.0"
|
||||
});
|
||||
|
||||
Assert.Equal(ZastavaContractVersions.AdmissionDecision.ToString(), negotiated.ToString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NegotiateRuntimeEvent_FallsBackToLocalWhenNoMatch()
|
||||
{
|
||||
var negotiated = ZastavaContractVersions.NegotiateRuntimeEvent(new[]
|
||||
{
|
||||
"zastava.runtime.event@v2.0",
|
||||
"zastava.admission.decision@v2.0"
|
||||
});
|
||||
|
||||
Assert.Equal(ZastavaContractVersions.RuntimeEvent.ToString(), negotiated.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
Assert.Equal(ZastavaContractVersions.RuntimeEvent.ToString(), negotiated.ToString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NegotiateAdmissionDecision_FallsBackToLocalWhenNoMatch()
|
||||
{
|
||||
var negotiated = ZastavaContractVersions.NegotiateAdmissionDecision(new[]
|
||||
{
|
||||
"zastava.admission.decision@v2.0",
|
||||
"zastava.runtime.event@v2.0"
|
||||
});
|
||||
|
||||
Assert.Equal(ZastavaContractVersions.AdmissionDecision.ToString(), negotiated.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Zastava.Core.Configuration;
|
||||
using StellaOps.Zastava.Core.Diagnostics;
|
||||
using StellaOps.Zastava.Core.Security;
|
||||
|
||||
namespace StellaOps.Zastava.Core.Tests.DependencyInjection;
|
||||
|
||||
public sealed class ZastavaServiceCollectionExtensionsTests
|
||||
{
|
||||
[Fact]
|
||||
public void AddZastavaRuntimeCore_BindsOptionsAndProvidesDiagnostics()
|
||||
{
|
||||
var configuration = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["zastava:runtime:tenant"] = "tenant-42",
|
||||
["zastava:runtime:environment"] = "prod",
|
||||
["zastava:runtime:deployment"] = "cluster-a",
|
||||
["zastava:runtime:metrics:meterName"] = "stellaops.zastava.runtime",
|
||||
["zastava:runtime:metrics:meterVersion"] = "2.0.0",
|
||||
["zastava:runtime:metrics:commonTags:cluster"] = "prod-cluster",
|
||||
["zastava:runtime:logging:staticScope:plane"] = "runtime",
|
||||
["zastava:runtime:authority:clientId"] = "zastava-observer",
|
||||
["zastava:runtime:authority:audience:0"] = "scanner",
|
||||
["zastava:runtime:authority:audience:1"] = "zastava",
|
||||
["zastava:runtime:authority:scopes:0"] = "aud:scanner",
|
||||
["zastava:runtime:authority:scopes:1"] = "api:scanner.runtime.write",
|
||||
["zastava:runtime:authority:allowStaticTokenFallback"] = "false"
|
||||
})
|
||||
.Build();
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging();
|
||||
services.AddZastavaRuntimeCore(configuration, componentName: "observer");
|
||||
|
||||
using var provider = services.BuildServiceProvider();
|
||||
|
||||
var runtimeOptions = provider.GetRequiredService<IOptions<ZastavaRuntimeOptions>>().Value;
|
||||
Assert.Equal("tenant-42", runtimeOptions.Tenant);
|
||||
Assert.Equal("prod", runtimeOptions.Environment);
|
||||
Assert.Equal("observer", runtimeOptions.Component);
|
||||
Assert.Equal("cluster-a", runtimeOptions.Deployment);
|
||||
Assert.Equal("stellaops.zastava.runtime", runtimeOptions.Metrics.MeterName);
|
||||
Assert.Equal("2.0.0", runtimeOptions.Metrics.MeterVersion);
|
||||
Assert.Equal("runtime", runtimeOptions.Logging.StaticScope["plane"]);
|
||||
Assert.Equal("zastava-observer", runtimeOptions.Authority.ClientId);
|
||||
Assert.Contains("scanner", runtimeOptions.Authority.Audience);
|
||||
Assert.Contains("zastava", runtimeOptions.Authority.Audience);
|
||||
Assert.Equal(new[] { "aud:scanner", "api:scanner.runtime.write" }, runtimeOptions.Authority.Scopes);
|
||||
Assert.False(runtimeOptions.Authority.AllowStaticTokenFallback);
|
||||
|
||||
var scopeBuilder = provider.GetRequiredService<IZastavaLogScopeBuilder>();
|
||||
var scope = scopeBuilder.BuildScope(
|
||||
correlationId: "corr-1",
|
||||
node: "node-1",
|
||||
workload: "payments/api",
|
||||
eventId: "evt-123",
|
||||
additional: new Dictionary<string, string>
|
||||
{
|
||||
["pod"] = "api-12345"
|
||||
});
|
||||
|
||||
Assert.Equal("tenant-42", scope["tenant"]);
|
||||
Assert.Equal("observer", scope["component"]);
|
||||
Assert.Equal("prod", scope["environment"]);
|
||||
Assert.Equal("cluster-a", scope["deployment"]);
|
||||
Assert.Equal("runtime", scope["plane"]);
|
||||
Assert.Equal("corr-1", scope["correlationId"]);
|
||||
Assert.Equal("node-1", scope["node"]);
|
||||
Assert.Equal("payments/api", scope["workload"]);
|
||||
Assert.Equal("evt-123", scope["eventId"]);
|
||||
Assert.Equal("api-12345", scope["pod"]);
|
||||
|
||||
var metrics = provider.GetRequiredService<IZastavaRuntimeMetrics>();
|
||||
Assert.Equal("stellaops.zastava.runtime", metrics.Meter.Name);
|
||||
Assert.Equal("2.0.0", metrics.Meter.Version);
|
||||
|
||||
var authorityProvider = provider.GetRequiredService<IZastavaAuthorityTokenProvider>();
|
||||
Assert.NotNull(authorityProvider);
|
||||
|
||||
var defaultTags = metrics.DefaultTags.ToArray();
|
||||
Assert.Contains(defaultTags, kvp => kvp.Key == "tenant" && (string?)kvp.Value == "tenant-42");
|
||||
Assert.Contains(defaultTags, kvp => kvp.Key == "component" && (string?)kvp.Value == "observer");
|
||||
Assert.Contains(defaultTags, kvp => kvp.Key == "environment" && (string?)kvp.Value == "prod");
|
||||
Assert.Contains(defaultTags, kvp => kvp.Key == "deployment" && (string?)kvp.Value == "cluster-a");
|
||||
Assert.Contains(defaultTags, kvp => kvp.Key == "cluster" && (string?)kvp.Value == "prod-cluster");
|
||||
|
||||
metrics.RuntimeEvents.Add(1, defaultTags);
|
||||
metrics.AdmissionDecisions.Add(1, defaultTags);
|
||||
metrics.BackendLatencyMs.Record(12.5, defaultTags);
|
||||
|
||||
var loggerFactoryOptions = provider.GetRequiredService<IOptionsMonitor<LoggerFactoryOptions>>().CurrentValue;
|
||||
Assert.True(loggerFactoryOptions.ActivityTrackingOptions.HasFlag(ActivityTrackingOptions.TraceId));
|
||||
Assert.True(loggerFactoryOptions.ActivityTrackingOptions.HasFlag(ActivityTrackingOptions.SpanId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AddZastavaRuntimeCore_ThrowsForInvalidTenant()
|
||||
{
|
||||
var configuration = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["zastava:runtime:tenant"] = "",
|
||||
["zastava:runtime:environment"] = "prod"
|
||||
})
|
||||
.Build();
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging();
|
||||
services.AddZastavaRuntimeCore(configuration, "observer");
|
||||
|
||||
Assert.Throws<OptionsValidationException>(() =>
|
||||
{
|
||||
using var provider = services.BuildServiceProvider();
|
||||
_ = provider.GetRequiredService<IOptions<ZastavaRuntimeOptions>>().Value;
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Microsoft.IdentityModel.Tokens;
|
||||
using StellaOps.Auth.Client;
|
||||
using StellaOps.Zastava.Core.Configuration;
|
||||
using StellaOps.Zastava.Core.Diagnostics;
|
||||
using StellaOps.Zastava.Core.Security;
|
||||
|
||||
namespace StellaOps.Zastava.Core.Tests.Security;
|
||||
|
||||
public sealed class ZastavaAuthorityTokenProviderTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task GetAsync_UsesCacheUntilRefreshWindow()
|
||||
{
|
||||
var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2025-10-23T12:00:00Z"));
|
||||
var runtimeOptions = CreateRuntimeOptions(refreshSkewSeconds: 120);
|
||||
|
||||
var tokenClient = new StubTokenClient();
|
||||
tokenClient.EnqueueToken(new StellaOpsTokenResult(
|
||||
"token-1",
|
||||
"DPoP",
|
||||
timeProvider.GetUtcNow() + TimeSpan.FromMinutes(10),
|
||||
new[] { "aud:scanner", "api:scanner.runtime.write" }));
|
||||
|
||||
tokenClient.EnqueueToken(new StellaOpsTokenResult(
|
||||
"token-2",
|
||||
"DPoP",
|
||||
timeProvider.GetUtcNow() + TimeSpan.FromMinutes(10),
|
||||
new[] { "aud:scanner", "api:scanner.runtime.write" }));
|
||||
|
||||
var provider = CreateProvider(runtimeOptions, tokenClient, timeProvider);
|
||||
|
||||
var tokenA = await provider.GetAsync("scanner");
|
||||
Assert.Equal("token-1", tokenA.AccessToken);
|
||||
Assert.Equal(1, tokenClient.RequestCount);
|
||||
|
||||
// Move time forward but still before refresh window (refresh skew = 2 minutes)
|
||||
timeProvider.Advance(TimeSpan.FromMinutes(5));
|
||||
var tokenB = await provider.GetAsync("scanner");
|
||||
Assert.Equal("token-1", tokenB.AccessToken);
|
||||
Assert.Equal(1, tokenClient.RequestCount);
|
||||
|
||||
// Cross refresh window to trigger renewal
|
||||
timeProvider.Advance(TimeSpan.FromMinutes(5));
|
||||
var tokenC = await provider.GetAsync("scanner");
|
||||
Assert.Equal("token-2", tokenC.AccessToken);
|
||||
Assert.Equal(2, tokenClient.RequestCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetAsync_ThrowsWhenMissingAudienceScope()
|
||||
{
|
||||
var runtimeOptions = CreateRuntimeOptions();
|
||||
var tokenClient = new StubTokenClient();
|
||||
tokenClient.EnqueueToken(new StellaOpsTokenResult(
|
||||
"token",
|
||||
"DPoP",
|
||||
DateTimeOffset.UtcNow + TimeSpan.FromMinutes(5),
|
||||
new[] { "api:scanner.runtime.write" }));
|
||||
|
||||
var provider = CreateProvider(runtimeOptions, tokenClient, new TestTimeProvider(DateTimeOffset.UtcNow));
|
||||
|
||||
var ex = await Assert.ThrowsAsync<InvalidOperationException>(() => provider.GetAsync("scanner").AsTask());
|
||||
Assert.Contains("audience scope", ex.Message, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetAsync_StaticFallbackUsedWhenEnabled()
|
||||
{
|
||||
var runtimeOptions = CreateRuntimeOptions(allowFallback: true, staticToken: "static-token", requireDpop: false);
|
||||
|
||||
var tokenClient = new StubTokenClient();
|
||||
tokenClient.FailWith(new InvalidOperationException("offline"));
|
||||
|
||||
var provider = CreateProvider(runtimeOptions, tokenClient, new TestTimeProvider(DateTimeOffset.UtcNow));
|
||||
|
||||
var token = await provider.GetAsync("scanner");
|
||||
Assert.Equal("static-token", token.AccessToken);
|
||||
Assert.Null(token.ExpiresAtUtc);
|
||||
Assert.Equal(0, tokenClient.RequestCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetAsync_ThrowsWhenDpopRequiredButTokenTypeIsBearer()
|
||||
{
|
||||
var runtimeOptions = CreateRuntimeOptions(requireDpop: true);
|
||||
|
||||
var tokenClient = new StubTokenClient();
|
||||
tokenClient.EnqueueToken(new StellaOpsTokenResult(
|
||||
"token",
|
||||
"Bearer",
|
||||
DateTimeOffset.UtcNow + TimeSpan.FromMinutes(5),
|
||||
new[] { "aud:scanner" }));
|
||||
|
||||
var provider = CreateProvider(runtimeOptions, tokenClient, new TestTimeProvider(DateTimeOffset.UtcNow));
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() => provider.GetAsync("scanner").AsTask());
|
||||
}
|
||||
|
||||
private static ZastavaRuntimeOptions CreateRuntimeOptions(
|
||||
double refreshSkewSeconds = 60,
|
||||
bool allowFallback = false,
|
||||
string? staticToken = null,
|
||||
bool requireDpop = true)
|
||||
=> new()
|
||||
{
|
||||
Tenant = "tenant-x",
|
||||
Environment = "test",
|
||||
Component = "observer",
|
||||
Authority = new ZastavaAuthorityOptions
|
||||
{
|
||||
Issuer = new Uri("https://authority.internal"),
|
||||
ClientId = "zastava-runtime",
|
||||
Audience = new[] { "scanner" },
|
||||
Scopes = new[] { "api:scanner.runtime.write" },
|
||||
RefreshSkewSeconds = refreshSkewSeconds,
|
||||
RequireDpop = requireDpop,
|
||||
RequireMutualTls = true,
|
||||
AllowStaticTokenFallback = allowFallback,
|
||||
StaticTokenValue = staticToken
|
||||
}
|
||||
};
|
||||
|
||||
private static ZastavaAuthorityTokenProvider CreateProvider(
|
||||
ZastavaRuntimeOptions runtimeOptions,
|
||||
IStellaOpsTokenClient tokenClient,
|
||||
TimeProvider timeProvider)
|
||||
{
|
||||
var optionsMonitor = new StaticOptionsMonitor<ZastavaRuntimeOptions>(runtimeOptions);
|
||||
var scopeBuilder = new ZastavaLogScopeBuilder(Options.Create(runtimeOptions));
|
||||
return new ZastavaAuthorityTokenProvider(
|
||||
tokenClient,
|
||||
optionsMonitor,
|
||||
scopeBuilder,
|
||||
timeProvider,
|
||||
NullLogger<ZastavaAuthorityTokenProvider>.Instance);
|
||||
}
|
||||
|
||||
private sealed class StubTokenClient : IStellaOpsTokenClient
|
||||
{
|
||||
private readonly Queue<Func<CancellationToken, Task<StellaOpsTokenResult>>> responses = new();
|
||||
private Exception? failure;
|
||||
|
||||
public int RequestCount { get; private set; }
|
||||
|
||||
public void EnqueueToken(StellaOpsTokenResult result)
|
||||
=> responses.Enqueue(_ => Task.FromResult(result));
|
||||
|
||||
public void FailWith(Exception exception)
|
||||
=> failure = exception;
|
||||
|
||||
public Task<StellaOpsTokenResult> RequestClientCredentialsTokenAsync(string? scope = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
RequestCount++;
|
||||
|
||||
if (failure is not null)
|
||||
{
|
||||
throw failure;
|
||||
}
|
||||
|
||||
if (responses.TryDequeue(out var factory))
|
||||
{
|
||||
return factory(cancellationToken);
|
||||
}
|
||||
|
||||
throw new InvalidOperationException("No token responses queued.");
|
||||
}
|
||||
|
||||
public Task<StellaOpsTokenResult> RequestPasswordTokenAsync(string username, string password, string? scope = null, CancellationToken cancellationToken = default)
|
||||
=> throw new NotImplementedException();
|
||||
|
||||
public Task<JsonWebKeySet> GetJsonWebKeySetAsync(CancellationToken cancellationToken = default)
|
||||
=> throw new NotImplementedException();
|
||||
|
||||
public ValueTask<StellaOpsTokenCacheEntry?> GetCachedTokenAsync(string key, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.FromResult<StellaOpsTokenCacheEntry?>(null);
|
||||
|
||||
public ValueTask CacheTokenAsync(string key, StellaOpsTokenCacheEntry entry, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask ClearCachedTokenAsync(string key, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
private sealed class StaticOptionsMonitor<T> : IOptionsMonitor<T>
|
||||
{
|
||||
public StaticOptionsMonitor(T value)
|
||||
{
|
||||
CurrentValue = value;
|
||||
}
|
||||
|
||||
public T CurrentValue { get; }
|
||||
|
||||
public T Get(string? name) => CurrentValue;
|
||||
|
||||
public IDisposable OnChange(Action<T, string> listener) => NullDisposable.Instance;
|
||||
|
||||
private sealed class NullDisposable : IDisposable
|
||||
{
|
||||
public static readonly NullDisposable Instance = new();
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class TestTimeProvider : TimeProvider
|
||||
{
|
||||
private DateTimeOffset current;
|
||||
|
||||
public TestTimeProvider(DateTimeOffset initial)
|
||||
{
|
||||
current = initial;
|
||||
}
|
||||
|
||||
public override DateTimeOffset GetUtcNow() => current;
|
||||
|
||||
public void Advance(TimeSpan delta)
|
||||
{
|
||||
current = current.Add(delta);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
using System;
|
||||
using System.Text;
|
||||
using System.Security.Cryptography;
|
||||
using StellaOps.Zastava.Core.Contracts;
|
||||
@@ -163,43 +164,32 @@ public sealed class ZastavaCanonicalJsonSerializerTests
|
||||
[Fact]
|
||||
public void ComputeMultihash_ProducesStableBase64UrlDigest()
|
||||
{
|
||||
var decision = AdmissionDecisionEnvelope.Create(
|
||||
new AdmissionDecision
|
||||
{
|
||||
AdmissionId = "admission-123",
|
||||
Namespace = "payments",
|
||||
PodSpecDigest = "sha256:deadbeef",
|
||||
Images = new[]
|
||||
{
|
||||
new AdmissionImageVerdict
|
||||
{
|
||||
Name = "ghcr.io/acme/api:1.2.3",
|
||||
Resolved = "ghcr.io/acme/api@sha256:abcd",
|
||||
Signed = true,
|
||||
HasSbomReferrers = true,
|
||||
PolicyVerdict = PolicyVerdict.Pass,
|
||||
Reasons = Array.Empty<string>(),
|
||||
Rekor = new AdmissionRekorEvidence
|
||||
{
|
||||
Uuid = "xyz",
|
||||
Verified = true
|
||||
}
|
||||
}
|
||||
},
|
||||
Decision = AdmissionDecisionOutcome.Allow,
|
||||
TtlSeconds = 300
|
||||
},
|
||||
ZastavaContractVersions.AdmissionDecision);
|
||||
|
||||
var canonicalJson = ZastavaCanonicalJsonSerializer.Serialize(decision);
|
||||
var expectedDigestBytes = SHA256.HashData(Encoding.UTF8.GetBytes(canonicalJson));
|
||||
var payloadBytes = Encoding.UTF8.GetBytes("{\"value\":42}");
|
||||
var expectedDigestBytes = SHA256.HashData(payloadBytes);
|
||||
var expected = $"sha256-{Convert.ToBase64String(expectedDigestBytes).TrimEnd('=').Replace('+', '-').Replace('/', '_')}";
|
||||
|
||||
var hash = ZastavaHashing.ComputeMultihash(decision);
|
||||
var hash = ZastavaHashing.ComputeMultihash(new ReadOnlySpan<byte>(payloadBytes));
|
||||
|
||||
Assert.Equal(expected, hash);
|
||||
|
||||
var sha512 = ZastavaHashing.ComputeMultihash(Encoding.UTF8.GetBytes(canonicalJson), "sha512");
|
||||
var sha512 = ZastavaHashing.ComputeMultihash(new ReadOnlySpan<byte>(payloadBytes), "sha512");
|
||||
Assert.StartsWith("sha512-", sha512, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ComputeMultihash_NormalizesAlgorithmAliases()
|
||||
{
|
||||
var bytes = Encoding.UTF8.GetBytes("sample");
|
||||
var digestDefault = ZastavaHashing.ComputeMultihash(new ReadOnlySpan<byte>(bytes));
|
||||
var digestAlias = ZastavaHashing.ComputeMultihash(new ReadOnlySpan<byte>(bytes), "sha-256");
|
||||
|
||||
Assert.Equal(digestDefault, digestAlias);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ComputeMultihash_UnknownAlgorithm_Throws()
|
||||
{
|
||||
var ex = Assert.Throws<NotSupportedException>(() => ZastavaHashing.ComputeMultihash(new ReadOnlySpan<byte>(Array.Empty<byte>()), "unsupported"));
|
||||
Assert.Contains("unsupported", ex.Message, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
|
||||
namespace StellaOps.Zastava.Core.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Authority client configuration shared by Zastava runtime components.
|
||||
/// </summary>
|
||||
public sealed class ZastavaAuthorityOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Authority issuer URL.
|
||||
/// </summary>
|
||||
[Required]
|
||||
public Uri Issuer { get; set; } = new("https://authority.internal");
|
||||
|
||||
/// <summary>
|
||||
/// OAuth client identifier used by runtime services.
|
||||
/// </summary>
|
||||
[Required(AllowEmptyStrings = false)]
|
||||
public string ClientId { get; set; } = "zastava-runtime";
|
||||
|
||||
/// <summary>
|
||||
/// Optional client secret when using confidential clients.
|
||||
/// </summary>
|
||||
public string? ClientSecret { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Audience claims required on issued tokens.
|
||||
/// </summary>
|
||||
[MinLength(1)]
|
||||
public string[] Audience { get; set; } = new[] { "scanner" };
|
||||
|
||||
/// <summary>
|
||||
/// Additional scopes requested for the runtime plane.
|
||||
/// </summary>
|
||||
public string[] Scopes { get; set; } = Array.Empty<string>();
|
||||
|
||||
/// <summary>
|
||||
/// Seconds before expiry when a cached token should be refreshed.
|
||||
/// </summary>
|
||||
[Range(typeof(double), "0", "3600")]
|
||||
public double RefreshSkewSeconds { get; set; } = 120;
|
||||
|
||||
/// <summary>
|
||||
/// Require the Authority to issue DPoP (proof-of-possession) tokens.
|
||||
/// </summary>
|
||||
public bool RequireDpop { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Require the Authority client to present mTLS during token acquisition.
|
||||
/// </summary>
|
||||
public bool RequireMutualTls { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Allow falling back to static tokens when Authority is unavailable.
|
||||
/// </summary>
|
||||
public bool AllowStaticTokenFallback { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional path to a static fallback token (PEM/plain text).
|
||||
/// </summary>
|
||||
public string? StaticTokenPath { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional literal static token (test/bootstrap only). Takes precedence over <see cref="StaticTokenPath"/>.
|
||||
/// </summary>
|
||||
public string? StaticTokenValue { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
|
||||
namespace StellaOps.Zastava.Core.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Common runtime configuration shared by Zastava components (observer, webhook, agent).
|
||||
/// </summary>
|
||||
public sealed class ZastavaRuntimeOptions
|
||||
{
|
||||
public const string SectionName = "zastava:runtime";
|
||||
|
||||
/// <summary>
|
||||
/// Tenant identifier used for scoping logs and metrics.
|
||||
/// </summary>
|
||||
[Required(AllowEmptyStrings = false)]
|
||||
public string Tenant { get; set; } = "default";
|
||||
|
||||
/// <summary>
|
||||
/// Deployment environment (prod, staging, etc.) used in telemetry dimensions.
|
||||
/// </summary>
|
||||
[Required(AllowEmptyStrings = false)]
|
||||
public string Environment { get; set; } = "local";
|
||||
|
||||
/// <summary>
|
||||
/// Component name (observer/webhook/agent) injected into scopes and metrics.
|
||||
/// </summary>
|
||||
public string? Component { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional deployment identifier (cluster, region, etc.).
|
||||
/// </summary>
|
||||
public string? Deployment { get; set; }
|
||||
|
||||
[Required]
|
||||
public ZastavaRuntimeLoggingOptions Logging { get; set; } = new();
|
||||
|
||||
[Required]
|
||||
public ZastavaRuntimeMetricsOptions Metrics { get; set; } = new();
|
||||
|
||||
[Required]
|
||||
public ZastavaAuthorityOptions Authority { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class ZastavaRuntimeLoggingOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether scopes should be enabled on the logger factory.
|
||||
/// </summary>
|
||||
public bool IncludeScopes { get; init; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Whether activity tracking metadata (TraceId/SpanId) should be captured.
|
||||
/// </summary>
|
||||
public bool IncludeActivityTracking { get; init; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Optional static key/value pairs appended to every log scope.
|
||||
/// </summary>
|
||||
public IDictionary<string, string> StaticScope { get; init; } = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
public sealed class ZastavaRuntimeMetricsOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Enables metrics emission.
|
||||
/// </summary>
|
||||
public bool Enabled { get; init; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Meter name used for all runtime instrumentation.
|
||||
/// </summary>
|
||||
[Required(AllowEmptyStrings = false)]
|
||||
public string MeterName { get; init; } = "StellaOps.Zastava";
|
||||
|
||||
/// <summary>
|
||||
/// Optional meter semantic version.
|
||||
/// </summary>
|
||||
public string? MeterVersion { get; init; } = "1.0.0";
|
||||
|
||||
/// <summary>
|
||||
/// Common dimensions attached to every metric emitted by the runtime plane.
|
||||
/// </summary>
|
||||
public IDictionary<string, string> CommonTags { get; init; } = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Auth.Client;
|
||||
using StellaOps.Zastava.Core.Configuration;
|
||||
using StellaOps.Zastava.Core.Diagnostics;
|
||||
using StellaOps.Zastava.Core.Security;
|
||||
|
||||
namespace Microsoft.Extensions.DependencyInjection;
|
||||
|
||||
public static class ZastavaServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddZastavaRuntimeCore(
|
||||
this IServiceCollection services,
|
||||
IConfiguration configuration,
|
||||
string componentName)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
if (string.IsNullOrWhiteSpace(componentName))
|
||||
{
|
||||
throw new ArgumentException("Component name is required.", nameof(componentName));
|
||||
}
|
||||
|
||||
services.AddOptions<ZastavaRuntimeOptions>()
|
||||
.Bind(configuration.GetSection(ZastavaRuntimeOptions.SectionName))
|
||||
.ValidateDataAnnotations()
|
||||
.Validate(static options => !string.IsNullOrWhiteSpace(options.Tenant), "Tenant is required.")
|
||||
.Validate(static options => !string.IsNullOrWhiteSpace(options.Environment), "Environment is required.")
|
||||
.PostConfigure(options =>
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(options.Component))
|
||||
{
|
||||
options.Component = componentName;
|
||||
}
|
||||
})
|
||||
.ValidateOnStart();
|
||||
|
||||
services.TryAddEnumerable(ServiceDescriptor.Singleton<IConfigureOptions<LoggerFactoryOptions>, ZastavaLoggerFactoryOptionsConfigurator>());
|
||||
services.TryAddSingleton<IZastavaLogScopeBuilder, ZastavaLogScopeBuilder>();
|
||||
services.TryAddSingleton<IZastavaRuntimeMetrics, ZastavaRuntimeMetrics>();
|
||||
ConfigureAuthorityServices(services, configuration);
|
||||
services.TryAddSingleton<IZastavaAuthorityTokenProvider, ZastavaAuthorityTokenProvider>();
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void ConfigureAuthorityServices(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
var authoritySection = configuration.GetSection($"{ZastavaRuntimeOptions.SectionName}:authority");
|
||||
var authorityOptions = new ZastavaAuthorityOptions();
|
||||
authoritySection.Bind(authorityOptions);
|
||||
|
||||
services.AddStellaOpsAuthClient(options =>
|
||||
{
|
||||
options.Authority = authorityOptions.Issuer.ToString();
|
||||
options.ClientId = authorityOptions.ClientId;
|
||||
options.ClientSecret = authorityOptions.ClientSecret;
|
||||
options.AllowOfflineCacheFallback = authorityOptions.AllowStaticTokenFallback;
|
||||
options.ExpirationSkew = TimeSpan.FromSeconds(Math.Clamp(authorityOptions.RefreshSkewSeconds, 0, 300));
|
||||
|
||||
options.DefaultScopes.Clear();
|
||||
var normalized = new SortedSet<string>(StringComparer.Ordinal);
|
||||
|
||||
if (authorityOptions.Audience is not null)
|
||||
{
|
||||
foreach (var audience in authorityOptions.Audience)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(audience))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
normalized.Add($"aud:{audience.Trim().ToLowerInvariant()}");
|
||||
}
|
||||
}
|
||||
|
||||
if (authorityOptions.Scopes is not null)
|
||||
{
|
||||
foreach (var scope in authorityOptions.Scopes)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(scope))
|
||||
{
|
||||
normalized.Add(scope.Trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var scope in normalized)
|
||||
{
|
||||
options.DefaultScopes.Add(scope);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Zastava.Core.Configuration;
|
||||
|
||||
namespace StellaOps.Zastava.Core.Diagnostics;
|
||||
|
||||
public interface IZastavaLogScopeBuilder
|
||||
{
|
||||
/// <summary>
|
||||
/// Builds a deterministic logging scope containing tenant/component metadata.
|
||||
/// </summary>
|
||||
IReadOnlyDictionary<string, object?> BuildScope(
|
||||
string? correlationId = null,
|
||||
string? node = null,
|
||||
string? workload = null,
|
||||
string? eventId = null,
|
||||
IReadOnlyDictionary<string, string>? additional = null);
|
||||
}
|
||||
|
||||
internal sealed class ZastavaLogScopeBuilder : IZastavaLogScopeBuilder
|
||||
{
|
||||
private readonly ZastavaRuntimeOptions options;
|
||||
private readonly IReadOnlyDictionary<string, string> staticScope;
|
||||
|
||||
public ZastavaLogScopeBuilder(IOptions<ZastavaRuntimeOptions> options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
this.options = options.Value;
|
||||
staticScope = (this.options.Logging.StaticScope ?? new Dictionary<string, string>(StringComparer.Ordinal))
|
||||
.ToImmutableDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
public IReadOnlyDictionary<string, object?> BuildScope(
|
||||
string? correlationId = null,
|
||||
string? node = null,
|
||||
string? workload = null,
|
||||
string? eventId = null,
|
||||
IReadOnlyDictionary<string, string>? additional = null)
|
||||
{
|
||||
var scope = new Dictionary<string, object?>(StringComparer.Ordinal)
|
||||
{
|
||||
["tenant"] = options.Tenant,
|
||||
["component"] = options.Component,
|
||||
["environment"] = options.Environment
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(options.Deployment))
|
||||
{
|
||||
scope["deployment"] = options.Deployment;
|
||||
}
|
||||
|
||||
foreach (var pair in staticScope)
|
||||
{
|
||||
scope[pair.Key] = pair.Value;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(correlationId))
|
||||
{
|
||||
scope["correlationId"] = correlationId;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(node))
|
||||
{
|
||||
scope["node"] = node;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(workload))
|
||||
{
|
||||
scope["workload"] = workload;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(eventId))
|
||||
{
|
||||
scope["eventId"] = eventId;
|
||||
}
|
||||
|
||||
if (additional is not null)
|
||||
{
|
||||
foreach (var pair in additional)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(pair.Key))
|
||||
{
|
||||
scope[pair.Key] = pair.Value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return scope.ToImmutableDictionary(StringComparer.Ordinal);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Zastava.Core.Configuration;
|
||||
|
||||
namespace StellaOps.Zastava.Core.Diagnostics;
|
||||
|
||||
internal sealed class ZastavaLoggerFactoryOptionsConfigurator : IConfigureOptions<LoggerFactoryOptions>
|
||||
{
|
||||
private readonly IOptions<ZastavaRuntimeOptions> options;
|
||||
|
||||
public ZastavaLoggerFactoryOptionsConfigurator(IOptions<ZastavaRuntimeOptions> options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
public void Configure(LoggerFactoryOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
var runtimeOptions = this.options.Value;
|
||||
if (runtimeOptions.Logging.IncludeActivityTracking)
|
||||
{
|
||||
options.ActivityTrackingOptions |= ActivityTrackingOptions.TraceId | ActivityTrackingOptions.SpanId | ActivityTrackingOptions.ParentId;
|
||||
}
|
||||
else if (runtimeOptions.Logging.IncludeScopes)
|
||||
{
|
||||
options.ActivityTrackingOptions |= ActivityTrackingOptions.TraceId | ActivityTrackingOptions.SpanId;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Zastava.Core.Configuration;
|
||||
|
||||
namespace StellaOps.Zastava.Core.Diagnostics;
|
||||
|
||||
public interface IZastavaRuntimeMetrics : IDisposable
|
||||
{
|
||||
Meter Meter { get; }
|
||||
Counter<long> RuntimeEvents { get; }
|
||||
Counter<long> AdmissionDecisions { get; }
|
||||
Histogram<double> BackendLatencyMs { get; }
|
||||
IReadOnlyList<KeyValuePair<string, object?>> DefaultTags { get; }
|
||||
}
|
||||
|
||||
internal sealed class ZastavaRuntimeMetrics : IZastavaRuntimeMetrics
|
||||
{
|
||||
private readonly Meter meter;
|
||||
private readonly IReadOnlyList<KeyValuePair<string, object?>> defaultTags;
|
||||
private readonly bool enabled;
|
||||
|
||||
public ZastavaRuntimeMetrics(IOptions<ZastavaRuntimeOptions> options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
var runtimeOptions = options.Value;
|
||||
var metrics = runtimeOptions.Metrics ?? new ZastavaRuntimeMetricsOptions();
|
||||
enabled = metrics.Enabled;
|
||||
|
||||
meter = new Meter(metrics.MeterName, metrics.MeterVersion);
|
||||
|
||||
RuntimeEvents = meter.CreateCounter<long>("zastava.runtime.events.total", unit: "1", description: "Total runtime events emitted by observers.");
|
||||
AdmissionDecisions = meter.CreateCounter<long>("zastava.admission.decisions.total", unit: "1", description: "Total admission decisions returned by the webhook.");
|
||||
BackendLatencyMs = meter.CreateHistogram<double>("zastava.runtime.backend.latency.ms", unit: "ms", description: "Round-trip latency to Scanner backend APIs.");
|
||||
|
||||
var baseline = new List<KeyValuePair<string, object?>>
|
||||
{
|
||||
new("tenant", runtimeOptions.Tenant),
|
||||
new("component", runtimeOptions.Component),
|
||||
new("environment", runtimeOptions.Environment)
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(runtimeOptions.Deployment))
|
||||
{
|
||||
baseline.Add(new("deployment", runtimeOptions.Deployment));
|
||||
}
|
||||
|
||||
if (metrics.CommonTags is not null)
|
||||
{
|
||||
foreach (var pair in metrics.CommonTags)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(pair.Key))
|
||||
{
|
||||
baseline.Add(new(pair.Key, pair.Value));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
defaultTags = baseline.ToImmutableArray();
|
||||
}
|
||||
|
||||
public Meter Meter => meter;
|
||||
|
||||
public Counter<long> RuntimeEvents { get; }
|
||||
|
||||
public Counter<long> AdmissionDecisions { get; }
|
||||
|
||||
public Histogram<double> BackendLatencyMs { get; }
|
||||
|
||||
public IReadOnlyList<KeyValuePair<string, object?>> DefaultTags => defaultTags;
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (enabled)
|
||||
{
|
||||
meter.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
3
src/StellaOps.Zastava.Core/Properties/AssemblyInfo.cs
Normal file
3
src/StellaOps.Zastava.Core/Properties/AssemblyInfo.cs
Normal file
@@ -0,0 +1,3 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("StellaOps.Zastava.Core.Tests")]
|
||||
@@ -0,0 +1,14 @@
|
||||
namespace StellaOps.Zastava.Core.Security;
|
||||
|
||||
public interface IZastavaAuthorityTokenProvider
|
||||
{
|
||||
ValueTask<ZastavaOperationalToken> GetAsync(
|
||||
string audience,
|
||||
IEnumerable<string>? additionalScopes = null,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
ValueTask InvalidateAsync(
|
||||
string audience,
|
||||
IEnumerable<string>? additionalScopes = null,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Auth.Client;
|
||||
using StellaOps.Zastava.Core.Configuration;
|
||||
using StellaOps.Zastava.Core.Diagnostics;
|
||||
|
||||
namespace StellaOps.Zastava.Core.Security;
|
||||
|
||||
internal sealed class ZastavaAuthorityTokenProvider : IZastavaAuthorityTokenProvider
|
||||
{
|
||||
private readonly IStellaOpsTokenClient tokenClient;
|
||||
private readonly IOptionsMonitor<ZastavaRuntimeOptions> optionsMonitor;
|
||||
private readonly IZastavaLogScopeBuilder scopeBuilder;
|
||||
private readonly TimeProvider timeProvider;
|
||||
private readonly ILogger<ZastavaAuthorityTokenProvider> logger;
|
||||
|
||||
private readonly ConcurrentDictionary<string, CacheEntry> cache = new(StringComparer.Ordinal);
|
||||
private readonly ConcurrentDictionary<string, SemaphoreSlim> locks = new(StringComparer.Ordinal);
|
||||
private readonly object guardrailLock = new();
|
||||
private bool guardrailsLogged;
|
||||
private ZastavaOperationalToken? staticFallbackToken;
|
||||
|
||||
public ZastavaAuthorityTokenProvider(
|
||||
IStellaOpsTokenClient tokenClient,
|
||||
IOptionsMonitor<ZastavaRuntimeOptions> optionsMonitor,
|
||||
IZastavaLogScopeBuilder scopeBuilder,
|
||||
TimeProvider? timeProvider = null,
|
||||
ILogger<ZastavaAuthorityTokenProvider>? logger = null)
|
||||
{
|
||||
this.tokenClient = tokenClient ?? throw new ArgumentNullException(nameof(tokenClient));
|
||||
this.optionsMonitor = optionsMonitor ?? throw new ArgumentNullException(nameof(optionsMonitor));
|
||||
this.scopeBuilder = scopeBuilder ?? throw new ArgumentNullException(nameof(scopeBuilder));
|
||||
this.timeProvider = timeProvider ?? TimeProvider.System;
|
||||
this.logger = logger ?? NullLogger<ZastavaAuthorityTokenProvider>.Instance;
|
||||
}
|
||||
|
||||
public async ValueTask<ZastavaOperationalToken> GetAsync(
|
||||
string audience,
|
||||
IEnumerable<string>? additionalScopes = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(audience);
|
||||
|
||||
var options = optionsMonitor.CurrentValue.Authority;
|
||||
EnsureGuardrails(options);
|
||||
|
||||
if (options.AllowStaticTokenFallback && TryGetStaticToken(options) is { } staticToken)
|
||||
{
|
||||
return staticToken;
|
||||
}
|
||||
|
||||
var normalizedAudience = NormalizeAudience(audience);
|
||||
var normalizedScopes = BuildScopes(options, normalizedAudience, additionalScopes);
|
||||
var cacheKey = BuildCacheKey(normalizedAudience, normalizedScopes);
|
||||
var refreshSkew = GetRefreshSkew(options);
|
||||
|
||||
if (cache.TryGetValue(cacheKey, out var cached) && !cached.Token.IsExpired(timeProvider, refreshSkew))
|
||||
{
|
||||
return cached.Token;
|
||||
}
|
||||
|
||||
var mutex = locks.GetOrAdd(cacheKey, static _ => new SemaphoreSlim(1, 1));
|
||||
await mutex.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
try
|
||||
{
|
||||
if (cache.TryGetValue(cacheKey, out cached) && !cached.Token.IsExpired(timeProvider, refreshSkew))
|
||||
{
|
||||
return cached.Token;
|
||||
}
|
||||
|
||||
var scopeString = string.Join(' ', normalizedScopes);
|
||||
var tokenResult = await tokenClient.RequestClientCredentialsTokenAsync(scopeString, cancellationToken).ConfigureAwait(false);
|
||||
ValidateToken(tokenResult, options, normalizedAudience);
|
||||
|
||||
var token = ZastavaOperationalToken.FromResult(
|
||||
tokenResult.AccessToken,
|
||||
tokenResult.TokenType,
|
||||
tokenResult.ExpiresAtUtc,
|
||||
tokenResult.Scopes);
|
||||
|
||||
cache[cacheKey] = new CacheEntry(token);
|
||||
|
||||
var scope = scopeBuilder.BuildScope(
|
||||
correlationId: null,
|
||||
node: null,
|
||||
workload: null,
|
||||
eventId: "authority.token.issue",
|
||||
additional: new Dictionary<string, string>
|
||||
{
|
||||
["audience"] = normalizedAudience,
|
||||
["expiresAt"] = token.ExpiresAtUtc?.ToString("O", CultureInfo.InvariantCulture) ?? "static",
|
||||
["scopes"] = scopeString
|
||||
});
|
||||
|
||||
using (logger.BeginScope(scope))
|
||||
{
|
||||
logger.LogInformation("Issued runtime OpTok for {Audience} (scopes: {Scopes}).", normalizedAudience, scopeString);
|
||||
}
|
||||
|
||||
return token;
|
||||
}
|
||||
catch (Exception ex) when (options.AllowStaticTokenFallback && TryGetStaticToken(options) is { } fallback)
|
||||
{
|
||||
var scope = scopeBuilder.BuildScope(
|
||||
eventId: "authority.token.fallback",
|
||||
additional: new Dictionary<string, string>
|
||||
{
|
||||
["audience"] = audience
|
||||
});
|
||||
|
||||
using (logger.BeginScope(scope))
|
||||
{
|
||||
logger.LogWarning(ex, "Authority token acquisition failed; using static fallback token.");
|
||||
}
|
||||
|
||||
return fallback;
|
||||
}
|
||||
finally
|
||||
{
|
||||
mutex.Release();
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask InvalidateAsync(
|
||||
string audience,
|
||||
IEnumerable<string>? additionalScopes = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(audience);
|
||||
|
||||
var normalizedAudience = NormalizeAudience(audience);
|
||||
var normalizedScopes = BuildScopes(optionsMonitor.CurrentValue.Authority, normalizedAudience, additionalScopes);
|
||||
var cacheKey = BuildCacheKey(normalizedAudience, normalizedScopes);
|
||||
|
||||
cache.TryRemove(cacheKey, out _);
|
||||
if (locks.TryRemove(cacheKey, out var mutex))
|
||||
{
|
||||
mutex.Dispose();
|
||||
}
|
||||
|
||||
var scope = scopeBuilder.BuildScope(
|
||||
eventId: "authority.token.invalidate",
|
||||
additional: new Dictionary<string, string>
|
||||
{
|
||||
["audience"] = normalizedAudience,
|
||||
["cacheKey"] = cacheKey
|
||||
});
|
||||
|
||||
using (logger.BeginScope(scope))
|
||||
{
|
||||
logger.LogInformation("Invalidated runtime OpTok cache entry.");
|
||||
}
|
||||
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
private void EnsureGuardrails(ZastavaAuthorityOptions options)
|
||||
{
|
||||
if (guardrailsLogged)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
lock (guardrailLock)
|
||||
{
|
||||
if (guardrailsLogged)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var scope = scopeBuilder.BuildScope(eventId: "authority.guardrails");
|
||||
using (logger.BeginScope(scope))
|
||||
{
|
||||
if (!options.RequireMutualTls)
|
||||
{
|
||||
logger.LogWarning("Mutual TLS requirement disabled for Authority token acquisition. This should only be used in controlled test environments.");
|
||||
}
|
||||
|
||||
if (!options.RequireDpop)
|
||||
{
|
||||
logger.LogWarning("DPoP requirement disabled for runtime plane. Tokens will be issued without proof-of-possession.");
|
||||
}
|
||||
|
||||
if (options.AllowStaticTokenFallback)
|
||||
{
|
||||
logger.LogWarning("Static Authority token fallback enabled. Ensure bootstrap tokens are rotated frequently.");
|
||||
}
|
||||
}
|
||||
|
||||
guardrailsLogged = true;
|
||||
}
|
||||
}
|
||||
|
||||
private ZastavaOperationalToken? TryGetStaticToken(ZastavaAuthorityOptions options)
|
||||
{
|
||||
if (!options.AllowStaticTokenFallback)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (options.StaticTokenValue is null && options.StaticTokenPath is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (staticFallbackToken is { } cached)
|
||||
{
|
||||
return cached;
|
||||
}
|
||||
|
||||
lock (guardrailLock)
|
||||
{
|
||||
if (staticFallbackToken is { } existing)
|
||||
{
|
||||
return existing;
|
||||
}
|
||||
|
||||
var tokenValue = options.StaticTokenValue;
|
||||
if (string.IsNullOrWhiteSpace(tokenValue) && !string.IsNullOrWhiteSpace(options.StaticTokenPath))
|
||||
{
|
||||
if (!File.Exists(options.StaticTokenPath))
|
||||
{
|
||||
throw new FileNotFoundException("Static Authority token file not found.", options.StaticTokenPath);
|
||||
}
|
||||
|
||||
tokenValue = File.ReadAllText(options.StaticTokenPath);
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(tokenValue))
|
||||
{
|
||||
throw new InvalidOperationException("Static Authority token fallback is enabled but no token value/path is configured.");
|
||||
}
|
||||
|
||||
staticFallbackToken = ZastavaOperationalToken.FromResult(
|
||||
tokenValue.Trim(),
|
||||
tokenType: "Bearer",
|
||||
expiresAtUtc: null,
|
||||
scopes: Array.Empty<string>());
|
||||
|
||||
return staticFallbackToken;
|
||||
}
|
||||
}
|
||||
|
||||
private void ValidateToken(StellaOpsTokenResult tokenResult, ZastavaAuthorityOptions options, string normalizedAudience)
|
||||
{
|
||||
if (options.RequireDpop && !string.Equals(tokenResult.TokenType, "DPoP", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException("Authority returned a token without DPoP token type while RequireDpop is enabled.");
|
||||
}
|
||||
|
||||
if (tokenResult.Scopes is not null)
|
||||
{
|
||||
var audienceScope = $"aud:{normalizedAudience}";
|
||||
if (!tokenResult.Scopes.Contains(audienceScope, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException($"Authority token missing required audience scope '{audienceScope}'.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static string NormalizeAudience(string audience)
|
||||
=> audience.Trim().ToLowerInvariant();
|
||||
|
||||
private static IReadOnlyList<string> BuildScopes(
|
||||
ZastavaAuthorityOptions options,
|
||||
string normalizedAudience,
|
||||
IEnumerable<string>? additionalScopes)
|
||||
{
|
||||
var scopeSet = new SortedSet<string>(StringComparer.Ordinal)
|
||||
{
|
||||
$"aud:{normalizedAudience}"
|
||||
};
|
||||
|
||||
if (options.Scopes is not null)
|
||||
{
|
||||
foreach (var scope in options.Scopes)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(scope))
|
||||
{
|
||||
scopeSet.Add(scope.Trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (additionalScopes is not null)
|
||||
{
|
||||
foreach (var scope in additionalScopes)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(scope))
|
||||
{
|
||||
scopeSet.Add(scope.Trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return scopeSet.ToArray();
|
||||
}
|
||||
|
||||
private static string BuildCacheKey(string audience, IReadOnlyList<string> scopes)
|
||||
=> Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes($"{audience}|{string.Join(' ', scopes)}")));
|
||||
|
||||
private static TimeSpan GetRefreshSkew(ZastavaAuthorityOptions options)
|
||||
{
|
||||
var seconds = Math.Clamp(options.RefreshSkewSeconds, 0, 3600);
|
||||
return TimeSpan.FromSeconds(seconds);
|
||||
}
|
||||
|
||||
private readonly record struct CacheEntry(ZastavaOperationalToken Token);
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
using System.Collections.ObjectModel;
|
||||
using System.Linq;
|
||||
|
||||
namespace StellaOps.Zastava.Core.Security;
|
||||
|
||||
public readonly record struct ZastavaOperationalToken(
|
||||
string AccessToken,
|
||||
string TokenType,
|
||||
DateTimeOffset? ExpiresAtUtc,
|
||||
IReadOnlyList<string> Scopes)
|
||||
{
|
||||
public bool IsExpired(TimeProvider timeProvider, TimeSpan refreshSkew)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(timeProvider);
|
||||
|
||||
if (ExpiresAtUtc is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return timeProvider.GetUtcNow() >= ExpiresAtUtc.Value - refreshSkew;
|
||||
}
|
||||
|
||||
public static ZastavaOperationalToken FromResult(
|
||||
string accessToken,
|
||||
string tokenType,
|
||||
DateTimeOffset? expiresAtUtc,
|
||||
IEnumerable<string> scopes)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(accessToken);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tokenType);
|
||||
|
||||
IReadOnlyList<string> normalized = scopes switch
|
||||
{
|
||||
null => Array.Empty<string>(),
|
||||
IReadOnlyList<string> readOnly => readOnly.Count == 0 ? Array.Empty<string>() : readOnly,
|
||||
ICollection<string> collection => NormalizeCollection(collection),
|
||||
_ => NormalizeEnumerable(scopes)
|
||||
};
|
||||
|
||||
return new ZastavaOperationalToken(
|
||||
accessToken,
|
||||
tokenType,
|
||||
expiresAtUtc,
|
||||
normalized);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> NormalizeCollection(ICollection<string> collection)
|
||||
{
|
||||
if (collection.Count == 0)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
if (collection is IReadOnlyList<string> readOnly)
|
||||
{
|
||||
return readOnly;
|
||||
}
|
||||
|
||||
var buffer = new string[collection.Count];
|
||||
collection.CopyTo(buffer, 0);
|
||||
return new ReadOnlyCollection<string>(buffer);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> NormalizeEnumerable(IEnumerable<string> scopes)
|
||||
{
|
||||
var buffer = scopes.ToArray();
|
||||
return buffer.Length == 0 ? Array.Empty<string>() : new ReadOnlyCollection<string>(buffer);
|
||||
}
|
||||
}
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| ZASTAVA-CORE-12-201 | DOING (2025-10-19) | Zastava Core Guild | — | Define runtime event/admission DTOs, hashing helpers, and versioning strategy. | DTOs cover runtime events and admission verdict envelopes with canonical JSON schema; hashing helpers accept payloads and yield deterministic multihash outputs; version negotiation rules documented and exercised by serialization tests. |
|
||||
| ZASTAVA-CORE-12-202 | DOING (2025-10-19) | Zastava Core Guild | — | Provide configuration/logging/metrics utilities shared by Observer/Webhook. | Shared options bind from configuration with validation; logging scopes/metrics exporters registered via reusable DI extension; integration test host demonstrates Observer/Webhook consumption with deterministic instrumentation. |
|
||||
| ZASTAVA-CORE-12-203 | DOING (2025-10-19) | Zastava Core Guild | — | Authority client helpers, OpTok caching, and security guardrails for runtime services. | Typed Authority client surfaces OpTok retrieval + renewal with configurable cache; guardrails enforce DPoP/mTLS expectations and emit structured audit logs; negative-path tests cover expired/invalid tokens and configuration toggles. |
|
||||
| ZASTAVA-OPS-12-204 | DOING (2025-10-19) | Zastava Core Guild | — | Operational runbooks, alert rules, and dashboard exports for runtime plane. | Runbooks capture install/upgrade/rollback + incident handling; alert rules and dashboard JSON exported for Prometheus/Grafana bundle; docs reference Offline Kit packaging and verification checklist. |
|
||||
| ZASTAVA-CORE-12-201 | DONE (2025-10-23) | Zastava Core Guild | — | Define runtime event/admission DTOs, hashing helpers, and versioning strategy. | DTOs cover runtime events and admission verdict envelopes with canonical JSON schema; hashing helpers accept payloads and yield deterministic multihash outputs; version negotiation rules documented and exercised by serialization tests. |
|
||||
| ZASTAVA-CORE-12-202 | DONE (2025-10-23) | Zastava Core Guild | — | Provide configuration/logging/metrics utilities shared by Observer/Webhook. | Shared options bind from configuration with validation; logging scopes/metrics exporters registered via reusable DI extension; integration test host demonstrates Observer/Webhook consumption with deterministic instrumentation. |
|
||||
| ZASTAVA-CORE-12-203 | DONE (2025-10-23) | Zastava Core Guild | — | Authority client helpers, OpTok caching, and security guardrails for runtime services. | Typed Authority client surfaces OpTok retrieval + renewal with configurable cache; guardrails enforce DPoP/mTLS expectations and emit structured audit logs; negative-path tests cover expired/invalid tokens and configuration toggles. |
|
||||
| ZASTAVA-OPS-12-204 | DONE (2025-10-23) | Zastava Core Guild | — | Operational runbooks, alert rules, and dashboard exports for runtime plane. | Runbooks capture install/upgrade/rollback + incident handling; alert rules and dashboard JSON exported for Prometheus/Grafana bundle; docs reference Offline Kit packaging and verification checklist. |
|
||||
|
||||
> Remark (2025-10-19): Prerequisites reviewed—none outstanding. ZASTAVA-CORE-12-201, ZASTAVA-CORE-12-202, ZASTAVA-CORE-12-203, and ZASTAVA-OPS-12-204 moved to DOING for Wave 0 kickoff.
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
|
||||
namespace StellaOps.Zastava.Observer.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Observer-specific configuration applied on top of the shared runtime options.
|
||||
/// </summary>
|
||||
public sealed class ZastavaObserverOptions
|
||||
{
|
||||
public const string SectionName = "zastava:observer";
|
||||
|
||||
private const string DefaultContainerdSocket = "unix:///run/containerd/containerd.sock";
|
||||
|
||||
/// <summary>
|
||||
/// Logical node identifier emitted with runtime events (defaults to environment hostname).
|
||||
/// </summary>
|
||||
[Required(AllowEmptyStrings = false)]
|
||||
public string NodeName { get; set; } =
|
||||
Environment.GetEnvironmentVariable("ZASTAVA_NODE_NAME")
|
||||
?? Environment.GetEnvironmentVariable("KUBERNETES_NODE_NAME")
|
||||
?? Environment.MachineName;
|
||||
|
||||
/// <summary>
|
||||
/// Baseline polling interval when watching CRI runtimes.
|
||||
/// </summary>
|
||||
[Range(typeof(TimeSpan), "00:00:01", "00:10:00")]
|
||||
public TimeSpan PollInterval { get; set; } = TimeSpan.FromSeconds(2);
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of runtime events held in the in-memory buffer.
|
||||
/// </summary>
|
||||
[Range(16, 65536)]
|
||||
public int MaxInMemoryBuffer { get; set; } = 2048;
|
||||
|
||||
/// <summary>
|
||||
/// Number of runtime events drained in one batch by downstream publishers.
|
||||
/// </summary>
|
||||
[Range(1, 512)]
|
||||
public int PublishBatchSize { get; set; } = 32;
|
||||
|
||||
/// <summary>
|
||||
/// Connectivity/backoff settings applied when CRI endpoints fail temporarily.
|
||||
/// </summary>
|
||||
[Required]
|
||||
public ObserverBackoffOptions Backoff { get; set; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// CRI runtime endpoints to monitor.
|
||||
/// </summary>
|
||||
[Required]
|
||||
public IList<ContainerRuntimeEndpointOptions> Runtimes { get; set; } = new List<ContainerRuntimeEndpointOptions>
|
||||
{
|
||||
new()
|
||||
{
|
||||
Name = "containerd",
|
||||
Engine = ContainerRuntimeEngine.Containerd,
|
||||
Endpoint = DefaultContainerdSocket,
|
||||
Enabled = true
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public sealed class ObserverBackoffOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Initial backoff delay applied after the first failure.
|
||||
/// </summary>
|
||||
[Range(typeof(TimeSpan), "00:00:01", "00:05:00")]
|
||||
public TimeSpan Initial { get; set; } = TimeSpan.FromSeconds(1);
|
||||
|
||||
/// <summary>
|
||||
/// Maximum backoff delay after repeated failures.
|
||||
/// </summary>
|
||||
[Range(typeof(TimeSpan), "00:00:01", "00:10:00")]
|
||||
public TimeSpan Max { get; set; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>
|
||||
/// Jitter ratio applied to the computed delay (0 disables jitter).
|
||||
/// </summary>
|
||||
[Range(0.0, 0.5)]
|
||||
public double JitterRatio { get; set; } = 0.2;
|
||||
}
|
||||
|
||||
public sealed class ContainerRuntimeEndpointOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Friendly name used for logging/metrics (defaults to engine identifier).
|
||||
/// </summary>
|
||||
public string? Name { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Runtime engine backing the endpoint.
|
||||
/// </summary>
|
||||
public ContainerRuntimeEngine Engine { get; set; } = ContainerRuntimeEngine.Containerd;
|
||||
|
||||
/// <summary>
|
||||
/// Endpoint URI (unix:///run/containerd/containerd.sock, npipe://./pipe/dockershim, https://127.0.0.1:1234, ...).
|
||||
/// </summary>
|
||||
[Required(AllowEmptyStrings = false)]
|
||||
public string Endpoint { get; set; } = "unix:///run/containerd/containerd.sock";
|
||||
|
||||
/// <summary>
|
||||
/// Optional explicit polling interval for this endpoint (falls back to global PollInterval).
|
||||
/// </summary>
|
||||
[Range(typeof(TimeSpan), "00:00:01", "00:10:00")]
|
||||
public TimeSpan? PollInterval { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional connection timeout override.
|
||||
/// </summary>
|
||||
[Range(typeof(TimeSpan), "00:00:01", "00:01:00")]
|
||||
public TimeSpan? ConnectTimeout { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Flag to allow disabling endpoints without removing configuration entries.
|
||||
/// </summary>
|
||||
public bool Enabled { get; set; } = true;
|
||||
|
||||
public string ResolveName()
|
||||
=> string.IsNullOrWhiteSpace(Name) ? Engine.ToString().ToLowerInvariant() : Name!;
|
||||
}
|
||||
|
||||
public enum ContainerRuntimeEngine
|
||||
{
|
||||
Containerd,
|
||||
CriO,
|
||||
Docker
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
using StellaOps.Zastava.Observer.ContainerRuntime.Cri;
|
||||
|
||||
namespace StellaOps.Zastava.Observer.ContainerRuntime;
|
||||
|
||||
internal sealed class ContainerStateTracker
|
||||
{
|
||||
private readonly Dictionary<string, ContainerStateEntry> entries = new(StringComparer.Ordinal);
|
||||
|
||||
public void BeginCycle()
|
||||
{
|
||||
foreach (var entry in entries.Values)
|
||||
{
|
||||
entry.SeenInCycle = false;
|
||||
}
|
||||
}
|
||||
|
||||
public ContainerLifecycleEvent? MarkRunning(CriContainerInfo snapshot, DateTimeOffset fallbackTimestamp)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(snapshot);
|
||||
var timestamp = snapshot.StartedAt ?? snapshot.CreatedAt;
|
||||
if (timestamp <= DateTimeOffset.MinValue)
|
||||
{
|
||||
timestamp = fallbackTimestamp;
|
||||
}
|
||||
|
||||
if (!entries.TryGetValue(snapshot.Id, out var entry))
|
||||
{
|
||||
entry = new ContainerStateEntry(snapshot);
|
||||
entries[snapshot.Id] = entry;
|
||||
entry.SeenInCycle = true;
|
||||
entry.State = ContainerLifecycleState.Running;
|
||||
entry.LastStart = timestamp;
|
||||
entry.LastSnapshot = snapshot;
|
||||
return new ContainerLifecycleEvent(ContainerLifecycleEventKind.Start, timestamp, snapshot);
|
||||
}
|
||||
|
||||
entry.SeenInCycle = true;
|
||||
|
||||
if (timestamp > entry.LastStart)
|
||||
{
|
||||
entry.LastStart = timestamp;
|
||||
entry.State = ContainerLifecycleState.Running;
|
||||
entry.LastSnapshot = snapshot;
|
||||
return new ContainerLifecycleEvent(ContainerLifecycleEventKind.Start, timestamp, snapshot);
|
||||
}
|
||||
|
||||
entry.State = ContainerLifecycleState.Running;
|
||||
entry.LastSnapshot = snapshot;
|
||||
return null;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<ContainerLifecycleEvent>> CompleteCycleAsync(
|
||||
Func<string, Task<CriContainerInfo?>> statusProvider,
|
||||
DateTimeOffset fallbackTimestamp,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(statusProvider);
|
||||
|
||||
var events = new List<ContainerLifecycleEvent>();
|
||||
foreach (var (containerId, entry) in entries.ToArray())
|
||||
{
|
||||
if (entry.SeenInCycle)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
CriContainerInfo? status = null;
|
||||
if (entry.LastSnapshot is not null && entry.LastSnapshot.FinishedAt is not null)
|
||||
{
|
||||
status = entry.LastSnapshot;
|
||||
}
|
||||
else
|
||||
{
|
||||
status = await statusProvider(containerId).ConfigureAwait(false) ?? entry.LastSnapshot;
|
||||
}
|
||||
|
||||
var stopTimestamp = status?.FinishedAt ?? fallbackTimestamp;
|
||||
if (stopTimestamp <= DateTimeOffset.MinValue)
|
||||
{
|
||||
stopTimestamp = fallbackTimestamp;
|
||||
}
|
||||
|
||||
if (entry.LastStop is not null && stopTimestamp <= entry.LastStop)
|
||||
{
|
||||
entries.Remove(containerId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var snapshot = status ?? entry.LastSnapshot ?? entry.MetadataFallback;
|
||||
var stopEvent = new ContainerLifecycleEvent(ContainerLifecycleEventKind.Stop, stopTimestamp, snapshot);
|
||||
events.Add(stopEvent);
|
||||
|
||||
entry.LastStop = stopTimestamp;
|
||||
entry.State = ContainerLifecycleState.Stopped;
|
||||
entries.Remove(containerId);
|
||||
}
|
||||
|
||||
return events
|
||||
.OrderBy(static e => e.Timestamp)
|
||||
.ThenBy(static e => e.Snapshot.Id, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private sealed class ContainerStateEntry
|
||||
{
|
||||
public ContainerStateEntry(CriContainerInfo seed)
|
||||
{
|
||||
MetadataFallback = seed;
|
||||
LastSnapshot = seed;
|
||||
}
|
||||
|
||||
public ContainerLifecycleState State { get; set; } = ContainerLifecycleState.Unknown;
|
||||
public bool SeenInCycle { get; set; }
|
||||
public DateTimeOffset LastStart { get; set; } = DateTimeOffset.MinValue;
|
||||
public DateTimeOffset? LastStop { get; set; }
|
||||
public CriContainerInfo MetadataFallback { get; }
|
||||
public CriContainerInfo? LastSnapshot { get; set; }
|
||||
}
|
||||
}
|
||||
|
||||
internal enum ContainerLifecycleState
|
||||
{
|
||||
Unknown,
|
||||
Running,
|
||||
Stopped
|
||||
}
|
||||
|
||||
internal sealed record ContainerLifecycleEvent(ContainerLifecycleEventKind Kind, DateTimeOffset Timestamp, CriContainerInfo Snapshot);
|
||||
|
||||
internal enum ContainerLifecycleEventKind
|
||||
{
|
||||
Start,
|
||||
Stop
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
using StellaOps.Zastava.Observer.Cri;
|
||||
|
||||
namespace StellaOps.Zastava.Observer.ContainerRuntime.Cri;
|
||||
|
||||
internal static class CriConversions
|
||||
{
|
||||
private const long NanosecondsPerTick = 100;
|
||||
|
||||
public static CriContainerInfo ToContainerInfo(Container container)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(container);
|
||||
|
||||
return new CriContainerInfo(
|
||||
Id: container.Id ?? string.Empty,
|
||||
PodSandboxId: container.PodSandboxId ?? string.Empty,
|
||||
Name: container.Metadata?.Name ?? string.Empty,
|
||||
Attempt: container.Metadata?.Attempt ?? 0,
|
||||
Image: container.Image?.Image,
|
||||
ImageRef: container.ImageRef,
|
||||
Labels: container.Labels?.ToDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.Ordinal) ?? new Dictionary<string, string>(StringComparer.Ordinal),
|
||||
Annotations: container.Annotations?.ToDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.Ordinal) ?? new Dictionary<string, string>(StringComparer.Ordinal),
|
||||
CreatedAt: FromUnixNanoseconds(container.CreatedAt),
|
||||
StartedAt: null,
|
||||
FinishedAt: null,
|
||||
ExitCode: null,
|
||||
Reason: null,
|
||||
Message: null);
|
||||
}
|
||||
|
||||
public static CriContainerInfo MergeStatus(CriContainerInfo baseline, ContainerStatus? status)
|
||||
{
|
||||
if (status is null)
|
||||
{
|
||||
return baseline;
|
||||
}
|
||||
|
||||
var labels = status.Labels?.ToDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.Ordinal)
|
||||
?? baseline.Labels;
|
||||
var annotations = status.Annotations?.ToDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.Ordinal)
|
||||
?? baseline.Annotations;
|
||||
|
||||
return baseline with
|
||||
{
|
||||
CreatedAt = status.CreatedAt > 0 ? FromUnixNanoseconds(status.CreatedAt) : baseline.CreatedAt,
|
||||
StartedAt = status.StartedAt > 0 ? FromUnixNanoseconds(status.StartedAt) : baseline.StartedAt,
|
||||
FinishedAt = status.FinishedAt > 0 ? FromUnixNanoseconds(status.FinishedAt) : baseline.FinishedAt,
|
||||
ExitCode = status.ExitCode != 0 ? status.ExitCode : baseline.ExitCode,
|
||||
Reason = string.IsNullOrWhiteSpace(status.Reason) ? baseline.Reason : status.Reason,
|
||||
Message = string.IsNullOrWhiteSpace(status.Message) ? baseline.Message : status.Message,
|
||||
Image: status.Image?.Image ?? baseline.Image,
|
||||
ImageRef: string.IsNullOrWhiteSpace(status.ImageRef) ? baseline.ImageRef : status.ImageRef,
|
||||
Labels = labels,
|
||||
Annotations = annotations
|
||||
};
|
||||
}
|
||||
|
||||
public static DateTimeOffset FromUnixNanoseconds(long nanoseconds)
|
||||
{
|
||||
if (nanoseconds <= 0)
|
||||
{
|
||||
return DateTimeOffset.MinValue;
|
||||
}
|
||||
|
||||
var seconds = Math.DivRem(nanoseconds, 1_000_000_000, out var remainder);
|
||||
var ticks = remainder / NanosecondsPerTick;
|
||||
try
|
||||
{
|
||||
var baseTime = DateTimeOffset.FromUnixTimeSeconds(seconds);
|
||||
return baseTime.AddTicks(ticks);
|
||||
}
|
||||
catch (ArgumentOutOfRangeException)
|
||||
{
|
||||
return DateTimeOffset.UnixEpoch;
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user