Add unit and integration tests for VexCandidateEmitter and SmartDiff repositories
- Implemented comprehensive unit tests for VexCandidateEmitter to validate candidate emission logic based on various scenarios including absent and present APIs, confidence thresholds, and rate limiting. - Added integration tests for SmartDiff PostgreSQL repositories, covering snapshot storage and retrieval, candidate storage, and material risk change handling. - Ensured tests validate correct behavior for storing, retrieving, and querying snapshots and candidates, including edge cases and expected outcomes.
This commit is contained in:
master
@@ -17,7 +17,7 @@ Operate the StellaOps Attestor service: accept signed DSSE envelopes from the Si
|
||||
## Key Directories
|
||||
- `src/Attestor/StellaOps.Attestor/StellaOps.Attestor.WebService/` — Minimal API host and HTTP surface.
|
||||
- `src/Attestor/StellaOps.Attestor/StellaOps.Attestor.Core/` — Domain contracts, submission/verification pipelines.
|
||||
- `src/Attestor/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/` — Mongo, Redis, Rekor, and archival implementations.
|
||||
- `src/Attestor/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/` — PostgreSQL, Redis, Rekor, and archival implementations.
|
||||
- `src/Attestor/StellaOps.Attestor/StellaOps.Attestor.Tests/` — Unit and integration tests.
|
||||
|
||||
---
|
||||
|
||||
@@ -37,6 +37,29 @@ public sealed class AttestorMetrics : IDisposable
|
||||
RekorOfflineVerifyTotal = _meter.CreateCounter<long>("attestor.rekor_offline_verify_total", description: "Rekor offline mode verification attempts grouped by result.");
|
||||
RekorCheckpointCacheHits = _meter.CreateCounter<long>("attestor.rekor_checkpoint_cache_hits", description: "Rekor checkpoint cache hits.");
|
||||
RekorCheckpointCacheMisses = _meter.CreateCounter<long>("attestor.rekor_checkpoint_cache_misses", description: "Rekor checkpoint cache misses.");
|
||||
|
||||
// SPRINT_3000_0001_0002 - Rekor retry queue metrics
|
||||
RekorQueueDepth = _meter.CreateObservableGauge("attestor.rekor_queue_depth",
|
||||
() => _queueDepthCallback?.Invoke() ?? 0,
|
||||
description: "Current Rekor queue depth (pending + retrying items).");
|
||||
RekorRetryAttemptsTotal = _meter.CreateCounter<long>("attestor.rekor_retry_attempts_total", description: "Total Rekor retry attempts grouped by backend and attempt number.");
|
||||
RekorSubmissionStatusTotal = _meter.CreateCounter<long>("attestor.rekor_submission_status_total", description: "Total Rekor submission status changes grouped by status and backend.");
|
||||
RekorQueueWaitTime = _meter.CreateHistogram<double>("attestor.rekor_queue_wait_seconds", unit: "s", description: "Time items spend waiting in the Rekor queue in seconds.");
|
||||
RekorDeadLetterTotal = _meter.CreateCounter<long>("attestor.rekor_dead_letter_total", description: "Total dead letter items grouped by backend.");
|
||||
|
||||
// SPRINT_3000_0001_0003 - Time skew validation metrics
|
||||
TimeSkewDetectedTotal = _meter.CreateCounter<long>("attestor.time_skew_detected_total", description: "Time skew anomalies detected grouped by severity and action.");
|
||||
TimeSkewSeconds = _meter.CreateHistogram<double>("attestor.time_skew_seconds", unit: "s", description: "Distribution of time skew values in seconds.");
|
||||
}
|
||||
|
||||
private Func<int>? _queueDepthCallback;
|
||||
|
||||
/// <summary>
|
||||
/// Register a callback to provide the current queue depth.
|
||||
/// </summary>
|
||||
public void RegisterQueueDepthCallback(Func<int> callback)
|
||||
{
|
||||
_queueDepthCallback = callback;
|
||||
}
|
||||
|
||||
public Counter<long> SubmitTotal { get; }
|
||||
@@ -107,6 +130,43 @@ public sealed class AttestorMetrics : IDisposable
|
||||
/// </summary>
|
||||
public Counter<long> RekorCheckpointCacheMisses { get; }
|
||||
|
||||
// SPRINT_3000_0001_0002 - Rekor retry queue metrics
|
||||
/// <summary>
|
||||
/// Current Rekor queue depth (pending + retrying items).
|
||||
/// </summary>
|
||||
public ObservableGauge<int> RekorQueueDepth { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Total Rekor retry attempts grouped by backend and attempt number.
|
||||
/// </summary>
|
||||
public Counter<long> RekorRetryAttemptsTotal { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Total Rekor submission status changes grouped by status and backend.
|
||||
/// </summary>
|
||||
public Counter<long> RekorSubmissionStatusTotal { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Time items spend waiting in the Rekor queue in seconds.
|
||||
/// </summary>
|
||||
public Histogram<double> RekorQueueWaitTime { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Total dead letter items grouped by backend.
|
||||
/// </summary>
|
||||
public Counter<long> RekorDeadLetterTotal { get; }
|
||||
|
||||
// SPRINT_3000_0001_0003 - Time skew validation metrics
|
||||
/// <summary>
|
||||
/// Time skew anomalies detected grouped by severity and action.
|
||||
/// </summary>
|
||||
public Counter<long> TimeSkewDetectedTotal { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Distribution of time skew values in seconds.
|
||||
/// </summary>
|
||||
public Histogram<double> TimeSkewSeconds { get; }
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed)
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System.Collections.Generic;
|
||||
using StellaOps.Attestor.Core.Verification;
|
||||
using StellaOps.Cryptography;
|
||||
|
||||
namespace StellaOps.Attestor.Core.Options;
|
||||
@@ -32,6 +33,11 @@ public sealed class AttestorOptions
|
||||
public TransparencyWitnessOptions TransparencyWitness { get; set; } = new();
|
||||
public VerificationOptions Verification { get; set; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Time skew validation options per SPRINT_3000_0001_0003.
|
||||
/// </summary>
|
||||
public TimeSkewOptions TimeSkew { get; set; } = new();
|
||||
|
||||
|
||||
public sealed class SecurityOptions
|
||||
{
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// IRekorSubmissionQueue.cs
|
||||
// Sprint: SPRINT_3000_0001_0002_rekor_retry_queue_metrics
|
||||
// Task: T3
|
||||
// Description: Interface for the Rekor submission queue
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
namespace StellaOps.Attestor.Core.Queue;
|
||||
|
||||
/// <summary>
|
||||
/// Interface for the durable Rekor submission queue.
|
||||
/// </summary>
|
||||
public interface IRekorSubmissionQueue
|
||||
{
|
||||
/// <summary>
|
||||
/// Enqueue a DSSE envelope for Rekor submission.
|
||||
/// </summary>
|
||||
/// <param name="tenantId">Tenant identifier.</param>
|
||||
/// <param name="bundleSha256">SHA-256 hash of the bundle being attested.</param>
|
||||
/// <param name="dssePayload">Serialized DSSE envelope payload.</param>
|
||||
/// <param name="backend">Target Rekor backend ('primary' or 'mirror').</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>The ID of the created queue item.</returns>
|
||||
Task<Guid> EnqueueAsync(
|
||||
string tenantId,
|
||||
string bundleSha256,
|
||||
byte[] dssePayload,
|
||||
string backend,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Dequeue items ready for submission/retry.
|
||||
/// Items are atomically transitioned to Submitting status.
|
||||
/// </summary>
|
||||
/// <param name="batchSize">Maximum number of items to dequeue.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>List of items ready for processing.</returns>
|
||||
Task<IReadOnlyList<RekorQueueItem>> DequeueAsync(
|
||||
int batchSize,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Mark item as successfully submitted.
|
||||
/// </summary>
|
||||
/// <param name="id">Queue item ID.</param>
|
||||
/// <param name="rekorUuid">UUID from Rekor.</param>
|
||||
/// <param name="logIndex">Log index from Rekor.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
Task MarkSubmittedAsync(
|
||||
Guid id,
|
||||
string rekorUuid,
|
||||
long? logIndex,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Mark item for retry with exponential backoff.
|
||||
/// </summary>
|
||||
/// <param name="id">Queue item ID.</param>
|
||||
/// <param name="error">Error message from the failed attempt.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
Task MarkRetryAsync(
|
||||
Guid id,
|
||||
string error,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Move item to dead letter after max retries.
|
||||
/// </summary>
|
||||
/// <param name="id">Queue item ID.</param>
|
||||
/// <param name="error">Error message from the final failed attempt.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
Task MarkDeadLetterAsync(
|
||||
Guid id,
|
||||
string error,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Get item by ID.
|
||||
/// </summary>
|
||||
/// <param name="id">Queue item ID.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>The queue item, or null if not found.</returns>
|
||||
Task<RekorQueueItem?> GetByIdAsync(
|
||||
Guid id,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Get current queue depth by status.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Snapshot of queue depth.</returns>
|
||||
Task<QueueDepthSnapshot> GetQueueDepthAsync(
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Purge dead letter items older than the retention period.
|
||||
/// </summary>
|
||||
/// <param name="retentionDays">Items older than this are purged.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Number of items purged.</returns>
|
||||
Task<int> PurgeDeadLetterAsync(
|
||||
int retentionDays,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Re-enqueue a dead letter item for retry.
|
||||
/// </summary>
|
||||
/// <param name="id">Queue item ID.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>True if the item was re-enqueued.</returns>
|
||||
Task<bool> RequeueDeadLetterAsync(
|
||||
Guid id,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
@@ -10,34 +10,47 @@ namespace StellaOps.Attestor.Core.Queue;
|
||||
/// <summary>
|
||||
/// Represents an item in the Rekor submission queue.
|
||||
/// </summary>
|
||||
/// <param name="Id">Unique identifier for the queue item.</param>
|
||||
/// <param name="TenantId">Tenant identifier.</param>
|
||||
/// <param name="BundleSha256">SHA-256 hash of the bundle being attested.</param>
|
||||
/// <param name="DssePayload">Serialized DSSE envelope payload.</param>
|
||||
/// <param name="Backend">Target Rekor backend ('primary' or 'mirror').</param>
|
||||
/// <param name="Status">Current submission status.</param>
|
||||
/// <param name="AttemptCount">Number of submission attempts made.</param>
|
||||
/// <param name="MaxAttempts">Maximum allowed attempts before dead-lettering.</param>
|
||||
/// <param name="LastAttemptAt">Timestamp of the last submission attempt.</param>
|
||||
/// <param name="LastError">Error message from the last failed attempt.</param>
|
||||
/// <param name="NextRetryAt">Scheduled time for the next retry attempt.</param>
|
||||
/// <param name="RekorUuid">UUID from Rekor after successful submission.</param>
|
||||
/// <param name="RekorLogIndex">Log index from Rekor after successful submission.</param>
|
||||
/// <param name="CreatedAt">Timestamp when the item was created.</param>
|
||||
/// <param name="UpdatedAt">Timestamp when the item was last updated.</param>
|
||||
public sealed record RekorQueueItem(
|
||||
Guid Id,
|
||||
string TenantId,
|
||||
string BundleSha256,
|
||||
byte[] DssePayload,
|
||||
string Backend,
|
||||
RekorSubmissionStatus Status,
|
||||
int AttemptCount,
|
||||
int MaxAttempts,
|
||||
DateTimeOffset? LastAttemptAt,
|
||||
string? LastError,
|
||||
DateTimeOffset? NextRetryAt,
|
||||
string? RekorUuid,
|
||||
long? RekorLogIndex,
|
||||
DateTimeOffset CreatedAt,
|
||||
DateTimeOffset UpdatedAt);
|
||||
public sealed class RekorQueueItem
|
||||
{
|
||||
/// <summary>Unique identifier for the queue item.</summary>
|
||||
public required Guid Id { get; init; }
|
||||
|
||||
/// <summary>Tenant identifier.</summary>
|
||||
public required string TenantId { get; init; }
|
||||
|
||||
/// <summary>SHA-256 hash of the bundle being attested.</summary>
|
||||
public required string BundleSha256 { get; init; }
|
||||
|
||||
/// <summary>Serialized DSSE envelope payload.</summary>
|
||||
public required byte[] DssePayload { get; init; }
|
||||
|
||||
/// <summary>Target Rekor backend ('primary' or 'mirror').</summary>
|
||||
public required string Backend { get; init; }
|
||||
|
||||
/// <summary>Current submission status.</summary>
|
||||
public required RekorSubmissionStatus Status { get; init; }
|
||||
|
||||
/// <summary>Number of submission attempts made.</summary>
|
||||
public required int AttemptCount { get; init; }
|
||||
|
||||
/// <summary>Maximum allowed attempts before dead-lettering.</summary>
|
||||
public required int MaxAttempts { get; init; }
|
||||
|
||||
/// <summary>Scheduled time for the next retry attempt.</summary>
|
||||
public DateTimeOffset? NextRetryAt { get; init; }
|
||||
|
||||
/// <summary>Timestamp when the item was created.</summary>
|
||||
public required DateTimeOffset CreatedAt { get; init; }
|
||||
|
||||
/// <summary>Timestamp when the item was last updated.</summary>
|
||||
public required DateTimeOffset UpdatedAt { get; init; }
|
||||
|
||||
/// <summary>Error message from the last failed attempt.</summary>
|
||||
public string? LastError { get; init; }
|
||||
|
||||
/// <summary>UUID from Rekor after successful submission.</summary>
|
||||
public string? RekorUuid { get; init; }
|
||||
|
||||
/// <summary>Log index from Rekor after successful submission.</summary>
|
||||
public long? RekorIndex { get; init; }
|
||||
}
|
||||
|
||||
@@ -92,6 +92,20 @@ public sealed class AttestorEntry
|
||||
public string Url { get; init; } = string.Empty;
|
||||
|
||||
public string? LogId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Unix timestamp when entry was integrated into the Rekor log.
|
||||
/// Used for time skew validation (SPRINT_3000_0001_0003).
|
||||
/// </summary>
|
||||
public long? IntegratedTime { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the integrated time as UTC DateTimeOffset.
|
||||
/// </summary>
|
||||
public DateTimeOffset? IntegratedTimeUtc =>
|
||||
IntegratedTime.HasValue
|
||||
? DateTimeOffset.FromUnixTimeSeconds(IntegratedTime.Value)
|
||||
: null;
|
||||
}
|
||||
|
||||
public sealed class SignerIdentityDescriptor
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// InstrumentedTimeSkewValidator.cs
|
||||
// Sprint: SPRINT_3000_0001_0003_rekor_time_skew_validation
|
||||
// Task: T7, T8
|
||||
// Description: Time skew validator with metrics and structured logging
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Attestor.Core.Observability;
|
||||
|
||||
namespace StellaOps.Attestor.Core.Verification;
|
||||
|
||||
/// <summary>
|
||||
/// Time skew validator with integrated metrics and structured logging.
|
||||
/// Wraps the base TimeSkewValidator with observability.
|
||||
/// </summary>
|
||||
public sealed class InstrumentedTimeSkewValidator : ITimeSkewValidator
|
||||
{
|
||||
private readonly TimeSkewValidator _inner;
|
||||
private readonly AttestorMetrics _metrics;
|
||||
private readonly ILogger<InstrumentedTimeSkewValidator> _logger;
|
||||
|
||||
public InstrumentedTimeSkewValidator(
|
||||
TimeSkewOptions options,
|
||||
AttestorMetrics metrics,
|
||||
ILogger<InstrumentedTimeSkewValidator> logger)
|
||||
{
|
||||
_inner = new TimeSkewValidator(options ?? throw new ArgumentNullException(nameof(options)));
|
||||
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public TimeSkewValidationResult Validate(DateTimeOffset? integratedTime, DateTimeOffset? localTime = null)
|
||||
{
|
||||
var result = _inner.Validate(integratedTime, localTime);
|
||||
|
||||
// Record skew distribution for all validations (except skipped)
|
||||
if (result.Status != TimeSkewStatus.Skipped)
|
||||
{
|
||||
_metrics.TimeSkewSeconds.Record(Math.Abs(result.SkewSeconds));
|
||||
}
|
||||
|
||||
// Record anomalies and log structured events
|
||||
switch (result.Status)
|
||||
{
|
||||
case TimeSkewStatus.Warning:
|
||||
_metrics.TimeSkewDetectedTotal.Add(1,
|
||||
new("severity", "warning"),
|
||||
new("action", "warned"));
|
||||
|
||||
_logger.LogWarning(
|
||||
"Time skew warning detected: IntegratedTime={IntegratedTime:O}, LocalTime={LocalTime:O}, SkewSeconds={SkewSeconds:F1}, Status={Status}",
|
||||
result.IntegratedTime,
|
||||
result.LocalTime,
|
||||
result.SkewSeconds,
|
||||
result.Status);
|
||||
break;
|
||||
|
||||
case TimeSkewStatus.Rejected:
|
||||
_metrics.TimeSkewDetectedTotal.Add(1,
|
||||
new("severity", "rejected"),
|
||||
new("action", "rejected"));
|
||||
|
||||
_logger.LogError(
|
||||
"Time skew rejected: IntegratedTime={IntegratedTime:O}, LocalTime={LocalTime:O}, SkewSeconds={SkewSeconds:F1}, Status={Status}, Message={Message}",
|
||||
result.IntegratedTime,
|
||||
result.LocalTime,
|
||||
result.SkewSeconds,
|
||||
result.Status,
|
||||
result.Message);
|
||||
break;
|
||||
|
||||
case TimeSkewStatus.FutureTimestamp:
|
||||
_metrics.TimeSkewDetectedTotal.Add(1,
|
||||
new("severity", "future"),
|
||||
new("action", "rejected"));
|
||||
|
||||
_logger.LogError(
|
||||
"Future timestamp detected (potential tampering): IntegratedTime={IntegratedTime:O}, LocalTime={LocalTime:O}, SkewSeconds={SkewSeconds:F1}, Status={Status}",
|
||||
result.IntegratedTime,
|
||||
result.LocalTime,
|
||||
result.SkewSeconds,
|
||||
result.Status);
|
||||
break;
|
||||
|
||||
case TimeSkewStatus.Ok:
|
||||
_logger.LogDebug(
|
||||
"Time skew validation passed: IntegratedTime={IntegratedTime:O}, LocalTime={LocalTime:O}, SkewSeconds={SkewSeconds:F1}",
|
||||
result.IntegratedTime,
|
||||
result.LocalTime,
|
||||
result.SkewSeconds);
|
||||
break;
|
||||
|
||||
case TimeSkewStatus.Skipped:
|
||||
_logger.LogDebug("Time skew validation skipped: {Message}", result.Message);
|
||||
break;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
namespace StellaOps.Attestor.Core.Verification;
|
||||
|
||||
/// <summary>
|
||||
/// Exception thrown when time skew validation fails and is configured to reject.
|
||||
/// Per SPRINT_3000_0001_0003.
|
||||
/// </summary>
|
||||
public sealed class TimeSkewValidationException : Exception
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the validation result that caused the exception.
|
||||
/// </summary>
|
||||
public TimeSkewValidationResult ValidationResult { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the time skew in seconds.
|
||||
/// </summary>
|
||||
public double SkewSeconds => ValidationResult.SkewSeconds;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the validation status.
|
||||
/// </summary>
|
||||
public TimeSkewStatus Status => ValidationResult.Status;
|
||||
|
||||
public TimeSkewValidationException(TimeSkewValidationResult result)
|
||||
: base(result.Message)
|
||||
{
|
||||
ValidationResult = result;
|
||||
}
|
||||
|
||||
public TimeSkewValidationException(TimeSkewValidationResult result, Exception innerException)
|
||||
: base(result.Message, innerException)
|
||||
{
|
||||
ValidationResult = result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- Migration: 20251216_001_create_rekor_submission_queue.sql
|
||||
-- Sprint: SPRINT_3000_0001_0002_rekor_retry_queue_metrics
|
||||
-- Task: T1
|
||||
-- Description: Create the Rekor submission queue table for durable retry
|
||||
-- -----------------------------------------------------------------------------
|
||||
|
||||
-- Create attestor schema if not exists
|
||||
CREATE SCHEMA IF NOT EXISTS attestor;
|
||||
|
||||
-- Create the queue table
|
||||
CREATE TABLE IF NOT EXISTS attestor.rekor_submission_queue (
|
||||
id UUID PRIMARY KEY,
|
||||
tenant_id TEXT NOT NULL,
|
||||
bundle_sha256 TEXT NOT NULL,
|
||||
dsse_payload BYTEA NOT NULL,
|
||||
backend TEXT NOT NULL DEFAULT 'primary',
|
||||
|
||||
-- Status lifecycle: pending -> submitting -> submitted | retrying -> dead_letter
|
||||
status TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (status IN ('pending', 'submitting', 'retrying', 'submitted', 'dead_letter')),
|
||||
|
||||
attempt_count INTEGER NOT NULL DEFAULT 0,
|
||||
max_attempts INTEGER NOT NULL DEFAULT 5,
|
||||
next_retry_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
|
||||
-- Populated on success
|
||||
rekor_uuid TEXT,
|
||||
rekor_index BIGINT,
|
||||
|
||||
-- Populated on failure
|
||||
last_error TEXT
|
||||
);
|
||||
|
||||
-- Comments
|
||||
COMMENT ON TABLE attestor.rekor_submission_queue IS
|
||||
'Durable retry queue for Rekor transparency log submissions';
|
||||
COMMENT ON COLUMN attestor.rekor_submission_queue.status IS
|
||||
'Submission lifecycle: pending -> submitting -> (submitted | retrying -> dead_letter)';
|
||||
COMMENT ON COLUMN attestor.rekor_submission_queue.backend IS
|
||||
'Target Rekor backend (primary or mirror)';
|
||||
COMMENT ON COLUMN attestor.rekor_submission_queue.dsse_payload IS
|
||||
'Serialized DSSE envelope to submit';
|
||||
|
||||
-- Index for dequeue operations (status + next_retry_at for SKIP LOCKED queries)
|
||||
CREATE INDEX IF NOT EXISTS idx_rekor_queue_dequeue
|
||||
ON attestor.rekor_submission_queue (status, next_retry_at)
|
||||
WHERE status IN ('pending', 'retrying');
|
||||
|
||||
-- Index for tenant-scoped queries
|
||||
CREATE INDEX IF NOT EXISTS idx_rekor_queue_tenant
|
||||
ON attestor.rekor_submission_queue (tenant_id);
|
||||
|
||||
-- Index for bundle lookup (deduplication check)
|
||||
CREATE INDEX IF NOT EXISTS idx_rekor_queue_bundle
|
||||
ON attestor.rekor_submission_queue (tenant_id, bundle_sha256);
|
||||
|
||||
-- Index for dead letter management
|
||||
CREATE INDEX IF NOT EXISTS idx_rekor_queue_dead_letter
|
||||
ON attestor.rekor_submission_queue (status, updated_at)
|
||||
WHERE status = 'dead_letter';
|
||||
|
||||
-- Index for cleanup of completed submissions
|
||||
CREATE INDEX IF NOT EXISTS idx_rekor_queue_completed
|
||||
ON attestor.rekor_submission_queue (status, updated_at)
|
||||
WHERE status = 'submitted';
|
||||
@@ -0,0 +1,524 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// PostgresRekorSubmissionQueue.cs
|
||||
// Sprint: SPRINT_3000_0001_0002_rekor_retry_queue_metrics
|
||||
// Task: T3
|
||||
// Description: PostgreSQL implementation of the Rekor submission queue
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Npgsql;
|
||||
using NpgsqlTypes;
|
||||
using StellaOps.Attestor.Core.Observability;
|
||||
using StellaOps.Attestor.Core.Options;
|
||||
using StellaOps.Attestor.Core.Queue;
|
||||
|
||||
namespace StellaOps.Attestor.Infrastructure.Queue;
|
||||
|
||||
/// <summary>
|
||||
/// PostgreSQL implementation of the Rekor submission queue.
|
||||
/// Uses a dedicated table for queue persistence with optimistic locking.
|
||||
/// </summary>
|
||||
public sealed class PostgresRekorSubmissionQueue : IRekorSubmissionQueue
|
||||
{
|
||||
private readonly NpgsqlDataSource _dataSource;
|
||||
private readonly RekorQueueOptions _options;
|
||||
private readonly AttestorMetrics _metrics;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<PostgresRekorSubmissionQueue> _logger;
|
||||
|
||||
private const int DefaultCommandTimeoutSeconds = 30;
|
||||
|
||||
public PostgresRekorSubmissionQueue(
|
||||
NpgsqlDataSource dataSource,
|
||||
IOptions<RekorQueueOptions> options,
|
||||
AttestorMetrics metrics,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<PostgresRekorSubmissionQueue> logger)
|
||||
{
|
||||
_dataSource = dataSource ?? throw new ArgumentNullException(nameof(dataSource));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<Guid> EnqueueAsync(
|
||||
string tenantId,
|
||||
string bundleSha256,
|
||||
byte[] dssePayload,
|
||||
string backend,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var id = Guid.NewGuid();
|
||||
|
||||
const string sql = """
|
||||
INSERT INTO attestor.rekor_submission_queue (
|
||||
id, tenant_id, bundle_sha256, dsse_payload, backend,
|
||||
status, attempt_count, max_attempts, next_retry_at,
|
||||
created_at, updated_at
|
||||
)
|
||||
VALUES (
|
||||
@id, @tenant_id, @bundle_sha256, @dsse_payload, @backend,
|
||||
@status, 0, @max_attempts, @next_retry_at,
|
||||
@created_at, @updated_at
|
||||
)
|
||||
""";
|
||||
|
||||
await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken);
|
||||
await using var command = new NpgsqlCommand(sql, connection)
|
||||
{
|
||||
CommandTimeout = DefaultCommandTimeoutSeconds
|
||||
};
|
||||
|
||||
command.Parameters.AddWithValue("@id", id);
|
||||
command.Parameters.AddWithValue("@tenant_id", tenantId);
|
||||
command.Parameters.AddWithValue("@bundle_sha256", bundleSha256);
|
||||
command.Parameters.AddWithValue("@dsse_payload", dssePayload);
|
||||
command.Parameters.AddWithValue("@backend", backend);
|
||||
command.Parameters.AddWithValue("@status", RekorSubmissionStatus.Pending.ToString().ToLowerInvariant());
|
||||
command.Parameters.AddWithValue("@max_attempts", _options.MaxAttempts);
|
||||
command.Parameters.AddWithValue("@next_retry_at", now);
|
||||
command.Parameters.AddWithValue("@created_at", now);
|
||||
command.Parameters.AddWithValue("@updated_at", now);
|
||||
|
||||
await command.ExecuteNonQueryAsync(cancellationToken);
|
||||
|
||||
_metrics.RekorSubmissionStatusTotal.Add(1,
|
||||
new("status", "pending"),
|
||||
new("backend", backend));
|
||||
|
||||
_logger.LogDebug(
|
||||
"Enqueued Rekor submission {Id} for bundle {BundleSha256} to {Backend}",
|
||||
id, bundleSha256, backend);
|
||||
|
||||
return id;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IReadOnlyList<RekorQueueItem>> DequeueAsync(
|
||||
int batchSize,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
// Use FOR UPDATE SKIP LOCKED for concurrent-safe dequeue
|
||||
const string sql = """
|
||||
UPDATE attestor.rekor_submission_queue
|
||||
SET status = 'submitting', updated_at = @now
|
||||
WHERE id IN (
|
||||
SELECT id FROM attestor.rekor_submission_queue
|
||||
WHERE status IN ('pending', 'retrying')
|
||||
AND next_retry_at <= @now
|
||||
ORDER BY next_retry_at ASC
|
||||
LIMIT @batch_size
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
RETURNING id, tenant_id, bundle_sha256, dsse_payload, backend,
|
||||
status, attempt_count, max_attempts, next_retry_at,
|
||||
created_at, updated_at, last_error
|
||||
""";
|
||||
|
||||
await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken);
|
||||
await using var command = new NpgsqlCommand(sql, connection)
|
||||
{
|
||||
CommandTimeout = DefaultCommandTimeoutSeconds
|
||||
};
|
||||
|
||||
command.Parameters.AddWithValue("@now", now);
|
||||
command.Parameters.AddWithValue("@batch_size", batchSize);
|
||||
|
||||
var results = new List<RekorQueueItem>();
|
||||
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken);
|
||||
while (await reader.ReadAsync(cancellationToken))
|
||||
{
|
||||
var queuedAt = reader.GetDateTime(reader.GetOrdinal("created_at"));
|
||||
var waitTime = (now - queuedAt).TotalSeconds;
|
||||
_metrics.RekorQueueWaitTime.Record(waitTime);
|
||||
|
||||
results.Add(ReadQueueItem(reader));
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task MarkSubmittedAsync(
|
||||
Guid id,
|
||||
string rekorUuid,
|
||||
long? rekorIndex,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
const string sql = """
|
||||
UPDATE attestor.rekor_submission_queue
|
||||
SET status = 'submitted',
|
||||
rekor_uuid = @rekor_uuid,
|
||||
rekor_index = @rekor_index,
|
||||
updated_at = @updated_at,
|
||||
last_error = NULL
|
||||
WHERE id = @id
|
||||
RETURNING backend
|
||||
""";
|
||||
|
||||
await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken);
|
||||
await using var command = new NpgsqlCommand(sql, connection)
|
||||
{
|
||||
CommandTimeout = DefaultCommandTimeoutSeconds
|
||||
};
|
||||
|
||||
command.Parameters.AddWithValue("@id", id);
|
||||
command.Parameters.AddWithValue("@rekor_uuid", rekorUuid);
|
||||
command.Parameters.AddWithValue("@rekor_index", (object?)rekorIndex ?? DBNull.Value);
|
||||
command.Parameters.AddWithValue("@updated_at", now);
|
||||
|
||||
var backend = await command.ExecuteScalarAsync(cancellationToken) as string ?? "unknown";
|
||||
|
||||
_metrics.RekorSubmissionStatusTotal.Add(1,
|
||||
new("status", "submitted"),
|
||||
new("backend", backend));
|
||||
|
||||
_logger.LogInformation(
|
||||
"Marked Rekor submission {Id} as submitted with UUID {RekorUuid}",
|
||||
id, rekorUuid);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task MarkFailedAsync(
|
||||
Guid id,
|
||||
string errorMessage,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
// Fetch current state to determine next action
|
||||
const string fetchSql = """
|
||||
SELECT attempt_count, max_attempts, backend
|
||||
FROM attestor.rekor_submission_queue
|
||||
WHERE id = @id
|
||||
FOR UPDATE
|
||||
""";
|
||||
|
||||
await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken);
|
||||
await using var transaction = await connection.BeginTransactionAsync(cancellationToken);
|
||||
|
||||
int attemptCount;
|
||||
int maxAttempts;
|
||||
string backend;
|
||||
|
||||
await using (var fetchCommand = new NpgsqlCommand(fetchSql, connection, transaction))
|
||||
{
|
||||
fetchCommand.Parameters.AddWithValue("@id", id);
|
||||
|
||||
await using var reader = await fetchCommand.ExecuteReaderAsync(cancellationToken);
|
||||
if (!await reader.ReadAsync(cancellationToken))
|
||||
{
|
||||
_logger.LogWarning("Attempted to mark non-existent queue item {Id} as failed", id);
|
||||
return;
|
||||
}
|
||||
|
||||
attemptCount = reader.GetInt32(0);
|
||||
maxAttempts = reader.GetInt32(1);
|
||||
backend = reader.GetString(2);
|
||||
}
|
||||
|
||||
attemptCount++;
|
||||
var isDeadLetter = attemptCount >= maxAttempts;
|
||||
|
||||
if (isDeadLetter)
|
||||
{
|
||||
const string deadLetterSql = """
|
||||
UPDATE attestor.rekor_submission_queue
|
||||
SET status = 'dead_letter',
|
||||
attempt_count = @attempt_count,
|
||||
last_error = @last_error,
|
||||
updated_at = @updated_at
|
||||
WHERE id = @id
|
||||
""";
|
||||
|
||||
await using var command = new NpgsqlCommand(deadLetterSql, connection, transaction);
|
||||
command.Parameters.AddWithValue("@id", id);
|
||||
command.Parameters.AddWithValue("@attempt_count", attemptCount);
|
||||
command.Parameters.AddWithValue("@last_error", errorMessage);
|
||||
command.Parameters.AddWithValue("@updated_at", now);
|
||||
|
||||
await command.ExecuteNonQueryAsync(cancellationToken);
|
||||
|
||||
_metrics.RekorSubmissionStatusTotal.Add(1,
|
||||
new("status", "dead_letter"),
|
||||
new("backend", backend));
|
||||
_metrics.RekorDeadLetterTotal.Add(1, new("backend", backend));
|
||||
|
||||
_logger.LogError(
|
||||
"Moved Rekor submission {Id} to dead letter after {Attempts} attempts: {Error}",
|
||||
id, attemptCount, errorMessage);
|
||||
}
|
||||
else
|
||||
{
|
||||
var nextRetryAt = CalculateNextRetryTime(now, attemptCount);
|
||||
|
||||
const string retrySql = """
|
||||
UPDATE attestor.rekor_submission_queue
|
||||
SET status = 'retrying',
|
||||
attempt_count = @attempt_count,
|
||||
next_retry_at = @next_retry_at,
|
||||
last_error = @last_error,
|
||||
updated_at = @updated_at
|
||||
WHERE id = @id
|
||||
""";
|
||||
|
||||
await using var command = new NpgsqlCommand(retrySql, connection, transaction);
|
||||
command.Parameters.AddWithValue("@id", id);
|
||||
command.Parameters.AddWithValue("@attempt_count", attemptCount);
|
||||
command.Parameters.AddWithValue("@next_retry_at", nextRetryAt);
|
||||
command.Parameters.AddWithValue("@last_error", errorMessage);
|
||||
command.Parameters.AddWithValue("@updated_at", now);
|
||||
|
||||
await command.ExecuteNonQueryAsync(cancellationToken);
|
||||
|
||||
_metrics.RekorSubmissionStatusTotal.Add(1,
|
||||
new("status", "retrying"),
|
||||
new("backend", backend));
|
||||
_metrics.RekorRetryAttemptsTotal.Add(1,
|
||||
new("backend", backend),
|
||||
new("attempt", attemptCount.ToString()));
|
||||
|
||||
_logger.LogWarning(
|
||||
"Marked Rekor submission {Id} for retry (attempt {Attempt}/{Max}): {Error}",
|
||||
id, attemptCount, maxAttempts, errorMessage);
|
||||
}
|
||||
|
||||
await transaction.CommitAsync(cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<RekorQueueItem?> GetByIdAsync(
|
||||
Guid id,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
const string sql = """
|
||||
SELECT id, tenant_id, bundle_sha256, dsse_payload, backend,
|
||||
status, attempt_count, max_attempts, next_retry_at,
|
||||
created_at, updated_at, last_error, rekor_uuid, rekor_index
|
||||
FROM attestor.rekor_submission_queue
|
||||
WHERE id = @id
|
||||
""";
|
||||
|
||||
await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken);
|
||||
await using var command = new NpgsqlCommand(sql, connection)
|
||||
{
|
||||
CommandTimeout = DefaultCommandTimeoutSeconds
|
||||
};
|
||||
|
||||
command.Parameters.AddWithValue("@id", id);
|
||||
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken);
|
||||
if (!await reader.ReadAsync(cancellationToken))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return ReadQueueItem(reader);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IReadOnlyList<RekorQueueItem>> GetByBundleShaAsync(
|
||||
string tenantId,
|
||||
string bundleSha256,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
const string sql = """
|
||||
SELECT id, tenant_id, bundle_sha256, dsse_payload, backend,
|
||||
status, attempt_count, max_attempts, next_retry_at,
|
||||
created_at, updated_at, last_error, rekor_uuid, rekor_index
|
||||
FROM attestor.rekor_submission_queue
|
||||
WHERE tenant_id = @tenant_id AND bundle_sha256 = @bundle_sha256
|
||||
ORDER BY created_at DESC
|
||||
""";
|
||||
|
||||
await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken);
|
||||
await using var command = new NpgsqlCommand(sql, connection)
|
||||
{
|
||||
CommandTimeout = DefaultCommandTimeoutSeconds
|
||||
};
|
||||
|
||||
command.Parameters.AddWithValue("@tenant_id", tenantId);
|
||||
command.Parameters.AddWithValue("@bundle_sha256", bundleSha256);
|
||||
|
||||
var results = new List<RekorQueueItem>();
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken);
|
||||
while (await reader.ReadAsync(cancellationToken))
|
||||
{
|
||||
results.Add(ReadQueueItem(reader));
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<int> GetQueueDepthAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
const string sql = """
|
||||
SELECT COUNT(*)
|
||||
FROM attestor.rekor_submission_queue
|
||||
WHERE status IN ('pending', 'retrying', 'submitting')
|
||||
""";
|
||||
|
||||
await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken);
|
||||
await using var command = new NpgsqlCommand(sql, connection)
|
||||
{
|
||||
CommandTimeout = DefaultCommandTimeoutSeconds
|
||||
};
|
||||
|
||||
var result = await command.ExecuteScalarAsync(cancellationToken);
|
||||
return Convert.ToInt32(result);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IReadOnlyList<RekorQueueItem>> GetDeadLetterItemsAsync(
|
||||
int limit,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
const string sql = """
|
||||
SELECT id, tenant_id, bundle_sha256, dsse_payload, backend,
|
||||
status, attempt_count, max_attempts, next_retry_at,
|
||||
created_at, updated_at, last_error, rekor_uuid, rekor_index
|
||||
FROM attestor.rekor_submission_queue
|
||||
WHERE status = 'dead_letter'
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT @limit
|
||||
""";
|
||||
|
||||
await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken);
|
||||
await using var command = new NpgsqlCommand(sql, connection)
|
||||
{
|
||||
CommandTimeout = DefaultCommandTimeoutSeconds
|
||||
};
|
||||
|
||||
command.Parameters.AddWithValue("@limit", limit);
|
||||
|
||||
var results = new List<RekorQueueItem>();
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken);
|
||||
while (await reader.ReadAsync(cancellationToken))
|
||||
{
|
||||
results.Add(ReadQueueItem(reader));
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<bool> RequeueDeadLetterAsync(
|
||||
Guid id,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
const string sql = """
|
||||
UPDATE attestor.rekor_submission_queue
|
||||
SET status = 'pending',
|
||||
attempt_count = 0,
|
||||
next_retry_at = @now,
|
||||
last_error = NULL,
|
||||
updated_at = @now
|
||||
WHERE id = @id AND status = 'dead_letter'
|
||||
RETURNING backend
|
||||
""";
|
||||
|
||||
await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken);
|
||||
await using var command = new NpgsqlCommand(sql, connection)
|
||||
{
|
||||
CommandTimeout = DefaultCommandTimeoutSeconds
|
||||
};
|
||||
|
||||
command.Parameters.AddWithValue("@id", id);
|
||||
command.Parameters.AddWithValue("@now", now);
|
||||
|
||||
var backend = await command.ExecuteScalarAsync(cancellationToken) as string;
|
||||
|
||||
if (backend is not null)
|
||||
{
|
||||
_metrics.RekorSubmissionStatusTotal.Add(1,
|
||||
new("status", "pending"),
|
||||
new("backend", backend));
|
||||
|
||||
_logger.LogInformation("Requeued dead letter item {Id} for retry", id);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<int> PurgeSubmittedAsync(
|
||||
TimeSpan olderThan,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var cutoff = _timeProvider.GetUtcNow().Add(-olderThan);
|
||||
|
||||
const string sql = """
|
||||
DELETE FROM attestor.rekor_submission_queue
|
||||
WHERE status = 'submitted' AND updated_at < @cutoff
|
||||
""";
|
||||
|
||||
await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken);
|
||||
await using var command = new NpgsqlCommand(sql, connection)
|
||||
{
|
||||
CommandTimeout = DefaultCommandTimeoutSeconds
|
||||
};
|
||||
|
||||
command.Parameters.AddWithValue("@cutoff", cutoff);
|
||||
|
||||
var deleted = await command.ExecuteNonQueryAsync(cancellationToken);
|
||||
|
||||
if (deleted > 0)
|
||||
{
|
||||
_logger.LogInformation("Purged {Count} submitted queue items older than {Cutoff}", deleted, cutoff);
|
||||
}
|
||||
|
||||
return deleted;
|
||||
}
|
||||
|
||||
private DateTimeOffset CalculateNextRetryTime(DateTimeOffset now, int attemptCount)
|
||||
{
|
||||
// Exponential backoff: baseDelay * 2^attempt, capped at maxDelay
|
||||
var delay = TimeSpan.FromSeconds(
|
||||
Math.Min(
|
||||
_options.BaseRetryDelaySeconds * Math.Pow(2, attemptCount - 1),
|
||||
_options.MaxRetryDelaySeconds));
|
||||
|
||||
return now.Add(delay);
|
||||
}
|
||||
|
||||
private static RekorQueueItem ReadQueueItem(NpgsqlDataReader reader)
|
||||
{
|
||||
return new RekorQueueItem
|
||||
{
|
||||
Id = reader.GetGuid(reader.GetOrdinal("id")),
|
||||
TenantId = reader.GetString(reader.GetOrdinal("tenant_id")),
|
||||
BundleSha256 = reader.GetString(reader.GetOrdinal("bundle_sha256")),
|
||||
DssePayload = reader.GetFieldValue<byte[]>(reader.GetOrdinal("dsse_payload")),
|
||||
Backend = reader.GetString(reader.GetOrdinal("backend")),
|
||||
Status = Enum.Parse<RekorSubmissionStatus>(reader.GetString(reader.GetOrdinal("status")), ignoreCase: true),
|
||||
AttemptCount = reader.GetInt32(reader.GetOrdinal("attempt_count")),
|
||||
MaxAttempts = reader.GetInt32(reader.GetOrdinal("max_attempts")),
|
||||
NextRetryAt = reader.GetDateTime(reader.GetOrdinal("next_retry_at")),
|
||||
CreatedAt = reader.GetDateTime(reader.GetOrdinal("created_at")),
|
||||
UpdatedAt = reader.GetDateTime(reader.GetOrdinal("updated_at")),
|
||||
LastError = reader.IsDBNull(reader.GetOrdinal("last_error"))
|
||||
? null
|
||||
: reader.GetString(reader.GetOrdinal("last_error")),
|
||||
RekorUuid = reader.IsDBNull(reader.GetOrdinal("rekor_uuid"))
|
||||
? null
|
||||
: reader.GetString(reader.GetOrdinal("rekor_uuid")),
|
||||
RekorIndex = reader.IsDBNull(reader.GetOrdinal("rekor_index"))
|
||||
? null
|
||||
: reader.GetInt64(reader.GetOrdinal("rekor_index"))
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -29,6 +29,7 @@ internal sealed class AttestorSubmissionService : IAttestorSubmissionService
|
||||
private readonly IAttestorArchiveStore _archiveStore;
|
||||
private readonly IAttestorAuditSink _auditSink;
|
||||
private readonly IAttestorVerificationCache _verificationCache;
|
||||
private readonly ITimeSkewValidator _timeSkewValidator;
|
||||
private readonly ILogger<AttestorSubmissionService> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly AttestorOptions _options;
|
||||
@@ -43,6 +44,7 @@ internal sealed class AttestorSubmissionService : IAttestorSubmissionService
|
||||
IAttestorArchiveStore archiveStore,
|
||||
IAttestorAuditSink auditSink,
|
||||
IAttestorVerificationCache verificationCache,
|
||||
ITimeSkewValidator timeSkewValidator,
|
||||
IOptions<AttestorOptions> options,
|
||||
ILogger<AttestorSubmissionService> logger,
|
||||
TimeProvider timeProvider,
|
||||
@@ -56,6 +58,7 @@ internal sealed class AttestorSubmissionService : IAttestorSubmissionService
|
||||
_archiveStore = archiveStore;
|
||||
_auditSink = auditSink;
|
||||
_verificationCache = verificationCache;
|
||||
_timeSkewValidator = timeSkewValidator ?? throw new ArgumentNullException(nameof(timeSkewValidator));
|
||||
_logger = logger;
|
||||
_timeProvider = timeProvider;
|
||||
_options = options.Value;
|
||||
@@ -139,6 +142,20 @@ internal sealed class AttestorSubmissionService : IAttestorSubmissionService
|
||||
throw new InvalidOperationException("No Rekor submission outcome was produced.");
|
||||
}
|
||||
|
||||
// Validate time skew between Rekor integrated time and local time (SPRINT_3000_0001_0003 T5)
|
||||
var timeSkewResult = ValidateSubmissionTimeSkew(canonicalOutcome);
|
||||
if (!timeSkewResult.IsValid && _options.TimeSkew.FailOnReject)
|
||||
{
|
||||
_logger.LogError(
|
||||
"Submission rejected due to time skew: BundleSha={BundleSha}, IntegratedTime={IntegratedTime:O}, LocalTime={LocalTime:O}, SkewSeconds={SkewSeconds:F1}, Status={Status}",
|
||||
request.Meta.BundleSha256,
|
||||
timeSkewResult.IntegratedTime,
|
||||
timeSkewResult.LocalTime,
|
||||
timeSkewResult.SkewSeconds,
|
||||
timeSkewResult.Status);
|
||||
throw new TimeSkewValidationException(timeSkewResult);
|
||||
}
|
||||
|
||||
var entry = CreateEntry(request, context, canonicalOutcome, mirrorOutcome);
|
||||
await _repository.SaveAsync(entry, cancellationToken).ConfigureAwait(false);
|
||||
await InvalidateVerificationCacheAsync(cacheSubject, cancellationToken).ConfigureAwait(false);
|
||||
@@ -490,6 +507,23 @@ internal sealed class AttestorSubmissionService : IAttestorSubmissionService
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates time skew between Rekor integrated time and local time.
|
||||
/// Per SPRINT_3000_0001_0003 T5.
|
||||
/// </summary>
|
||||
private TimeSkewValidationResult ValidateSubmissionTimeSkew(SubmissionOutcome outcome)
|
||||
{
|
||||
if (outcome.Submission is null)
|
||||
{
|
||||
return TimeSkewValidationResult.Skipped("No submission response available");
|
||||
}
|
||||
|
||||
var integratedTime = outcome.Submission.IntegratedTimeUtc;
|
||||
var localTime = _timeProvider.GetUtcNow();
|
||||
|
||||
return _timeSkewValidator.Validate(integratedTime, localTime);
|
||||
}
|
||||
|
||||
private async Task ArchiveAsync(
|
||||
AttestorEntry entry,
|
||||
byte[] canonicalBundle,
|
||||
|
||||
@@ -25,6 +25,7 @@ internal sealed class AttestorVerificationService : IAttestorVerificationService
|
||||
private readonly IRekorClient _rekorClient;
|
||||
private readonly ITransparencyWitnessClient _witnessClient;
|
||||
private readonly IAttestorVerificationEngine _engine;
|
||||
private readonly ITimeSkewValidator _timeSkewValidator;
|
||||
private readonly ILogger<AttestorVerificationService> _logger;
|
||||
private readonly AttestorOptions _options;
|
||||
private readonly AttestorMetrics _metrics;
|
||||
@@ -37,6 +38,7 @@ internal sealed class AttestorVerificationService : IAttestorVerificationService
|
||||
IRekorClient rekorClient,
|
||||
ITransparencyWitnessClient witnessClient,
|
||||
IAttestorVerificationEngine engine,
|
||||
ITimeSkewValidator timeSkewValidator,
|
||||
IOptions<AttestorOptions> options,
|
||||
ILogger<AttestorVerificationService> logger,
|
||||
AttestorMetrics metrics,
|
||||
@@ -48,6 +50,7 @@ internal sealed class AttestorVerificationService : IAttestorVerificationService
|
||||
_rekorClient = rekorClient ?? throw new ArgumentNullException(nameof(rekorClient));
|
||||
_witnessClient = witnessClient ?? throw new ArgumentNullException(nameof(witnessClient));
|
||||
_engine = engine ?? throw new ArgumentNullException(nameof(engine));
|
||||
_timeSkewValidator = timeSkewValidator ?? throw new ArgumentNullException(nameof(timeSkewValidator));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
|
||||
_activitySource = activitySource ?? throw new ArgumentNullException(nameof(activitySource));
|
||||
@@ -72,13 +75,38 @@ internal sealed class AttestorVerificationService : IAttestorVerificationService
|
||||
using var activity = _activitySource.StartVerification(subjectTag, issuerTag, policyId);
|
||||
|
||||
var evaluationTime = _timeProvider.GetUtcNow();
|
||||
|
||||
// Validate time skew between entry's integrated time and evaluation time (SPRINT_3000_0001_0003 T6)
|
||||
var timeSkewResult = ValidateVerificationTimeSkew(entry, evaluationTime);
|
||||
var additionalIssues = new List<string>();
|
||||
if (!timeSkewResult.IsValid)
|
||||
{
|
||||
var issue = $"time_skew_rejected: {timeSkewResult.Message}";
|
||||
_logger.LogWarning(
|
||||
"Verification time skew issue for entry {Uuid}: IntegratedTime={IntegratedTime:O}, EvaluationTime={EvaluationTime:O}, SkewSeconds={SkewSeconds:F1}, Status={Status}",
|
||||
entry.RekorUuid,
|
||||
timeSkewResult.IntegratedTime,
|
||||
evaluationTime,
|
||||
timeSkewResult.SkewSeconds,
|
||||
timeSkewResult.Status);
|
||||
|
||||
if (_options.TimeSkew.FailOnReject)
|
||||
{
|
||||
additionalIssues.Add(issue);
|
||||
}
|
||||
}
|
||||
|
||||
var report = await _engine.EvaluateAsync(entry, request.Bundle, evaluationTime, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var result = report.Succeeded ? "ok" : "failed";
|
||||
// Merge any time skew issues with the report
|
||||
var allIssues = report.Issues.Concat(additionalIssues).ToArray();
|
||||
var succeeded = report.Succeeded && additionalIssues.Count == 0;
|
||||
|
||||
var result = succeeded ? "ok" : "failed";
|
||||
activity?.SetTag(AttestorTelemetryTags.Result, result);
|
||||
if (!report.Succeeded)
|
||||
if (!succeeded)
|
||||
{
|
||||
activity?.SetStatus(ActivityStatusCode.Error, string.Join(",", report.Issues));
|
||||
activity?.SetStatus(ActivityStatusCode.Error, string.Join(",", allIssues));
|
||||
}
|
||||
|
||||
_metrics.VerifyTotal.Add(
|
||||
@@ -98,17 +126,27 @@ internal sealed class AttestorVerificationService : IAttestorVerificationService
|
||||
|
||||
return new AttestorVerificationResult
|
||||
{
|
||||
Ok = report.Succeeded,
|
||||
Ok = succeeded,
|
||||
Uuid = entry.RekorUuid,
|
||||
Index = entry.Index,
|
||||
LogUrl = entry.Log.Url,
|
||||
Status = entry.Status,
|
||||
Issues = report.Issues,
|
||||
Issues = allIssues,
|
||||
CheckedAt = evaluationTime,
|
||||
Report = report
|
||||
Report = report with { Succeeded = succeeded, Issues = allIssues }
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates time skew between entry's integrated time and evaluation time.
|
||||
/// Per SPRINT_3000_0001_0003 T6.
|
||||
/// </summary>
|
||||
private TimeSkewValidationResult ValidateVerificationTimeSkew(AttestorEntry entry, DateTimeOffset evaluationTime)
|
||||
{
|
||||
var integratedTime = entry.Log.IntegratedTimeUtc;
|
||||
return _timeSkewValidator.Validate(integratedTime, evaluationTime);
|
||||
}
|
||||
|
||||
public Task<AttestorEntry?> GetEntryAsync(string rekorUuid, bool refreshProof, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(rekorUuid))
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// RekorRetryWorker.cs
|
||||
// Sprint: SPRINT_3000_0001_0002_rekor_retry_queue_metrics
|
||||
// Task: T7
|
||||
// Description: Background service for processing the Rekor retry queue
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Attestor.Core.Observability;
|
||||
using StellaOps.Attestor.Core.Options;
|
||||
using StellaOps.Attestor.Core.Queue;
|
||||
using StellaOps.Attestor.Core.Rekor;
|
||||
using StellaOps.Attestor.Core.Submission;
|
||||
|
||||
namespace StellaOps.Attestor.Infrastructure.Workers;
|
||||
|
||||
/// <summary>
|
||||
/// Background service for processing the Rekor submission retry queue.
|
||||
/// </summary>
|
||||
public sealed class RekorRetryWorker : BackgroundService
|
||||
{
|
||||
private readonly IRekorSubmissionQueue _queue;
|
||||
private readonly IRekorClient _rekorClient;
|
||||
private readonly RekorQueueOptions _options;
|
||||
private readonly AttestorOptions _attestorOptions;
|
||||
private readonly AttestorMetrics _metrics;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<RekorRetryWorker> _logger;
|
||||
|
||||
public RekorRetryWorker(
|
||||
IRekorSubmissionQueue queue,
|
||||
IRekorClient rekorClient,
|
||||
IOptions<RekorQueueOptions> queueOptions,
|
||||
IOptions<AttestorOptions> attestorOptions,
|
||||
AttestorMetrics metrics,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<RekorRetryWorker> logger)
|
||||
{
|
||||
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
|
||||
_rekorClient = rekorClient ?? throw new ArgumentNullException(nameof(rekorClient));
|
||||
_options = queueOptions?.Value ?? throw new ArgumentNullException(nameof(queueOptions));
|
||||
_attestorOptions = attestorOptions?.Value ?? throw new ArgumentNullException(nameof(attestorOptions));
|
||||
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
|
||||
// Register queue depth callback for metrics
|
||||
_metrics.RegisterQueueDepthCallback(GetCurrentQueueDepth);
|
||||
}
|
||||
|
||||
private int _lastKnownQueueDepth;
|
||||
|
||||
private int GetCurrentQueueDepth() => _lastKnownQueueDepth;
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
if (!_options.Enabled)
|
||||
{
|
||||
_logger.LogInformation("Rekor retry queue is disabled");
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Rekor retry worker started with batch size {BatchSize}, poll interval {PollIntervalMs}ms",
|
||||
_options.BatchSize, _options.PollIntervalMs);
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
await ProcessBatchAsync(stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Rekor retry worker error during batch processing");
|
||||
_metrics.ErrorTotal.Add(1, new("type", "rekor_retry_worker"));
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await Task.Delay(_options.PollIntervalMs, stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogInformation("Rekor retry worker stopped");
|
||||
}
|
||||
|
||||
private async Task ProcessBatchAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
// Update queue depth gauge
|
||||
var depth = await _queue.GetQueueDepthAsync(stoppingToken);
|
||||
_lastKnownQueueDepth = depth.TotalWaiting;
|
||||
|
||||
if (depth.TotalWaiting == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogDebug(
|
||||
"Queue depth: pending={Pending}, submitting={Submitting}, retrying={Retrying}, dead_letter={DeadLetter}",
|
||||
depth.Pending, depth.Submitting, depth.Retrying, depth.DeadLetter);
|
||||
|
||||
// Process batch
|
||||
var items = await _queue.DequeueAsync(_options.BatchSize, stoppingToken);
|
||||
|
||||
if (items.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogDebug("Processing {Count} items from Rekor queue", items.Count);
|
||||
|
||||
foreach (var item in items)
|
||||
{
|
||||
if (stoppingToken.IsCancellationRequested)
|
||||
break;
|
||||
|
||||
await ProcessItemAsync(item, stoppingToken);
|
||||
}
|
||||
|
||||
// Purge old dead letter items periodically
|
||||
if (_options.DeadLetterRetentionDays > 0 && depth.DeadLetter > 0)
|
||||
{
|
||||
await _queue.PurgeDeadLetterAsync(_options.DeadLetterRetentionDays, stoppingToken);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ProcessItemAsync(RekorQueueItem item, CancellationToken ct)
|
||||
{
|
||||
var attemptNumber = item.AttemptCount + 1;
|
||||
|
||||
_logger.LogDebug(
|
||||
"Processing Rekor queue item {Id}, attempt {Attempt}/{MaxAttempts}, backend={Backend}",
|
||||
item.Id, attemptNumber, item.MaxAttempts, item.Backend);
|
||||
|
||||
_metrics.RekorRetryAttemptsTotal.Add(1,
|
||||
new("backend", item.Backend),
|
||||
new("attempt", attemptNumber.ToString()));
|
||||
|
||||
try
|
||||
{
|
||||
var backend = ResolveBackend(item.Backend);
|
||||
var request = BuildSubmissionRequest(item);
|
||||
|
||||
var response = await _rekorClient.SubmitAsync(request, backend, ct);
|
||||
|
||||
await _queue.MarkSubmittedAsync(
|
||||
item.Id,
|
||||
response.Uuid ?? string.Empty,
|
||||
response.Index,
|
||||
ct);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Rekor queue item {Id} successfully submitted: UUID={RekorUuid}, Index={LogIndex}",
|
||||
item.Id, response.Uuid, response.Index);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"Rekor queue item {Id} submission failed on attempt {Attempt}: {Message}",
|
||||
item.Id, attemptNumber, ex.Message);
|
||||
|
||||
if (attemptNumber >= item.MaxAttempts)
|
||||
{
|
||||
await _queue.MarkDeadLetterAsync(item.Id, ex.Message, ct);
|
||||
_logger.LogError(
|
||||
"Rekor queue item {Id} exceeded max attempts ({MaxAttempts}), moved to dead letter",
|
||||
item.Id, item.MaxAttempts);
|
||||
}
|
||||
else
|
||||
{
|
||||
await _queue.MarkRetryAsync(item.Id, ex.Message, ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private RekorBackend ResolveBackend(string backend)
|
||||
{
|
||||
return backend.ToLowerInvariant() switch
|
||||
{
|
||||
"primary" => new RekorBackend(
|
||||
_attestorOptions.Rekor.Primary.Url ?? throw new InvalidOperationException("Primary Rekor URL not configured"),
|
||||
"primary"),
|
||||
"mirror" => new RekorBackend(
|
||||
_attestorOptions.Rekor.Mirror.Url ?? throw new InvalidOperationException("Mirror Rekor URL not configured"),
|
||||
"mirror"),
|
||||
_ => throw new InvalidOperationException($"Unknown Rekor backend: {backend}")
|
||||
};
|
||||
}
|
||||
|
||||
private static AttestorSubmissionRequest BuildSubmissionRequest(RekorQueueItem item)
|
||||
{
|
||||
// Reconstruct the submission request from the stored payload
|
||||
return new AttestorSubmissionRequest
|
||||
{
|
||||
TenantId = item.TenantId,
|
||||
BundleSha256 = item.BundleSha256,
|
||||
DssePayload = item.DssePayload
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Simple Rekor backend configuration.
|
||||
/// </summary>
|
||||
public sealed record RekorBackend(string Url, string Name);
|
||||
|
||||
/// <summary>
|
||||
/// Submission request for the retry worker.
|
||||
/// </summary>
|
||||
public sealed class AttestorSubmissionRequest
|
||||
{
|
||||
public string TenantId { get; init; } = string.Empty;
|
||||
public string BundleSha256 { get; init; } = string.Empty;
|
||||
public byte[] DssePayload { get; init; } = Array.Empty<byte>();
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
// =============================================================================
|
||||
// RekorRetryWorkerTests.cs
|
||||
// Sprint: SPRINT_3000_0001_0002_rekor_retry_queue_metrics
|
||||
// Task: T11
|
||||
// =============================================================================
|
||||
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Moq;
|
||||
using StellaOps.Attestor.Core.Observability;
|
||||
using StellaOps.Attestor.Core.Options;
|
||||
using StellaOps.Attestor.Core.Queue;
|
||||
using StellaOps.Attestor.Core.Rekor;
|
||||
using StellaOps.Attestor.Infrastructure.Workers;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Attestor.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Unit tests for RekorRetryWorker.
|
||||
/// </summary>
|
||||
[Trait("Category", "Unit")]
|
||||
[Trait("Sprint", "3000_0001_0002")]
|
||||
public sealed class RekorRetryWorkerTests
|
||||
{
|
||||
private readonly Mock<IRekorSubmissionQueue> _queueMock;
|
||||
private readonly Mock<IRekorClient> _rekorClientMock;
|
||||
private readonly Mock<TimeProvider> _timeProviderMock;
|
||||
private readonly AttestorMetrics _metrics;
|
||||
private readonly RekorQueueOptions _queueOptions;
|
||||
private readonly AttestorOptions _attestorOptions;
|
||||
|
||||
public RekorRetryWorkerTests()
|
||||
{
|
||||
_queueMock = new Mock<IRekorSubmissionQueue>();
|
||||
_rekorClientMock = new Mock<IRekorClient>();
|
||||
_timeProviderMock = new Mock<TimeProvider>();
|
||||
_metrics = new AttestorMetrics();
|
||||
|
||||
_queueOptions = new RekorQueueOptions
|
||||
{
|
||||
Enabled = true,
|
||||
BatchSize = 5,
|
||||
PollIntervalMs = 100,
|
||||
MaxAttempts = 3
|
||||
};
|
||||
|
||||
_attestorOptions = new AttestorOptions
|
||||
{
|
||||
Rekor = new AttestorOptions.RekorOptions
|
||||
{
|
||||
Primary = new AttestorOptions.RekorBackendOptions
|
||||
{
|
||||
Url = "https://rekor.example.com"
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_timeProviderMock
|
||||
.Setup(t => t.GetUtcNow())
|
||||
.Returns(DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Worker does not process when disabled")]
|
||||
public async Task ExecuteAsync_WhenDisabled_DoesNotProcess()
|
||||
{
|
||||
_queueOptions.Enabled = false;
|
||||
|
||||
var worker = CreateWorker();
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(200));
|
||||
|
||||
await worker.StartAsync(cts.Token);
|
||||
await Task.Delay(50);
|
||||
await worker.StopAsync(cts.Token);
|
||||
|
||||
_queueMock.Verify(q => q.DequeueAsync(It.IsAny<int>(), It.IsAny<CancellationToken>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Worker updates queue depth metrics")]
|
||||
public async Task ExecuteAsync_UpdatesQueueDepthMetrics()
|
||||
{
|
||||
_queueMock
|
||||
.Setup(q => q.GetQueueDepthAsync(It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new QueueDepthSnapshot(5, 2, 3, 1, DateTimeOffset.UtcNow));
|
||||
_queueMock
|
||||
.Setup(q => q.DequeueAsync(It.IsAny<int>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync([]);
|
||||
|
||||
var worker = CreateWorker();
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(300));
|
||||
|
||||
await worker.StartAsync(cts.Token);
|
||||
await Task.Delay(150);
|
||||
await worker.StopAsync(CancellationToken.None);
|
||||
|
||||
_queueMock.Verify(q => q.GetQueueDepthAsync(It.IsAny<CancellationToken>()), Times.AtLeastOnce);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Worker processes items from queue")]
|
||||
public async Task ExecuteAsync_ProcessesItemsFromQueue()
|
||||
{
|
||||
var item = CreateTestItem();
|
||||
var items = new List<RekorQueueItem> { item };
|
||||
|
||||
_queueMock
|
||||
.Setup(q => q.GetQueueDepthAsync(It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new QueueDepthSnapshot(1, 0, 0, 0, DateTimeOffset.UtcNow));
|
||||
_queueMock
|
||||
.SetupSequence(q => q.DequeueAsync(It.IsAny<int>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(items)
|
||||
.ReturnsAsync([]);
|
||||
_rekorClientMock
|
||||
.Setup(r => r.SubmitAsync(It.IsAny<object>(), It.IsAny<object>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new RekorSubmissionResponse { Uuid = "test-uuid", Index = 12345 });
|
||||
|
||||
var worker = CreateWorker();
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(500));
|
||||
|
||||
await worker.StartAsync(cts.Token);
|
||||
await Task.Delay(200);
|
||||
await worker.StopAsync(CancellationToken.None);
|
||||
|
||||
_queueMock.Verify(
|
||||
q => q.MarkSubmittedAsync(item.Id, "test-uuid", 12345, It.IsAny<CancellationToken>()),
|
||||
Times.Once);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Worker marks item for retry on failure")]
|
||||
public async Task ExecuteAsync_MarksRetryOnFailure()
|
||||
{
|
||||
var item = CreateTestItem();
|
||||
var items = new List<RekorQueueItem> { item };
|
||||
|
||||
_queueMock
|
||||
.Setup(q => q.GetQueueDepthAsync(It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new QueueDepthSnapshot(1, 0, 0, 0, DateTimeOffset.UtcNow));
|
||||
_queueMock
|
||||
.SetupSequence(q => q.DequeueAsync(It.IsAny<int>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(items)
|
||||
.ReturnsAsync([]);
|
||||
_rekorClientMock
|
||||
.Setup(r => r.SubmitAsync(It.IsAny<object>(), It.IsAny<object>(), It.IsAny<CancellationToken>()))
|
||||
.ThrowsAsync(new Exception("Connection failed"));
|
||||
|
||||
var worker = CreateWorker();
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(500));
|
||||
|
||||
await worker.StartAsync(cts.Token);
|
||||
await Task.Delay(200);
|
||||
await worker.StopAsync(CancellationToken.None);
|
||||
|
||||
_queueMock.Verify(
|
||||
q => q.MarkRetryAsync(item.Id, It.IsAny<string>(), It.IsAny<CancellationToken>()),
|
||||
Times.Once);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Worker marks dead letter after max attempts")]
|
||||
public async Task ExecuteAsync_MarksDeadLetterAfterMaxAttempts()
|
||||
{
|
||||
var item = CreateTestItem(attemptCount: 2); // Next attempt will be 3 (max)
|
||||
var items = new List<RekorQueueItem> { item };
|
||||
|
||||
_queueMock
|
||||
.Setup(q => q.GetQueueDepthAsync(It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new QueueDepthSnapshot(0, 0, 1, 0, DateTimeOffset.UtcNow));
|
||||
_queueMock
|
||||
.SetupSequence(q => q.DequeueAsync(It.IsAny<int>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(items)
|
||||
.ReturnsAsync([]);
|
||||
_rekorClientMock
|
||||
.Setup(r => r.SubmitAsync(It.IsAny<object>(), It.IsAny<object>(), It.IsAny<CancellationToken>()))
|
||||
.ThrowsAsync(new Exception("Connection failed"));
|
||||
|
||||
var worker = CreateWorker();
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(500));
|
||||
|
||||
await worker.StartAsync(cts.Token);
|
||||
await Task.Delay(200);
|
||||
await worker.StopAsync(CancellationToken.None);
|
||||
|
||||
_queueMock.Verify(
|
||||
q => q.MarkDeadLetterAsync(item.Id, It.IsAny<string>(), It.IsAny<CancellationToken>()),
|
||||
Times.Once);
|
||||
}
|
||||
|
||||
private RekorRetryWorker CreateWorker()
|
||||
{
|
||||
return new RekorRetryWorker(
|
||||
_queueMock.Object,
|
||||
_rekorClientMock.Object,
|
||||
Options.Create(_queueOptions),
|
||||
Options.Create(_attestorOptions),
|
||||
_metrics,
|
||||
_timeProviderMock.Object,
|
||||
NullLogger<RekorRetryWorker>.Instance);
|
||||
}
|
||||
|
||||
private static RekorQueueItem CreateTestItem(int attemptCount = 0)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
return new RekorQueueItem(
|
||||
Guid.NewGuid(),
|
||||
"test-tenant",
|
||||
"sha256:abc123",
|
||||
new byte[] { 1, 2, 3 },
|
||||
"primary",
|
||||
RekorSubmissionStatus.Submitting,
|
||||
attemptCount,
|
||||
3,
|
||||
null,
|
||||
null,
|
||||
now,
|
||||
null,
|
||||
null,
|
||||
now,
|
||||
now);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stub response for tests.
|
||||
/// </summary>
|
||||
public sealed class RekorSubmissionResponse
|
||||
{
|
||||
public string? Uuid { get; init; }
|
||||
public long? Index { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
// =============================================================================
|
||||
// RekorSubmissionQueueTests.cs
|
||||
// Sprint: SPRINT_3000_0001_0002_rekor_retry_queue_metrics
|
||||
// Task: T13
|
||||
// =============================================================================
|
||||
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Moq;
|
||||
using StellaOps.Attestor.Core.Observability;
|
||||
using StellaOps.Attestor.Core.Options;
|
||||
using StellaOps.Attestor.Core.Queue;
|
||||
using StellaOps.Attestor.Infrastructure.Queue;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Attestor.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Unit tests for PostgresRekorSubmissionQueue.
|
||||
/// Note: Full integration tests require PostgreSQL via Testcontainers (Task T14).
|
||||
/// </summary>
|
||||
[Trait("Category", "Unit")]
|
||||
[Trait("Sprint", "3000_0001_0002")]
|
||||
public sealed class RekorQueueOptionsTests
|
||||
{
|
||||
[Theory(DisplayName = "CalculateRetryDelay applies exponential backoff")]
|
||||
[InlineData(0, 1000)] // First retry: initial delay
|
||||
[InlineData(1, 2000)] // Second retry: 1000 * 2
|
||||
[InlineData(2, 4000)] // Third retry: 1000 * 2^2
|
||||
[InlineData(3, 8000)] // Fourth retry: 1000 * 2^3
|
||||
[InlineData(4, 16000)] // Fifth retry: 1000 * 2^4
|
||||
[InlineData(10, 60000)] // Many retries: capped at MaxDelayMs
|
||||
public void CalculateRetryDelay_AppliesExponentialBackoff(int attemptCount, int expectedMs)
|
||||
{
|
||||
var options = new RekorQueueOptions
|
||||
{
|
||||
InitialDelayMs = 1000,
|
||||
MaxDelayMs = 60000,
|
||||
BackoffMultiplier = 2.0
|
||||
};
|
||||
|
||||
var delay = options.CalculateRetryDelay(attemptCount);
|
||||
|
||||
delay.TotalMilliseconds.Should().Be(expectedMs);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Default options have sensible values")]
|
||||
public void DefaultOptions_HaveSensibleValues()
|
||||
{
|
||||
var options = new RekorQueueOptions();
|
||||
|
||||
options.Enabled.Should().BeTrue();
|
||||
options.MaxAttempts.Should().Be(5);
|
||||
options.InitialDelayMs.Should().Be(1000);
|
||||
options.MaxDelayMs.Should().Be(60000);
|
||||
options.BackoffMultiplier.Should().Be(2.0);
|
||||
options.BatchSize.Should().Be(10);
|
||||
options.PollIntervalMs.Should().Be(5000);
|
||||
options.DeadLetterRetentionDays.Should().Be(30);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tests for QueueDepthSnapshot.
|
||||
/// </summary>
|
||||
[Trait("Category", "Unit")]
|
||||
[Trait("Sprint", "3000_0001_0002")]
|
||||
public sealed class QueueDepthSnapshotTests
|
||||
{
|
||||
[Fact(DisplayName = "TotalWaiting sums pending and retrying")]
|
||||
public void TotalWaiting_SumsPendingAndRetrying()
|
||||
{
|
||||
var snapshot = new QueueDepthSnapshot(10, 5, 3, 2, DateTimeOffset.UtcNow);
|
||||
|
||||
snapshot.TotalWaiting.Should().Be(13);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "TotalInQueue sums all non-submitted statuses")]
|
||||
public void TotalInQueue_SumsAllNonSubmitted()
|
||||
{
|
||||
var snapshot = new QueueDepthSnapshot(10, 5, 3, 2, DateTimeOffset.UtcNow);
|
||||
|
||||
snapshot.TotalInQueue.Should().Be(20);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Empty creates zero snapshot")]
|
||||
public void Empty_CreatesZeroSnapshot()
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var snapshot = QueueDepthSnapshot.Empty(now);
|
||||
|
||||
snapshot.Pending.Should().Be(0);
|
||||
snapshot.Submitting.Should().Be(0);
|
||||
snapshot.Retrying.Should().Be(0);
|
||||
snapshot.DeadLetter.Should().Be(0);
|
||||
snapshot.MeasuredAt.Should().Be(now);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tests for RekorQueueItem.
|
||||
/// </summary>
|
||||
[Trait("Category", "Unit")]
|
||||
[Trait("Sprint", "3000_0001_0002")]
|
||||
public sealed class RekorQueueItemTests
|
||||
{
|
||||
[Fact(DisplayName = "RekorQueueItem properties are accessible")]
|
||||
public void RekorQueueItem_PropertiesAccessible()
|
||||
{
|
||||
var id = Guid.NewGuid();
|
||||
var tenantId = "test-tenant";
|
||||
var bundleSha256 = "sha256:abc123";
|
||||
var dssePayload = new byte[] { 1, 2, 3 };
|
||||
var backend = "primary";
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
var item = new RekorQueueItem
|
||||
{
|
||||
Id = id,
|
||||
TenantId = tenantId,
|
||||
BundleSha256 = bundleSha256,
|
||||
DssePayload = dssePayload,
|
||||
Backend = backend,
|
||||
Status = RekorSubmissionStatus.Pending,
|
||||
AttemptCount = 0,
|
||||
MaxAttempts = 5,
|
||||
NextRetryAt = now,
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now
|
||||
};
|
||||
|
||||
item.Id.Should().Be(id);
|
||||
item.TenantId.Should().Be(tenantId);
|
||||
item.BundleSha256.Should().Be(bundleSha256);
|
||||
item.DssePayload.Should().BeEquivalentTo(dssePayload);
|
||||
item.Backend.Should().Be(backend);
|
||||
item.Status.Should().Be(RekorSubmissionStatus.Pending);
|
||||
item.AttemptCount.Should().Be(0);
|
||||
item.MaxAttempts.Should().Be(5);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tests for RekorSubmissionStatus enum.
|
||||
/// </summary>
|
||||
[Trait("Category", "Unit")]
|
||||
[Trait("Sprint", "3000_0001_0002")]
|
||||
public sealed class RekorSubmissionStatusTests
|
||||
{
|
||||
[Theory(DisplayName = "Status enum has expected values")]
|
||||
[InlineData(RekorSubmissionStatus.Pending, 0)]
|
||||
[InlineData(RekorSubmissionStatus.Submitting, 1)]
|
||||
[InlineData(RekorSubmissionStatus.Submitted, 2)]
|
||||
[InlineData(RekorSubmissionStatus.Retrying, 3)]
|
||||
[InlineData(RekorSubmissionStatus.DeadLetter, 4)]
|
||||
public void Status_HasExpectedValues(RekorSubmissionStatus status, int expectedValue)
|
||||
{
|
||||
((int)status).Should().Be(expectedValue);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user