feat(rate-limiting): Implement core rate limiting functionality with configuration, decision-making, metrics, middleware, and service registration

- Add RateLimitConfig for configuration management with YAML binding support.
- Introduce RateLimitDecision to encapsulate the result of rate limit checks.
- Implement RateLimitMetrics for OpenTelemetry metrics tracking.
- Create RateLimitMiddleware for enforcing rate limits on incoming requests.
- Develop RateLimitService to orchestrate instance and environment rate limit checks.
- Add RateLimitServiceCollectionExtensions for dependency injection registration.
This commit is contained in:
master
2025-12-17 18:02:37 +02:00
parent 394b57f6bf
commit 8bbfe4d2d2
211 changed files with 47179 additions and 1590 deletions

View File

@@ -0,0 +1,362 @@
// -----------------------------------------------------------------------------
// FeedChangeRescoreJob.cs
// Sprint: SPRINT_3401_0002_0001_score_replay_proof_bundle
// Task: SCORE-REPLAY-011 - Add scheduled job to rescore when feed snapshots change
// Description: Background job that detects feed changes and triggers rescoring
// -----------------------------------------------------------------------------
using System.Diagnostics;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Scanner.WebService.Services;
namespace StellaOps.Scanner.WebService.Services;
/// <summary>
/// Options for the feed change rescore job.
/// </summary>
public sealed class FeedChangeRescoreOptions
{
/// <summary>
/// Whether the job is enabled. Default: true.
/// </summary>
public bool Enabled { get; set; } = true;
/// <summary>
/// Interval between feed change checks. Default: 15 minutes.
/// </summary>
public TimeSpan CheckInterval { get; set; } = TimeSpan.FromMinutes(15);
/// <summary>
/// Maximum scans to rescore per cycle. Default: 100.
/// </summary>
public int MaxScansPerCycle { get; set; } = 100;
/// <summary>
/// Time window for considering scans for rescoring. Default: 7 days.
/// </summary>
public TimeSpan ScanAgeLimit { get; set; } = TimeSpan.FromDays(7);
/// <summary>
/// Concurrency limit for rescoring operations. Default: 4.
/// </summary>
public int RescoreConcurrency { get; set; } = 4;
}
/// <summary>
/// Background job that monitors feed snapshot changes and triggers rescoring for affected scans.
/// Per Sprint 3401.0002.0001 - Score Replay & Proof Bundle.
/// </summary>
public sealed class FeedChangeRescoreJob : BackgroundService
{
private readonly IFeedSnapshotTracker _feedTracker;
private readonly IScanManifestRepository _manifestRepository;
private readonly IScoreReplayService _replayService;
private readonly IOptions<FeedChangeRescoreOptions> _options;
private readonly ILogger<FeedChangeRescoreJob> _logger;
private readonly ActivitySource _activitySource = new("StellaOps.Scanner.FeedChangeRescore");
private string? _lastConcelierSnapshot;
private string? _lastExcititorSnapshot;
private string? _lastPolicySnapshot;
public FeedChangeRescoreJob(
IFeedSnapshotTracker feedTracker,
IScanManifestRepository manifestRepository,
IScoreReplayService replayService,
IOptions<FeedChangeRescoreOptions> options,
ILogger<FeedChangeRescoreJob> logger)
{
_feedTracker = feedTracker ?? throw new ArgumentNullException(nameof(feedTracker));
_manifestRepository = manifestRepository ?? throw new ArgumentNullException(nameof(manifestRepository));
_replayService = replayService ?? throw new ArgumentNullException(nameof(replayService));
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Feed change rescore job started");
// Initial delay to let the system stabilize
await Task.Delay(TimeSpan.FromSeconds(30), stoppingToken);
// Initialize snapshot tracking
await InitializeSnapshotsAsync(stoppingToken);
while (!stoppingToken.IsCancellationRequested)
{
var opts = _options.Value;
if (!opts.Enabled)
{
_logger.LogDebug("Feed change rescore job is disabled");
await Task.Delay(opts.CheckInterval, stoppingToken);
continue;
}
using var activity = _activitySource.StartActivity("feedchange.rescore.cycle", ActivityKind.Internal);
try
{
await CheckAndRescoreAsync(opts, stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
catch (Exception ex)
{
_logger.LogError(ex, "Feed change rescore cycle failed");
activity?.SetStatus(ActivityStatusCode.Error, ex.Message);
FeedChangeRescoreMetrics.RecordError("cycle_failed");
}
await Task.Delay(opts.CheckInterval, stoppingToken);
}
_logger.LogInformation("Feed change rescore job stopped");
}
private async Task InitializeSnapshotsAsync(CancellationToken ct)
{
try
{
var snapshots = await _feedTracker.GetCurrentSnapshotsAsync(ct);
_lastConcelierSnapshot = snapshots.ConcelierHash;
_lastExcititorSnapshot = snapshots.ExcititorHash;
_lastPolicySnapshot = snapshots.PolicyHash;
_logger.LogInformation(
"Initialized feed snapshots: Concelier={ConcelierHash}, Excititor={ExcititorHash}, Policy={PolicyHash}",
_lastConcelierSnapshot?[..12] ?? "null",
_lastExcititorSnapshot?[..12] ?? "null",
_lastPolicySnapshot?[..12] ?? "null");
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to initialize feed snapshots, will retry on next cycle");
}
}
private async Task CheckAndRescoreAsync(FeedChangeRescoreOptions opts, CancellationToken ct)
{
var sw = Stopwatch.StartNew();
// Get current feed snapshots
var currentSnapshots = await _feedTracker.GetCurrentSnapshotsAsync(ct);
// Check for changes
var changes = DetectChanges(currentSnapshots);
if (changes.Count == 0)
{
_logger.LogDebug("No feed changes detected");
return;
}
_logger.LogInformation("Feed changes detected: {Changes}", string.Join(", ", changes));
FeedChangeRescoreMetrics.RecordFeedChange(changes);
// Find scans affected by the changes
var affectedScans = await FindAffectedScansAsync(changes, opts, ct);
if (affectedScans.Count == 0)
{
_logger.LogDebug("No affected scans found");
UpdateSnapshots(currentSnapshots);
return;
}
_logger.LogInformation("Found {Count} scans to rescore", affectedScans.Count);
// Rescore affected scans with concurrency limit
var rescored = 0;
var semaphore = new SemaphoreSlim(opts.RescoreConcurrency);
var tasks = affectedScans.Select(async scanId =>
{
await semaphore.WaitAsync(ct);
try
{
await RescoreScanAsync(scanId, ct);
Interlocked.Increment(ref rescored);
}
finally
{
semaphore.Release();
}
});
await Task.WhenAll(tasks);
// Update tracked snapshots
UpdateSnapshots(currentSnapshots);
sw.Stop();
_logger.LogInformation(
"Feed change rescore cycle completed in {ElapsedMs}ms: {Rescored}/{Total} scans rescored",
sw.ElapsedMilliseconds, rescored, affectedScans.Count);
FeedChangeRescoreMetrics.RecordCycle(sw.Elapsed.TotalMilliseconds, rescored);
}
private List<string> DetectChanges(FeedSnapshots current)
{
var changes = new List<string>();
if (_lastConcelierSnapshot != current.ConcelierHash)
changes.Add("concelier");
if (_lastExcititorSnapshot != current.ExcititorHash)
changes.Add("excititor");
if (_lastPolicySnapshot != current.PolicyHash)
changes.Add("policy");
return changes;
}
private async Task<List<string>> FindAffectedScansAsync(
List<string> changes,
FeedChangeRescoreOptions opts,
CancellationToken ct)
{
var cutoff = DateTimeOffset.UtcNow - opts.ScanAgeLimit;
// Find scans using the old snapshot hashes
var query = new AffectedScansQuery
{
ChangedFeeds = changes,
OldConcelierHash = changes.Contains("concelier") ? _lastConcelierSnapshot : null,
OldExcititorHash = changes.Contains("excititor") ? _lastExcititorSnapshot : null,
OldPolicyHash = changes.Contains("policy") ? _lastPolicySnapshot : null,
MinCreatedAt = cutoff,
Limit = opts.MaxScansPerCycle
};
return await _manifestRepository.FindAffectedScansAsync(query, ct);
}
private async Task RescoreScanAsync(string scanId, CancellationToken ct)
{
try
{
_logger.LogDebug("Rescoring scan {ScanId}", scanId);
var result = await _replayService.ReplayScoreAsync(scanId, cancellationToken: ct);
if (result is not null)
{
_logger.LogDebug(
"Rescored scan {ScanId}: Score={Score}, RootHash={RootHash}",
scanId, result.Score, result.RootHash[..12]);
FeedChangeRescoreMetrics.RecordRescore(result.Deterministic);
}
else
{
_logger.LogWarning("Failed to rescore scan {ScanId}: manifest not found", scanId);
FeedChangeRescoreMetrics.RecordError("manifest_not_found");
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to rescore scan {ScanId}", scanId);
FeedChangeRescoreMetrics.RecordError("rescore_failed");
}
}
private void UpdateSnapshots(FeedSnapshots current)
{
_lastConcelierSnapshot = current.ConcelierHash;
_lastExcititorSnapshot = current.ExcititorHash;
_lastPolicySnapshot = current.PolicyHash;
}
}
/// <summary>
/// Current feed snapshot hashes.
/// </summary>
public sealed record FeedSnapshots(
string ConcelierHash,
string ExcititorHash,
string PolicyHash);
/// <summary>
/// Query for finding affected scans.
/// </summary>
public sealed record AffectedScansQuery
{
public required List<string> ChangedFeeds { get; init; }
public string? OldConcelierHash { get; init; }
public string? OldExcititorHash { get; init; }
public string? OldPolicyHash { get; init; }
public DateTimeOffset MinCreatedAt { get; init; }
public int Limit { get; init; }
}
/// <summary>
/// Interface for tracking feed snapshots.
/// </summary>
public interface IFeedSnapshotTracker
{
/// <summary>
/// Get current feed snapshot hashes.
/// </summary>
Task<FeedSnapshots> GetCurrentSnapshotsAsync(CancellationToken cancellationToken = default);
}
/// <summary>
/// Interface for scan manifest repository operations.
/// </summary>
public interface IScanManifestRepository
{
/// <summary>
/// Find scans affected by feed changes.
/// </summary>
Task<List<string>> FindAffectedScansAsync(AffectedScansQuery query, CancellationToken cancellationToken = default);
}
/// <summary>
/// Metrics for feed change rescore operations.
/// </summary>
public static class FeedChangeRescoreMetrics
{
private static readonly System.Diagnostics.Metrics.Meter Meter =
new("StellaOps.Scanner.FeedChangeRescore", "1.0.0");
private static readonly System.Diagnostics.Metrics.Counter<int> FeedChanges =
Meter.CreateCounter<int>("stellaops.scanner.feed_changes", description: "Number of feed changes detected");
private static readonly System.Diagnostics.Metrics.Counter<int> Rescores =
Meter.CreateCounter<int>("stellaops.scanner.rescores", description: "Number of scans rescored");
private static readonly System.Diagnostics.Metrics.Counter<int> Errors =
Meter.CreateCounter<int>("stellaops.scanner.rescore_errors", description: "Number of rescore errors");
private static readonly System.Diagnostics.Metrics.Histogram<double> CycleDuration =
Meter.CreateHistogram<double>("stellaops.scanner.rescore_cycle_duration_ms", description: "Duration of rescore cycle in ms");
public static void RecordFeedChange(List<string> changes)
{
foreach (var change in changes)
{
FeedChanges.Add(1, new System.Diagnostics.TagList { { "feed", change } });
}
}
public static void RecordRescore(bool deterministic)
{
Rescores.Add(1, new System.Diagnostics.TagList { { "deterministic", deterministic.ToString().ToLowerInvariant() } });
}
public static void RecordError(string context)
{
Errors.Add(1, new System.Diagnostics.TagList { { "context", context } });
}
public static void RecordCycle(double durationMs, int rescored)
{
CycleDuration.Record(durationMs);
}
}

View File

@@ -0,0 +1,97 @@
// -----------------------------------------------------------------------------
// IScoreReplayService.cs
// Sprint: SPRINT_3401_0002_0001_score_replay_proof_bundle
// Task: SCORE-REPLAY-010 - Implement score replay service
// Description: Service interface for score replay operations
// -----------------------------------------------------------------------------
using StellaOps.Scanner.Core;
namespace StellaOps.Scanner.WebService.Services;
/// <summary>
/// Service for replaying scores and managing proof bundles.
/// </summary>
public interface IScoreReplayService
{
/// <summary>
/// Replay scoring for a previous scan using frozen inputs.
/// </summary>
/// <param name="scanId">The scan ID to replay.</param>
/// <param name="manifestHash">Optional specific manifest hash to use.</param>
/// <param name="freezeTimestamp">Optional freeze timestamp for deterministic replay.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Replay result or null if scan not found.</returns>
Task<ScoreReplayResult?> ReplayScoreAsync(
string scanId,
string? manifestHash = null,
DateTimeOffset? freezeTimestamp = null,
CancellationToken cancellationToken = default);
/// <summary>
/// Get a proof bundle for a scan.
/// </summary>
/// <param name="scanId">The scan ID.</param>
/// <param name="rootHash">Optional specific root hash to retrieve.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>The proof bundle or null if not found.</returns>
Task<ProofBundle?> GetBundleAsync(
string scanId,
string? rootHash = null,
CancellationToken cancellationToken = default);
/// <summary>
/// Verify a proof bundle against expected root hash.
/// </summary>
/// <param name="scanId">The scan ID.</param>
/// <param name="expectedRootHash">The expected root hash.</param>
/// <param name="bundleUri">Optional specific bundle URI to verify.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Verification result.</returns>
Task<BundleVerifyResult> VerifyBundleAsync(
string scanId,
string expectedRootHash,
string? bundleUri = null,
CancellationToken cancellationToken = default);
}
/// <summary>
/// Result of a score replay operation.
/// </summary>
/// <param name="Score">The computed score (0.0 - 1.0).</param>
/// <param name="RootHash">Root hash of the proof ledger.</param>
/// <param name="BundleUri">URI to the proof bundle.</param>
/// <param name="ManifestHash">Hash of the manifest used.</param>
/// <param name="ReplayedAt">When the replay was performed.</param>
/// <param name="Deterministic">Whether the replay was deterministic.</param>
public sealed record ScoreReplayResult(
double Score,
string RootHash,
string BundleUri,
string ManifestHash,
DateTimeOffset ReplayedAt,
bool Deterministic);
/// <summary>
/// Result of bundle verification.
/// </summary>
/// <param name="Valid">Whether the bundle is valid.</param>
/// <param name="ComputedRootHash">The computed root hash.</param>
/// <param name="ManifestValid">Whether the manifest signature is valid.</param>
/// <param name="LedgerValid">Whether the ledger integrity is valid.</param>
/// <param name="VerifiedAt">When verification was performed.</param>
/// <param name="ErrorMessage">Error message if verification failed.</param>
public sealed record BundleVerifyResult(
bool Valid,
string ComputedRootHash,
bool ManifestValid,
bool LedgerValid,
DateTimeOffset VerifiedAt,
string? ErrorMessage = null)
{
public static BundleVerifyResult Success(string computedRootHash) =>
new(true, computedRootHash, true, true, DateTimeOffset.UtcNow);
public static BundleVerifyResult Failure(string error, string computedRootHash = "") =>
new(false, computedRootHash, false, false, DateTimeOffset.UtcNow, error);
}

View File

@@ -0,0 +1,206 @@
// -----------------------------------------------------------------------------
// ScoreReplayService.cs
// Sprint: SPRINT_3401_0002_0001_score_replay_proof_bundle
// Task: SCORE-REPLAY-010 - Implement score replay service
// Description: Service implementation for score replay operations
// -----------------------------------------------------------------------------
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Policy.Scoring;
using StellaOps.Scanner.Core;
namespace StellaOps.Scanner.WebService.Services;
/// <summary>
/// Default implementation of IScoreReplayService.
/// </summary>
public sealed class ScoreReplayService : IScoreReplayService
{
private readonly IScanManifestRepository _manifestRepository;
private readonly IProofBundleRepository _bundleRepository;
private readonly IProofBundleWriter _bundleWriter;
private readonly IScanManifestSigner _manifestSigner;
private readonly IScoringService _scoringService;
private readonly ILogger<ScoreReplayService> _logger;
public ScoreReplayService(
IScanManifestRepository manifestRepository,
IProofBundleRepository bundleRepository,
IProofBundleWriter bundleWriter,
IScanManifestSigner manifestSigner,
IScoringService scoringService,
ILogger<ScoreReplayService> logger)
{
_manifestRepository = manifestRepository ?? throw new ArgumentNullException(nameof(manifestRepository));
_bundleRepository = bundleRepository ?? throw new ArgumentNullException(nameof(bundleRepository));
_bundleWriter = bundleWriter ?? throw new ArgumentNullException(nameof(bundleWriter));
_manifestSigner = manifestSigner ?? throw new ArgumentNullException(nameof(manifestSigner));
_scoringService = scoringService ?? throw new ArgumentNullException(nameof(scoringService));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
/// <inheritdoc />
public async Task<ScoreReplayResult?> ReplayScoreAsync(
string scanId,
string? manifestHash = null,
DateTimeOffset? freezeTimestamp = null,
CancellationToken cancellationToken = default)
{
_logger.LogInformation("Starting score replay for scan {ScanId}", scanId);
// Get the manifest
var signedManifest = await _manifestRepository.GetManifestAsync(scanId, manifestHash, cancellationToken);
if (signedManifest is null)
{
_logger.LogWarning("Manifest not found for scan {ScanId}", scanId);
return null;
}
// Verify manifest signature
var verifyResult = await _manifestSigner.VerifyAsync(signedManifest, cancellationToken);
if (!verifyResult.IsValid)
{
throw new InvalidOperationException($"Manifest signature verification failed: {verifyResult.ErrorMessage}");
}
var manifest = signedManifest.Manifest;
// Replay scoring with frozen inputs
var ledger = new ProofLedger();
var score = await _scoringService.ReplayScoreAsync(
manifest.ScanId,
manifest.ConcelierSnapshotHash,
manifest.ExcititorSnapshotHash,
manifest.LatticePolicyHash,
manifest.Seed,
freezeTimestamp ?? manifest.CreatedAtUtc,
ledger,
cancellationToken);
// Create proof bundle
var bundle = await _bundleWriter.CreateBundleAsync(signedManifest, ledger, cancellationToken);
// Store bundle reference
await _bundleRepository.SaveBundleAsync(bundle, cancellationToken);
_logger.LogInformation(
"Score replay complete for scan {ScanId}: score={Score}, rootHash={RootHash}",
scanId, score, bundle.RootHash);
return new ScoreReplayResult(
Score: score,
RootHash: bundle.RootHash,
BundleUri: bundle.BundleUri,
ManifestHash: manifest.ComputeHash(),
ReplayedAt: DateTimeOffset.UtcNow,
Deterministic: manifest.Deterministic);
}
/// <inheritdoc />
public async Task<ProofBundle?> GetBundleAsync(
string scanId,
string? rootHash = null,
CancellationToken cancellationToken = default)
{
return await _bundleRepository.GetBundleAsync(scanId, rootHash, cancellationToken);
}
/// <inheritdoc />
public async Task<BundleVerifyResult> VerifyBundleAsync(
string scanId,
string expectedRootHash,
string? bundleUri = null,
CancellationToken cancellationToken = default)
{
_logger.LogInformation("Verifying bundle for scan {ScanId}, expected hash {ExpectedHash}", scanId, expectedRootHash);
try
{
// Get bundle URI if not provided
if (string.IsNullOrEmpty(bundleUri))
{
var bundle = await _bundleRepository.GetBundleAsync(scanId, expectedRootHash, cancellationToken);
if (bundle is null)
{
return BundleVerifyResult.Failure($"Bundle not found for scan {scanId}");
}
bundleUri = bundle.BundleUri;
}
// Read and verify bundle
var contents = await _bundleWriter.ReadBundleAsync(bundleUri, cancellationToken);
// Verify manifest signature
var manifestVerify = await _manifestSigner.VerifyAsync(contents.SignedManifest, cancellationToken);
// Verify ledger integrity
var ledgerValid = contents.ProofLedger.VerifyIntegrity();
// Compute and compare root hash
var computedRootHash = contents.ProofLedger.RootHash();
var hashMatch = computedRootHash.Equals(expectedRootHash, StringComparison.Ordinal);
if (!manifestVerify.IsValid || !ledgerValid || !hashMatch)
{
var errors = new List<string>();
if (!manifestVerify.IsValid) errors.Add($"Manifest: {manifestVerify.ErrorMessage}");
if (!ledgerValid) errors.Add("Ledger integrity check failed");
if (!hashMatch) errors.Add($"Root hash mismatch: expected {expectedRootHash}, got {computedRootHash}");
return new BundleVerifyResult(
Valid: false,
ComputedRootHash: computedRootHash,
ManifestValid: manifestVerify.IsValid,
LedgerValid: ledgerValid,
VerifiedAt: DateTimeOffset.UtcNow,
ErrorMessage: string.Join("; ", errors));
}
_logger.LogInformation("Bundle verification successful for scan {ScanId}", scanId);
return BundleVerifyResult.Success(computedRootHash);
}
catch (Exception ex)
{
_logger.LogError(ex, "Bundle verification failed for scan {ScanId}", scanId);
return BundleVerifyResult.Failure(ex.Message);
}
}
}
/// <summary>
/// Repository interface for scan manifests.
/// </summary>
public interface IScanManifestRepository
{
Task<SignedScanManifest?> GetManifestAsync(string scanId, string? manifestHash = null, CancellationToken cancellationToken = default);
Task SaveManifestAsync(SignedScanManifest manifest, CancellationToken cancellationToken = default);
}
/// <summary>
/// Repository interface for proof bundles.
/// </summary>
public interface IProofBundleRepository
{
Task<ProofBundle?> GetBundleAsync(string scanId, string? rootHash = null, CancellationToken cancellationToken = default);
Task SaveBundleAsync(ProofBundle bundle, CancellationToken cancellationToken = default);
}
/// <summary>
/// Scoring service interface for replay.
/// </summary>
public interface IScoringService
{
/// <summary>
/// Replay scoring with frozen inputs.
/// </summary>
Task<double> ReplayScoreAsync(
string scanId,
string concelierSnapshotHash,
string excititorSnapshotHash,
string latticePolicyHash,
byte[] seed,
DateTimeOffset freezeTimestamp,
ProofLedger ledger,
CancellationToken cancellationToken = default);
}