538 lines
18 KiB
C#
538 lines
18 KiB
C#
using System.Collections.Concurrent;
|
|
using System.Collections.Immutable;
|
|
using System.Diagnostics;
|
|
using System.Globalization;
|
|
using System.Security.Cryptography;
|
|
using System.Text;
|
|
|
|
namespace StellaOps.Policy.Engine.IncrementalOrchestrator;
|
|
|
|
/// <summary>
|
|
/// Configuration options for the incremental policy orchestrator.
|
|
/// </summary>
|
|
public sealed record IncrementalOrchestratorOptions
|
|
{
|
|
/// <summary>
|
|
/// How often to poll for new change events.
|
|
/// </summary>
|
|
public TimeSpan PollInterval { get; init; } = TimeSpan.FromSeconds(5);
|
|
|
|
/// <summary>
|
|
/// How long to wait before batching events together.
|
|
/// </summary>
|
|
public TimeSpan BatchWindow { get; init; } = TimeSpan.FromSeconds(10);
|
|
|
|
/// <summary>
|
|
/// Maximum events per batch.
|
|
/// </summary>
|
|
public int MaxBatchSize { get; init; } = 100;
|
|
|
|
/// <summary>
|
|
/// Maximum retry attempts for failed processing.
|
|
/// </summary>
|
|
public int MaxRetryAttempts { get; init; } = 3;
|
|
|
|
/// <summary>
|
|
/// Delay between retry attempts.
|
|
/// </summary>
|
|
public TimeSpan RetryBackoff { get; init; } = TimeSpan.FromSeconds(5);
|
|
|
|
/// <summary>
|
|
/// Whether to enable deduplication within batch window.
|
|
/// </summary>
|
|
public bool EnableDeduplication { get; init; } = true;
|
|
|
|
/// <summary>
|
|
/// Maximum age of events to process (older events are skipped).
|
|
/// </summary>
|
|
public TimeSpan MaxEventAge { get; init; } = TimeSpan.FromHours(24);
|
|
|
|
/// <summary>
|
|
/// Default options.
|
|
/// </summary>
|
|
public static IncrementalOrchestratorOptions Default { get; } = new();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Interface for reading change events from a source.
|
|
/// </summary>
|
|
public interface IPolicyChangeEventSource
|
|
{
|
|
/// <summary>
|
|
/// Reads pending change events.
|
|
/// </summary>
|
|
IAsyncEnumerable<PolicyChangeEvent> ReadAsync(CancellationToken cancellationToken);
|
|
|
|
/// <summary>
|
|
/// Acknowledges that an event has been processed.
|
|
/// </summary>
|
|
Task AcknowledgeAsync(string eventId, CancellationToken cancellationToken);
|
|
|
|
/// <summary>
|
|
/// Marks an event as failed for retry.
|
|
/// </summary>
|
|
Task MarkFailedAsync(string eventId, string error, CancellationToken cancellationToken);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Interface for submitting policy re-evaluation jobs.
|
|
/// </summary>
|
|
public interface IPolicyReEvaluationSubmitter
|
|
{
|
|
/// <summary>
|
|
/// Submits a batch for re-evaluation.
|
|
/// </summary>
|
|
Task<PolicyReEvaluationResult> SubmitAsync(
|
|
PolicyChangeBatch batch,
|
|
CancellationToken cancellationToken);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Interface for idempotency tracking.
|
|
/// </summary>
|
|
public interface IPolicyChangeIdempotencyStore
|
|
{
|
|
/// <summary>
|
|
/// Checks if an event has already been processed.
|
|
/// </summary>
|
|
Task<bool> HasSeenAsync(string eventId, CancellationToken cancellationToken);
|
|
|
|
/// <summary>
|
|
/// Marks an event as processed.
|
|
/// </summary>
|
|
Task MarkSeenAsync(string eventId, DateTimeOffset processedAt, CancellationToken cancellationToken);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Result of a policy re-evaluation submission.
|
|
/// </summary>
|
|
public sealed record PolicyReEvaluationResult
|
|
{
|
|
/// <summary>
|
|
/// Whether the submission succeeded.
|
|
/// </summary>
|
|
public required bool Succeeded { get; init; }
|
|
|
|
/// <summary>
|
|
/// Job ID(s) created for the re-evaluation.
|
|
/// </summary>
|
|
public required ImmutableArray<string> JobIds { get; init; }
|
|
|
|
/// <summary>
|
|
/// Number of findings that will be re-evaluated.
|
|
/// </summary>
|
|
public int EstimatedFindingsCount { get; init; }
|
|
|
|
/// <summary>
|
|
/// Error message if failed.
|
|
/// </summary>
|
|
public string? Error { get; init; }
|
|
|
|
/// <summary>
|
|
/// Processing duration.
|
|
/// </summary>
|
|
public long ProcessingTimeMs { get; init; }
|
|
}
|
|
|
|
/// <summary>
|
|
/// Orchestrates incremental policy re-evaluations in response to
|
|
/// advisory, VEX, and SBOM change streams.
|
|
/// </summary>
|
|
public sealed class IncrementalPolicyOrchestrator
|
|
{
|
|
private readonly IPolicyChangeEventSource _eventSource;
|
|
private readonly IPolicyReEvaluationSubmitter _submitter;
|
|
private readonly IPolicyChangeIdempotencyStore _idempotencyStore;
|
|
private readonly IncrementalOrchestratorOptions _options;
|
|
private readonly TimeProvider _timeProvider;
|
|
|
|
public IncrementalPolicyOrchestrator(
|
|
IPolicyChangeEventSource eventSource,
|
|
IPolicyReEvaluationSubmitter submitter,
|
|
IPolicyChangeIdempotencyStore idempotencyStore,
|
|
IncrementalOrchestratorOptions? options = null,
|
|
TimeProvider? timeProvider = null)
|
|
{
|
|
_eventSource = eventSource ?? throw new ArgumentNullException(nameof(eventSource));
|
|
_submitter = submitter ?? throw new ArgumentNullException(nameof(submitter));
|
|
_idempotencyStore = idempotencyStore ?? throw new ArgumentNullException(nameof(idempotencyStore));
|
|
_options = options ?? IncrementalOrchestratorOptions.Default;
|
|
_timeProvider = timeProvider ?? TimeProvider.System;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Processes a single batch of pending events.
|
|
/// </summary>
|
|
public async Task<OrchestratorProcessResult> ProcessAsync(CancellationToken cancellationToken)
|
|
{
|
|
var stopwatch = Stopwatch.StartNew();
|
|
var now = _timeProvider.GetUtcNow();
|
|
var cutoffTime = now - _options.MaxEventAge;
|
|
|
|
var eventsByTenant = new Dictionary<string, List<PolicyChangeEvent>>(StringComparer.OrdinalIgnoreCase);
|
|
var skippedOld = 0;
|
|
var skippedDuplicate = 0;
|
|
var totalRead = 0;
|
|
|
|
// Read and group events by tenant
|
|
await foreach (var evt in _eventSource.ReadAsync(cancellationToken))
|
|
{
|
|
totalRead++;
|
|
|
|
// Skip events older than max age
|
|
if (evt.OccurredAt < cutoffTime)
|
|
{
|
|
skippedOld++;
|
|
await _eventSource.AcknowledgeAsync(evt.EventId, cancellationToken).ConfigureAwait(false);
|
|
continue;
|
|
}
|
|
|
|
// Check idempotency
|
|
if (_options.EnableDeduplication &&
|
|
await _idempotencyStore.HasSeenAsync(evt.EventId, cancellationToken).ConfigureAwait(false))
|
|
{
|
|
skippedDuplicate++;
|
|
await _eventSource.AcknowledgeAsync(evt.EventId, cancellationToken).ConfigureAwait(false);
|
|
continue;
|
|
}
|
|
|
|
if (!eventsByTenant.TryGetValue(evt.TenantId, out var tenantEvents))
|
|
{
|
|
tenantEvents = new List<PolicyChangeEvent>();
|
|
eventsByTenant[evt.TenantId] = tenantEvents;
|
|
}
|
|
|
|
tenantEvents.Add(evt);
|
|
|
|
// Limit total events per processing cycle
|
|
if (totalRead >= _options.MaxBatchSize * 10)
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
|
|
var batchesProcessed = 0;
|
|
var batchesFailed = 0;
|
|
var jobsCreated = new List<string>();
|
|
|
|
// Process each tenant's events
|
|
foreach (var (tenantId, events) in eventsByTenant.OrderBy(kvp => kvp.Key, StringComparer.Ordinal))
|
|
{
|
|
var batches = CreateBatches(tenantId, events, now);
|
|
|
|
foreach (var batch in batches)
|
|
{
|
|
var attempts = 0;
|
|
var success = false;
|
|
|
|
while (attempts < _options.MaxRetryAttempts && !success)
|
|
{
|
|
try
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
|
|
var result = await _submitter.SubmitAsync(batch, cancellationToken).ConfigureAwait(false);
|
|
|
|
if (result.Succeeded)
|
|
{
|
|
success = true;
|
|
batchesProcessed++;
|
|
jobsCreated.AddRange(result.JobIds);
|
|
|
|
// Mark all events in batch as seen
|
|
foreach (var evt in batch.Events)
|
|
{
|
|
await _idempotencyStore.MarkSeenAsync(evt.EventId, now, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
await _eventSource.AcknowledgeAsync(evt.EventId, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
attempts++;
|
|
if (attempts < _options.MaxRetryAttempts)
|
|
{
|
|
await Task.Delay(_options.RetryBackoff, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
}
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
throw;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
attempts++;
|
|
if (attempts >= _options.MaxRetryAttempts)
|
|
{
|
|
batchesFailed++;
|
|
foreach (var evt in batch.Events)
|
|
{
|
|
await _eventSource.MarkFailedAsync(evt.EventId, ex.Message, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
await Task.Delay(_options.RetryBackoff, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
stopwatch.Stop();
|
|
|
|
return new OrchestratorProcessResult
|
|
{
|
|
TotalEventsRead = totalRead,
|
|
EventsSkippedOld = skippedOld,
|
|
EventsSkippedDuplicate = skippedDuplicate,
|
|
BatchesProcessed = batchesProcessed,
|
|
BatchesFailed = batchesFailed,
|
|
JobsCreated = jobsCreated.ToImmutableArray(),
|
|
ProcessingTimeMs = stopwatch.ElapsedMilliseconds
|
|
};
|
|
}
|
|
|
|
/// <summary>
|
|
/// Creates deterministically ordered batches from events.
|
|
/// </summary>
|
|
private IReadOnlyList<PolicyChangeBatch> CreateBatches(
|
|
string tenantId,
|
|
IReadOnlyList<PolicyChangeEvent> events,
|
|
DateTimeOffset now)
|
|
{
|
|
// Sort by priority (highest first), then by occurred time
|
|
var ordered = events
|
|
.OrderByDescending(e => (int)e.Priority)
|
|
.ThenBy(e => e.OccurredAt)
|
|
.ThenBy(e => e.EventId, StringComparer.Ordinal)
|
|
.ToList();
|
|
|
|
var batches = new List<PolicyChangeBatch>();
|
|
var currentBatch = new List<PolicyChangeEvent>();
|
|
var currentPriority = PolicyChangePriority.Normal;
|
|
|
|
foreach (var evt in ordered)
|
|
{
|
|
// Start new batch if priority changes or batch is full
|
|
if (currentBatch.Count > 0 &&
|
|
(evt.Priority != currentPriority || currentBatch.Count >= _options.MaxBatchSize))
|
|
{
|
|
batches.Add(CreateBatchFromEvents(tenantId, currentBatch, currentPriority, now));
|
|
currentBatch = new List<PolicyChangeEvent>();
|
|
}
|
|
|
|
currentBatch.Add(evt);
|
|
currentPriority = evt.Priority;
|
|
}
|
|
|
|
// Add final batch
|
|
if (currentBatch.Count > 0)
|
|
{
|
|
batches.Add(CreateBatchFromEvents(tenantId, currentBatch, currentPriority, now));
|
|
}
|
|
|
|
return batches;
|
|
}
|
|
|
|
private static PolicyChangeBatch CreateBatchFromEvents(
|
|
string tenantId,
|
|
IReadOnlyList<PolicyChangeEvent> events,
|
|
PolicyChangePriority priority,
|
|
DateTimeOffset createdAt)
|
|
{
|
|
var batchId = CreateBatchId(tenantId, events, createdAt);
|
|
|
|
// Aggregate all affected items
|
|
var allPurls = events
|
|
.SelectMany(e => e.AffectedPurls)
|
|
.Where(p => !string.IsNullOrWhiteSpace(p))
|
|
.Distinct(StringComparer.Ordinal)
|
|
.OrderBy(p => p, StringComparer.Ordinal)
|
|
.ToImmutableArray();
|
|
|
|
var allProductKeys = events
|
|
.SelectMany(e => e.AffectedProductKeys)
|
|
.Where(k => !string.IsNullOrWhiteSpace(k))
|
|
.Distinct(StringComparer.OrdinalIgnoreCase)
|
|
.OrderBy(k => k, StringComparer.OrdinalIgnoreCase)
|
|
.ToImmutableArray();
|
|
|
|
var allSbomIds = events
|
|
.SelectMany(e => e.AffectedSbomIds)
|
|
.Where(id => !string.IsNullOrWhiteSpace(id))
|
|
.Distinct(StringComparer.Ordinal)
|
|
.OrderBy(id => id, StringComparer.Ordinal)
|
|
.ToImmutableArray();
|
|
|
|
var allVulnIds = events
|
|
.Select(e => e.VulnerabilityId)
|
|
.Where(v => !string.IsNullOrWhiteSpace(v))
|
|
.Cast<string>()
|
|
.Distinct(StringComparer.OrdinalIgnoreCase)
|
|
.OrderBy(v => v, StringComparer.OrdinalIgnoreCase)
|
|
.ToImmutableArray();
|
|
|
|
return new PolicyChangeBatch
|
|
{
|
|
BatchId = batchId,
|
|
TenantId = tenantId,
|
|
Events = events.ToImmutableArray(),
|
|
Priority = priority,
|
|
CreatedAt = createdAt,
|
|
AffectedPurls = allPurls,
|
|
AffectedProductKeys = allProductKeys,
|
|
AffectedSbomIds = allSbomIds,
|
|
VulnerabilityIds = allVulnIds
|
|
};
|
|
}
|
|
|
|
private static string CreateBatchId(
|
|
string tenantId,
|
|
IReadOnlyList<PolicyChangeEvent> events,
|
|
DateTimeOffset createdAt)
|
|
{
|
|
var builder = new StringBuilder();
|
|
builder.Append(tenantId).Append('|');
|
|
builder.Append(createdAt.ToString("O", CultureInfo.InvariantCulture)).Append('|');
|
|
|
|
foreach (var evt in events.OrderBy(e => e.EventId, StringComparer.Ordinal))
|
|
{
|
|
builder.Append(evt.EventId).Append('|');
|
|
}
|
|
|
|
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(builder.ToString()));
|
|
return $"pcb-{Convert.ToHexStringLower(bytes)[..16]}";
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Result of an orchestrator processing cycle.
|
|
/// </summary>
|
|
public sealed record OrchestratorProcessResult
|
|
{
|
|
/// <summary>
|
|
/// Total events read from source.
|
|
/// </summary>
|
|
public required int TotalEventsRead { get; init; }
|
|
|
|
/// <summary>
|
|
/// Events skipped due to age.
|
|
/// </summary>
|
|
public required int EventsSkippedOld { get; init; }
|
|
|
|
/// <summary>
|
|
/// Events skipped due to deduplication.
|
|
/// </summary>
|
|
public required int EventsSkippedDuplicate { get; init; }
|
|
|
|
/// <summary>
|
|
/// Batches successfully processed.
|
|
/// </summary>
|
|
public required int BatchesProcessed { get; init; }
|
|
|
|
/// <summary>
|
|
/// Batches that failed after retries.
|
|
/// </summary>
|
|
public required int BatchesFailed { get; init; }
|
|
|
|
/// <summary>
|
|
/// Job IDs created during processing.
|
|
/// </summary>
|
|
public required ImmutableArray<string> JobsCreated { get; init; }
|
|
|
|
/// <summary>
|
|
/// Total processing time in milliseconds.
|
|
/// </summary>
|
|
public required long ProcessingTimeMs { get; init; }
|
|
|
|
/// <summary>
|
|
/// Whether any work was done.
|
|
/// </summary>
|
|
public bool HasWork => TotalEventsRead > 0;
|
|
|
|
/// <summary>
|
|
/// Whether all batches succeeded.
|
|
/// </summary>
|
|
public bool AllSucceeded => BatchesFailed == 0;
|
|
}
|
|
|
|
/// <summary>
|
|
/// In-memory implementation of policy change event source for testing.
|
|
/// </summary>
|
|
public sealed class InMemoryPolicyChangeEventSource : IPolicyChangeEventSource
|
|
{
|
|
private readonly ConcurrentQueue<PolicyChangeEvent> _pending = new();
|
|
private readonly ConcurrentDictionary<string, PolicyChangeEvent> _failed = new();
|
|
private readonly ConcurrentDictionary<string, PolicyChangeEvent> _acknowledged = new();
|
|
|
|
public void Enqueue(PolicyChangeEvent evt)
|
|
{
|
|
_pending.Enqueue(evt);
|
|
}
|
|
|
|
public void EnqueueRange(IEnumerable<PolicyChangeEvent> events)
|
|
{
|
|
foreach (var evt in events)
|
|
{
|
|
_pending.Enqueue(evt);
|
|
}
|
|
}
|
|
|
|
public async IAsyncEnumerable<PolicyChangeEvent> ReadAsync(
|
|
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
|
{
|
|
while (_pending.TryDequeue(out var evt))
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
yield return evt;
|
|
}
|
|
|
|
await Task.CompletedTask;
|
|
}
|
|
|
|
public Task AcknowledgeAsync(string eventId, CancellationToken cancellationToken)
|
|
{
|
|
// Remove from failed if retrying
|
|
_failed.TryRemove(eventId, out _);
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
public Task MarkFailedAsync(string eventId, string error, CancellationToken cancellationToken)
|
|
{
|
|
// Events could be tracked for retry
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
public int PendingCount => _pending.Count;
|
|
|
|
public IReadOnlyCollection<PolicyChangeEvent> GetAcknowledged() =>
|
|
_acknowledged.Values.ToList();
|
|
}
|
|
|
|
/// <summary>
|
|
/// In-memory implementation of idempotency store for testing.
|
|
/// </summary>
|
|
public sealed class InMemoryPolicyChangeIdempotencyStore : IPolicyChangeIdempotencyStore
|
|
{
|
|
private readonly ConcurrentDictionary<string, DateTimeOffset> _seen = new(StringComparer.Ordinal);
|
|
|
|
public Task<bool> HasSeenAsync(string eventId, CancellationToken cancellationToken)
|
|
{
|
|
return Task.FromResult(_seen.ContainsKey(eventId));
|
|
}
|
|
|
|
public Task MarkSeenAsync(string eventId, DateTimeOffset processedAt, CancellationToken cancellationToken)
|
|
{
|
|
_seen[eventId] = processedAt;
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
public int SeenCount => _seen.Count;
|
|
|
|
public void Clear() => _seen.Clear();
|
|
}
|