stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Details
AOC Guard CI / aoc-guard (push) Has been cancelled
Details
AOC Guard CI / aoc-verify (push) Has been cancelled
Details
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Details
Export Center CI / export-ci (push) Has been cancelled
Details
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Details
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Details
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Details
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Details
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Details
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Details
Signals CI & Image / signals-ci (push) Has been cancelled
Details
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Details
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Details

Browse Source
This commit is contained in:
StellaOps Bot
2025-12-13 00:20:26 +02:00
parent e1f1bef4c1
commit 564df71bfb
2376 changed files with 334389 additions and 328032 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
src/Excititor/StellaOps.Excititor.Worker/Options/VexWorkerOptions.cs
View File

@@ -1,16 +1,16 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using StellaOps.Excititor.Worker.Scheduling;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Worker.Options;
public sealed class VexWorkerOptions
{
public TimeSpan DefaultInterval { get; set; } = TimeSpan.FromHours(1);
public TimeSpan OfflineInterval { get; set; } = TimeSpan.FromHours(6);
using System.Collections.Generic;
using System.Collections.Immutable;
using StellaOps.Excititor.Worker.Scheduling;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Worker.Options;
public sealed class VexWorkerOptions
{
public TimeSpan DefaultInterval { get; set; } = TimeSpan.FromHours(1);
public TimeSpan OfflineInterval { get; set; } = TimeSpan.FromHours(6);
public TimeSpan DefaultInitialDelay { get; set; } = TimeSpan.FromMinutes(5);
public bool OfflineMode { get; set; }
@@ -23,55 +23,55 @@ public sealed class VexWorkerOptions
public VexWorkerRetryOptions Retry { get; } = new();
public VexWorkerRefreshOptions Refresh { get; } = new();
internal IReadOnlyList<VexWorkerSchedule> ResolveSchedules()
{
var schedules = new List<VexWorkerSchedule>();
foreach (var provider in Providers)
{
if (!provider.Enabled)
{
continue;
}
var providerId = provider.ProviderId?.Trim();
if (string.IsNullOrWhiteSpace(providerId))
{
continue;
}
var interval = provider.Interval ?? (OfflineMode ? OfflineInterval : DefaultInterval);
if (interval <= TimeSpan.Zero)
{
continue;
}
var initialDelay = provider.InitialDelay ?? DefaultInitialDelay;
if (initialDelay < TimeSpan.Zero)
{
initialDelay = TimeSpan.Zero;
}
var connectorSettings = provider.Settings.Count == 0
? VexConnectorSettings.Empty
: new VexConnectorSettings(provider.Settings.ToImmutableDictionary(StringComparer.Ordinal));
schedules.Add(new VexWorkerSchedule(providerId, interval, initialDelay, connectorSettings));
}
return schedules;
}
}
public sealed class VexWorkerProviderOptions
{
public string ProviderId { get; set; } = string.Empty;
public bool Enabled { get; set; } = true;
public TimeSpan? Interval { get; set; }
public TimeSpan? InitialDelay { get; set; }
public IDictionary<string, string> Settings { get; } = new Dictionary<string, string>(StringComparer.Ordinal);
}
internal IReadOnlyList<VexWorkerSchedule> ResolveSchedules()
{
var schedules = new List<VexWorkerSchedule>();
foreach (var provider in Providers)
{
if (!provider.Enabled)
{
continue;
}
var providerId = provider.ProviderId?.Trim();
if (string.IsNullOrWhiteSpace(providerId))
{
continue;
}
var interval = provider.Interval ?? (OfflineMode ? OfflineInterval : DefaultInterval);
if (interval <= TimeSpan.Zero)
{
continue;
}
var initialDelay = provider.InitialDelay ?? DefaultInitialDelay;
if (initialDelay < TimeSpan.Zero)
{
initialDelay = TimeSpan.Zero;
}
var connectorSettings = provider.Settings.Count == 0
? VexConnectorSettings.Empty
: new VexConnectorSettings(provider.Settings.ToImmutableDictionary(StringComparer.Ordinal));
schedules.Add(new VexWorkerSchedule(providerId, interval, initialDelay, connectorSettings));
}
return schedules;
}
}
public sealed class VexWorkerProviderOptions
{
public string ProviderId { get; set; } = string.Empty;
public bool Enabled { get; set; } = true;
public TimeSpan? Interval { get; set; }
public TimeSpan? InitialDelay { get; set; }
public IDictionary<string, string> Settings { get; } = new Dictionary<string, string>(StringComparer.Ordinal);
}

42
src/Excititor/StellaOps.Excititor.Worker/Options/VexWorkerPluginOptions.cs
View File

@@ -1,21 +1,21 @@
using System;
using System.IO;
namespace StellaOps.Excititor.Worker.Options;
public sealed class VexWorkerPluginOptions
{
public string? Directory { get; set; }
public string? SearchPattern { get; set; }
internal string ResolveDirectory()
=> string.IsNullOrWhiteSpace(Directory)
? Path.Combine(AppContext.BaseDirectory, "plugins")
: Path.GetFullPath(Directory);
internal string ResolveSearchPattern()
=> string.IsNullOrWhiteSpace(SearchPattern)
? "StellaOps.Excititor.Connectors.*.dll"
: SearchPattern!;
}
using System;
using System.IO;
namespace StellaOps.Excititor.Worker.Options;
public sealed class VexWorkerPluginOptions
{
public string? Directory { get; set; }
public string? SearchPattern { get; set; }
internal string ResolveDirectory()
=> string.IsNullOrWhiteSpace(Directory)
? Path.Combine(AppContext.BaseDirectory, "plugins")
: Path.GetFullPath(Directory);
internal string ResolveSearchPattern()
=> string.IsNullOrWhiteSpace(SearchPattern)
? "StellaOps.Excititor.Connectors.*.dll"
: SearchPattern!;
}

180
src/Excititor/StellaOps.Excititor.Worker/Options/VexWorkerRefreshOptions.cs
View File

@@ -1,90 +1,90 @@
using System.Collections.Generic;
using System.Linq;
namespace StellaOps.Excititor.Worker.Options;
public sealed class VexWorkerRefreshOptions
{
private static readonly TimeSpan DefaultScanInterval = TimeSpan.FromMinutes(10);
private static readonly TimeSpan DefaultConsensusTtl = TimeSpan.FromHours(2);
public bool Enabled { get; set; } = true;
public TimeSpan ScanInterval { get; set; } = DefaultScanInterval;
public TimeSpan ConsensusTtl { get; set; } = DefaultConsensusTtl;
public int ScanBatchSize { get; set; } = 250;
public VexStabilityDamperOptions Damper { get; } = new();
}
public sealed class VexStabilityDamperOptions
{
private static readonly TimeSpan DefaultMinimum = TimeSpan.FromHours(24);
private static readonly TimeSpan DefaultMaximum = TimeSpan.FromHours(48);
private static readonly TimeSpan DefaultDurationBaseline = TimeSpan.FromHours(36);
public TimeSpan Minimum { get; set; } = DefaultMinimum;
public TimeSpan Maximum { get; set; } = DefaultMaximum;
public TimeSpan DefaultDuration { get; set; } = DefaultDurationBaseline;
public IList<VexStabilityDamperRule> Rules { get; } = new List<VexStabilityDamperRule>
{
new() { MinWeight = 0.9, Duration = TimeSpan.FromHours(24) },
new() { MinWeight = 0.75, Duration = TimeSpan.FromHours(30) },
new() { MinWeight = 0.5, Duration = TimeSpan.FromHours(36) },
};
internal TimeSpan ClampDuration(TimeSpan duration)
{
if (duration < Minimum)
{
return Minimum;
}
if (duration > Maximum)
{
return Maximum;
}
return duration;
}
public TimeSpan ResolveDuration(double weight)
{
if (double.IsNaN(weight) || double.IsInfinity(weight) || weight < 0)
{
return ClampDuration(DefaultDuration);
}
if (Rules.Count == 0)
{
return ClampDuration(DefaultDuration);
}
// Evaluate highest weight threshold first.
TimeSpan? selected = null;
foreach (var rule in Rules.OrderByDescending(static r => r.MinWeight))
{
if (weight >= rule.MinWeight)
{
selected = rule.Duration;
break;
}
}
return ClampDuration(selected ?? DefaultDuration);
}
}
public sealed class VexStabilityDamperRule
{
public double MinWeight { get; set; }
= 1.0;
public TimeSpan Duration { get; set; }
= TimeSpan.FromHours(24);
}
using System.Collections.Generic;
using System.Linq;
namespace StellaOps.Excititor.Worker.Options;
public sealed class VexWorkerRefreshOptions
{
private static readonly TimeSpan DefaultScanInterval = TimeSpan.FromMinutes(10);
private static readonly TimeSpan DefaultConsensusTtl = TimeSpan.FromHours(2);
public bool Enabled { get; set; } = true;
public TimeSpan ScanInterval { get; set; } = DefaultScanInterval;
public TimeSpan ConsensusTtl { get; set; } = DefaultConsensusTtl;
public int ScanBatchSize { get; set; } = 250;
public VexStabilityDamperOptions Damper { get; } = new();
}
public sealed class VexStabilityDamperOptions
{
private static readonly TimeSpan DefaultMinimum = TimeSpan.FromHours(24);
private static readonly TimeSpan DefaultMaximum = TimeSpan.FromHours(48);
private static readonly TimeSpan DefaultDurationBaseline = TimeSpan.FromHours(36);
public TimeSpan Minimum { get; set; } = DefaultMinimum;
public TimeSpan Maximum { get; set; } = DefaultMaximum;
public TimeSpan DefaultDuration { get; set; } = DefaultDurationBaseline;
public IList<VexStabilityDamperRule> Rules { get; } = new List<VexStabilityDamperRule>
{
new() { MinWeight = 0.9, Duration = TimeSpan.FromHours(24) },
new() { MinWeight = 0.75, Duration = TimeSpan.FromHours(30) },
new() { MinWeight = 0.5, Duration = TimeSpan.FromHours(36) },
};
internal TimeSpan ClampDuration(TimeSpan duration)
{
if (duration < Minimum)
{
return Minimum;
}
if (duration > Maximum)
{
return Maximum;
}
return duration;
}
public TimeSpan ResolveDuration(double weight)
{
if (double.IsNaN(weight) || double.IsInfinity(weight) || weight < 0)
{
return ClampDuration(DefaultDuration);
}
if (Rules.Count == 0)
{
return ClampDuration(DefaultDuration);
}
// Evaluate highest weight threshold first.
TimeSpan? selected = null;
foreach (var rule in Rules.OrderByDescending(static r => r.MinWeight))
{
if (weight >= rule.MinWeight)
{
selected = rule.Duration;
break;
}
}
return ClampDuration(selected ?? DefaultDuration);
}
}
public sealed class VexStabilityDamperRule
{
public double MinWeight { get; set; }
= 1.0;
public TimeSpan Duration { get; set; }
= TimeSpan.FromHours(24);
}

40
src/Excititor/StellaOps.Excititor.Worker/Options/VexWorkerRetryOptions.cs
View File

@@ -1,20 +1,20 @@
using System.ComponentModel.DataAnnotations;
namespace StellaOps.Excititor.Worker.Options;
public sealed class VexWorkerRetryOptions
{
[Range(1, int.MaxValue)]
public int FailureThreshold { get; set; } = 3;
[Range(typeof(double), "0.0", "1.0")]
public double JitterRatio { get; set; } = 0.2;
public TimeSpan BaseDelay { get; set; } = TimeSpan.FromMinutes(5);
public TimeSpan MaxDelay { get; set; } = TimeSpan.FromHours(6);
public TimeSpan QuarantineDuration { get; set; } = TimeSpan.FromHours(12);
public TimeSpan RetryCap { get; set; } = TimeSpan.FromHours(24);
}
using System.ComponentModel.DataAnnotations;
namespace StellaOps.Excititor.Worker.Options;
public sealed class VexWorkerRetryOptions
{
[Range(1, int.MaxValue)]
public int FailureThreshold { get; set; } = 3;
[Range(typeof(double), "0.0", "1.0")]
public double JitterRatio { get; set; } = 0.2;
public TimeSpan BaseDelay { get; set; } = TimeSpan.FromMinutes(5);
public TimeSpan MaxDelay { get; set; } = TimeSpan.FromHours(6);
public TimeSpan QuarantineDuration { get; set; } = TimeSpan.FromHours(12);
public TimeSpan RetryCap { get; set; } = TimeSpan.FromHours(24);
}

6
src/Excititor/StellaOps.Excititor.Worker/Properties/AssemblyInfo.cs
View File

@@ -1,3 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Excititor.Worker.Tests")]
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Excititor.Worker.Tests")]

764
src/Excititor/StellaOps.Excititor.Worker/Scheduling/DefaultVexProviderRunner.cs
View File

@@ -1,382 +1,382 @@
using System;
using System.Collections.Immutable;
using System.Linq;
using System.Security.Cryptography;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Plugin;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Orchestration;
using StellaOps.Excititor.Worker.Options;
using StellaOps.Excititor.Worker.Orchestration;
using StellaOps.Excititor.Worker.Signature;
namespace StellaOps.Excititor.Worker.Scheduling;
internal sealed class DefaultVexProviderRunner : IVexProviderRunner
{
private readonly IServiceProvider _serviceProvider;
private readonly PluginCatalog _pluginCatalog;
private readonly IVexWorkerOrchestratorClient _orchestratorClient;
private readonly VexWorkerHeartbeatService _heartbeatService;
private readonly ILogger<DefaultVexProviderRunner> _logger;
private readonly TimeProvider _timeProvider;
private readonly VexWorkerRetryOptions _retryOptions;
private readonly VexWorkerOrchestratorOptions _orchestratorOptions;
public DefaultVexProviderRunner(
IServiceProvider serviceProvider,
PluginCatalog pluginCatalog,
IVexWorkerOrchestratorClient orchestratorClient,
VexWorkerHeartbeatService heartbeatService,
ILogger<DefaultVexProviderRunner> logger,
TimeProvider timeProvider,
IOptions<VexWorkerOptions> workerOptions,
IOptions<VexWorkerOrchestratorOptions> orchestratorOptions)
{
_serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
_pluginCatalog = pluginCatalog ?? throw new ArgumentNullException(nameof(pluginCatalog));
_orchestratorClient = orchestratorClient ?? throw new ArgumentNullException(nameof(orchestratorClient));
_heartbeatService = heartbeatService ?? throw new ArgumentNullException(nameof(heartbeatService));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
if (workerOptions is null)
{
throw new ArgumentNullException(nameof(workerOptions));
}
_retryOptions = workerOptions.Value?.Retry ?? throw new InvalidOperationException("VexWorkerOptions.Retry must be configured.");
_orchestratorOptions = orchestratorOptions?.Value ?? new VexWorkerOrchestratorOptions();
}
public async ValueTask RunAsync(VexWorkerSchedule schedule, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(schedule);
ArgumentException.ThrowIfNullOrWhiteSpace(schedule.ProviderId);
using var scope = _serviceProvider.CreateScope();
var availablePlugins = _pluginCatalog.GetAvailableConnectorPlugins(scope.ServiceProvider);
var matched = availablePlugins.FirstOrDefault(plugin =>
string.Equals(plugin.Name, schedule.ProviderId, StringComparison.OrdinalIgnoreCase));
if (matched is not null)
{
_logger.LogInformation(
"Connector plugin {PluginName} ({ProviderId}) is available. Execution hooks will be added in subsequent tasks.",
matched.Name,
schedule.ProviderId);
}
else
{
_logger.LogInformation("No legacy connector plugin registered for provider {ProviderId}; falling back to DI-managed connectors.", schedule.ProviderId);
}
var connectors = scope.ServiceProvider.GetServices<IVexConnector>();
var connector = connectors.FirstOrDefault(c => string.Equals(c.Id, schedule.ProviderId, StringComparison.OrdinalIgnoreCase));
if (connector is null)
{
_logger.LogWarning("No IVexConnector implementation registered for provider {ProviderId}; skipping run.", schedule.ProviderId);
return;
}
await ExecuteConnectorAsync(scope.ServiceProvider, connector, schedule.Settings, cancellationToken).ConfigureAwait(false);
}
private async Task ExecuteConnectorAsync(IServiceProvider scopeProvider, IVexConnector connector, VexConnectorSettings settings, CancellationToken cancellationToken)
{
var effectiveSettings = settings ?? VexConnectorSettings.Empty;
var rawStore = scopeProvider.GetRequiredService<IVexRawStore>();
var providerStore = scopeProvider.GetRequiredService<IVexProviderStore>();
var stateRepository = scopeProvider.GetRequiredService<IVexConnectorStateRepository>();
var normalizerRouter = scopeProvider.GetRequiredService<IVexNormalizerRouter>();
var signatureVerifier = scopeProvider.GetRequiredService<IVexSignatureVerifier>();
var descriptor = connector switch
{
VexConnectorBase baseConnector => baseConnector.Descriptor,
_ => new VexConnectorDescriptor(connector.Id, VexProviderKind.Vendor, connector.Id)
};
var provider = await providerStore.FindAsync(descriptor.Id, cancellationToken).ConfigureAwait(false)
?? new VexProvider(descriptor.Id, descriptor.DisplayName, descriptor.Kind);
await providerStore.SaveAsync(provider, cancellationToken).ConfigureAwait(false);
var stateBeforeRun = await stateRepository.GetAsync(descriptor.Id, cancellationToken).ConfigureAwait(false);
var now = _timeProvider.GetUtcNow();
if (stateBeforeRun?.NextEligibleRun is { } nextEligible && nextEligible > now)
{
_logger.LogInformation(
"Connector {ConnectorId} is in backoff until {NextEligible:O}; skipping run.",
connector.Id,
nextEligible);
return;
}
await connector.ValidateAsync(effectiveSettings, cancellationToken).ConfigureAwait(false);
var verifyingSink = new VerifyingVexRawDocumentSink(rawStore, signatureVerifier);
var connectorContext = new VexConnectorContext(
Since: stateBeforeRun?.LastUpdated,
Settings: effectiveSettings,
RawSink: verifyingSink,
SignatureVerifier: signatureVerifier,
Normalizers: normalizerRouter,
Services: scopeProvider,
ResumeTokens: stateBeforeRun?.ResumeTokens ?? ImmutableDictionary<string, string>.Empty);
// Start orchestrator job for heartbeat/progress tracking
var jobContext = await _orchestratorClient.StartJobAsync(
_orchestratorOptions.DefaultTenant,
connector.Id,
stateBeforeRun?.LastCheckpoint,
cancellationToken).ConfigureAwait(false);
var documentCount = 0;
string? lastArtifactHash = null;
string? lastArtifactKind = null;
var currentStatus = VexWorkerHeartbeatStatus.Running;
// Start heartbeat loop in background
using var heartbeatCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
var heartbeatTask = _heartbeatService.RunAsync(
jobContext,
() => currentStatus,
() => null, // Progress not tracked at document level
() => lastArtifactHash,
() => lastArtifactKind,
heartbeatCts.Token);
try
{
await foreach (var document in connector.FetchAsync(connectorContext, cancellationToken).ConfigureAwait(false))
{
documentCount++;
lastArtifactHash = document.Digest;
lastArtifactKind = "vex-raw-document";
// Record artifact for determinism tracking
if (_orchestratorOptions.Enabled)
{
var artifact = new VexWorkerArtifact(
document.Digest,
"vex-raw-document",
connector.Id,
document.Digest,
_timeProvider.GetUtcNow());
await _orchestratorClient.RecordArtifactAsync(jobContext, artifact, cancellationToken).ConfigureAwait(false);
}
}
// Stop heartbeat loop
currentStatus = VexWorkerHeartbeatStatus.Succeeded;
await heartbeatCts.CancelAsync().ConfigureAwait(false);
await SafeWaitForTaskAsync(heartbeatTask).ConfigureAwait(false);
_logger.LogInformation(
"Connector {ConnectorId} persisted {DocumentCount} raw document(s) this run.",
connector.Id,
documentCount);
// Complete orchestrator job
var completedAt = _timeProvider.GetUtcNow();
var result = new VexWorkerJobResult(
documentCount,
ClaimsGenerated: 0, // Claims generated in separate normalization pass
lastArtifactHash,
lastArtifactHash,
completedAt);
await _orchestratorClient.CompleteJobAsync(jobContext, result, cancellationToken).ConfigureAwait(false);
await UpdateSuccessStateAsync(stateRepository, descriptor.Id, completedAt, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
currentStatus = VexWorkerHeartbeatStatus.Failed;
await heartbeatCts.CancelAsync().ConfigureAwait(false);
await SafeWaitForTaskAsync(heartbeatTask).ConfigureAwait(false);
var error = VexWorkerError.Cancelled("Operation cancelled by host");
await _orchestratorClient.FailJobAsync(jobContext, error, CancellationToken.None).ConfigureAwait(false);
throw;
}
catch (Exception ex)
{
currentStatus = VexWorkerHeartbeatStatus.Failed;
await heartbeatCts.CancelAsync().ConfigureAwait(false);
await SafeWaitForTaskAsync(heartbeatTask).ConfigureAwait(false);
// Classify the error for appropriate retry handling
var classifiedError = VexWorkerError.FromException(ex, stage: "fetch");
// Apply backoff delay for retryable errors
var retryDelay = classifiedError.Retryable
? (int)CalculateDelayWithJitter(1).TotalSeconds
: (int?)null;
var errorWithRetry = classifiedError.Retryable && retryDelay.HasValue
? new VexWorkerError(
classifiedError.Code,
classifiedError.Category,
classifiedError.Message,
classifiedError.Retryable,
retryDelay,
classifiedError.Stage,
classifiedError.Details)
: classifiedError;
await _orchestratorClient.FailJobAsync(jobContext, errorWithRetry, CancellationToken.None).ConfigureAwait(false);
await UpdateFailureStateAsync(stateRepository, descriptor.Id, _timeProvider.GetUtcNow(), ex, classifiedError.Retryable, cancellationToken).ConfigureAwait(false);
throw;
}
}
private static async Task SafeWaitForTaskAsync(Task task)
{
try
{
await task.ConfigureAwait(false);
}
catch (OperationCanceledException)
{
// Expected when cancellation is requested
}
}
private async Task UpdateSuccessStateAsync(
IVexConnectorStateRepository stateRepository,
string connectorId,
DateTimeOffset completedAt,
CancellationToken cancellationToken)
{
var current = await stateRepository.GetAsync(connectorId, cancellationToken).ConfigureAwait(false)
?? new VexConnectorState(connectorId, null, ImmutableArray<string>.Empty);
var updated = current with
{
LastSuccessAt = completedAt,
FailureCount = 0,
NextEligibleRun = null,
LastFailureReason = null
};
await stateRepository.SaveAsync(updated, cancellationToken).ConfigureAwait(false);
}
private async Task UpdateFailureStateAsync(
IVexConnectorStateRepository stateRepository,
string connectorId,
DateTimeOffset failureTime,
Exception exception,
bool retryable,
CancellationToken cancellationToken)
{
var current = await stateRepository.GetAsync(connectorId, cancellationToken).ConfigureAwait(false)
?? new VexConnectorState(connectorId, null, ImmutableArray<string>.Empty);
var failureCount = current.FailureCount + 1;
DateTimeOffset? nextEligible;
if (retryable)
{
// Apply exponential backoff for retryable errors
var delay = CalculateDelayWithJitter(failureCount);
nextEligible = failureTime + delay;
if (failureCount >= _retryOptions.FailureThreshold)
{
var quarantineUntil = failureTime + _retryOptions.QuarantineDuration;
if (quarantineUntil > nextEligible)
{
nextEligible = quarantineUntil;
}
}
var retryCap = failureTime + _retryOptions.RetryCap;
if (nextEligible > retryCap)
{
nextEligible = retryCap;
}
if (nextEligible < failureTime)
{
nextEligible = failureTime;
}
}
else
{
// Non-retryable errors: apply quarantine immediately
nextEligible = failureTime + _retryOptions.QuarantineDuration;
}
var updated = current with
{
FailureCount = failureCount,
NextEligibleRun = nextEligible,
LastFailureReason = Truncate(exception.Message, 512)
};
await stateRepository.SaveAsync(updated, cancellationToken).ConfigureAwait(false);
_logger.LogWarning(
exception,
"Connector {ConnectorId} failed (attempt {Attempt}, retryable={Retryable}). Next eligible run at {NextEligible:O}.",
connectorId,
failureCount,
retryable,
nextEligible);
}
private TimeSpan CalculateDelayWithJitter(int failureCount)
{
var exponent = Math.Max(0, failureCount - 1);
var factor = Math.Pow(2, exponent);
var baselineTicks = (long)Math.Min(_retryOptions.BaseDelay.Ticks * factor, _retryOptions.MaxDelay.Ticks);
if (_retryOptions.JitterRatio <= 0)
{
return TimeSpan.FromTicks(baselineTicks);
}
var minFactor = 1.0 - _retryOptions.JitterRatio;
var maxFactor = 1.0 + _retryOptions.JitterRatio;
Span<byte> buffer = stackalloc byte[8];
RandomNumberGenerator.Fill(buffer);
var sample = BitConverter.ToUInt64(buffer) / (double)ulong.MaxValue;
var jitterFactor = minFactor + (maxFactor - minFactor) * sample;
var jitteredTicks = (long)Math.Round(baselineTicks * jitterFactor);
if (jitteredTicks < _retryOptions.BaseDelay.Ticks)
{
jitteredTicks = _retryOptions.BaseDelay.Ticks;
}
if (jitteredTicks > _retryOptions.MaxDelay.Ticks)
{
jitteredTicks = _retryOptions.MaxDelay.Ticks;
}
return TimeSpan.FromTicks(jitteredTicks);
}
private static string Truncate(string? value, int maxLength)
{
if (string.IsNullOrEmpty(value))
{
return string.Empty;
}
return value.Length <= maxLength
? value
: value[..maxLength];
}
}
using System;
using System.Collections.Immutable;
using System.Linq;
using System.Security.Cryptography;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Plugin;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Orchestration;
using StellaOps.Excititor.Worker.Options;
using StellaOps.Excititor.Worker.Orchestration;
using StellaOps.Excititor.Worker.Signature;
namespace StellaOps.Excititor.Worker.Scheduling;
internal sealed class DefaultVexProviderRunner : IVexProviderRunner
{
private readonly IServiceProvider _serviceProvider;
private readonly PluginCatalog _pluginCatalog;
private readonly IVexWorkerOrchestratorClient _orchestratorClient;
private readonly VexWorkerHeartbeatService _heartbeatService;
private readonly ILogger<DefaultVexProviderRunner> _logger;
private readonly TimeProvider _timeProvider;
private readonly VexWorkerRetryOptions _retryOptions;
private readonly VexWorkerOrchestratorOptions _orchestratorOptions;
public DefaultVexProviderRunner(
IServiceProvider serviceProvider,
PluginCatalog pluginCatalog,
IVexWorkerOrchestratorClient orchestratorClient,
VexWorkerHeartbeatService heartbeatService,
ILogger<DefaultVexProviderRunner> logger,
TimeProvider timeProvider,
IOptions<VexWorkerOptions> workerOptions,
IOptions<VexWorkerOrchestratorOptions> orchestratorOptions)
{
_serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
_pluginCatalog = pluginCatalog ?? throw new ArgumentNullException(nameof(pluginCatalog));
_orchestratorClient = orchestratorClient ?? throw new ArgumentNullException(nameof(orchestratorClient));
_heartbeatService = heartbeatService ?? throw new ArgumentNullException(nameof(heartbeatService));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
if (workerOptions is null)
{
throw new ArgumentNullException(nameof(workerOptions));
}
_retryOptions = workerOptions.Value?.Retry ?? throw new InvalidOperationException("VexWorkerOptions.Retry must be configured.");
_orchestratorOptions = orchestratorOptions?.Value ?? new VexWorkerOrchestratorOptions();
}
public async ValueTask RunAsync(VexWorkerSchedule schedule, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(schedule);
ArgumentException.ThrowIfNullOrWhiteSpace(schedule.ProviderId);
using var scope = _serviceProvider.CreateScope();
var availablePlugins = _pluginCatalog.GetAvailableConnectorPlugins(scope.ServiceProvider);
var matched = availablePlugins.FirstOrDefault(plugin =>
string.Equals(plugin.Name, schedule.ProviderId, StringComparison.OrdinalIgnoreCase));
if (matched is not null)
{
_logger.LogInformation(
"Connector plugin {PluginName} ({ProviderId}) is available. Execution hooks will be added in subsequent tasks.",
matched.Name,
schedule.ProviderId);
}
else
{
_logger.LogInformation("No legacy connector plugin registered for provider {ProviderId}; falling back to DI-managed connectors.", schedule.ProviderId);
}
var connectors = scope.ServiceProvider.GetServices<IVexConnector>();
var connector = connectors.FirstOrDefault(c => string.Equals(c.Id, schedule.ProviderId, StringComparison.OrdinalIgnoreCase));
if (connector is null)
{
_logger.LogWarning("No IVexConnector implementation registered for provider {ProviderId}; skipping run.", schedule.ProviderId);
return;
}
await ExecuteConnectorAsync(scope.ServiceProvider, connector, schedule.Settings, cancellationToken).ConfigureAwait(false);
}
private async Task ExecuteConnectorAsync(IServiceProvider scopeProvider, IVexConnector connector, VexConnectorSettings settings, CancellationToken cancellationToken)
{
var effectiveSettings = settings ?? VexConnectorSettings.Empty;
var rawStore = scopeProvider.GetRequiredService<IVexRawStore>();
var providerStore = scopeProvider.GetRequiredService<IVexProviderStore>();
var stateRepository = scopeProvider.GetRequiredService<IVexConnectorStateRepository>();
var normalizerRouter = scopeProvider.GetRequiredService<IVexNormalizerRouter>();
var signatureVerifier = scopeProvider.GetRequiredService<IVexSignatureVerifier>();
var descriptor = connector switch
{
VexConnectorBase baseConnector => baseConnector.Descriptor,
_ => new VexConnectorDescriptor(connector.Id, VexProviderKind.Vendor, connector.Id)
};
var provider = await providerStore.FindAsync(descriptor.Id, cancellationToken).ConfigureAwait(false)
?? new VexProvider(descriptor.Id, descriptor.DisplayName, descriptor.Kind);
await providerStore.SaveAsync(provider, cancellationToken).ConfigureAwait(false);
var stateBeforeRun = await stateRepository.GetAsync(descriptor.Id, cancellationToken).ConfigureAwait(false);
var now = _timeProvider.GetUtcNow();
if (stateBeforeRun?.NextEligibleRun is { } nextEligible && nextEligible > now)
{
_logger.LogInformation(
"Connector {ConnectorId} is in backoff until {NextEligible:O}; skipping run.",
connector.Id,
nextEligible);
return;
}
await connector.ValidateAsync(effectiveSettings, cancellationToken).ConfigureAwait(false);
var verifyingSink = new VerifyingVexRawDocumentSink(rawStore, signatureVerifier);
var connectorContext = new VexConnectorContext(
Since: stateBeforeRun?.LastUpdated,
Settings: effectiveSettings,
RawSink: verifyingSink,
SignatureVerifier: signatureVerifier,
Normalizers: normalizerRouter,
Services: scopeProvider,
ResumeTokens: stateBeforeRun?.ResumeTokens ?? ImmutableDictionary<string, string>.Empty);
// Start orchestrator job for heartbeat/progress tracking
var jobContext = await _orchestratorClient.StartJobAsync(
_orchestratorOptions.DefaultTenant,
connector.Id,
stateBeforeRun?.LastCheckpoint,
cancellationToken).ConfigureAwait(false);
var documentCount = 0;
string? lastArtifactHash = null;
string? lastArtifactKind = null;
var currentStatus = VexWorkerHeartbeatStatus.Running;
// Start heartbeat loop in background
using var heartbeatCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
var heartbeatTask = _heartbeatService.RunAsync(
jobContext,
() => currentStatus,
() => null, // Progress not tracked at document level
() => lastArtifactHash,
() => lastArtifactKind,
heartbeatCts.Token);
try
{
await foreach (var document in connector.FetchAsync(connectorContext, cancellationToken).ConfigureAwait(false))
{
documentCount++;
lastArtifactHash = document.Digest;
lastArtifactKind = "vex-raw-document";
// Record artifact for determinism tracking
if (_orchestratorOptions.Enabled)
{
var artifact = new VexWorkerArtifact(
document.Digest,
"vex-raw-document",
connector.Id,
document.Digest,
_timeProvider.GetUtcNow());
await _orchestratorClient.RecordArtifactAsync(jobContext, artifact, cancellationToken).ConfigureAwait(false);
}
}
// Stop heartbeat loop
currentStatus = VexWorkerHeartbeatStatus.Succeeded;
await heartbeatCts.CancelAsync().ConfigureAwait(false);
await SafeWaitForTaskAsync(heartbeatTask).ConfigureAwait(false);
_logger.LogInformation(
"Connector {ConnectorId} persisted {DocumentCount} raw document(s) this run.",
connector.Id,
documentCount);
// Complete orchestrator job
var completedAt = _timeProvider.GetUtcNow();
var result = new VexWorkerJobResult(
documentCount,
ClaimsGenerated: 0, // Claims generated in separate normalization pass
lastArtifactHash,
lastArtifactHash,
completedAt);
await _orchestratorClient.CompleteJobAsync(jobContext, result, cancellationToken).ConfigureAwait(false);
await UpdateSuccessStateAsync(stateRepository, descriptor.Id, completedAt, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
currentStatus = VexWorkerHeartbeatStatus.Failed;
await heartbeatCts.CancelAsync().ConfigureAwait(false);
await SafeWaitForTaskAsync(heartbeatTask).ConfigureAwait(false);
var error = VexWorkerError.Cancelled("Operation cancelled by host");
await _orchestratorClient.FailJobAsync(jobContext, error, CancellationToken.None).ConfigureAwait(false);
throw;
}
catch (Exception ex)
{
currentStatus = VexWorkerHeartbeatStatus.Failed;
await heartbeatCts.CancelAsync().ConfigureAwait(false);
await SafeWaitForTaskAsync(heartbeatTask).ConfigureAwait(false);
// Classify the error for appropriate retry handling
var classifiedError = VexWorkerError.FromException(ex, stage: "fetch");
// Apply backoff delay for retryable errors
var retryDelay = classifiedError.Retryable
? (int)CalculateDelayWithJitter(1).TotalSeconds
: (int?)null;
var errorWithRetry = classifiedError.Retryable && retryDelay.HasValue
? new VexWorkerError(
classifiedError.Code,
classifiedError.Category,
classifiedError.Message,
classifiedError.Retryable,
retryDelay,
classifiedError.Stage,
classifiedError.Details)
: classifiedError;
await _orchestratorClient.FailJobAsync(jobContext, errorWithRetry, CancellationToken.None).ConfigureAwait(false);
await UpdateFailureStateAsync(stateRepository, descriptor.Id, _timeProvider.GetUtcNow(), ex, classifiedError.Retryable, cancellationToken).ConfigureAwait(false);
throw;
}
}
private static async Task SafeWaitForTaskAsync(Task task)
{
try
{
await task.ConfigureAwait(false);
}
catch (OperationCanceledException)
{
// Expected when cancellation is requested
}
}
private async Task UpdateSuccessStateAsync(
IVexConnectorStateRepository stateRepository,
string connectorId,
DateTimeOffset completedAt,
CancellationToken cancellationToken)
{
var current = await stateRepository.GetAsync(connectorId, cancellationToken).ConfigureAwait(false)
?? new VexConnectorState(connectorId, null, ImmutableArray<string>.Empty);
var updated = current with
{
LastSuccessAt = completedAt,
FailureCount = 0,
NextEligibleRun = null,
LastFailureReason = null
};
await stateRepository.SaveAsync(updated, cancellationToken).ConfigureAwait(false);
}
private async Task UpdateFailureStateAsync(
IVexConnectorStateRepository stateRepository,
string connectorId,
DateTimeOffset failureTime,
Exception exception,
bool retryable,
CancellationToken cancellationToken)
{
var current = await stateRepository.GetAsync(connectorId, cancellationToken).ConfigureAwait(false)
?? new VexConnectorState(connectorId, null, ImmutableArray<string>.Empty);
var failureCount = current.FailureCount + 1;
DateTimeOffset? nextEligible;
if (retryable)
{
// Apply exponential backoff for retryable errors
var delay = CalculateDelayWithJitter(failureCount);
nextEligible = failureTime + delay;
if (failureCount >= _retryOptions.FailureThreshold)
{
var quarantineUntil = failureTime + _retryOptions.QuarantineDuration;
if (quarantineUntil > nextEligible)
{
nextEligible = quarantineUntil;
}
}
var retryCap = failureTime + _retryOptions.RetryCap;
if (nextEligible > retryCap)
{
nextEligible = retryCap;
}
if (nextEligible < failureTime)
{
nextEligible = failureTime;
}
}
else
{
// Non-retryable errors: apply quarantine immediately
nextEligible = failureTime + _retryOptions.QuarantineDuration;
}
var updated = current with
{
FailureCount = failureCount,
NextEligibleRun = nextEligible,
LastFailureReason = Truncate(exception.Message, 512)
};
await stateRepository.SaveAsync(updated, cancellationToken).ConfigureAwait(false);
_logger.LogWarning(
exception,
"Connector {ConnectorId} failed (attempt {Attempt}, retryable={Retryable}). Next eligible run at {NextEligible:O}.",
connectorId,
failureCount,
retryable,
nextEligible);
}
private TimeSpan CalculateDelayWithJitter(int failureCount)
{
var exponent = Math.Max(0, failureCount - 1);
var factor = Math.Pow(2, exponent);
var baselineTicks = (long)Math.Min(_retryOptions.BaseDelay.Ticks * factor, _retryOptions.MaxDelay.Ticks);
if (_retryOptions.JitterRatio <= 0)
{
return TimeSpan.FromTicks(baselineTicks);
}
var minFactor = 1.0 - _retryOptions.JitterRatio;
var maxFactor = 1.0 + _retryOptions.JitterRatio;
Span<byte> buffer = stackalloc byte[8];
RandomNumberGenerator.Fill(buffer);
var sample = BitConverter.ToUInt64(buffer) / (double)ulong.MaxValue;
var jitterFactor = minFactor + (maxFactor - minFactor) * sample;
var jitteredTicks = (long)Math.Round(baselineTicks * jitterFactor);
if (jitteredTicks < _retryOptions.BaseDelay.Ticks)
{
jitteredTicks = _retryOptions.BaseDelay.Ticks;
}
if (jitteredTicks > _retryOptions.MaxDelay.Ticks)
{
jitteredTicks = _retryOptions.MaxDelay.Ticks;
}
return TimeSpan.FromTicks(jitteredTicks);
}
private static string Truncate(string? value, int maxLength)
{
if (string.IsNullOrEmpty(value))
{
return string.Empty;
}
return value.Length <= maxLength
? value
: value[..maxLength];
}
}

12
src/Excititor/StellaOps.Excititor.Worker/Scheduling/IVexConsensusRefreshScheduler.cs
View File

@@ -1,6 +1,6 @@
namespace StellaOps.Excititor.Worker.Scheduling;
public interface IVexConsensusRefreshScheduler
{
void ScheduleRefresh(string vulnerabilityId, string productKey);
}
namespace StellaOps.Excititor.Worker.Scheduling;
public interface IVexConsensusRefreshScheduler
{
void ScheduleRefresh(string vulnerabilityId, string productKey);
}

12
src/Excititor/StellaOps.Excititor.Worker/Scheduling/IVexProviderRunner.cs
View File

@@ -1,6 +1,6 @@
namespace StellaOps.Excititor.Worker.Scheduling;
internal interface IVexProviderRunner
{
ValueTask RunAsync(VexWorkerSchedule schedule, CancellationToken cancellationToken);
}
namespace StellaOps.Excititor.Worker.Scheduling;
internal interface IVexProviderRunner
{
ValueTask RunAsync(VexWorkerSchedule schedule, CancellationToken cancellationToken);
}

220
src/Excititor/StellaOps.Excititor.Worker/Scheduling/VexWorkerHostedService.cs
View File

@@ -1,110 +1,110 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Worker.Options;
namespace StellaOps.Excititor.Worker.Scheduling;
internal sealed class VexWorkerHostedService : BackgroundService
{
private readonly IOptions<VexWorkerOptions> _options;
private readonly IVexProviderRunner _runner;
private readonly ILogger<VexWorkerHostedService> _logger;
private readonly TimeProvider _timeProvider;
public VexWorkerHostedService(
IOptions<VexWorkerOptions> options,
IVexProviderRunner runner,
ILogger<VexWorkerHostedService> logger,
TimeProvider timeProvider)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
_runner = runner ?? throw new ArgumentNullException(nameof(runner));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var schedules = _options.Value.ResolveSchedules();
if (schedules.Count == 0)
{
_logger.LogWarning("Excititor worker has no configured provider schedules; the service will remain idle.");
await Task.CompletedTask;
return;
}
_logger.LogInformation("Excititor worker starting with {ProviderCount} provider schedule(s).", schedules.Count);
var tasks = new List<Task>(schedules.Count);
foreach (var schedule in schedules)
{
tasks.Add(RunScheduleAsync(schedule, stoppingToken));
}
await Task.WhenAll(tasks);
}
private async Task RunScheduleAsync(VexWorkerSchedule schedule, CancellationToken cancellationToken)
{
try
{
if (schedule.InitialDelay > TimeSpan.Zero)
{
_logger.LogInformation(
"Provider {ProviderId} initial delay of {InitialDelay} before first execution.",
schedule.ProviderId,
schedule.InitialDelay);
await Task.Delay(schedule.InitialDelay, cancellationToken).ConfigureAwait(false);
}
using var timer = new PeriodicTimer(schedule.Interval);
do
{
var startedAt = _timeProvider.GetUtcNow();
_logger.LogInformation(
"Provider {ProviderId} run started at {StartedAt}. Interval={Interval}.",
schedule.ProviderId,
startedAt,
schedule.Interval);
try
{
await _runner.RunAsync(schedule, cancellationToken).ConfigureAwait(false);
var completedAt = _timeProvider.GetUtcNow();
var elapsed = completedAt - startedAt;
_logger.LogInformation(
"Provider {ProviderId} run completed at {CompletedAt} (duration {Duration}).",
schedule.ProviderId,
completedAt,
elapsed);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
_logger.LogInformation("Provider {ProviderId} run cancelled.", schedule.ProviderId);
break;
}
catch (Exception ex)
{
_logger.LogError(
ex,
"Provider {ProviderId} run failed: {Message}",
schedule.ProviderId,
ex.Message);
}
}
while (await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false));
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
_logger.LogInformation("Provider {ProviderId} schedule cancelled.", schedule.ProviderId);
}
}
}
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Worker.Options;
namespace StellaOps.Excititor.Worker.Scheduling;
internal sealed class VexWorkerHostedService : BackgroundService
{
private readonly IOptions<VexWorkerOptions> _options;
private readonly IVexProviderRunner _runner;
private readonly ILogger<VexWorkerHostedService> _logger;
private readonly TimeProvider _timeProvider;
public VexWorkerHostedService(
IOptions<VexWorkerOptions> options,
IVexProviderRunner runner,
ILogger<VexWorkerHostedService> logger,
TimeProvider timeProvider)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
_runner = runner ?? throw new ArgumentNullException(nameof(runner));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var schedules = _options.Value.ResolveSchedules();
if (schedules.Count == 0)
{
_logger.LogWarning("Excititor worker has no configured provider schedules; the service will remain idle.");
await Task.CompletedTask;
return;
}
_logger.LogInformation("Excititor worker starting with {ProviderCount} provider schedule(s).", schedules.Count);
var tasks = new List<Task>(schedules.Count);
foreach (var schedule in schedules)
{
tasks.Add(RunScheduleAsync(schedule, stoppingToken));
}
await Task.WhenAll(tasks);
}
private async Task RunScheduleAsync(VexWorkerSchedule schedule, CancellationToken cancellationToken)
{
try
{
if (schedule.InitialDelay > TimeSpan.Zero)
{
_logger.LogInformation(
"Provider {ProviderId} initial delay of {InitialDelay} before first execution.",
schedule.ProviderId,
schedule.InitialDelay);
await Task.Delay(schedule.InitialDelay, cancellationToken).ConfigureAwait(false);
}
using var timer = new PeriodicTimer(schedule.Interval);
do
{
var startedAt = _timeProvider.GetUtcNow();
_logger.LogInformation(
"Provider {ProviderId} run started at {StartedAt}. Interval={Interval}.",
schedule.ProviderId,
startedAt,
schedule.Interval);
try
{
await _runner.RunAsync(schedule, cancellationToken).ConfigureAwait(false);
var completedAt = _timeProvider.GetUtcNow();
var elapsed = completedAt - startedAt;
_logger.LogInformation(
"Provider {ProviderId} run completed at {CompletedAt} (duration {Duration}).",
schedule.ProviderId,
completedAt,
elapsed);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
_logger.LogInformation("Provider {ProviderId} run cancelled.", schedule.ProviderId);
break;
}
catch (Exception ex)
{
_logger.LogError(
ex,
"Provider {ProviderId} run failed: {Message}",
schedule.ProviderId,
ex.Message);
}
}
while (await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false));
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
_logger.LogInformation("Provider {ProviderId} schedule cancelled.", schedule.ProviderId);
}
}
}

36
src/Excititor/StellaOps.Excititor.Worker/Scheduling/VexWorkerSchedule.cs
View File

@@ -1,18 +1,18 @@
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Worker.Scheduling;
/// <summary>
/// Schedule configuration for a VEX provider worker.
/// </summary>
/// <param name="ProviderId">The provider identifier.</param>
/// <param name="Interval">The interval between runs.</param>
/// <param name="InitialDelay">The initial delay before the first run.</param>
/// <param name="Settings">The connector settings.</param>
/// <param name="Tenant">The tenant identifier (optional; defaults to global).</param>
internal sealed record VexWorkerSchedule(
string ProviderId,
TimeSpan Interval,
TimeSpan InitialDelay,
VexConnectorSettings Settings,
string? Tenant = null);
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Worker.Scheduling;
/// <summary>
/// Schedule configuration for a VEX provider worker.
/// </summary>
/// <param name="ProviderId">The provider identifier.</param>
/// <param name="Interval">The interval between runs.</param>
/// <param name="InitialDelay">The initial delay before the first run.</param>
/// <param name="Settings">The connector settings.</param>
/// <param name="Tenant">The tenant identifier (optional; defaults to global).</param>
internal sealed record VexWorkerSchedule(
string ProviderId,
TimeSpan Interval,
TimeSpan InitialDelay,
VexConnectorSettings Settings,
string? Tenant = null);

532
src/Excititor/StellaOps.Excititor.Worker/Signature/WorkerSignatureVerifier.cs
View File

@@ -1,14 +1,14 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics.Metrics;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.Logging;
using StellaOps.Aoc;
using StellaOps.Excititor.Attestation.Dsse;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics.Metrics;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.Logging;
using StellaOps.Aoc;
using StellaOps.Excititor.Attestation.Dsse;
using StellaOps.Excititor.Attestation.Models;
using StellaOps.Excititor.Attestation.Verification;
using StellaOps.Excititor.Core;
@@ -16,35 +16,35 @@ using StellaOps.Excititor.Core.Aoc;
using StellaOps.IssuerDirectory.Client;
namespace StellaOps.Excititor.Worker.Signature;
/// <summary>
/// Enforces checksum validation and records signature verification metadata.
/// </summary>
internal sealed class WorkerSignatureVerifier : IVexSignatureVerifier
{
private static readonly Meter Meter = new("StellaOps.Excititor.Worker", "1.0");
private static readonly Counter<long> SignatureVerificationCounter = Meter.CreateCounter<long>(
"ingestion_signature_verified_total",
description: "Counts signature and checksum verification results for Excititor worker ingestion.");
/// <summary>
/// Enforces checksum validation and records signature verification metadata.
/// </summary>
internal sealed class WorkerSignatureVerifier : IVexSignatureVerifier
{
private static readonly Meter Meter = new("StellaOps.Excititor.Worker", "1.0");
private static readonly Counter<long> SignatureVerificationCounter = Meter.CreateCounter<long>(
"ingestion_signature_verified_total",
description: "Counts signature and checksum verification results for Excititor worker ingestion.");
private readonly ILogger<WorkerSignatureVerifier> _logger;
private readonly IVexAttestationVerifier? _attestationVerifier;
private readonly TimeProvider _timeProvider;
private readonly IIssuerDirectoryClient? _issuerDirectoryClient;
private static readonly JsonSerializerOptions EnvelopeSerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
};
private static readonly JsonSerializerOptions StatementSerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.Never,
Converters = { new JsonStringEnumConverter(JsonNamingPolicy.CamelCase) },
};
private static readonly JsonSerializerOptions EnvelopeSerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
};
private static readonly JsonSerializerOptions StatementSerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.Never,
Converters = { new JsonStringEnumConverter(JsonNamingPolicy.CamelCase) },
};
public WorkerSignatureVerifier(
ILogger<WorkerSignatureVerifier> logger,
IVexAttestationVerifier? attestationVerifier = null,
@@ -56,34 +56,34 @@ internal sealed class WorkerSignatureVerifier : IVexSignatureVerifier
_timeProvider = timeProvider ?? TimeProvider.System;
_issuerDirectoryClient = issuerDirectoryClient;
}
public async ValueTask<VexSignatureMetadata?> VerifyAsync(VexRawDocument document, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(document);
var metadata = document.Metadata ?? ImmutableDictionary<string, string>.Empty;
var expectedDigest = NormalizeDigest(document.Digest);
var computedDigest = ComputeDigest(document.Content.Span);
if (!string.Equals(expectedDigest, computedDigest, StringComparison.OrdinalIgnoreCase))
{
RecordVerification(document.ProviderId, metadata, "fail");
_logger.LogError(
"Checksum mismatch for provider {ProviderId} (expected={ExpectedDigest}, computed={ComputedDigest}, uri={SourceUri})",
document.ProviderId,
expectedDigest,
computedDigest,
document.SourceUri);
var violation = AocViolation.Create(
AocViolationCode.SignatureInvalid,
"/upstream/content_hash",
$"Content hash mismatch. Expected {expectedDigest}, computed {computedDigest}.");
throw new ExcititorAocGuardException(AocGuardResult.FromViolations(new[] { violation }));
}
public async ValueTask<VexSignatureMetadata?> VerifyAsync(VexRawDocument document, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(document);
var metadata = document.Metadata ?? ImmutableDictionary<string, string>.Empty;
var expectedDigest = NormalizeDigest(document.Digest);
var computedDigest = ComputeDigest(document.Content.Span);
if (!string.Equals(expectedDigest, computedDigest, StringComparison.OrdinalIgnoreCase))
{
RecordVerification(document.ProviderId, metadata, "fail");
_logger.LogError(
"Checksum mismatch for provider {ProviderId} (expected={ExpectedDigest}, computed={ComputedDigest}, uri={SourceUri})",
document.ProviderId,
expectedDigest,
computedDigest,
document.SourceUri);
var violation = AocViolation.Create(
AocViolationCode.SignatureInvalid,
"/upstream/content_hash",
$"Content hash mismatch. Expected {expectedDigest}, computed {computedDigest}.");
throw new ExcititorAocGuardException(AocGuardResult.FromViolations(new[] { violation }));
}
VexSignatureMetadata? signatureMetadata = null;
VexAttestationDiagnostics? attestationDiagnostics = null;
if (document.Format == VexDocumentFormat.OciAttestation && _attestationVerifier is not null)
@@ -111,12 +111,12 @@ internal sealed class WorkerSignatureVerifier : IVexSignatureVerifier
if (resultLabel == "skipped")
{
_logger.LogDebug(
"Signature verification skipped for provider {ProviderId} (no signature metadata).",
document.ProviderId);
}
else
{
_logger.LogDebug(
"Signature verification skipped for provider {ProviderId} (no signature metadata).",
document.ProviderId);
}
else
{
_logger.LogInformation(
"Signature metadata recorded for provider {ProviderId} (type={SignatureType}, subject={Subject}, issuer={Issuer}, result={Result}).",
document.ProviderId,
@@ -133,31 +133,31 @@ internal sealed class WorkerSignatureVerifier : IVexSignatureVerifier
VexRawDocument document,
ImmutableDictionary<string, string> metadata,
CancellationToken cancellationToken)
{
try
{
var envelopeJson = Encoding.UTF8.GetString(document.Content.Span);
var envelope = JsonSerializer.Deserialize<DsseEnvelope>(envelopeJson, EnvelopeSerializerOptions)
?? throw new InvalidOperationException("DSSE envelope deserialized to null.");
var payloadBytes = Convert.FromBase64String(envelope.Payload);
var statement = JsonSerializer.Deserialize<VexInTotoStatement>(payloadBytes, StatementSerializerOptions)
?? throw new InvalidOperationException("DSSE statement deserialized to null.");
if (statement.Subject is null || statement.Subject.Count == 0)
{
throw new InvalidOperationException("DSSE statement subject is missing.");
}
var predicate = statement.Predicate ?? throw new InvalidOperationException("DSSE predicate is missing.");
var request = BuildAttestationRequest(statement, predicate);
var attestationMetadata = BuildAttestationMetadata(statement, envelope, metadata);
var verificationRequest = new VexAttestationVerificationRequest(
request,
attestationMetadata,
envelopeJson);
{
try
{
var envelopeJson = Encoding.UTF8.GetString(document.Content.Span);
var envelope = JsonSerializer.Deserialize<DsseEnvelope>(envelopeJson, EnvelopeSerializerOptions)
?? throw new InvalidOperationException("DSSE envelope deserialized to null.");
var payloadBytes = Convert.FromBase64String(envelope.Payload);
var statement = JsonSerializer.Deserialize<VexInTotoStatement>(payloadBytes, StatementSerializerOptions)
?? throw new InvalidOperationException("DSSE statement deserialized to null.");
if (statement.Subject is null || statement.Subject.Count == 0)
{
throw new InvalidOperationException("DSSE statement subject is missing.");
}
var predicate = statement.Predicate ?? throw new InvalidOperationException("DSSE predicate is missing.");
var request = BuildAttestationRequest(statement, predicate);
var attestationMetadata = BuildAttestationMetadata(statement, envelope, metadata);
var verificationRequest = new VexAttestationVerificationRequest(
request,
attestationMetadata,
envelopeJson);
var verification = await _attestationVerifier!
.VerifyAsync(verificationRequest, cancellationToken)
.ConfigureAwait(false);
@@ -200,196 +200,196 @@ internal sealed class WorkerSignatureVerifier : IVexSignatureVerifier
}
catch (ExcititorAocGuardException)
{
throw;
}
catch (Exception ex)
{
_logger.LogError(
ex,
"Failed to verify attestation for provider {ProviderId} (uri={SourceUri})",
document.ProviderId,
document.SourceUri);
var violation = AocViolation.Create(
AocViolationCode.SignatureInvalid,
"/upstream/signature",
$"Attestation verification encountered an error: {ex.Message}");
throw;
}
catch (Exception ex)
{
_logger.LogError(
ex,
"Failed to verify attestation for provider {ProviderId} (uri={SourceUri})",
document.ProviderId,
document.SourceUri);
var violation = AocViolation.Create(
AocViolationCode.SignatureInvalid,
"/upstream/signature",
$"Attestation verification encountered an error: {ex.Message}");
RecordVerification(document.ProviderId, metadata, "error");
throw new ExcititorAocGuardException(AocGuardResult.FromViolations(new[] { violation }));
}
}
private VexAttestationRequest BuildAttestationRequest(VexInTotoStatement statement, VexAttestationPredicate predicate)
{
var subject = statement.Subject!.First();
var exportId = predicate.ExportId ?? subject.Name ?? throw new InvalidOperationException("Attestation export ID missing.");
var querySignature = new VexQuerySignature(predicate.QuerySignature ?? throw new InvalidOperationException("Attestation query signature missing."));
if (string.IsNullOrWhiteSpace(predicate.ArtifactAlgorithm) || string.IsNullOrWhiteSpace(predicate.ArtifactDigest))
{
throw new InvalidOperationException("Attestation artifact metadata is incomplete.");
}
var artifact = new VexContentAddress(predicate.ArtifactAlgorithm, predicate.ArtifactDigest);
var sourceProviders = predicate.SourceProviders?.ToImmutableArray() ?? ImmutableArray<string>.Empty;
var metadata = predicate.Metadata?.ToImmutableDictionary(StringComparer.Ordinal) ?? ImmutableDictionary<string, string>.Empty;
return new VexAttestationRequest(
exportId,
querySignature,
artifact,
predicate.Format,
predicate.CreatedAt,
sourceProviders,
metadata);
}
private VexAttestationMetadata BuildAttestationMetadata(
VexInTotoStatement statement,
DsseEnvelope envelope,
ImmutableDictionary<string, string> metadata)
{
VexRekorReference? rekor = null;
if (metadata.TryGetValue("vex.signature.transparencyLogReference", out var rekorValue) && !string.IsNullOrWhiteSpace(rekorValue))
{
rekor = new VexRekorReference("0.1", rekorValue);
}
DateTimeOffset signedAt;
if (metadata.TryGetValue("vex.signature.verifiedAt", out var signedAtRaw)
&& DateTimeOffset.TryParse(signedAtRaw, out var parsedSignedAt))
{
signedAt = parsedSignedAt;
}
else
{
signedAt = _timeProvider.GetUtcNow();
}
return new VexAttestationMetadata(
statement.PredicateType ?? "https://stella-ops.org/attestations/vex-export",
rekor,
VexDsseBuilder.ComputeEnvelopeDigest(envelope),
signedAt);
}
private VexAttestationRequest BuildAttestationRequest(VexInTotoStatement statement, VexAttestationPredicate predicate)
{
var subject = statement.Subject!.First();
var exportId = predicate.ExportId ?? subject.Name ?? throw new InvalidOperationException("Attestation export ID missing.");
var querySignature = new VexQuerySignature(predicate.QuerySignature ?? throw new InvalidOperationException("Attestation query signature missing."));
if (string.IsNullOrWhiteSpace(predicate.ArtifactAlgorithm) || string.IsNullOrWhiteSpace(predicate.ArtifactDigest))
{
throw new InvalidOperationException("Attestation artifact metadata is incomplete.");
}
var artifact = new VexContentAddress(predicate.ArtifactAlgorithm, predicate.ArtifactDigest);
var sourceProviders = predicate.SourceProviders?.ToImmutableArray() ?? ImmutableArray<string>.Empty;
var metadata = predicate.Metadata?.ToImmutableDictionary(StringComparer.Ordinal) ?? ImmutableDictionary<string, string>.Empty;
return new VexAttestationRequest(
exportId,
querySignature,
artifact,
predicate.Format,
predicate.CreatedAt,
sourceProviders,
metadata);
}
private VexAttestationMetadata BuildAttestationMetadata(
VexInTotoStatement statement,
DsseEnvelope envelope,
ImmutableDictionary<string, string> metadata)
{
VexRekorReference? rekor = null;
if (metadata.TryGetValue("vex.signature.transparencyLogReference", out var rekorValue) && !string.IsNullOrWhiteSpace(rekorValue))
{
rekor = new VexRekorReference("0.1", rekorValue);
}
DateTimeOffset signedAt;
if (metadata.TryGetValue("vex.signature.verifiedAt", out var signedAtRaw)
&& DateTimeOffset.TryParse(signedAtRaw, out var parsedSignedAt))
{
signedAt = parsedSignedAt;
}
else
{
signedAt = _timeProvider.GetUtcNow();
}
return new VexAttestationMetadata(
statement.PredicateType ?? "https://stella-ops.org/attestations/vex-export",
rekor,
VexDsseBuilder.ComputeEnvelopeDigest(envelope),
signedAt);
}
private VexSignatureMetadata BuildSignatureMetadata(
VexInTotoStatement statement,
ImmutableDictionary<string, string> metadata,
VexAttestationMetadata attestationMetadata,
VexAttestationDiagnostics diagnostics)
{
metadata.TryGetValue("vex.signature.type", out var type);
metadata.TryGetValue("vex.provenance.cosign.subject", out var subject);
metadata.TryGetValue("vex.provenance.cosign.issuer", out var issuer);
metadata.TryGetValue("vex.signature.keyId", out var keyId);
metadata.TryGetValue("vex.signature.transparencyLogReference", out var transparencyReference);
if (string.IsNullOrWhiteSpace(type))
{
type = statement.PredicateType?.Contains("attest", StringComparison.OrdinalIgnoreCase) == true
? "cosign"
: "attestation";
}
if (string.IsNullOrWhiteSpace(subject) && statement.Subject is { Count: > 0 })
{
subject = statement.Subject[0].Name;
}
if (string.IsNullOrWhiteSpace(transparencyReference) && attestationMetadata.Rekor is not null)
{
transparencyReference = attestationMetadata.Rekor.Location;
}
if (string.IsNullOrWhiteSpace(issuer)
&& diagnostics.TryGetValue("verification.issuer", out var diagnosticIssuer)
&& !string.IsNullOrWhiteSpace(diagnosticIssuer))
{
issuer = diagnosticIssuer;
}
if (string.IsNullOrWhiteSpace(keyId)
&& diagnostics.TryGetValue("verification.keyId", out var diagnosticKeyId)
&& !string.IsNullOrWhiteSpace(diagnosticKeyId))
{
keyId = diagnosticKeyId;
}
var verifiedAt = attestationMetadata.SignedAt ?? _timeProvider.GetUtcNow();
return new VexSignatureMetadata(
type!,
subject,
issuer,
keyId,
verifiedAt,
transparencyReference);
}
private static string NormalizeDigest(string digest)
{
if (string.IsNullOrWhiteSpace(digest))
{
return string.Empty;
}
return digest.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)
? digest
: $"sha256:{digest}";
}
private static string ComputeDigest(ReadOnlySpan<byte> content)
{
Span<byte> buffer = stackalloc byte[32];
if (!SHA256.TryHashData(content, buffer, out _))
{
var hash = SHA256.HashData(content.ToArray());
return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant();
}
return "sha256:" + Convert.ToHexString(buffer).ToLowerInvariant();
}
{
metadata.TryGetValue("vex.signature.type", out var type);
metadata.TryGetValue("vex.provenance.cosign.subject", out var subject);
metadata.TryGetValue("vex.provenance.cosign.issuer", out var issuer);
metadata.TryGetValue("vex.signature.keyId", out var keyId);
metadata.TryGetValue("vex.signature.transparencyLogReference", out var transparencyReference);
if (string.IsNullOrWhiteSpace(type))
{
type = statement.PredicateType?.Contains("attest", StringComparison.OrdinalIgnoreCase) == true
? "cosign"
: "attestation";
}
if (string.IsNullOrWhiteSpace(subject) && statement.Subject is { Count: > 0 })
{
subject = statement.Subject[0].Name;
}
if (string.IsNullOrWhiteSpace(transparencyReference) && attestationMetadata.Rekor is not null)
{
transparencyReference = attestationMetadata.Rekor.Location;
}
if (string.IsNullOrWhiteSpace(issuer)
&& diagnostics.TryGetValue("verification.issuer", out var diagnosticIssuer)
&& !string.IsNullOrWhiteSpace(diagnosticIssuer))
{
issuer = diagnosticIssuer;
}
if (string.IsNullOrWhiteSpace(keyId)
&& diagnostics.TryGetValue("verification.keyId", out var diagnosticKeyId)
&& !string.IsNullOrWhiteSpace(diagnosticKeyId))
{
keyId = diagnosticKeyId;
}
var verifiedAt = attestationMetadata.SignedAt ?? _timeProvider.GetUtcNow();
return new VexSignatureMetadata(
type!,
subject,
issuer,
keyId,
verifiedAt,
transparencyReference);
}
private static string NormalizeDigest(string digest)
{
if (string.IsNullOrWhiteSpace(digest))
{
return string.Empty;
}
return digest.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)
? digest
: $"sha256:{digest}";
}
private static string ComputeDigest(ReadOnlySpan<byte> content)
{
Span<byte> buffer = stackalloc byte[32];
if (!SHA256.TryHashData(content, buffer, out _))
{
var hash = SHA256.HashData(content.ToArray());
return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant();
}
return "sha256:" + Convert.ToHexString(buffer).ToLowerInvariant();
}
private static VexSignatureMetadata? ExtractSignatureMetadata(ImmutableDictionary<string, string> metadata)
{
if (!metadata.TryGetValue("vex.signature.type", out var type) || string.IsNullOrWhiteSpace(type))
{
return null;
}
metadata.TryGetValue("vex.signature.subject", out var subject);
metadata.TryGetValue("vex.signature.issuer", out var issuer);
metadata.TryGetValue("vex.signature.keyId", out var keyId);
metadata.TryGetValue("vex.signature.verifiedAt", out var verifiedAtRaw);
metadata.TryGetValue("vex.signature.transparencyLogReference", out var tlog);
DateTimeOffset? verifiedAt = null;
if (!string.IsNullOrWhiteSpace(verifiedAtRaw) && DateTimeOffset.TryParse(verifiedAtRaw, out var parsed))
{
verifiedAt = parsed;
}
}
metadata.TryGetValue("vex.signature.subject", out var subject);
metadata.TryGetValue("vex.signature.issuer", out var issuer);
metadata.TryGetValue("vex.signature.keyId", out var keyId);
metadata.TryGetValue("vex.signature.verifiedAt", out var verifiedAtRaw);
metadata.TryGetValue("vex.signature.transparencyLogReference", out var tlog);
DateTimeOffset? verifiedAt = null;
if (!string.IsNullOrWhiteSpace(verifiedAtRaw) && DateTimeOffset.TryParse(verifiedAtRaw, out var parsed))
{
verifiedAt = parsed;
}
return new VexSignatureMetadata(type, subject, issuer, keyId, verifiedAt, tlog);
}
private static void RecordVerification(string providerId, ImmutableDictionary<string, string> metadata, string result)
{
var tags = new List<KeyValuePair<string, object?>>(3)
{
new("source", providerId),
new("result", result),
};
if (!metadata.TryGetValue("tenant", out var tenant) || string.IsNullOrWhiteSpace(tenant))
{
tenant = "tenant-default";
}
tags.Add(new KeyValuePair<string, object?>("tenant", tenant));
var tags = new List<KeyValuePair<string, object?>>(3)
{
new("source", providerId),
new("result", result),
};
if (!metadata.TryGetValue("tenant", out var tenant) || string.IsNullOrWhiteSpace(tenant))
{
tenant = "tenant-default";
}
tags.Add(new KeyValuePair<string, object?>("tenant", tenant));
SignatureVerificationCounter.Add(1, tags.ToArray());
}